?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
dm thin metadata: fix __udivdi3 undefined on 32-bit
[linux/fpc-iii.git] / drivers / iommu / intel-iommu.c
blob49b266433f4c3a2be43fde5d9ed0f2bdd2e2433b
1 /*
2 * Copyright © 2006-2014 Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
18 * Joerg Roedel <jroedel@suse.de>
19 */
20
21 #define pr_fmt(fmt) "DMAR: " fmt
22
23 #include <linux/init.h>
24 #include <linux/bitmap.h>
25 #include <linux/debugfs.h>
26 #include <linux/export.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/memory.h>
36 #include <linux/timer.h>
37 #include <linux/io.h>
38 #include <linux/iova.h>
39 #include <linux/iommu.h>
40 #include <linux/intel-iommu.h>
41 #include <linux/syscore_ops.h>
42 #include <linux/tboot.h>
43 #include <linux/dmi.h>
44 #include <linux/pci-ats.h>
45 #include <linux/memblock.h>
46 #include <linux/dma-contiguous.h>
47 #include <linux/crash_dump.h>
48 #include <asm/irq_remapping.h>
49 #include <asm/cacheflush.h>
50 #include <asm/iommu.h>
51
52 #include "irq_remapping.h"
53
54 #define ROOT_SIZE VTD_PAGE_SIZE
55 #define CONTEXT_SIZE VTD_PAGE_SIZE
56
57 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
58 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
59 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
60 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
61
62 #define IOAPIC_RANGE_START (0xfee00000)
63 #define IOAPIC_RANGE_END (0xfeefffff)
64 #define IOVA_START_ADDR (0x1000)
65
66 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
67
68 #define MAX_AGAW_WIDTH 64
69 #define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
70
71 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
72 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
73
74 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
75 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
76 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
77 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
78 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
79
80 /* IO virtual address start page frame number */
81 #define IOVA_START_PFN (1)
82
83 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
84 #define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
85 #define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
86
87 /* page table handling */
88 #define LEVEL_STRIDE (9)
89 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
90
91 /*
92 * This bitmap is used to advertise the page sizes our hardware support
93 * to the IOMMU core, which will then use this information to split
94 * physically contiguous memory regions it is mapping into page sizes
95 * that we support.
96 *
97 * Traditionally the IOMMU core just handed us the mappings directly,
98 * after making sure the size is an order of a 4KiB page and that the
99 * mapping has natural alignment.
100 *
101 * To retain this behavior, we currently advertise that we support
102 * all page sizes that are an order of 4KiB.
103 *
104 * If at some point we'd like to utilize the IOMMU core's new behavior,
105 * we could change this to advertise the real page sizes we support.
106 */
107 #define INTEL_IOMMU_PGSIZES (~0xFFFUL)
108
109 static inline int agaw_to_level(int agaw)
110 {
111 return agaw + 2;
112 }
113
114 static inline int agaw_to_width(int agaw)
115 {
116 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
117 }
118
119 static inline int width_to_agaw(int width)
120 {
121 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
122 }
123
124 static inline unsigned int level_to_offset_bits(int level)
125 {
126 return (level - 1) * LEVEL_STRIDE;
127 }
128
129 static inline int pfn_level_offset(unsigned long pfn, int level)
130 {
131 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
132 }
133
134 static inline unsigned long level_mask(int level)
135 {
136 return -1UL << level_to_offset_bits(level);
137 }
138
139 static inline unsigned long level_size(int level)
140 {
141 return 1UL << level_to_offset_bits(level);
142 }
143
144 static inline unsigned long align_to_level(unsigned long pfn, int level)
145 {
146 return (pfn + level_size(level) - 1) & level_mask(level);
147 }
148
149 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
150 {
151 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
152 }
153
154 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
155 are never going to work. */
156 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
157 {
158 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
159 }
160
161 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
162 {
163 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
164 }
165 static inline unsigned long page_to_dma_pfn(struct page *pg)
166 {
167 return mm_to_dma_pfn(page_to_pfn(pg));
168 }
169 static inline unsigned long virt_to_dma_pfn(void *p)
170 {
171 return page_to_dma_pfn(virt_to_page(p));
172 }
173
174 /* global iommu list, set NULL for ignored DMAR units */
175 static struct intel_iommu **g_iommus;
176
177 static void __init check_tylersburg_isoch(void);
178 static int rwbf_quirk;
179
180 /*
181 * set to 1 to panic kernel if can't successfully enable VT-d
182 * (used when kernel is launched w/ TXT)
183 */
184 static int force_on = 0;
185
186 /*
187 * 0: Present
188 * 1-11: Reserved
189 * 12-63: Context Ptr (12 - (haw-1))
190 * 64-127: Reserved
191 */
192 struct root_entry {
193 u64 lo;
194 u64 hi;
195 };
196 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
197
198 /*
199 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
200 * if marked present.
201 */
202 static phys_addr_t root_entry_lctp(struct root_entry *re)
203 {
204 if (!(re->lo & 1))
205 return 0;
206
207 return re->lo & VTD_PAGE_MASK;
208 }
209
210 /*
211 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
212 * if marked present.
213 */
214 static phys_addr_t root_entry_uctp(struct root_entry *re)
215 {
216 if (!(re->hi & 1))
217 return 0;
218
219 return re->hi & VTD_PAGE_MASK;
220 }
221 /*
222 * low 64 bits:
223 * 0: present
224 * 1: fault processing disable
225 * 2-3: translation type
226 * 12-63: address space root
227 * high 64 bits:
228 * 0-2: address width
229 * 3-6: aval
230 * 8-23: domain id
231 */
232 struct context_entry {
233 u64 lo;
234 u64 hi;
235 };
236
237 static inline void context_clear_pasid_enable(struct context_entry *context)
238 {
239 context->lo &= ~(1ULL << 11);
240 }
241
242 static inline bool context_pasid_enabled(struct context_entry *context)
243 {
244 return !!(context->lo & (1ULL << 11));
245 }
246
247 static inline void context_set_copied(struct context_entry *context)
248 {
249 context->hi |= (1ull << 3);
250 }
251
252 static inline bool context_copied(struct context_entry *context)
253 {
254 return !!(context->hi & (1ULL << 3));
255 }
256
257 static inline bool __context_present(struct context_entry *context)
258 {
259 return (context->lo & 1);
260 }
261
262 static inline bool context_present(struct context_entry *context)
263 {
264 return context_pasid_enabled(context) ?
265 __context_present(context) :
266 __context_present(context) && !context_copied(context);
267 }
268
269 static inline void context_set_present(struct context_entry *context)
270 {
271 context->lo |= 1;
272 }
273
274 static inline void context_set_fault_enable(struct context_entry *context)
275 {
276 context->lo &= (((u64)-1) << 2) | 1;
277 }
278
279 static inline void context_set_translation_type(struct context_entry *context,
280 unsigned long value)
281 {
282 context->lo &= (((u64)-1) << 4) | 3;
283 context->lo |= (value & 3) << 2;
284 }
285
286 static inline void context_set_address_root(struct context_entry *context,
287 unsigned long value)
288 {
289 context->lo &= ~VTD_PAGE_MASK;
290 context->lo |= value & VTD_PAGE_MASK;
291 }
292
293 static inline void context_set_address_width(struct context_entry *context,
294 unsigned long value)
295 {
296 context->hi |= value & 7;
297 }
298
299 static inline void context_set_domain_id(struct context_entry *context,
300 unsigned long value)
301 {
302 context->hi |= (value & ((1 << 16) - 1)) << 8;
303 }
304
305 static inline int context_domain_id(struct context_entry *c)
306 {
307 return((c->hi >> 8) & 0xffff);
308 }
309
310 static inline void context_clear_entry(struct context_entry *context)
311 {
312 context->lo = 0;
313 context->hi = 0;
314 }
315
316 /*
317 * 0: readable
318 * 1: writable
319 * 2-6: reserved
320 * 7: super page
321 * 8-10: available
322 * 11: snoop behavior
323 * 12-63: Host physcial address
324 */
325 struct dma_pte {
326 u64 val;
327 };
328
329 static inline void dma_clear_pte(struct dma_pte *pte)
330 {
331 pte->val = 0;
332 }
333
334 static inline u64 dma_pte_addr(struct dma_pte *pte)
335 {
336 #ifdef CONFIG_64BIT
337 return pte->val & VTD_PAGE_MASK;
338 #else
339 /* Must have a full atomic 64-bit read */
340 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
341 #endif
342 }
343
344 static inline bool dma_pte_present(struct dma_pte *pte)
345 {
346 return (pte->val & 3) != 0;
347 }
348
349 static inline bool dma_pte_superpage(struct dma_pte *pte)
350 {
351 return (pte->val & DMA_PTE_LARGE_PAGE);
352 }
353
354 static inline int first_pte_in_page(struct dma_pte *pte)
355 {
356 return !((unsigned long)pte & ~VTD_PAGE_MASK);
357 }
358
359 /*
360 * This domain is a statically identity mapping domain.
361 * 1. This domain creats a static 1:1 mapping to all usable memory.
362 * 2. It maps to each iommu if successful.
363 * 3. Each iommu mapps to this domain if successful.
364 */
365 static struct dmar_domain *si_domain;
366 static int hw_pass_through = 1;
367
368 /*
369 * Domain represents a virtual machine, more than one devices
370 * across iommus may be owned in one domain, e.g. kvm guest.
371 */
372 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
373
374 /* si_domain contains mulitple devices */
375 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
376
377 #define for_each_domain_iommu(idx, domain) \
378 for (idx = 0; idx < g_num_of_iommus; idx++) \
379 if (domain->iommu_refcnt[idx])
380
381 struct dmar_domain {
382 int nid; /* node id */
383
384 unsigned iommu_refcnt[DMAR_UNITS_SUPPORTED];
385 /* Refcount of devices per iommu */
386
387
388 u16 iommu_did[DMAR_UNITS_SUPPORTED];
389 /* Domain ids per IOMMU. Use u16 since
390 * domain ids are 16 bit wide according
391 * to VT-d spec, section 9.3 */
392
393 struct list_head devices; /* all devices' list */
394 struct iova_domain iovad; /* iova's that belong to this domain */
395
396 struct dma_pte *pgd; /* virtual address */
397 int gaw; /* max guest address width */
398
399 /* adjusted guest address width, 0 is level 2 30-bit */
400 int agaw;
401
402 int flags; /* flags to find out type of domain */
403
404 int iommu_coherency;/* indicate coherency of iommu access */
405 int iommu_snooping; /* indicate snooping control feature*/
406 int iommu_count; /* reference count of iommu */
407 int iommu_superpage;/* Level of superpages supported:
408 0 == 4KiB (no superpages), 1 == 2MiB,
409 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
410 u64 max_addr; /* maximum mapped address */
411
412 struct iommu_domain domain; /* generic domain data structure for
413 iommu core */
414 };
415
416 /* PCI domain-device relationship */
417 struct device_domain_info {
418 struct list_head link; /* link to domain siblings */
419 struct list_head global; /* link to global list */
420 u8 bus; /* PCI bus number */
421 u8 devfn; /* PCI devfn number */
422 u16 pfsid; /* SRIOV physical function source ID */
423 u8 pasid_supported:3;
424 u8 pasid_enabled:1;
425 u8 pri_supported:1;
426 u8 pri_enabled:1;
427 u8 ats_supported:1;
428 u8 ats_enabled:1;
429 u8 ats_qdep;
430 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
431 struct intel_iommu *iommu; /* IOMMU used by this device */
432 struct dmar_domain *domain; /* pointer to domain */
433 };
434
435 struct dmar_rmrr_unit {
436 struct list_head list; /* list of rmrr units */
437 struct acpi_dmar_header *hdr; /* ACPI header */
438 u64 base_address; /* reserved base address*/
439 u64 end_address; /* reserved end address */
440 struct dmar_dev_scope *devices; /* target devices */
441 int devices_cnt; /* target device count */
442 };
443
444 struct dmar_atsr_unit {
445 struct list_head list; /* list of ATSR units */
446 struct acpi_dmar_header *hdr; /* ACPI header */
447 struct dmar_dev_scope *devices; /* target devices */
448 int devices_cnt; /* target device count */
449 u8 include_all:1; /* include all ports */
450 };
451
452 static LIST_HEAD(dmar_atsr_units);
453 static LIST_HEAD(dmar_rmrr_units);
454
455 #define for_each_rmrr_units(rmrr) \
456 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
457
458 static void flush_unmaps_timeout(unsigned long data);
459
460 static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
461
462 #define HIGH_WATER_MARK 250
463 struct deferred_flush_tables {
464 int next;
465 struct iova *iova[HIGH_WATER_MARK];
466 struct dmar_domain *domain[HIGH_WATER_MARK];
467 struct page *freelist[HIGH_WATER_MARK];
468 };
469
470 static struct deferred_flush_tables *deferred_flush;
471
472 /* bitmap for indexing intel_iommus */
473 static int g_num_of_iommus;
474
475 static DEFINE_SPINLOCK(async_umap_flush_lock);
476 static LIST_HEAD(unmaps_to_do);
477
478 static int timer_on;
479 static long list_size;
480
481 static void domain_exit(struct dmar_domain *domain);
482 static void domain_remove_dev_info(struct dmar_domain *domain);
483 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
484 struct device *dev);
485 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
486 static void domain_context_clear(struct intel_iommu *iommu,
487 struct device *dev);
488 static int domain_detach_iommu(struct dmar_domain *domain,
489 struct intel_iommu *iommu);
490
491 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
492 int dmar_disabled = 0;
493 #else
494 int dmar_disabled = 1;
495 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
496
497 int intel_iommu_enabled = 0;
498 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
499
500 static int dmar_map_gfx = 1;
501 static int dmar_forcedac;
502 static int intel_iommu_strict;
503 static int intel_iommu_superpage = 1;
504 static int intel_iommu_ecs = 1;
505 static int intel_iommu_pasid28;
506 static int iommu_identity_mapping;
507
508 #define IDENTMAP_ALL 1
509 #define IDENTMAP_GFX 2
510 #define IDENTMAP_AZALIA 4
511
512 /* Broadwell and Skylake have broken ECS support — normal so-called "second
513 * level" translation of DMA requests-without-PASID doesn't actually happen
514 * unless you also set the NESTE bit in an extended context-entry. Which of
515 * course means that SVM doesn't work because it's trying to do nested
516 * translation of the physical addresses it finds in the process page tables,
517 * through the IOVA->phys mapping found in the "second level" page tables.
518 *
519 * The VT-d specification was retroactively changed to change the definition
520 * of the capability bits and pretend that Broadwell/Skylake never happened...
521 * but unfortunately the wrong bit was changed. It's ECS which is broken, but
522 * for some reason it was the PASID capability bit which was redefined (from
523 * bit 28 on BDW/SKL to bit 40 in future).
524 *
525 * So our test for ECS needs to eschew those implementations which set the old
526 * PASID capabiity bit 28, since those are the ones on which ECS is broken.
527 * Unless we are working around the 'pasid28' limitations, that is, by putting
528 * the device into passthrough mode for normal DMA and thus masking the bug.
529 */
530 #define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
531 (intel_iommu_pasid28 || !ecap_broken_pasid(iommu->ecap)))
532 /* PASID support is thus enabled if ECS is enabled and *either* of the old
533 * or new capability bits are set. */
534 #define pasid_enabled(iommu) (ecs_enabled(iommu) && \
535 (ecap_pasid(iommu->ecap) || ecap_broken_pasid(iommu->ecap)))
536
537 int intel_iommu_gfx_mapped;
538 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
539
540 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
541 static DEFINE_SPINLOCK(device_domain_lock);
542 static LIST_HEAD(device_domain_list);
543
544 static const struct iommu_ops intel_iommu_ops;
545
546 static bool translation_pre_enabled(struct intel_iommu *iommu)
547 {
548 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
549 }
550
551 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
552 {
553 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
554 }
555
556 static void init_translation_status(struct intel_iommu *iommu)
557 {
558 u32 gsts;
559
560 gsts = readl(iommu->reg + DMAR_GSTS_REG);
561 if (gsts & DMA_GSTS_TES)
562 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
563 }
564
565 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
566 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
567 {
568 return container_of(dom, struct dmar_domain, domain);
569 }
570
571 static int __init intel_iommu_setup(char *str)
572 {
573 if (!str)
574 return -EINVAL;
575 while (*str) {
576 if (!strncmp(str, "on", 2)) {
577 dmar_disabled = 0;
578 pr_info("IOMMU enabled\n");
579 } else if (!strncmp(str, "off", 3)) {
580 dmar_disabled = 1;
581 pr_info("IOMMU disabled\n");
582 } else if (!strncmp(str, "igfx_off", 8)) {
583 dmar_map_gfx = 0;
584 pr_info("Disable GFX device mapping\n");
585 } else if (!strncmp(str, "forcedac", 8)) {
586 pr_info("Forcing DAC for PCI devices\n");
587 dmar_forcedac = 1;
588 } else if (!strncmp(str, "strict", 6)) {
589 pr_info("Disable batched IOTLB flush\n");
590 intel_iommu_strict = 1;
591 } else if (!strncmp(str, "sp_off", 6)) {
592 pr_info("Disable supported super page\n");
593 intel_iommu_superpage = 0;
594 } else if (!strncmp(str, "ecs_off", 7)) {
595 printk(KERN_INFO
596 "Intel-IOMMU: disable extended context table support\n");
597 intel_iommu_ecs = 0;
598 } else if (!strncmp(str, "pasid28", 7)) {
599 printk(KERN_INFO
600 "Intel-IOMMU: enable pre-production PASID support\n");
601 intel_iommu_pasid28 = 1;
602 iommu_identity_mapping |= IDENTMAP_GFX;
603 }
604
605 str += strcspn(str, ",");
606 while (*str == ',')
607 str++;
608 }
609 return 0;
610 }
611 __setup("intel_iommu=", intel_iommu_setup);
612
613 static struct kmem_cache *iommu_domain_cache;
614 static struct kmem_cache *iommu_devinfo_cache;
615
616 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
617 {
618 struct dmar_domain **domains;
619 int idx = did >> 8;
620
621 domains = iommu->domains[idx];
622 if (!domains)
623 return NULL;
624
625 return domains[did & 0xff];
626 }
627
628 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
629 struct dmar_domain *domain)
630 {
631 struct dmar_domain **domains;
632 int idx = did >> 8;
633
634 if (!iommu->domains[idx]) {
635 size_t size = 256 * sizeof(struct dmar_domain *);
636 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
637 }
638
639 domains = iommu->domains[idx];
640 if (WARN_ON(!domains))
641 return;
642 else
643 domains[did & 0xff] = domain;
644 }
645
646 static inline void *alloc_pgtable_page(int node)
647 {
648 struct page *page;
649 void *vaddr = NULL;
650
651 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
652 if (page)
653 vaddr = page_address(page);
654 return vaddr;
655 }
656
657 static inline void free_pgtable_page(void *vaddr)
658 {
659 free_page((unsigned long)vaddr);
660 }
661
662 static inline void *alloc_domain_mem(void)
663 {
664 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
665 }
666
667 static void free_domain_mem(void *vaddr)
668 {
669 kmem_cache_free(iommu_domain_cache, vaddr);
670 }
671
672 static inline void * alloc_devinfo_mem(void)
673 {
674 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
675 }
676
677 static inline void free_devinfo_mem(void *vaddr)
678 {
679 kmem_cache_free(iommu_devinfo_cache, vaddr);
680 }
681
682 static inline int domain_type_is_vm(struct dmar_domain *domain)
683 {
684 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
685 }
686
687 static inline int domain_type_is_si(struct dmar_domain *domain)
688 {
689 return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
690 }
691
692 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
693 {
694 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
695 DOMAIN_FLAG_STATIC_IDENTITY);
696 }
697
698 static inline int domain_pfn_supported(struct dmar_domain *domain,
699 unsigned long pfn)
700 {
701 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
702
703 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
704 }
705
706 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
707 {
708 unsigned long sagaw;
709 int agaw = -1;
710
711 sagaw = cap_sagaw(iommu->cap);
712 for (agaw = width_to_agaw(max_gaw);
713 agaw >= 0; agaw--) {
714 if (test_bit(agaw, &sagaw))
715 break;
716 }
717
718 return agaw;
719 }
720
721 /*
722 * Calculate max SAGAW for each iommu.
723 */
724 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
725 {
726 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
727 }
728
729 /*
730 * calculate agaw for each iommu.
731 * "SAGAW" may be different across iommus, use a default agaw, and
732 * get a supported less agaw for iommus that don't support the default agaw.
733 */
734 int iommu_calculate_agaw(struct intel_iommu *iommu)
735 {
736 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
737 }
738
739 /* This functionin only returns single iommu in a domain */
740 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
741 {
742 int iommu_id;
743
744 /* si_domain and vm domain should not get here. */
745 BUG_ON(domain_type_is_vm_or_si(domain));
746 for_each_domain_iommu(iommu_id, domain)
747 break;
748
749 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
750 return NULL;
751
752 return g_iommus[iommu_id];
753 }
754
755 static void domain_update_iommu_coherency(struct dmar_domain *domain)
756 {
757 struct dmar_drhd_unit *drhd;
758 struct intel_iommu *iommu;
759 bool found = false;
760 int i;
761
762 domain->iommu_coherency = 1;
763
764 for_each_domain_iommu(i, domain) {
765 found = true;
766 if (!ecap_coherent(g_iommus[i]->ecap)) {
767 domain->iommu_coherency = 0;
768 break;
769 }
770 }
771 if (found)
772 return;
773
774 /* No hardware attached; use lowest common denominator */
775 rcu_read_lock();
776 for_each_active_iommu(iommu, drhd) {
777 if (!ecap_coherent(iommu->ecap)) {
778 domain->iommu_coherency = 0;
779 break;
780 }
781 }
782 rcu_read_unlock();
783 }
784
785 static int domain_update_iommu_snooping(struct intel_iommu *skip)
786 {
787 struct dmar_drhd_unit *drhd;
788 struct intel_iommu *iommu;
789 int ret = 1;
790
791 rcu_read_lock();
792 for_each_active_iommu(iommu, drhd) {
793 if (iommu != skip) {
794 if (!ecap_sc_support(iommu->ecap)) {
795 ret = 0;
796 break;
797 }
798 }
799 }
800 rcu_read_unlock();
801
802 return ret;
803 }
804
805 static int domain_update_iommu_superpage(struct intel_iommu *skip)
806 {
807 struct dmar_drhd_unit *drhd;
808 struct intel_iommu *iommu;
809 int mask = 0xf;
810
811 if (!intel_iommu_superpage) {
812 return 0;
813 }
814
815 /* set iommu_superpage to the smallest common denominator */
816 rcu_read_lock();
817 for_each_active_iommu(iommu, drhd) {
818 if (iommu != skip) {
819 mask &= cap_super_page_val(iommu->cap);
820 if (!mask)
821 break;
822 }
823 }
824 rcu_read_unlock();
825
826 return fls(mask);
827 }
828
829 /* Some capabilities may be different across iommus */
830 static void domain_update_iommu_cap(struct dmar_domain *domain)
831 {
832 domain_update_iommu_coherency(domain);
833 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
834 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
835 }
836
837 static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
838 u8 bus, u8 devfn, int alloc)
839 {
840 struct root_entry *root = &iommu->root_entry[bus];
841 struct context_entry *context;
842 u64 *entry;
843
844 entry = &root->lo;
845 if (ecs_enabled(iommu)) {
846 if (devfn >= 0x80) {
847 devfn -= 0x80;
848 entry = &root->hi;
849 }
850 devfn *= 2;
851 }
852 if (*entry & 1)
853 context = phys_to_virt(*entry & VTD_PAGE_MASK);
854 else {
855 unsigned long phy_addr;
856 if (!alloc)
857 return NULL;
858
859 context = alloc_pgtable_page(iommu->node);
860 if (!context)
861 return NULL;
862
863 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
864 phy_addr = virt_to_phys((void *)context);
865 *entry = phy_addr | 1;
866 __iommu_flush_cache(iommu, entry, sizeof(*entry));
867 }
868 return &context[devfn];
869 }
870
871 static int iommu_dummy(struct device *dev)
872 {
873 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
874 }
875
876 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
877 {
878 struct dmar_drhd_unit *drhd = NULL;
879 struct intel_iommu *iommu;
880 struct device *tmp;
881 struct pci_dev *ptmp, *pdev = NULL;
882 u16 segment = 0;
883 int i;
884
885 if (iommu_dummy(dev))
886 return NULL;
887
888 if (dev_is_pci(dev)) {
889 struct pci_dev *pf_pdev;
890
891 pdev = to_pci_dev(dev);
892 /* VFs aren't listed in scope tables; we need to look up
893 * the PF instead to find the IOMMU. */
894 pf_pdev = pci_physfn(pdev);
895 dev = &pf_pdev->dev;
896 segment = pci_domain_nr(pdev->bus);
897 } else if (has_acpi_companion(dev))
898 dev = &ACPI_COMPANION(dev)->dev;
899
900 rcu_read_lock();
901 for_each_active_iommu(iommu, drhd) {
902 if (pdev && segment != drhd->segment)
903 continue;
904
905 for_each_active_dev_scope(drhd->devices,
906 drhd->devices_cnt, i, tmp) {
907 if (tmp == dev) {
908 /* For a VF use its original BDF# not that of the PF
909 * which we used for the IOMMU lookup. Strictly speaking
910 * we could do this for all PCI devices; we only need to
911 * get the BDF# from the scope table for ACPI matches. */
912 if (pdev && pdev->is_virtfn)
913 goto got_pdev;
914
915 *bus = drhd->devices[i].bus;
916 *devfn = drhd->devices[i].devfn;
917 goto out;
918 }
919
920 if (!pdev || !dev_is_pci(tmp))
921 continue;
922
923 ptmp = to_pci_dev(tmp);
924 if (ptmp->subordinate &&
925 ptmp->subordinate->number <= pdev->bus->number &&
926 ptmp->subordinate->busn_res.end >= pdev->bus->number)
927 goto got_pdev;
928 }
929
930 if (pdev && drhd->include_all) {
931 got_pdev:
932 *bus = pdev->bus->number;
933 *devfn = pdev->devfn;
934 goto out;
935 }
936 }
937 iommu = NULL;
938 out:
939 rcu_read_unlock();
940
941 return iommu;
942 }
943
944 static void domain_flush_cache(struct dmar_domain *domain,
945 void *addr, int size)
946 {
947 if (!domain->iommu_coherency)
948 clflush_cache_range(addr, size);
949 }
950
951 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
952 {
953 struct context_entry *context;
954 int ret = 0;
955 unsigned long flags;
956
957 spin_lock_irqsave(&iommu->lock, flags);
958 context = iommu_context_addr(iommu, bus, devfn, 0);
959 if (context)
960 ret = context_present(context);
961 spin_unlock_irqrestore(&iommu->lock, flags);
962 return ret;
963 }
964
965 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
966 {
967 struct context_entry *context;
968 unsigned long flags;
969
970 spin_lock_irqsave(&iommu->lock, flags);
971 context = iommu_context_addr(iommu, bus, devfn, 0);
972 if (context) {
973 context_clear_entry(context);
974 __iommu_flush_cache(iommu, context, sizeof(*context));
975 }
976 spin_unlock_irqrestore(&iommu->lock, flags);
977 }
978
979 static void free_context_table(struct intel_iommu *iommu)
980 {
981 int i;
982 unsigned long flags;
983 struct context_entry *context;
984
985 spin_lock_irqsave(&iommu->lock, flags);
986 if (!iommu->root_entry) {
987 goto out;
988 }
989 for (i = 0; i < ROOT_ENTRY_NR; i++) {
990 context = iommu_context_addr(iommu, i, 0, 0);
991 if (context)
992 free_pgtable_page(context);
993
994 if (!ecs_enabled(iommu))
995 continue;
996
997 context = iommu_context_addr(iommu, i, 0x80, 0);
998 if (context)
999 free_pgtable_page(context);
1000
1001 }
1002 free_pgtable_page(iommu->root_entry);
1003 iommu->root_entry = NULL;
1004 out:
1005 spin_unlock_irqrestore(&iommu->lock, flags);
1006 }
1007
1008 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
1009 unsigned long pfn, int *target_level)
1010 {
1011 struct dma_pte *parent, *pte = NULL;
1012 int level = agaw_to_level(domain->agaw);
1013 int offset;
1014
1015 BUG_ON(!domain->pgd);
1016
1017 if (!domain_pfn_supported(domain, pfn))
1018 /* Address beyond IOMMU's addressing capabilities. */
1019 return NULL;
1020
1021 parent = domain->pgd;
1022
1023 while (1) {
1024 void *tmp_page;
1025
1026 offset = pfn_level_offset(pfn, level);
1027 pte = &parent[offset];
1028 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
1029 break;
1030 if (level == *target_level)
1031 break;
1032
1033 if (!dma_pte_present(pte)) {
1034 uint64_t pteval;
1035
1036 tmp_page = alloc_pgtable_page(domain->nid);
1037
1038 if (!tmp_page)
1039 return NULL;
1040
1041 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
1042 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
1043 if (cmpxchg64(&pte->val, 0ULL, pteval))
1044 /* Someone else set it while we were thinking; use theirs. */
1045 free_pgtable_page(tmp_page);
1046 else
1047 domain_flush_cache(domain, pte, sizeof(*pte));
1048 }
1049 if (level == 1)
1050 break;
1051
1052 parent = phys_to_virt(dma_pte_addr(pte));
1053 level--;
1054 }
1055
1056 if (!*target_level)
1057 *target_level = level;
1058
1059 return pte;
1060 }
1061
1062
1063 /* return address's pte at specific level */
1064 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1065 unsigned long pfn,
1066 int level, int *large_page)
1067 {
1068 struct dma_pte *parent, *pte = NULL;
1069 int total = agaw_to_level(domain->agaw);
1070 int offset;
1071
1072 parent = domain->pgd;
1073 while (level <= total) {
1074 offset = pfn_level_offset(pfn, total);
1075 pte = &parent[offset];
1076 if (level == total)
1077 return pte;
1078
1079 if (!dma_pte_present(pte)) {
1080 *large_page = total;
1081 break;
1082 }
1083
1084 if (dma_pte_superpage(pte)) {
1085 *large_page = total;
1086 return pte;
1087 }
1088
1089 parent = phys_to_virt(dma_pte_addr(pte));
1090 total--;
1091 }
1092 return NULL;
1093 }
1094
1095 /* clear last level pte, a tlb flush should be followed */
1096 static void dma_pte_clear_range(struct dmar_domain *domain,
1097 unsigned long start_pfn,
1098 unsigned long last_pfn)
1099 {
1100 unsigned int large_page = 1;
1101 struct dma_pte *first_pte, *pte;
1102
1103 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1104 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1105 BUG_ON(start_pfn > last_pfn);
1106
1107 /* we don't need lock here; nobody else touches the iova range */
1108 do {
1109 large_page = 1;
1110 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
1111 if (!pte) {
1112 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
1113 continue;
1114 }
1115 do {
1116 dma_clear_pte(pte);
1117 start_pfn += lvl_to_nr_pages(large_page);
1118 pte++;
1119 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1120
1121 domain_flush_cache(domain, first_pte,
1122 (void *)pte - (void *)first_pte);
1123
1124 } while (start_pfn && start_pfn <= last_pfn);
1125 }
1126
1127 static void dma_pte_free_level(struct dmar_domain *domain, int level,
1128 struct dma_pte *pte, unsigned long pfn,
1129 unsigned long start_pfn, unsigned long last_pfn)
1130 {
1131 pfn = max(start_pfn, pfn);
1132 pte = &pte[pfn_level_offset(pfn, level)];
1133
1134 do {
1135 unsigned long level_pfn;
1136 struct dma_pte *level_pte;
1137
1138 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1139 goto next;
1140
1141 level_pfn = pfn & level_mask(level);
1142 level_pte = phys_to_virt(dma_pte_addr(pte));
1143
1144 if (level > 2)
1145 dma_pte_free_level(domain, level - 1, level_pte,
1146 level_pfn, start_pfn, last_pfn);
1147
1148 /* If range covers entire pagetable, free it */
1149 if (!(start_pfn > level_pfn ||
1150 last_pfn < level_pfn + level_size(level) - 1)) {
1151 dma_clear_pte(pte);
1152 domain_flush_cache(domain, pte, sizeof(*pte));
1153 free_pgtable_page(level_pte);
1154 }
1155 next:
1156 pfn += level_size(level);
1157 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1158 }
1159
1160 /* free page table pages. last level pte should already be cleared */
1161 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1162 unsigned long start_pfn,
1163 unsigned long last_pfn)
1164 {
1165 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1166 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1167 BUG_ON(start_pfn > last_pfn);
1168
1169 dma_pte_clear_range(domain, start_pfn, last_pfn);
1170
1171 /* We don't need lock here; nobody else touches the iova range */
1172 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1173 domain->pgd, 0, start_pfn, last_pfn);
1174
1175 /* free pgd */
1176 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1177 free_pgtable_page(domain->pgd);
1178 domain->pgd = NULL;
1179 }
1180 }
1181
1182 /* When a page at a given level is being unlinked from its parent, we don't
1183 need to *modify* it at all. All we need to do is make a list of all the
1184 pages which can be freed just as soon as we've flushed the IOTLB and we
1185 know the hardware page-walk will no longer touch them.
1186 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1187 be freed. */
1188 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1189 int level, struct dma_pte *pte,
1190 struct page *freelist)
1191 {
1192 struct page *pg;
1193
1194 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1195 pg->freelist = freelist;
1196 freelist = pg;
1197
1198 if (level == 1)
1199 return freelist;
1200
1201 pte = page_address(pg);
1202 do {
1203 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1204 freelist = dma_pte_list_pagetables(domain, level - 1,
1205 pte, freelist);
1206 pte++;
1207 } while (!first_pte_in_page(pte));
1208
1209 return freelist;
1210 }
1211
1212 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1213 struct dma_pte *pte, unsigned long pfn,
1214 unsigned long start_pfn,
1215 unsigned long last_pfn,
1216 struct page *freelist)
1217 {
1218 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1219
1220 pfn = max(start_pfn, pfn);
1221 pte = &pte[pfn_level_offset(pfn, level)];
1222
1223 do {
1224 unsigned long level_pfn;
1225
1226 if (!dma_pte_present(pte))
1227 goto next;
1228
1229 level_pfn = pfn & level_mask(level);
1230
1231 /* If range covers entire pagetable, free it */
1232 if (start_pfn <= level_pfn &&
1233 last_pfn >= level_pfn + level_size(level) - 1) {
1234 /* These suborbinate page tables are going away entirely. Don't
1235 bother to clear them; we're just going to *free* them. */
1236 if (level > 1 && !dma_pte_superpage(pte))
1237 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1238
1239 dma_clear_pte(pte);
1240 if (!first_pte)
1241 first_pte = pte;
1242 last_pte = pte;
1243 } else if (level > 1) {
1244 /* Recurse down into a level that isn't *entirely* obsolete */
1245 freelist = dma_pte_clear_level(domain, level - 1,
1246 phys_to_virt(dma_pte_addr(pte)),
1247 level_pfn, start_pfn, last_pfn,
1248 freelist);
1249 }
1250 next:
1251 pfn += level_size(level);
1252 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1253
1254 if (first_pte)
1255 domain_flush_cache(domain, first_pte,
1256 (void *)++last_pte - (void *)first_pte);
1257
1258 return freelist;
1259 }
1260
1261 /* We can't just free the pages because the IOMMU may still be walking
1262 the page tables, and may have cached the intermediate levels. The
1263 pages can only be freed after the IOTLB flush has been done. */
1264 static struct page *domain_unmap(struct dmar_domain *domain,
1265 unsigned long start_pfn,
1266 unsigned long last_pfn)
1267 {
1268 struct page *freelist = NULL;
1269
1270 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1271 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1272 BUG_ON(start_pfn > last_pfn);
1273
1274 /* we don't need lock here; nobody else touches the iova range */
1275 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1276 domain->pgd, 0, start_pfn, last_pfn, NULL);
1277
1278 /* free pgd */
1279 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1280 struct page *pgd_page = virt_to_page(domain->pgd);
1281 pgd_page->freelist = freelist;
1282 freelist = pgd_page;
1283
1284 domain->pgd = NULL;
1285 }
1286
1287 return freelist;
1288 }
1289
1290 static void dma_free_pagelist(struct page *freelist)
1291 {
1292 struct page *pg;
1293
1294 while ((pg = freelist)) {
1295 freelist = pg->freelist;
1296 free_pgtable_page(page_address(pg));
1297 }
1298 }
1299
1300 /* iommu handling */
1301 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1302 {
1303 struct root_entry *root;
1304 unsigned long flags;
1305
1306 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1307 if (!root) {
1308 pr_err("Allocating root entry for %s failed\n",
1309 iommu->name);
1310 return -ENOMEM;
1311 }
1312
1313 __iommu_flush_cache(iommu, root, ROOT_SIZE);
1314
1315 spin_lock_irqsave(&iommu->lock, flags);
1316 iommu->root_entry = root;
1317 spin_unlock_irqrestore(&iommu->lock, flags);
1318
1319 return 0;
1320 }
1321
1322 static void iommu_set_root_entry(struct intel_iommu *iommu)
1323 {
1324 u64 addr;
1325 u32 sts;
1326 unsigned long flag;
1327
1328 addr = virt_to_phys(iommu->root_entry);
1329 if (ecs_enabled(iommu))
1330 addr |= DMA_RTADDR_RTT;
1331
1332 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1333 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1334
1335 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1336
1337 /* Make sure hardware complete it */
1338 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1339 readl, (sts & DMA_GSTS_RTPS), sts);
1340
1341 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1342 }
1343
1344 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1345 {
1346 u32 val;
1347 unsigned long flag;
1348
1349 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1350 return;
1351
1352 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1353 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1354
1355 /* Make sure hardware complete it */
1356 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1357 readl, (!(val & DMA_GSTS_WBFS)), val);
1358
1359 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1360 }
1361
1362 /* return value determine if we need a write buffer flush */
1363 static void __iommu_flush_context(struct intel_iommu *iommu,
1364 u16 did, u16 source_id, u8 function_mask,
1365 u64 type)
1366 {
1367 u64 val = 0;
1368 unsigned long flag;
1369
1370 switch (type) {
1371 case DMA_CCMD_GLOBAL_INVL:
1372 val = DMA_CCMD_GLOBAL_INVL;
1373 break;
1374 case DMA_CCMD_DOMAIN_INVL:
1375 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1376 break;
1377 case DMA_CCMD_DEVICE_INVL:
1378 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1379 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1380 break;
1381 default:
1382 BUG();
1383 }
1384 val |= DMA_CCMD_ICC;
1385
1386 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1387 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1388
1389 /* Make sure hardware complete it */
1390 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1391 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1392
1393 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1394 }
1395
1396 /* return value determine if we need a write buffer flush */
1397 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1398 u64 addr, unsigned int size_order, u64 type)
1399 {
1400 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1401 u64 val = 0, val_iva = 0;
1402 unsigned long flag;
1403
1404 switch (type) {
1405 case DMA_TLB_GLOBAL_FLUSH:
1406 /* global flush doesn't need set IVA_REG */
1407 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1408 break;
1409 case DMA_TLB_DSI_FLUSH:
1410 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1411 break;
1412 case DMA_TLB_PSI_FLUSH:
1413 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1414 /* IH bit is passed in as part of address */
1415 val_iva = size_order | addr;
1416 break;
1417 default:
1418 BUG();
1419 }
1420 /* Note: set drain read/write */
1421 #if 0
1422 /*
1423 * This is probably to be super secure.. Looks like we can
1424 * ignore it without any impact.
1425 */
1426 if (cap_read_drain(iommu->cap))
1427 val |= DMA_TLB_READ_DRAIN;
1428 #endif
1429 if (cap_write_drain(iommu->cap))
1430 val |= DMA_TLB_WRITE_DRAIN;
1431
1432 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1433 /* Note: Only uses first TLB reg currently */
1434 if (val_iva)
1435 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1436 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1437
1438 /* Make sure hardware complete it */
1439 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1440 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1441
1442 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1443
1444 /* check IOTLB invalidation granularity */
1445 if (DMA_TLB_IAIG(val) == 0)
1446 pr_err("Flush IOTLB failed\n");
1447 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1448 pr_debug("TLB flush request %Lx, actual %Lx\n",
1449 (unsigned long long)DMA_TLB_IIRG(type),
1450 (unsigned long long)DMA_TLB_IAIG(val));
1451 }
1452
1453 static struct device_domain_info *
1454 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1455 u8 bus, u8 devfn)
1456 {
1457 struct device_domain_info *info;
1458
1459 assert_spin_locked(&device_domain_lock);
1460
1461 if (!iommu->qi)
1462 return NULL;
1463
1464 list_for_each_entry(info, &domain->devices, link)
1465 if (info->iommu == iommu && info->bus == bus &&
1466 info->devfn == devfn) {
1467 if (info->ats_supported && info->dev)
1468 return info;
1469 break;
1470 }
1471
1472 return NULL;
1473 }
1474
1475 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1476 {
1477 struct pci_dev *pdev;
1478
1479 if (!info || !dev_is_pci(info->dev))
1480 return;
1481
1482 pdev = to_pci_dev(info->dev);
1483 /* For IOMMU that supports device IOTLB throttling (DIT), we assign
1484 * PFSID to the invalidation desc of a VF such that IOMMU HW can gauge
1485 * queue depth at PF level. If DIT is not set, PFSID will be treated as
1486 * reserved, which should be set to 0.
1487 */
1488 if (!ecap_dit(info->iommu->ecap))
1489 info->pfsid = 0;
1490 else {
1491 struct pci_dev *pf_pdev;
1492
1493 /* pdev will be returned if device is not a vf */
1494 pf_pdev = pci_physfn(pdev);
1495 info->pfsid = PCI_DEVID(pf_pdev->bus->number, pf_pdev->devfn);
1496 }
1497
1498 #ifdef CONFIG_INTEL_IOMMU_SVM
1499 /* The PCIe spec, in its wisdom, declares that the behaviour of
1500 the device if you enable PASID support after ATS support is
1501 undefined. So always enable PASID support on devices which
1502 have it, even if we can't yet know if we're ever going to
1503 use it. */
1504 if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1505 info->pasid_enabled = 1;
1506
1507 if (info->pri_supported && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1508 info->pri_enabled = 1;
1509 #endif
1510 if (info->ats_supported && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1511 info->ats_enabled = 1;
1512 info->ats_qdep = pci_ats_queue_depth(pdev);
1513 }
1514 }
1515
1516 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1517 {
1518 struct pci_dev *pdev;
1519
1520 if (!dev_is_pci(info->dev))
1521 return;
1522
1523 pdev = to_pci_dev(info->dev);
1524
1525 if (info->ats_enabled) {
1526 pci_disable_ats(pdev);
1527 info->ats_enabled = 0;
1528 }
1529 #ifdef CONFIG_INTEL_IOMMU_SVM
1530 if (info->pri_enabled) {
1531 pci_disable_pri(pdev);
1532 info->pri_enabled = 0;
1533 }
1534 if (info->pasid_enabled) {
1535 pci_disable_pasid(pdev);
1536 info->pasid_enabled = 0;
1537 }
1538 #endif
1539 }
1540
1541 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1542 u64 addr, unsigned mask)
1543 {
1544 u16 sid, qdep;
1545 unsigned long flags;
1546 struct device_domain_info *info;
1547
1548 spin_lock_irqsave(&device_domain_lock, flags);
1549 list_for_each_entry(info, &domain->devices, link) {
1550 if (!info->ats_enabled)
1551 continue;
1552
1553 sid = info->bus << 8 | info->devfn;
1554 qdep = info->ats_qdep;
1555 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
1556 qdep, addr, mask);
1557 }
1558 spin_unlock_irqrestore(&device_domain_lock, flags);
1559 }
1560
1561 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1562 struct dmar_domain *domain,
1563 unsigned long pfn, unsigned int pages,
1564 int ih, int map)
1565 {
1566 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1567 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1568 u16 did = domain->iommu_did[iommu->seq_id];
1569
1570 BUG_ON(pages == 0);
1571
1572 if (ih)
1573 ih = 1 << 6;
1574 /*
1575 * Fallback to domain selective flush if no PSI support or the size is
1576 * too big.
1577 * PSI requires page size to be 2 ^ x, and the base address is naturally
1578 * aligned to the size
1579 */
1580 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1581 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1582 DMA_TLB_DSI_FLUSH);
1583 else
1584 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1585 DMA_TLB_PSI_FLUSH);
1586
1587 /*
1588 * In caching mode, changes of pages from non-present to present require
1589 * flush. However, device IOTLB doesn't need to be flushed in this case.
1590 */
1591 if (!cap_caching_mode(iommu->cap) || !map)
1592 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1593 addr, mask);
1594 }
1595
1596 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1597 {
1598 u32 pmen;
1599 unsigned long flags;
1600
1601 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1602 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1603 pmen &= ~DMA_PMEN_EPM;
1604 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1605
1606 /* wait for the protected region status bit to clear */
1607 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1608 readl, !(pmen & DMA_PMEN_PRS), pmen);
1609
1610 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1611 }
1612
1613 static void iommu_enable_translation(struct intel_iommu *iommu)
1614 {
1615 u32 sts;
1616 unsigned long flags;
1617
1618 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1619 iommu->gcmd |= DMA_GCMD_TE;
1620 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1621
1622 /* Make sure hardware complete it */
1623 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1624 readl, (sts & DMA_GSTS_TES), sts);
1625
1626 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1627 }
1628
1629 static void iommu_disable_translation(struct intel_iommu *iommu)
1630 {
1631 u32 sts;
1632 unsigned long flag;
1633
1634 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1635 iommu->gcmd &= ~DMA_GCMD_TE;
1636 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1637
1638 /* Make sure hardware complete it */
1639 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1640 readl, (!(sts & DMA_GSTS_TES)), sts);
1641
1642 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1643 }
1644
1645
1646 static int iommu_init_domains(struct intel_iommu *iommu)
1647 {
1648 u32 ndomains, nlongs;
1649 size_t size;
1650
1651 ndomains = cap_ndoms(iommu->cap);
1652 pr_debug("%s: Number of Domains supported <%d>\n",
1653 iommu->name, ndomains);
1654 nlongs = BITS_TO_LONGS(ndomains);
1655
1656 spin_lock_init(&iommu->lock);
1657
1658 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1659 if (!iommu->domain_ids) {
1660 pr_err("%s: Allocating domain id array failed\n",
1661 iommu->name);
1662 return -ENOMEM;
1663 }
1664
1665 size = ((ndomains >> 8) + 1) * sizeof(struct dmar_domain **);
1666 iommu->domains = kzalloc(size, GFP_KERNEL);
1667
1668 if (iommu->domains) {
1669 size = 256 * sizeof(struct dmar_domain *);
1670 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1671 }
1672
1673 if (!iommu->domains || !iommu->domains[0]) {
1674 pr_err("%s: Allocating domain array failed\n",
1675 iommu->name);
1676 kfree(iommu->domain_ids);
1677 kfree(iommu->domains);
1678 iommu->domain_ids = NULL;
1679 iommu->domains = NULL;
1680 return -ENOMEM;
1681 }
1682
1683
1684
1685 /*
1686 * If Caching mode is set, then invalid translations are tagged
1687 * with domain-id 0, hence we need to pre-allocate it. We also
1688 * use domain-id 0 as a marker for non-allocated domain-id, so
1689 * make sure it is not used for a real domain.
1690 */
1691 set_bit(0, iommu->domain_ids);
1692
1693 return 0;
1694 }
1695
1696 static void disable_dmar_iommu(struct intel_iommu *iommu)
1697 {
1698 struct device_domain_info *info, *tmp;
1699 unsigned long flags;
1700
1701 if (!iommu->domains || !iommu->domain_ids)
1702 return;
1703
1704 again:
1705 spin_lock_irqsave(&device_domain_lock, flags);
1706 list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1707 struct dmar_domain *domain;
1708
1709 if (info->iommu != iommu)
1710 continue;
1711
1712 if (!info->dev || !info->domain)
1713 continue;
1714
1715 domain = info->domain;
1716
1717 __dmar_remove_one_dev_info(info);
1718
1719 if (!domain_type_is_vm_or_si(domain)) {
1720 /*
1721 * The domain_exit() function can't be called under
1722 * device_domain_lock, as it takes this lock itself.
1723 * So release the lock here and re-run the loop
1724 * afterwards.
1725 */
1726 spin_unlock_irqrestore(&device_domain_lock, flags);
1727 domain_exit(domain);
1728 goto again;
1729 }
1730 }
1731 spin_unlock_irqrestore(&device_domain_lock, flags);
1732
1733 if (iommu->gcmd & DMA_GCMD_TE)
1734 iommu_disable_translation(iommu);
1735 }
1736
1737 static void free_dmar_iommu(struct intel_iommu *iommu)
1738 {
1739 if ((iommu->domains) && (iommu->domain_ids)) {
1740 int elems = (cap_ndoms(iommu->cap) >> 8) + 1;
1741 int i;
1742
1743 for (i = 0; i < elems; i++)
1744 kfree(iommu->domains[i]);
1745 kfree(iommu->domains);
1746 kfree(iommu->domain_ids);
1747 iommu->domains = NULL;
1748 iommu->domain_ids = NULL;
1749 }
1750
1751 g_iommus[iommu->seq_id] = NULL;
1752
1753 /* free context mapping */
1754 free_context_table(iommu);
1755
1756 #ifdef CONFIG_INTEL_IOMMU_SVM
1757 if (pasid_enabled(iommu)) {
1758 if (ecap_prs(iommu->ecap))
1759 intel_svm_finish_prq(iommu);
1760 intel_svm_free_pasid_tables(iommu);
1761 }
1762 #endif
1763 }
1764
1765 static struct dmar_domain *alloc_domain(int flags)
1766 {
1767 struct dmar_domain *domain;
1768
1769 domain = alloc_domain_mem();
1770 if (!domain)
1771 return NULL;
1772
1773 memset(domain, 0, sizeof(*domain));
1774 domain->nid = -1;
1775 domain->flags = flags;
1776 INIT_LIST_HEAD(&domain->devices);
1777
1778 return domain;
1779 }
1780
1781 /* Must be called with iommu->lock */
1782 static int domain_attach_iommu(struct dmar_domain *domain,
1783 struct intel_iommu *iommu)
1784 {
1785 unsigned long ndomains;
1786 int num;
1787
1788 assert_spin_locked(&device_domain_lock);
1789 assert_spin_locked(&iommu->lock);
1790
1791 domain->iommu_refcnt[iommu->seq_id] += 1;
1792 domain->iommu_count += 1;
1793 if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1794 ndomains = cap_ndoms(iommu->cap);
1795 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1796
1797 if (num >= ndomains) {
1798 pr_err("%s: No free domain ids\n", iommu->name);
1799 domain->iommu_refcnt[iommu->seq_id] -= 1;
1800 domain->iommu_count -= 1;
1801 return -ENOSPC;
1802 }
1803
1804 set_bit(num, iommu->domain_ids);
1805 set_iommu_domain(iommu, num, domain);
1806
1807 domain->iommu_did[iommu->seq_id] = num;
1808 domain->nid = iommu->node;
1809
1810 domain_update_iommu_cap(domain);
1811 }
1812
1813 return 0;
1814 }
1815
1816 static int domain_detach_iommu(struct dmar_domain *domain,
1817 struct intel_iommu *iommu)
1818 {
1819 int num, count = INT_MAX;
1820
1821 assert_spin_locked(&device_domain_lock);
1822 assert_spin_locked(&iommu->lock);
1823
1824 domain->iommu_refcnt[iommu->seq_id] -= 1;
1825 count = --domain->iommu_count;
1826 if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1827 num = domain->iommu_did[iommu->seq_id];
1828 clear_bit(num, iommu->domain_ids);
1829 set_iommu_domain(iommu, num, NULL);
1830
1831 domain_update_iommu_cap(domain);
1832 domain->iommu_did[iommu->seq_id] = 0;
1833 }
1834
1835 return count;
1836 }
1837
1838 static struct iova_domain reserved_iova_list;
1839 static struct lock_class_key reserved_rbtree_key;
1840
1841 static int dmar_init_reserved_ranges(void)
1842 {
1843 struct pci_dev *pdev = NULL;
1844 struct iova *iova;
1845 int i;
1846
1847 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1848 DMA_32BIT_PFN);
1849
1850 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1851 &reserved_rbtree_key);
1852
1853 /* IOAPIC ranges shouldn't be accessed by DMA */
1854 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1855 IOVA_PFN(IOAPIC_RANGE_END));
1856 if (!iova) {
1857 pr_err("Reserve IOAPIC range failed\n");
1858 return -ENODEV;
1859 }
1860
1861 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1862 for_each_pci_dev(pdev) {
1863 struct resource *r;
1864
1865 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1866 r = &pdev->resource[i];
1867 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1868 continue;
1869 iova = reserve_iova(&reserved_iova_list,
1870 IOVA_PFN(r->start),
1871 IOVA_PFN(r->end));
1872 if (!iova) {
1873 pr_err("Reserve iova failed\n");
1874 return -ENODEV;
1875 }
1876 }
1877 }
1878 return 0;
1879 }
1880
1881 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1882 {
1883 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1884 }
1885
1886 static inline int guestwidth_to_adjustwidth(int gaw)
1887 {
1888 int agaw;
1889 int r = (gaw - 12) % 9;
1890
1891 if (r == 0)
1892 agaw = gaw;
1893 else
1894 agaw = gaw + 9 - r;
1895 if (agaw > 64)
1896 agaw = 64;
1897 return agaw;
1898 }
1899
1900 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1901 int guest_width)
1902 {
1903 int adjust_width, agaw;
1904 unsigned long sagaw;
1905
1906 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1907 DMA_32BIT_PFN);
1908 domain_reserve_special_ranges(domain);
1909
1910 /* calculate AGAW */
1911 if (guest_width > cap_mgaw(iommu->cap))
1912 guest_width = cap_mgaw(iommu->cap);
1913 domain->gaw = guest_width;
1914 adjust_width = guestwidth_to_adjustwidth(guest_width);
1915 agaw = width_to_agaw(adjust_width);
1916 sagaw = cap_sagaw(iommu->cap);
1917 if (!test_bit(agaw, &sagaw)) {
1918 /* hardware doesn't support it, choose a bigger one */
1919 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1920 agaw = find_next_bit(&sagaw, 5, agaw);
1921 if (agaw >= 5)
1922 return -ENODEV;
1923 }
1924 domain->agaw = agaw;
1925
1926 if (ecap_coherent(iommu->ecap))
1927 domain->iommu_coherency = 1;
1928 else
1929 domain->iommu_coherency = 0;
1930
1931 if (ecap_sc_support(iommu->ecap))
1932 domain->iommu_snooping = 1;
1933 else
1934 domain->iommu_snooping = 0;
1935
1936 if (intel_iommu_superpage)
1937 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1938 else
1939 domain->iommu_superpage = 0;
1940
1941 domain->nid = iommu->node;
1942
1943 /* always allocate the top pgd */
1944 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1945 if (!domain->pgd)
1946 return -ENOMEM;
1947 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1948 return 0;
1949 }
1950
1951 static void domain_exit(struct dmar_domain *domain)
1952 {
1953 struct page *freelist = NULL;
1954
1955 /* Domain 0 is reserved, so dont process it */
1956 if (!domain)
1957 return;
1958
1959 /* Flush any lazy unmaps that may reference this domain */
1960 if (!intel_iommu_strict)
1961 flush_unmaps_timeout(0);
1962
1963 /* Remove associated devices and clear attached or cached domains */
1964 rcu_read_lock();
1965 domain_remove_dev_info(domain);
1966 rcu_read_unlock();
1967
1968 /* destroy iovas */
1969 put_iova_domain(&domain->iovad);
1970
1971 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1972
1973 dma_free_pagelist(freelist);
1974
1975 free_domain_mem(domain);
1976 }
1977
1978 static int domain_context_mapping_one(struct dmar_domain *domain,
1979 struct intel_iommu *iommu,
1980 u8 bus, u8 devfn)
1981 {
1982 u16 did = domain->iommu_did[iommu->seq_id];
1983 int translation = CONTEXT_TT_MULTI_LEVEL;
1984 struct device_domain_info *info = NULL;
1985 struct context_entry *context;
1986 unsigned long flags;
1987 struct dma_pte *pgd;
1988 int ret, agaw;
1989
1990 WARN_ON(did == 0);
1991
1992 if (hw_pass_through && domain_type_is_si(domain))
1993 translation = CONTEXT_TT_PASS_THROUGH;
1994
1995 pr_debug("Set context mapping for %02x:%02x.%d\n",
1996 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1997
1998 BUG_ON(!domain->pgd);
1999
2000 spin_lock_irqsave(&device_domain_lock, flags);
2001 spin_lock(&iommu->lock);
2002
2003 ret = -ENOMEM;
2004 context = iommu_context_addr(iommu, bus, devfn, 1);
2005 if (!context)
2006 goto out_unlock;
2007
2008 ret = 0;
2009 if (context_present(context))
2010 goto out_unlock;
2011
2012 /*
2013 * For kdump cases, old valid entries may be cached due to the
2014 * in-flight DMA and copied pgtable, but there is no unmapping
2015 * behaviour for them, thus we need an explicit cache flush for
2016 * the newly-mapped device. For kdump, at this point, the device
2017 * is supposed to finish reset at its driver probe stage, so no
2018 * in-flight DMA will exist, and we don't need to worry anymore
2019 * hereafter.
2020 */
2021 if (context_copied(context)) {
2022 u16 did_old = context_domain_id(context);
2023
2024 if (did_old >= 0 && did_old < cap_ndoms(iommu->cap)) {
2025 iommu->flush.flush_context(iommu, did_old,
2026 (((u16)bus) << 8) | devfn,
2027 DMA_CCMD_MASK_NOBIT,
2028 DMA_CCMD_DEVICE_INVL);
2029 iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
2030 DMA_TLB_DSI_FLUSH);
2031 }
2032 }
2033
2034 pgd = domain->pgd;
2035
2036 context_clear_entry(context);
2037 context_set_domain_id(context, did);
2038
2039 /*
2040 * Skip top levels of page tables for iommu which has less agaw
2041 * than default. Unnecessary for PT mode.
2042 */
2043 if (translation != CONTEXT_TT_PASS_THROUGH) {
2044 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
2045 ret = -ENOMEM;
2046 pgd = phys_to_virt(dma_pte_addr(pgd));
2047 if (!dma_pte_present(pgd))
2048 goto out_unlock;
2049 }
2050
2051 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2052 if (info && info->ats_supported)
2053 translation = CONTEXT_TT_DEV_IOTLB;
2054 else
2055 translation = CONTEXT_TT_MULTI_LEVEL;
2056
2057 context_set_address_root(context, virt_to_phys(pgd));
2058 context_set_address_width(context, iommu->agaw);
2059 } else {
2060 /*
2061 * In pass through mode, AW must be programmed to
2062 * indicate the largest AGAW value supported by
2063 * hardware. And ASR is ignored by hardware.
2064 */
2065 context_set_address_width(context, iommu->msagaw);
2066 }
2067
2068 context_set_translation_type(context, translation);
2069 context_set_fault_enable(context);
2070 context_set_present(context);
2071 domain_flush_cache(domain, context, sizeof(*context));
2072
2073 /*
2074 * It's a non-present to present mapping. If hardware doesn't cache
2075 * non-present entry we only need to flush the write-buffer. If the
2076 * _does_ cache non-present entries, then it does so in the special
2077 * domain #0, which we have to flush:
2078 */
2079 if (cap_caching_mode(iommu->cap)) {
2080 iommu->flush.flush_context(iommu, 0,
2081 (((u16)bus) << 8) | devfn,
2082 DMA_CCMD_MASK_NOBIT,
2083 DMA_CCMD_DEVICE_INVL);
2084 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2085 } else {
2086 iommu_flush_write_buffer(iommu);
2087 }
2088 iommu_enable_dev_iotlb(info);
2089
2090 ret = 0;
2091
2092 out_unlock:
2093 spin_unlock(&iommu->lock);
2094 spin_unlock_irqrestore(&device_domain_lock, flags);
2095
2096 return ret;
2097 }
2098
2099 struct domain_context_mapping_data {
2100 struct dmar_domain *domain;
2101 struct intel_iommu *iommu;
2102 };
2103
2104 static int domain_context_mapping_cb(struct pci_dev *pdev,
2105 u16 alias, void *opaque)
2106 {
2107 struct domain_context_mapping_data *data = opaque;
2108
2109 return domain_context_mapping_one(data->domain, data->iommu,
2110 PCI_BUS_NUM(alias), alias & 0xff);
2111 }
2112
2113 static int
2114 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2115 {
2116 struct intel_iommu *iommu;
2117 u8 bus, devfn;
2118 struct domain_context_mapping_data data;
2119
2120 iommu = device_to_iommu(dev, &bus, &devfn);
2121 if (!iommu)
2122 return -ENODEV;
2123
2124 if (!dev_is_pci(dev))
2125 return domain_context_mapping_one(domain, iommu, bus, devfn);
2126
2127 data.domain = domain;
2128 data.iommu = iommu;
2129
2130 return pci_for_each_dma_alias(to_pci_dev(dev),
2131 &domain_context_mapping_cb, &data);
2132 }
2133
2134 static int domain_context_mapped_cb(struct pci_dev *pdev,
2135 u16 alias, void *opaque)
2136 {
2137 struct intel_iommu *iommu = opaque;
2138
2139 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2140 }
2141
2142 static int domain_context_mapped(struct device *dev)
2143 {
2144 struct intel_iommu *iommu;
2145 u8 bus, devfn;
2146
2147 iommu = device_to_iommu(dev, &bus, &devfn);
2148 if (!iommu)
2149 return -ENODEV;
2150
2151 if (!dev_is_pci(dev))
2152 return device_context_mapped(iommu, bus, devfn);
2153
2154 return !pci_for_each_dma_alias(to_pci_dev(dev),
2155 domain_context_mapped_cb, iommu);
2156 }
2157
2158 /* Returns a number of VTD pages, but aligned to MM page size */
2159 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2160 size_t size)
2161 {
2162 host_addr &= ~PAGE_MASK;
2163 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2164 }
2165
2166 /* Return largest possible superpage level for a given mapping */
2167 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2168 unsigned long iov_pfn,
2169 unsigned long phy_pfn,
2170 unsigned long pages)
2171 {
2172 int support, level = 1;
2173 unsigned long pfnmerge;
2174
2175 support = domain->iommu_superpage;
2176
2177 /* To use a large page, the virtual *and* physical addresses
2178 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2179 of them will mean we have to use smaller pages. So just
2180 merge them and check both at once. */
2181 pfnmerge = iov_pfn | phy_pfn;
2182
2183 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2184 pages >>= VTD_STRIDE_SHIFT;
2185 if (!pages)
2186 break;
2187 pfnmerge >>= VTD_STRIDE_SHIFT;
2188 level++;
2189 support--;
2190 }
2191 return level;
2192 }
2193
2194 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2195 struct scatterlist *sg, unsigned long phys_pfn,
2196 unsigned long nr_pages, int prot)
2197 {
2198 struct dma_pte *first_pte = NULL, *pte = NULL;
2199 phys_addr_t uninitialized_var(pteval);
2200 unsigned long sg_res = 0;
2201 unsigned int largepage_lvl = 0;
2202 unsigned long lvl_pages = 0;
2203
2204 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2205
2206 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2207 return -EINVAL;
2208
2209 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2210
2211 if (!sg) {
2212 sg_res = nr_pages;
2213 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2214 }
2215
2216 while (nr_pages > 0) {
2217 uint64_t tmp;
2218
2219 if (!sg_res) {
2220 unsigned int pgoff = sg->offset & ~PAGE_MASK;
2221
2222 sg_res = aligned_nrpages(sg->offset, sg->length);
2223 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + pgoff;
2224 sg->dma_length = sg->length;
2225 pteval = (sg_phys(sg) - pgoff) | prot;
2226 phys_pfn = pteval >> VTD_PAGE_SHIFT;
2227 }
2228
2229 if (!pte) {
2230 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2231
2232 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2233 if (!pte)
2234 return -ENOMEM;
2235 /* It is large page*/
2236 if (largepage_lvl > 1) {
2237 unsigned long nr_superpages, end_pfn;
2238
2239 pteval |= DMA_PTE_LARGE_PAGE;
2240 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2241
2242 nr_superpages = sg_res / lvl_pages;
2243 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2244
2245 /*
2246 * Ensure that old small page tables are
2247 * removed to make room for superpage(s).
2248 */
2249 dma_pte_free_pagetable(domain, iov_pfn, end_pfn);
2250 } else {
2251 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2252 }
2253
2254 }
2255 /* We don't need lock here, nobody else
2256 * touches the iova range
2257 */
2258 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2259 if (tmp) {
2260 static int dumps = 5;
2261 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2262 iov_pfn, tmp, (unsigned long long)pteval);
2263 if (dumps) {
2264 dumps--;
2265 debug_dma_dump_mappings(NULL);
2266 }
2267 WARN_ON(1);
2268 }
2269
2270 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2271
2272 BUG_ON(nr_pages < lvl_pages);
2273 BUG_ON(sg_res < lvl_pages);
2274
2275 nr_pages -= lvl_pages;
2276 iov_pfn += lvl_pages;
2277 phys_pfn += lvl_pages;
2278 pteval += lvl_pages * VTD_PAGE_SIZE;
2279 sg_res -= lvl_pages;
2280
2281 /* If the next PTE would be the first in a new page, then we
2282 need to flush the cache on the entries we've just written.
2283 And then we'll need to recalculate 'pte', so clear it and
2284 let it get set again in the if (!pte) block above.
2285
2286 If we're done (!nr_pages) we need to flush the cache too.
2287
2288 Also if we've been setting superpages, we may need to
2289 recalculate 'pte' and switch back to smaller pages for the
2290 end of the mapping, if the trailing size is not enough to
2291 use another superpage (i.e. sg_res < lvl_pages). */
2292 pte++;
2293 if (!nr_pages || first_pte_in_page(pte) ||
2294 (largepage_lvl > 1 && sg_res < lvl_pages)) {
2295 domain_flush_cache(domain, first_pte,
2296 (void *)pte - (void *)first_pte);
2297 pte = NULL;
2298 }
2299
2300 if (!sg_res && nr_pages)
2301 sg = sg_next(sg);
2302 }
2303 return 0;
2304 }
2305
2306 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2307 struct scatterlist *sg, unsigned long nr_pages,
2308 int prot)
2309 {
2310 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2311 }
2312
2313 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2314 unsigned long phys_pfn, unsigned long nr_pages,
2315 int prot)
2316 {
2317 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2318 }
2319
2320 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2321 {
2322 if (!iommu)
2323 return;
2324
2325 clear_context_table(iommu, bus, devfn);
2326 iommu->flush.flush_context(iommu, 0, 0, 0,
2327 DMA_CCMD_GLOBAL_INVL);
2328 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2329 }
2330
2331 static inline void unlink_domain_info(struct device_domain_info *info)
2332 {
2333 assert_spin_locked(&device_domain_lock);
2334 list_del(&info->link);
2335 list_del(&info->global);
2336 if (info->dev)
2337 info->dev->archdata.iommu = NULL;
2338 }
2339
2340 static void domain_remove_dev_info(struct dmar_domain *domain)
2341 {
2342 struct device_domain_info *info, *tmp;
2343 unsigned long flags;
2344
2345 spin_lock_irqsave(&device_domain_lock, flags);
2346 list_for_each_entry_safe(info, tmp, &domain->devices, link)
2347 __dmar_remove_one_dev_info(info);
2348 spin_unlock_irqrestore(&device_domain_lock, flags);
2349 }
2350
2351 /*
2352 * find_domain
2353 * Note: we use struct device->archdata.iommu stores the info
2354 */
2355 static struct dmar_domain *find_domain(struct device *dev)
2356 {
2357 struct device_domain_info *info;
2358
2359 /* No lock here, assumes no domain exit in normal case */
2360 info = dev->archdata.iommu;
2361 if (info)
2362 return info->domain;
2363 return NULL;
2364 }
2365
2366 static inline struct device_domain_info *
2367 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2368 {
2369 struct device_domain_info *info;
2370
2371 list_for_each_entry(info, &device_domain_list, global)
2372 if (info->iommu->segment == segment && info->bus == bus &&
2373 info->devfn == devfn)
2374 return info;
2375
2376 return NULL;
2377 }
2378
2379 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2380 int bus, int devfn,
2381 struct device *dev,
2382 struct dmar_domain *domain)
2383 {
2384 struct dmar_domain *found = NULL;
2385 struct device_domain_info *info;
2386 unsigned long flags;
2387 int ret;
2388
2389 info = alloc_devinfo_mem();
2390 if (!info)
2391 return NULL;
2392
2393 info->bus = bus;
2394 info->devfn = devfn;
2395 info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2396 info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2397 info->ats_qdep = 0;
2398 info->dev = dev;
2399 info->domain = domain;
2400 info->iommu = iommu;
2401
2402 if (dev && dev_is_pci(dev)) {
2403 struct pci_dev *pdev = to_pci_dev(info->dev);
2404
2405 if (ecap_dev_iotlb_support(iommu->ecap) &&
2406 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2407 dmar_find_matched_atsr_unit(pdev))
2408 info->ats_supported = 1;
2409
2410 if (ecs_enabled(iommu)) {
2411 if (pasid_enabled(iommu)) {
2412 int features = pci_pasid_features(pdev);
2413 if (features >= 0)
2414 info->pasid_supported = features | 1;
2415 }
2416
2417 if (info->ats_supported && ecap_prs(iommu->ecap) &&
2418 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2419 info->pri_supported = 1;
2420 }
2421 }
2422
2423 spin_lock_irqsave(&device_domain_lock, flags);
2424 if (dev)
2425 found = find_domain(dev);
2426
2427 if (!found) {
2428 struct device_domain_info *info2;
2429 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2430 if (info2) {
2431 found = info2->domain;
2432 info2->dev = dev;
2433 }
2434 }
2435
2436 if (found) {
2437 spin_unlock_irqrestore(&device_domain_lock, flags);
2438 free_devinfo_mem(info);
2439 /* Caller must free the original domain */
2440 return found;
2441 }
2442
2443 spin_lock(&iommu->lock);
2444 ret = domain_attach_iommu(domain, iommu);
2445 spin_unlock(&iommu->lock);
2446
2447 if (ret) {
2448 spin_unlock_irqrestore(&device_domain_lock, flags);
2449 free_devinfo_mem(info);
2450 return NULL;
2451 }
2452
2453 list_add(&info->link, &domain->devices);
2454 list_add(&info->global, &device_domain_list);
2455 if (dev)
2456 dev->archdata.iommu = info;
2457 spin_unlock_irqrestore(&device_domain_lock, flags);
2458
2459 if (dev && domain_context_mapping(domain, dev)) {
2460 pr_err("Domain context map for %s failed\n", dev_name(dev));
2461 dmar_remove_one_dev_info(domain, dev);
2462 return NULL;
2463 }
2464
2465 return domain;
2466 }
2467
2468 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2469 {
2470 *(u16 *)opaque = alias;
2471 return 0;
2472 }
2473
2474 /* domain is initialized */
2475 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2476 {
2477 struct device_domain_info *info = NULL;
2478 struct dmar_domain *domain, *tmp;
2479 struct intel_iommu *iommu;
2480 u16 req_id, dma_alias;
2481 unsigned long flags;
2482 u8 bus, devfn;
2483
2484 domain = find_domain(dev);
2485 if (domain)
2486 return domain;
2487
2488 iommu = device_to_iommu(dev, &bus, &devfn);
2489 if (!iommu)
2490 return NULL;
2491
2492 req_id = ((u16)bus << 8) | devfn;
2493
2494 if (dev_is_pci(dev)) {
2495 struct pci_dev *pdev = to_pci_dev(dev);
2496
2497 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2498
2499 spin_lock_irqsave(&device_domain_lock, flags);
2500 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2501 PCI_BUS_NUM(dma_alias),
2502 dma_alias & 0xff);
2503 if (info) {
2504 iommu = info->iommu;
2505 domain = info->domain;
2506 }
2507 spin_unlock_irqrestore(&device_domain_lock, flags);
2508
2509 /* DMA alias already has a domain, uses it */
2510 if (info)
2511 goto found_domain;
2512 }
2513
2514 /* Allocate and initialize new domain for the device */
2515 domain = alloc_domain(0);
2516 if (!domain)
2517 return NULL;
2518 if (domain_init(domain, iommu, gaw)) {
2519 domain_exit(domain);
2520 return NULL;
2521 }
2522
2523 /* register PCI DMA alias device */
2524 if (req_id != dma_alias && dev_is_pci(dev)) {
2525 tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2526 dma_alias & 0xff, NULL, domain);
2527
2528 if (!tmp || tmp != domain) {
2529 domain_exit(domain);
2530 domain = tmp;
2531 }
2532
2533 if (!domain)
2534 return NULL;
2535 }
2536
2537 found_domain:
2538 tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2539
2540 if (!tmp || tmp != domain) {
2541 domain_exit(domain);
2542 domain = tmp;
2543 }
2544
2545 return domain;
2546 }
2547
2548 static int iommu_domain_identity_map(struct dmar_domain *domain,
2549 unsigned long long start,
2550 unsigned long long end)
2551 {
2552 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2553 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2554
2555 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2556 dma_to_mm_pfn(last_vpfn))) {
2557 pr_err("Reserving iova failed\n");
2558 return -ENOMEM;
2559 }
2560
2561 pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2562 /*
2563 * RMRR range might have overlap with physical memory range,
2564 * clear it first
2565 */
2566 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2567
2568 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2569 last_vpfn - first_vpfn + 1,
2570 DMA_PTE_READ|DMA_PTE_WRITE);
2571 }
2572
2573 static int domain_prepare_identity_map(struct device *dev,
2574 struct dmar_domain *domain,
2575 unsigned long long start,
2576 unsigned long long end)
2577 {
2578 /* For _hardware_ passthrough, don't bother. But for software
2579 passthrough, we do it anyway -- it may indicate a memory
2580 range which is reserved in E820, so which didn't get set
2581 up to start with in si_domain */
2582 if (domain == si_domain && hw_pass_through) {
2583 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2584 dev_name(dev), start, end);
2585 return 0;
2586 }
2587
2588 pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2589 dev_name(dev), start, end);
2590
2591 if (end < start) {
2592 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2593 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2594 dmi_get_system_info(DMI_BIOS_VENDOR),
2595 dmi_get_system_info(DMI_BIOS_VERSION),
2596 dmi_get_system_info(DMI_PRODUCT_VERSION));
2597 return -EIO;
2598 }
2599
2600 if (end >> agaw_to_width(domain->agaw)) {
2601 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2602 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2603 agaw_to_width(domain->agaw),
2604 dmi_get_system_info(DMI_BIOS_VENDOR),
2605 dmi_get_system_info(DMI_BIOS_VERSION),
2606 dmi_get_system_info(DMI_PRODUCT_VERSION));
2607 return -EIO;
2608 }
2609
2610 return iommu_domain_identity_map(domain, start, end);
2611 }
2612
2613 static int iommu_prepare_identity_map(struct device *dev,
2614 unsigned long long start,
2615 unsigned long long end)
2616 {
2617 struct dmar_domain *domain;
2618 int ret;
2619
2620 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2621 if (!domain)
2622 return -ENOMEM;
2623
2624 ret = domain_prepare_identity_map(dev, domain, start, end);
2625 if (ret)
2626 domain_exit(domain);
2627
2628 return ret;
2629 }
2630
2631 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2632 struct device *dev)
2633 {
2634 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2635 return 0;
2636 return iommu_prepare_identity_map(dev, rmrr->base_address,
2637 rmrr->end_address);
2638 }
2639
2640 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2641 static inline void iommu_prepare_isa(void)
2642 {
2643 struct pci_dev *pdev;
2644 int ret;
2645
2646 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2647 if (!pdev)
2648 return;
2649
2650 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2651 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2652
2653 if (ret)
2654 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2655
2656 pci_dev_put(pdev);
2657 }
2658 #else
2659 static inline void iommu_prepare_isa(void)
2660 {
2661 return;
2662 }
2663 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2664
2665 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2666
2667 static int __init si_domain_init(int hw)
2668 {
2669 int nid, ret = 0;
2670
2671 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2672 if (!si_domain)
2673 return -EFAULT;
2674
2675 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2676 domain_exit(si_domain);
2677 return -EFAULT;
2678 }
2679
2680 pr_debug("Identity mapping domain allocated\n");
2681
2682 if (hw)
2683 return 0;
2684
2685 for_each_online_node(nid) {
2686 unsigned long start_pfn, end_pfn;
2687 int i;
2688
2689 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2690 ret = iommu_domain_identity_map(si_domain,
2691 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2692 if (ret)
2693 return ret;
2694 }
2695 }
2696
2697 return 0;
2698 }
2699
2700 static int identity_mapping(struct device *dev)
2701 {
2702 struct device_domain_info *info;
2703
2704 if (likely(!iommu_identity_mapping))
2705 return 0;
2706
2707 info = dev->archdata.iommu;
2708 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2709 return (info->domain == si_domain);
2710
2711 return 0;
2712 }
2713
2714 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2715 {
2716 struct dmar_domain *ndomain;
2717 struct intel_iommu *iommu;
2718 u8 bus, devfn;
2719
2720 iommu = device_to_iommu(dev, &bus, &devfn);
2721 if (!iommu)
2722 return -ENODEV;
2723
2724 ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2725 if (ndomain != domain)
2726 return -EBUSY;
2727
2728 return 0;
2729 }
2730
2731 static bool device_has_rmrr(struct device *dev)
2732 {
2733 struct dmar_rmrr_unit *rmrr;
2734 struct device *tmp;
2735 int i;
2736
2737 rcu_read_lock();
2738 for_each_rmrr_units(rmrr) {
2739 /*
2740 * Return TRUE if this RMRR contains the device that
2741 * is passed in.
2742 */
2743 for_each_active_dev_scope(rmrr->devices,
2744 rmrr->devices_cnt, i, tmp)
2745 if (tmp == dev) {
2746 rcu_read_unlock();
2747 return true;
2748 }
2749 }
2750 rcu_read_unlock();
2751 return false;
2752 }
2753
2754 /*
2755 * There are a couple cases where we need to restrict the functionality of
2756 * devices associated with RMRRs. The first is when evaluating a device for
2757 * identity mapping because problems exist when devices are moved in and out
2758 * of domains and their respective RMRR information is lost. This means that
2759 * a device with associated RMRRs will never be in a "passthrough" domain.
2760 * The second is use of the device through the IOMMU API. This interface
2761 * expects to have full control of the IOVA space for the device. We cannot
2762 * satisfy both the requirement that RMRR access is maintained and have an
2763 * unencumbered IOVA space. We also have no ability to quiesce the device's
2764 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2765 * We therefore prevent devices associated with an RMRR from participating in
2766 * the IOMMU API, which eliminates them from device assignment.
2767 *
2768 * In both cases we assume that PCI USB devices with RMRRs have them largely
2769 * for historical reasons and that the RMRR space is not actively used post
2770 * boot. This exclusion may change if vendors begin to abuse it.
2771 *
2772 * The same exception is made for graphics devices, with the requirement that
2773 * any use of the RMRR regions will be torn down before assigning the device
2774 * to a guest.
2775 */
2776 static bool device_is_rmrr_locked(struct device *dev)
2777 {
2778 if (!device_has_rmrr(dev))
2779 return false;
2780
2781 if (dev_is_pci(dev)) {
2782 struct pci_dev *pdev = to_pci_dev(dev);
2783
2784 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2785 return false;
2786 }
2787
2788 return true;
2789 }
2790
2791 static int iommu_should_identity_map(struct device *dev, int startup)
2792 {
2793
2794 if (dev_is_pci(dev)) {
2795 struct pci_dev *pdev = to_pci_dev(dev);
2796
2797 if (device_is_rmrr_locked(dev))
2798 return 0;
2799
2800 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2801 return 1;
2802
2803 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2804 return 1;
2805
2806 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2807 return 0;
2808
2809 /*
2810 * We want to start off with all devices in the 1:1 domain, and
2811 * take them out later if we find they can't access all of memory.
2812 *
2813 * However, we can't do this for PCI devices behind bridges,
2814 * because all PCI devices behind the same bridge will end up
2815 * with the same source-id on their transactions.
2816 *
2817 * Practically speaking, we can't change things around for these
2818 * devices at run-time, because we can't be sure there'll be no
2819 * DMA transactions in flight for any of their siblings.
2820 *
2821 * So PCI devices (unless they're on the root bus) as well as
2822 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2823 * the 1:1 domain, just in _case_ one of their siblings turns out
2824 * not to be able to map all of memory.
2825 */
2826 if (!pci_is_pcie(pdev)) {
2827 if (!pci_is_root_bus(pdev->bus))
2828 return 0;
2829 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2830 return 0;
2831 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2832 return 0;
2833 } else {
2834 if (device_has_rmrr(dev))
2835 return 0;
2836 }
2837
2838 /*
2839 * At boot time, we don't yet know if devices will be 64-bit capable.
2840 * Assume that they will — if they turn out not to be, then we can
2841 * take them out of the 1:1 domain later.
2842 */
2843 if (!startup) {
2844 /*
2845 * If the device's dma_mask is less than the system's memory
2846 * size then this is not a candidate for identity mapping.
2847 */
2848 u64 dma_mask = *dev->dma_mask;
2849
2850 if (dev->coherent_dma_mask &&
2851 dev->coherent_dma_mask < dma_mask)
2852 dma_mask = dev->coherent_dma_mask;
2853
2854 return dma_mask >= dma_get_required_mask(dev);
2855 }
2856
2857 return 1;
2858 }
2859
2860 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2861 {
2862 int ret;
2863
2864 if (!iommu_should_identity_map(dev, 1))
2865 return 0;
2866
2867 ret = domain_add_dev_info(si_domain, dev);
2868 if (!ret)
2869 pr_info("%s identity mapping for device %s\n",
2870 hw ? "Hardware" : "Software", dev_name(dev));
2871 else if (ret == -ENODEV)
2872 /* device not associated with an iommu */
2873 ret = 0;
2874
2875 return ret;
2876 }
2877
2878
2879 static int __init iommu_prepare_static_identity_mapping(int hw)
2880 {
2881 struct pci_dev *pdev = NULL;
2882 struct dmar_drhd_unit *drhd;
2883 struct intel_iommu *iommu;
2884 struct device *dev;
2885 int i;
2886 int ret = 0;
2887
2888 for_each_pci_dev(pdev) {
2889 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2890 if (ret)
2891 return ret;
2892 }
2893
2894 for_each_active_iommu(iommu, drhd)
2895 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2896 struct acpi_device_physical_node *pn;
2897 struct acpi_device *adev;
2898
2899 if (dev->bus != &acpi_bus_type)
2900 continue;
2901
2902 adev= to_acpi_device(dev);
2903 mutex_lock(&adev->physical_node_lock);
2904 list_for_each_entry(pn, &adev->physical_node_list, node) {
2905 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2906 if (ret)
2907 break;
2908 }
2909 mutex_unlock(&adev->physical_node_lock);
2910 if (ret)
2911 return ret;
2912 }
2913
2914 return 0;
2915 }
2916
2917 static void intel_iommu_init_qi(struct intel_iommu *iommu)
2918 {
2919 /*
2920 * Start from the sane iommu hardware state.
2921 * If the queued invalidation is already initialized by us
2922 * (for example, while enabling interrupt-remapping) then
2923 * we got the things already rolling from a sane state.
2924 */
2925 if (!iommu->qi) {
2926 /*
2927 * Clear any previous faults.
2928 */
2929 dmar_fault(-1, iommu);
2930 /*
2931 * Disable queued invalidation if supported and already enabled
2932 * before OS handover.
2933 */
2934 dmar_disable_qi(iommu);
2935 }
2936
2937 if (dmar_enable_qi(iommu)) {
2938 /*
2939 * Queued Invalidate not enabled, use Register Based Invalidate
2940 */
2941 iommu->flush.flush_context = __iommu_flush_context;
2942 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2943 pr_info("%s: Using Register based invalidation\n",
2944 iommu->name);
2945 } else {
2946 iommu->flush.flush_context = qi_flush_context;
2947 iommu->flush.flush_iotlb = qi_flush_iotlb;
2948 pr_info("%s: Using Queued invalidation\n", iommu->name);
2949 }
2950 }
2951
2952 static int copy_context_table(struct intel_iommu *iommu,
2953 struct root_entry *old_re,
2954 struct context_entry **tbl,
2955 int bus, bool ext)
2956 {
2957 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
2958 struct context_entry *new_ce = NULL, ce;
2959 struct context_entry *old_ce = NULL;
2960 struct root_entry re;
2961 phys_addr_t old_ce_phys;
2962
2963 tbl_idx = ext ? bus * 2 : bus;
2964 memcpy(&re, old_re, sizeof(re));
2965
2966 for (devfn = 0; devfn < 256; devfn++) {
2967 /* First calculate the correct index */
2968 idx = (ext ? devfn * 2 : devfn) % 256;
2969
2970 if (idx == 0) {
2971 /* First save what we may have and clean up */
2972 if (new_ce) {
2973 tbl[tbl_idx] = new_ce;
2974 __iommu_flush_cache(iommu, new_ce,
2975 VTD_PAGE_SIZE);
2976 pos = 1;
2977 }
2978
2979 if (old_ce)
2980 iounmap(old_ce);
2981
2982 ret = 0;
2983 if (devfn < 0x80)
2984 old_ce_phys = root_entry_lctp(&re);
2985 else
2986 old_ce_phys = root_entry_uctp(&re);
2987
2988 if (!old_ce_phys) {
2989 if (ext && devfn == 0) {
2990 /* No LCTP, try UCTP */
2991 devfn = 0x7f;
2992 continue;
2993 } else {
2994 goto out;
2995 }
2996 }
2997
2998 ret = -ENOMEM;
2999 old_ce = memremap(old_ce_phys, PAGE_SIZE,
3000 MEMREMAP_WB);
3001 if (!old_ce)
3002 goto out;
3003
3004 new_ce = alloc_pgtable_page(iommu->node);
3005 if (!new_ce)
3006 goto out_unmap;
3007
3008 ret = 0;
3009 }
3010
3011 /* Now copy the context entry */
3012 memcpy(&ce, old_ce + idx, sizeof(ce));
3013
3014 if (!__context_present(&ce))
3015 continue;
3016
3017 did = context_domain_id(&ce);
3018 if (did >= 0 && did < cap_ndoms(iommu->cap))
3019 set_bit(did, iommu->domain_ids);
3020
3021 /*
3022 * We need a marker for copied context entries. This
3023 * marker needs to work for the old format as well as
3024 * for extended context entries.
3025 *
3026 * Bit 67 of the context entry is used. In the old
3027 * format this bit is available to software, in the
3028 * extended format it is the PGE bit, but PGE is ignored
3029 * by HW if PASIDs are disabled (and thus still
3030 * available).
3031 *
3032 * So disable PASIDs first and then mark the entry
3033 * copied. This means that we don't copy PASID
3034 * translations from the old kernel, but this is fine as
3035 * faults there are not fatal.
3036 */
3037 context_clear_pasid_enable(&ce);
3038 context_set_copied(&ce);
3039
3040 new_ce[idx] = ce;
3041 }
3042
3043 tbl[tbl_idx + pos] = new_ce;
3044
3045 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3046
3047 out_unmap:
3048 memunmap(old_ce);
3049
3050 out:
3051 return ret;
3052 }
3053
3054 static int copy_translation_tables(struct intel_iommu *iommu)
3055 {
3056 struct context_entry **ctxt_tbls;
3057 struct root_entry *old_rt;
3058 phys_addr_t old_rt_phys;
3059 int ctxt_table_entries;
3060 unsigned long flags;
3061 u64 rtaddr_reg;
3062 int bus, ret;
3063 bool new_ext, ext;
3064
3065 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3066 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
3067 new_ext = !!ecap_ecs(iommu->ecap);
3068
3069 /*
3070 * The RTT bit can only be changed when translation is disabled,
3071 * but disabling translation means to open a window for data
3072 * corruption. So bail out and don't copy anything if we would
3073 * have to change the bit.
3074 */
3075 if (new_ext != ext)
3076 return -EINVAL;
3077
3078 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3079 if (!old_rt_phys)
3080 return -EINVAL;
3081
3082 old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
3083 if (!old_rt)
3084 return -ENOMEM;
3085
3086 /* This is too big for the stack - allocate it from slab */
3087 ctxt_table_entries = ext ? 512 : 256;
3088 ret = -ENOMEM;
3089 ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
3090 if (!ctxt_tbls)
3091 goto out_unmap;
3092
3093 for (bus = 0; bus < 256; bus++) {
3094 ret = copy_context_table(iommu, &old_rt[bus],
3095 ctxt_tbls, bus, ext);
3096 if (ret) {
3097 pr_err("%s: Failed to copy context table for bus %d\n",
3098 iommu->name, bus);
3099 continue;
3100 }
3101 }
3102
3103 spin_lock_irqsave(&iommu->lock, flags);
3104
3105 /* Context tables are copied, now write them to the root_entry table */
3106 for (bus = 0; bus < 256; bus++) {
3107 int idx = ext ? bus * 2 : bus;
3108 u64 val;
3109
3110 if (ctxt_tbls[idx]) {
3111 val = virt_to_phys(ctxt_tbls[idx]) | 1;
3112 iommu->root_entry[bus].lo = val;
3113 }
3114
3115 if (!ext || !ctxt_tbls[idx + 1])
3116 continue;
3117
3118 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3119 iommu->root_entry[bus].hi = val;
3120 }
3121
3122 spin_unlock_irqrestore(&iommu->lock, flags);
3123
3124 kfree(ctxt_tbls);
3125
3126 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3127
3128 ret = 0;
3129
3130 out_unmap:
3131 memunmap(old_rt);
3132
3133 return ret;
3134 }
3135
3136 static int __init init_dmars(void)
3137 {
3138 struct dmar_drhd_unit *drhd;
3139 struct dmar_rmrr_unit *rmrr;
3140 bool copied_tables = false;
3141 struct device *dev;
3142 struct intel_iommu *iommu;
3143 int i, ret;
3144
3145 /*
3146 * for each drhd
3147 * allocate root
3148 * initialize and program root entry to not present
3149 * endfor
3150 */
3151 for_each_drhd_unit(drhd) {
3152 /*
3153 * lock not needed as this is only incremented in the single
3154 * threaded kernel __init code path all other access are read
3155 * only
3156 */
3157 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3158 g_num_of_iommus++;
3159 continue;
3160 }
3161 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3162 }
3163
3164 /* Preallocate enough resources for IOMMU hot-addition */
3165 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3166 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3167
3168 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3169 GFP_KERNEL);
3170 if (!g_iommus) {
3171 pr_err("Allocating global iommu array failed\n");
3172 ret = -ENOMEM;
3173 goto error;
3174 }
3175
3176 deferred_flush = kzalloc(g_num_of_iommus *
3177 sizeof(struct deferred_flush_tables), GFP_KERNEL);
3178 if (!deferred_flush) {
3179 ret = -ENOMEM;
3180 goto free_g_iommus;
3181 }
3182
3183 for_each_active_iommu(iommu, drhd) {
3184 g_iommus[iommu->seq_id] = iommu;
3185
3186 intel_iommu_init_qi(iommu);
3187
3188 ret = iommu_init_domains(iommu);
3189 if (ret)
3190 goto free_iommu;
3191
3192 init_translation_status(iommu);
3193
3194 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3195 iommu_disable_translation(iommu);
3196 clear_translation_pre_enabled(iommu);
3197 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3198 iommu->name);
3199 }
3200
3201 /*
3202 * TBD:
3203 * we could share the same root & context tables
3204 * among all IOMMU's. Need to Split it later.
3205 */
3206 ret = iommu_alloc_root_entry(iommu);
3207 if (ret)
3208 goto free_iommu;
3209
3210 if (translation_pre_enabled(iommu)) {
3211 pr_info("Translation already enabled - trying to copy translation structures\n");
3212
3213 ret = copy_translation_tables(iommu);
3214 if (ret) {
3215 /*
3216 * We found the IOMMU with translation
3217 * enabled - but failed to copy over the
3218 * old root-entry table. Try to proceed
3219 * by disabling translation now and
3220 * allocating a clean root-entry table.
3221 * This might cause DMAR faults, but
3222 * probably the dump will still succeed.
3223 */
3224 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3225 iommu->name);
3226 iommu_disable_translation(iommu);
3227 clear_translation_pre_enabled(iommu);
3228 } else {
3229 pr_info("Copied translation tables from previous kernel for %s\n",
3230 iommu->name);
3231 copied_tables = true;
3232 }
3233 }
3234
3235 if (!ecap_pass_through(iommu->ecap))
3236 hw_pass_through = 0;
3237 #ifdef CONFIG_INTEL_IOMMU_SVM
3238 if (pasid_enabled(iommu))
3239 intel_svm_alloc_pasid_tables(iommu);
3240 #endif
3241 }
3242
3243 /*
3244 * Now that qi is enabled on all iommus, set the root entry and flush
3245 * caches. This is required on some Intel X58 chipsets, otherwise the
3246 * flush_context function will loop forever and the boot hangs.
3247 */
3248 for_each_active_iommu(iommu, drhd) {
3249 iommu_flush_write_buffer(iommu);
3250 iommu_set_root_entry(iommu);
3251 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3252 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3253 }
3254
3255 if (iommu_pass_through)
3256 iommu_identity_mapping |= IDENTMAP_ALL;
3257
3258 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3259 iommu_identity_mapping |= IDENTMAP_GFX;
3260 #endif
3261
3262 check_tylersburg_isoch();
3263
3264 if (iommu_identity_mapping) {
3265 ret = si_domain_init(hw_pass_through);
3266 if (ret)
3267 goto free_iommu;
3268 }
3269
3270
3271 /*
3272 * If we copied translations from a previous kernel in the kdump
3273 * case, we can not assign the devices to domains now, as that
3274 * would eliminate the old mappings. So skip this part and defer
3275 * the assignment to device driver initialization time.
3276 */
3277 if (copied_tables)
3278 goto domains_done;
3279
3280 /*
3281 * If pass through is not set or not enabled, setup context entries for
3282 * identity mappings for rmrr, gfx, and isa and may fall back to static
3283 * identity mapping if iommu_identity_mapping is set.
3284 */
3285 if (iommu_identity_mapping) {
3286 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3287 if (ret) {
3288 pr_crit("Failed to setup IOMMU pass-through\n");
3289 goto free_iommu;
3290 }
3291 }
3292 /*
3293 * For each rmrr
3294 * for each dev attached to rmrr
3295 * do
3296 * locate drhd for dev, alloc domain for dev
3297 * allocate free domain
3298 * allocate page table entries for rmrr
3299 * if context not allocated for bus
3300 * allocate and init context
3301 * set present in root table for this bus
3302 * init context with domain, translation etc
3303 * endfor
3304 * endfor
3305 */
3306 pr_info("Setting RMRR:\n");
3307 for_each_rmrr_units(rmrr) {
3308 /* some BIOS lists non-exist devices in DMAR table. */
3309 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3310 i, dev) {
3311 ret = iommu_prepare_rmrr_dev(rmrr, dev);
3312 if (ret)
3313 pr_err("Mapping reserved region failed\n");
3314 }
3315 }
3316
3317 iommu_prepare_isa();
3318
3319 domains_done:
3320
3321 /*
3322 * for each drhd
3323 * enable fault log
3324 * global invalidate context cache
3325 * global invalidate iotlb
3326 * enable translation
3327 */
3328 for_each_iommu(iommu, drhd) {
3329 if (drhd->ignored) {
3330 /*
3331 * we always have to disable PMRs or DMA may fail on
3332 * this device
3333 */
3334 if (force_on)
3335 iommu_disable_protect_mem_regions(iommu);
3336 continue;
3337 }
3338
3339 iommu_flush_write_buffer(iommu);
3340
3341 #ifdef CONFIG_INTEL_IOMMU_SVM
3342 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
3343 ret = intel_svm_enable_prq(iommu);
3344 if (ret)
3345 goto free_iommu;
3346 }
3347 #endif
3348 ret = dmar_set_interrupt(iommu);
3349 if (ret)
3350 goto free_iommu;
3351
3352 if (!translation_pre_enabled(iommu))
3353 iommu_enable_translation(iommu);
3354
3355 iommu_disable_protect_mem_regions(iommu);
3356 }
3357
3358 return 0;
3359
3360 free_iommu:
3361 for_each_active_iommu(iommu, drhd) {
3362 disable_dmar_iommu(iommu);
3363 free_dmar_iommu(iommu);
3364 }
3365 kfree(deferred_flush);
3366 free_g_iommus:
3367 kfree(g_iommus);
3368 error:
3369 return ret;
3370 }
3371
3372 /* This takes a number of _MM_ pages, not VTD pages */
3373 static struct iova *intel_alloc_iova(struct device *dev,
3374 struct dmar_domain *domain,
3375 unsigned long nrpages, uint64_t dma_mask)
3376 {
3377 struct iova *iova = NULL;
3378
3379 /* Restrict dma_mask to the width that the iommu can handle */
3380 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3381 /* Ensure we reserve the whole size-aligned region */
3382 nrpages = __roundup_pow_of_two(nrpages);
3383
3384 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3385 /*
3386 * First try to allocate an io virtual address in
3387 * DMA_BIT_MASK(32) and if that fails then try allocating
3388 * from higher range
3389 */
3390 iova = alloc_iova(&domain->iovad, nrpages,
3391 IOVA_PFN(DMA_BIT_MASK(32)), 1);
3392 if (iova)
3393 return iova;
3394 }
3395 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
3396 if (unlikely(!iova)) {
3397 pr_err("Allocating %ld-page iova for %s failed",
3398 nrpages, dev_name(dev));
3399 return NULL;
3400 }
3401
3402 return iova;
3403 }
3404
3405 static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
3406 {
3407 struct dmar_rmrr_unit *rmrr;
3408 struct dmar_domain *domain;
3409 struct device *i_dev;
3410 int i, ret;
3411
3412 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3413 if (!domain) {
3414 pr_err("Allocating domain for %s failed\n",
3415 dev_name(dev));
3416 return NULL;
3417 }
3418
3419 /* We have a new domain - setup possible RMRRs for the device */
3420 rcu_read_lock();
3421 for_each_rmrr_units(rmrr) {
3422 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3423 i, i_dev) {
3424 if (i_dev != dev)
3425 continue;
3426
3427 ret = domain_prepare_identity_map(dev, domain,
3428 rmrr->base_address,
3429 rmrr->end_address);
3430 if (ret)
3431 dev_err(dev, "Mapping reserved region failed\n");
3432 }
3433 }
3434 rcu_read_unlock();
3435
3436 return domain;
3437 }
3438
3439 static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3440 {
3441 struct device_domain_info *info;
3442
3443 /* No lock here, assumes no domain exit in normal case */
3444 info = dev->archdata.iommu;
3445 if (likely(info))
3446 return info->domain;
3447
3448 return __get_valid_domain_for_dev(dev);
3449 }
3450
3451 /* Check if the dev needs to go through non-identity map and unmap process.*/
3452 static int iommu_no_mapping(struct device *dev)
3453 {
3454 int found;
3455
3456 if (iommu_dummy(dev))
3457 return 1;
3458
3459 if (!iommu_identity_mapping)
3460 return 0;
3461
3462 found = identity_mapping(dev);
3463 if (found) {
3464 if (iommu_should_identity_map(dev, 0))
3465 return 1;
3466 else {
3467 /*
3468 * 32 bit DMA is removed from si_domain and fall back
3469 * to non-identity mapping.
3470 */
3471 dmar_remove_one_dev_info(si_domain, dev);
3472 pr_info("32bit %s uses non-identity mapping\n",
3473 dev_name(dev));
3474 return 0;
3475 }
3476 } else {
3477 /*
3478 * In case of a detached 64 bit DMA device from vm, the device
3479 * is put into si_domain for identity mapping.
3480 */
3481 if (iommu_should_identity_map(dev, 0)) {
3482 int ret;
3483 ret = domain_add_dev_info(si_domain, dev);
3484 if (!ret) {
3485 pr_info("64bit %s uses identity mapping\n",
3486 dev_name(dev));
3487 return 1;
3488 }
3489 }
3490 }
3491
3492 return 0;
3493 }
3494
3495 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3496 size_t size, int dir, u64 dma_mask)
3497 {
3498 struct dmar_domain *domain;
3499 phys_addr_t start_paddr;
3500 struct iova *iova;
3501 int prot = 0;
3502 int ret;
3503 struct intel_iommu *iommu;
3504 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3505
3506 BUG_ON(dir == DMA_NONE);
3507
3508 if (iommu_no_mapping(dev))
3509 return paddr;
3510
3511 domain = get_valid_domain_for_dev(dev);
3512 if (!domain)
3513 return 0;
3514
3515 iommu = domain_get_iommu(domain);
3516 size = aligned_nrpages(paddr, size);
3517
3518 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3519 if (!iova)
3520 goto error;
3521
3522 /*
3523 * Check if DMAR supports zero-length reads on write only
3524 * mappings..
3525 */
3526 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3527 !cap_zlr(iommu->cap))
3528 prot |= DMA_PTE_READ;
3529 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3530 prot |= DMA_PTE_WRITE;
3531 /*
3532 * paddr - (paddr + size) might be partial page, we should map the whole
3533 * page. Note: if two part of one page are separately mapped, we
3534 * might have two guest_addr mapping to the same host paddr, but this
3535 * is not a big problem
3536 */
3537 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
3538 mm_to_dma_pfn(paddr_pfn), size, prot);
3539 if (ret)
3540 goto error;
3541
3542 /* it's a non-present to present mapping. Only flush if caching mode */
3543 if (cap_caching_mode(iommu->cap))
3544 iommu_flush_iotlb_psi(iommu, domain,
3545 mm_to_dma_pfn(iova->pfn_lo),
3546 size, 0, 1);
3547 else
3548 iommu_flush_write_buffer(iommu);
3549
3550 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
3551 start_paddr += paddr & ~PAGE_MASK;
3552 return start_paddr;
3553
3554 error:
3555 if (iova)
3556 __free_iova(&domain->iovad, iova);
3557 pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
3558 dev_name(dev), size, (unsigned long long)paddr, dir);
3559 return 0;
3560 }
3561
3562 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3563 unsigned long offset, size_t size,
3564 enum dma_data_direction dir,
3565 struct dma_attrs *attrs)
3566 {
3567 return __intel_map_single(dev, page_to_phys(page) + offset, size,
3568 dir, *dev->dma_mask);
3569 }
3570
3571 static void flush_unmaps(void)
3572 {
3573 int i, j;
3574
3575 timer_on = 0;
3576
3577 /* just flush them all */
3578 for (i = 0; i < g_num_of_iommus; i++) {
3579 struct intel_iommu *iommu = g_iommus[i];
3580 if (!iommu)
3581 continue;
3582
3583 if (!deferred_flush[i].next)
3584 continue;
3585
3586 /* In caching mode, global flushes turn emulation expensive */
3587 if (!cap_caching_mode(iommu->cap))
3588 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3589 DMA_TLB_GLOBAL_FLUSH);
3590 for (j = 0; j < deferred_flush[i].next; j++) {
3591 unsigned long mask;
3592 struct iova *iova = deferred_flush[i].iova[j];
3593 struct dmar_domain *domain = deferred_flush[i].domain[j];
3594
3595 /* On real hardware multiple invalidations are expensive */
3596 if (cap_caching_mode(iommu->cap))
3597 iommu_flush_iotlb_psi(iommu, domain,
3598 iova->pfn_lo, iova_size(iova),
3599 !deferred_flush[i].freelist[j], 0);
3600 else {
3601 mask = ilog2(mm_to_dma_pfn(iova_size(iova)));
3602 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3603 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3604 }
3605 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
3606 if (deferred_flush[i].freelist[j])
3607 dma_free_pagelist(deferred_flush[i].freelist[j]);
3608 }
3609 deferred_flush[i].next = 0;
3610 }
3611
3612 list_size = 0;
3613 }
3614
3615 static void flush_unmaps_timeout(unsigned long data)
3616 {
3617 unsigned long flags;
3618
3619 spin_lock_irqsave(&async_umap_flush_lock, flags);
3620 flush_unmaps();
3621 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3622 }
3623
3624 static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
3625 {
3626 unsigned long flags;
3627 int next, iommu_id;
3628 struct intel_iommu *iommu;
3629
3630 spin_lock_irqsave(&async_umap_flush_lock, flags);
3631 if (list_size == HIGH_WATER_MARK)
3632 flush_unmaps();
3633
3634 iommu = domain_get_iommu(dom);
3635 iommu_id = iommu->seq_id;
3636
3637 next = deferred_flush[iommu_id].next;
3638 deferred_flush[iommu_id].domain[next] = dom;
3639 deferred_flush[iommu_id].iova[next] = iova;
3640 deferred_flush[iommu_id].freelist[next] = freelist;
3641 deferred_flush[iommu_id].next++;
3642
3643 if (!timer_on) {
3644 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3645 timer_on = 1;
3646 }
3647 list_size++;
3648 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3649 }
3650
3651 static void intel_unmap(struct device *dev, dma_addr_t dev_addr)
3652 {
3653 struct dmar_domain *domain;
3654 unsigned long start_pfn, last_pfn;
3655 struct iova *iova;
3656 struct intel_iommu *iommu;
3657 struct page *freelist;
3658
3659 if (iommu_no_mapping(dev))
3660 return;
3661
3662 domain = find_domain(dev);
3663 BUG_ON(!domain);
3664
3665 iommu = domain_get_iommu(domain);
3666
3667 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
3668 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3669 (unsigned long long)dev_addr))
3670 return;
3671
3672 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3673 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
3674
3675 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3676 dev_name(dev), start_pfn, last_pfn);
3677
3678 freelist = domain_unmap(domain, start_pfn, last_pfn);
3679
3680 if (intel_iommu_strict) {
3681 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3682 last_pfn - start_pfn + 1, !freelist, 0);
3683 /* free iova */
3684 __free_iova(&domain->iovad, iova);
3685 dma_free_pagelist(freelist);
3686 } else {
3687 add_unmap(domain, iova, freelist);
3688 /*
3689 * queue up the release of the unmap to save the 1/6th of the
3690 * cpu used up by the iotlb flush operation...
3691 */
3692 }
3693 }
3694
3695 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3696 size_t size, enum dma_data_direction dir,
3697 struct dma_attrs *attrs)
3698 {
3699 intel_unmap(dev, dev_addr);
3700 }
3701
3702 static void *intel_alloc_coherent(struct device *dev, size_t size,
3703 dma_addr_t *dma_handle, gfp_t flags,
3704 struct dma_attrs *attrs)
3705 {
3706 struct page *page = NULL;
3707 int order;
3708
3709 size = PAGE_ALIGN(size);
3710 order = get_order(size);
3711
3712 if (!iommu_no_mapping(dev))
3713 flags &= ~(GFP_DMA | GFP_DMA32);
3714 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3715 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3716 flags |= GFP_DMA;
3717 else
3718 flags |= GFP_DMA32;
3719 }
3720
3721 if (gfpflags_allow_blocking(flags)) {
3722 unsigned int count = size >> PAGE_SHIFT;
3723
3724 page = dma_alloc_from_contiguous(dev, count, order);
3725 if (page && iommu_no_mapping(dev) &&
3726 page_to_phys(page) + size > dev->coherent_dma_mask) {
3727 dma_release_from_contiguous(dev, page, count);
3728 page = NULL;
3729 }
3730 }
3731
3732 if (!page)
3733 page = alloc_pages(flags, order);
3734 if (!page)
3735 return NULL;
3736 memset(page_address(page), 0, size);
3737
3738 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3739 DMA_BIDIRECTIONAL,
3740 dev->coherent_dma_mask);
3741 if (*dma_handle)
3742 return page_address(page);
3743 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3744 __free_pages(page, order);
3745
3746 return NULL;
3747 }
3748
3749 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3750 dma_addr_t dma_handle, struct dma_attrs *attrs)
3751 {
3752 int order;
3753 struct page *page = virt_to_page(vaddr);
3754
3755 size = PAGE_ALIGN(size);
3756 order = get_order(size);
3757
3758 intel_unmap(dev, dma_handle);
3759 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3760 __free_pages(page, order);
3761 }
3762
3763 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3764 int nelems, enum dma_data_direction dir,
3765 struct dma_attrs *attrs)
3766 {
3767 intel_unmap(dev, sglist[0].dma_address);
3768 }
3769
3770 static int intel_nontranslate_map_sg(struct device *hddev,
3771 struct scatterlist *sglist, int nelems, int dir)
3772 {
3773 int i;
3774 struct scatterlist *sg;
3775
3776 for_each_sg(sglist, sg, nelems, i) {
3777 BUG_ON(!sg_page(sg));
3778 sg->dma_address = sg_phys(sg);
3779 sg->dma_length = sg->length;
3780 }
3781 return nelems;
3782 }
3783
3784 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3785 enum dma_data_direction dir, struct dma_attrs *attrs)
3786 {
3787 int i;
3788 struct dmar_domain *domain;
3789 size_t size = 0;
3790 int prot = 0;
3791 struct iova *iova = NULL;
3792 int ret;
3793 struct scatterlist *sg;
3794 unsigned long start_vpfn;
3795 struct intel_iommu *iommu;
3796
3797 BUG_ON(dir == DMA_NONE);
3798 if (iommu_no_mapping(dev))
3799 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3800
3801 domain = get_valid_domain_for_dev(dev);
3802 if (!domain)
3803 return 0;
3804
3805 iommu = domain_get_iommu(domain);
3806
3807 for_each_sg(sglist, sg, nelems, i)
3808 size += aligned_nrpages(sg->offset, sg->length);
3809
3810 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3811 *dev->dma_mask);
3812 if (!iova) {
3813 sglist->dma_length = 0;
3814 return 0;
3815 }
3816
3817 /*
3818 * Check if DMAR supports zero-length reads on write only
3819 * mappings..
3820 */
3821 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3822 !cap_zlr(iommu->cap))
3823 prot |= DMA_PTE_READ;
3824 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3825 prot |= DMA_PTE_WRITE;
3826
3827 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
3828
3829 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3830 if (unlikely(ret)) {
3831 dma_pte_free_pagetable(domain, start_vpfn,
3832 start_vpfn + size - 1);
3833 __free_iova(&domain->iovad, iova);
3834 return 0;
3835 }
3836
3837 /* it's a non-present to present mapping. Only flush if caching mode */
3838 if (cap_caching_mode(iommu->cap))
3839 iommu_flush_iotlb_psi(iommu, domain, start_vpfn, size, 0, 1);
3840 else
3841 iommu_flush_write_buffer(iommu);
3842
3843 return nelems;
3844 }
3845
3846 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3847 {
3848 return !dma_addr;
3849 }
3850
3851 struct dma_map_ops intel_dma_ops = {
3852 .alloc = intel_alloc_coherent,
3853 .free = intel_free_coherent,
3854 .map_sg = intel_map_sg,
3855 .unmap_sg = intel_unmap_sg,
3856 .map_page = intel_map_page,
3857 .unmap_page = intel_unmap_page,
3858 .mapping_error = intel_mapping_error,
3859 };
3860
3861 static inline int iommu_domain_cache_init(void)
3862 {
3863 int ret = 0;
3864
3865 iommu_domain_cache = kmem_cache_create("iommu_domain",
3866 sizeof(struct dmar_domain),
3867 0,
3868 SLAB_HWCACHE_ALIGN,
3869
3870 NULL);
3871 if (!iommu_domain_cache) {
3872 pr_err("Couldn't create iommu_domain cache\n");
3873 ret = -ENOMEM;
3874 }
3875
3876 return ret;
3877 }
3878
3879 static inline int iommu_devinfo_cache_init(void)
3880 {
3881 int ret = 0;
3882
3883 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3884 sizeof(struct device_domain_info),
3885 0,
3886 SLAB_HWCACHE_ALIGN,
3887 NULL);
3888 if (!iommu_devinfo_cache) {
3889 pr_err("Couldn't create devinfo cache\n");
3890 ret = -ENOMEM;
3891 }
3892
3893 return ret;
3894 }
3895
3896 static int __init iommu_init_mempool(void)
3897 {
3898 int ret;
3899 ret = iova_cache_get();
3900 if (ret)
3901 return ret;
3902
3903 ret = iommu_domain_cache_init();
3904 if (ret)
3905 goto domain_error;
3906
3907 ret = iommu_devinfo_cache_init();
3908 if (!ret)
3909 return ret;
3910
3911 kmem_cache_destroy(iommu_domain_cache);
3912 domain_error:
3913 iova_cache_put();
3914
3915 return -ENOMEM;
3916 }
3917
3918 static void __init iommu_exit_mempool(void)
3919 {
3920 kmem_cache_destroy(iommu_devinfo_cache);
3921 kmem_cache_destroy(iommu_domain_cache);
3922 iova_cache_put();
3923 }
3924
3925 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3926 {
3927 struct dmar_drhd_unit *drhd;
3928 u32 vtbar;
3929 int rc;
3930
3931 /* We know that this device on this chipset has its own IOMMU.
3932 * If we find it under a different IOMMU, then the BIOS is lying
3933 * to us. Hope that the IOMMU for this device is actually
3934 * disabled, and it needs no translation...
3935 */
3936 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3937 if (rc) {
3938 /* "can't" happen */
3939 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3940 return;
3941 }
3942 vtbar &= 0xffff0000;
3943
3944 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3945 drhd = dmar_find_matched_drhd_unit(pdev);
3946 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3947 TAINT_FIRMWARE_WORKAROUND,
3948 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3949 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3950 }
3951 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3952
3953 static void __init init_no_remapping_devices(void)
3954 {
3955 struct dmar_drhd_unit *drhd;
3956 struct device *dev;
3957 int i;
3958
3959 for_each_drhd_unit(drhd) {
3960 if (!drhd->include_all) {
3961 for_each_active_dev_scope(drhd->devices,
3962 drhd->devices_cnt, i, dev)
3963 break;
3964 /* ignore DMAR unit if no devices exist */
3965 if (i == drhd->devices_cnt)
3966 drhd->ignored = 1;
3967 }
3968 }
3969
3970 for_each_active_drhd_unit(drhd) {
3971 if (drhd->include_all)
3972 continue;
3973
3974 for_each_active_dev_scope(drhd->devices,
3975 drhd->devices_cnt, i, dev)
3976 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
3977 break;
3978 if (i < drhd->devices_cnt)
3979 continue;
3980
3981 /* This IOMMU has *only* gfx devices. Either bypass it or
3982 set the gfx_mapped flag, as appropriate */
3983 if (dmar_map_gfx) {
3984 intel_iommu_gfx_mapped = 1;
3985 } else {
3986 drhd->ignored = 1;
3987 for_each_active_dev_scope(drhd->devices,
3988 drhd->devices_cnt, i, dev)
3989 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3990 }
3991 }
3992 }
3993
3994 #ifdef CONFIG_SUSPEND
3995 static int init_iommu_hw(void)
3996 {
3997 struct dmar_drhd_unit *drhd;
3998 struct intel_iommu *iommu = NULL;
3999
4000 for_each_active_iommu(iommu, drhd)
4001 if (iommu->qi)
4002 dmar_reenable_qi(iommu);
4003
4004 for_each_iommu(iommu, drhd) {
4005 if (drhd->ignored) {
4006 /*
4007 * we always have to disable PMRs or DMA may fail on
4008 * this device
4009 */
4010 if (force_on)
4011 iommu_disable_protect_mem_regions(iommu);
4012 continue;
4013 }
4014
4015 iommu_flush_write_buffer(iommu);
4016
4017 iommu_set_root_entry(iommu);
4018
4019 iommu->flush.flush_context(iommu, 0, 0, 0,
4020 DMA_CCMD_GLOBAL_INVL);
4021 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4022 iommu_enable_translation(iommu);
4023 iommu_disable_protect_mem_regions(iommu);
4024 }
4025
4026 return 0;
4027 }
4028
4029 static void iommu_flush_all(void)
4030 {
4031 struct dmar_drhd_unit *drhd;
4032 struct intel_iommu *iommu;
4033
4034 for_each_active_iommu(iommu, drhd) {
4035 iommu->flush.flush_context(iommu, 0, 0, 0,
4036 DMA_CCMD_GLOBAL_INVL);
4037 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
4038 DMA_TLB_GLOBAL_FLUSH);
4039 }
4040 }
4041
4042 static int iommu_suspend(void)
4043 {
4044 struct dmar_drhd_unit *drhd;
4045 struct intel_iommu *iommu = NULL;
4046 unsigned long flag;
4047
4048 for_each_active_iommu(iommu, drhd) {
4049 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
4050 GFP_ATOMIC);
4051 if (!iommu->iommu_state)
4052 goto nomem;
4053 }
4054
4055 iommu_flush_all();
4056
4057 for_each_active_iommu(iommu, drhd) {
4058 iommu_disable_translation(iommu);
4059
4060 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4061
4062 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4063 readl(iommu->reg + DMAR_FECTL_REG);
4064 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4065 readl(iommu->reg + DMAR_FEDATA_REG);
4066 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4067 readl(iommu->reg + DMAR_FEADDR_REG);
4068 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4069 readl(iommu->reg + DMAR_FEUADDR_REG);
4070
4071 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4072 }
4073 return 0;
4074
4075 nomem:
4076 for_each_active_iommu(iommu, drhd)
4077 kfree(iommu->iommu_state);
4078
4079 return -ENOMEM;
4080 }
4081
4082 static void iommu_resume(void)
4083 {
4084 struct dmar_drhd_unit *drhd;
4085 struct intel_iommu *iommu = NULL;
4086 unsigned long flag;
4087
4088 if (init_iommu_hw()) {
4089 if (force_on)
4090 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4091 else
4092 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
4093 return;
4094 }
4095
4096 for_each_active_iommu(iommu, drhd) {
4097
4098 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4099
4100 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4101 iommu->reg + DMAR_FECTL_REG);
4102 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4103 iommu->reg + DMAR_FEDATA_REG);
4104 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4105 iommu->reg + DMAR_FEADDR_REG);
4106 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4107 iommu->reg + DMAR_FEUADDR_REG);
4108
4109 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4110 }
4111
4112 for_each_active_iommu(iommu, drhd)
4113 kfree(iommu->iommu_state);
4114 }
4115
4116 static struct syscore_ops iommu_syscore_ops = {
4117 .resume = iommu_resume,
4118 .suspend = iommu_suspend,
4119 };
4120
4121 static void __init init_iommu_pm_ops(void)
4122 {
4123 register_syscore_ops(&iommu_syscore_ops);
4124 }
4125
4126 #else
4127 static inline void init_iommu_pm_ops(void) {}
4128 #endif /* CONFIG_PM */
4129
4130
4131 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
4132 {
4133 struct acpi_dmar_reserved_memory *rmrr;
4134 struct dmar_rmrr_unit *rmrru;
4135
4136 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4137 if (!rmrru)
4138 return -ENOMEM;
4139
4140 rmrru->hdr = header;
4141 rmrr = (struct acpi_dmar_reserved_memory *)header;
4142 rmrru->base_address = rmrr->base_address;
4143 rmrru->end_address = rmrr->end_address;
4144 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4145 ((void *)rmrr) + rmrr->header.length,
4146 &rmrru->devices_cnt);
4147 if (rmrru->devices_cnt && rmrru->devices == NULL) {
4148 kfree(rmrru);
4149 return -ENOMEM;
4150 }
4151
4152 list_add(&rmrru->list, &dmar_rmrr_units);
4153
4154 return 0;
4155 }
4156
4157 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4158 {
4159 struct dmar_atsr_unit *atsru;
4160 struct acpi_dmar_atsr *tmp;
4161
4162 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4163 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4164 if (atsr->segment != tmp->segment)
4165 continue;
4166 if (atsr->header.length != tmp->header.length)
4167 continue;
4168 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4169 return atsru;
4170 }
4171
4172 return NULL;
4173 }
4174
4175 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4176 {
4177 struct acpi_dmar_atsr *atsr;
4178 struct dmar_atsr_unit *atsru;
4179
4180 if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
4181 return 0;
4182
4183 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4184 atsru = dmar_find_atsr(atsr);
4185 if (atsru)
4186 return 0;
4187
4188 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4189 if (!atsru)
4190 return -ENOMEM;
4191
4192 /*
4193 * If memory is allocated from slab by ACPI _DSM method, we need to
4194 * copy the memory content because the memory buffer will be freed
4195 * on return.
4196 */
4197 atsru->hdr = (void *)(atsru + 1);
4198 memcpy(atsru->hdr, hdr, hdr->length);
4199 atsru->include_all = atsr->flags & 0x1;
4200 if (!atsru->include_all) {
4201 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4202 (void *)atsr + atsr->header.length,
4203 &atsru->devices_cnt);
4204 if (atsru->devices_cnt && atsru->devices == NULL) {
4205 kfree(atsru);
4206 return -ENOMEM;
4207 }
4208 }
4209
4210 list_add_rcu(&atsru->list, &dmar_atsr_units);
4211
4212 return 0;
4213 }
4214
4215 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4216 {
4217 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4218 kfree(atsru);
4219 }
4220
4221 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4222 {
4223 struct acpi_dmar_atsr *atsr;
4224 struct dmar_atsr_unit *atsru;
4225
4226 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4227 atsru = dmar_find_atsr(atsr);
4228 if (atsru) {
4229 list_del_rcu(&atsru->list);
4230 synchronize_rcu();
4231 intel_iommu_free_atsr(atsru);
4232 }
4233
4234 return 0;
4235 }
4236
4237 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4238 {
4239 int i;
4240 struct device *dev;
4241 struct acpi_dmar_atsr *atsr;
4242 struct dmar_atsr_unit *atsru;
4243
4244 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4245 atsru = dmar_find_atsr(atsr);
4246 if (!atsru)
4247 return 0;
4248
4249 if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
4250 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4251 i, dev)
4252 return -EBUSY;
4253 }
4254
4255 return 0;
4256 }
4257
4258 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4259 {
4260 int sp, ret = 0;
4261 struct intel_iommu *iommu = dmaru->iommu;
4262
4263 if (g_iommus[iommu->seq_id])
4264 return 0;
4265
4266 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4267 pr_warn("%s: Doesn't support hardware pass through.\n",
4268 iommu->name);
4269 return -ENXIO;
4270 }
4271 if (!ecap_sc_support(iommu->ecap) &&
4272 domain_update_iommu_snooping(iommu)) {
4273 pr_warn("%s: Doesn't support snooping.\n",
4274 iommu->name);
4275 return -ENXIO;
4276 }
4277 sp = domain_update_iommu_superpage(iommu) - 1;
4278 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4279 pr_warn("%s: Doesn't support large page.\n",
4280 iommu->name);
4281 return -ENXIO;
4282 }
4283
4284 /*
4285 * Disable translation if already enabled prior to OS handover.
4286 */
4287 if (iommu->gcmd & DMA_GCMD_TE)
4288 iommu_disable_translation(iommu);
4289
4290 g_iommus[iommu->seq_id] = iommu;
4291 ret = iommu_init_domains(iommu);
4292 if (ret == 0)
4293 ret = iommu_alloc_root_entry(iommu);
4294 if (ret)
4295 goto out;
4296
4297 #ifdef CONFIG_INTEL_IOMMU_SVM
4298 if (pasid_enabled(iommu))
4299 intel_svm_alloc_pasid_tables(iommu);
4300 #endif
4301
4302 if (dmaru->ignored) {
4303 /*
4304 * we always have to disable PMRs or DMA may fail on this device
4305 */
4306 if (force_on)
4307 iommu_disable_protect_mem_regions(iommu);
4308 return 0;
4309 }
4310
4311 intel_iommu_init_qi(iommu);
4312 iommu_flush_write_buffer(iommu);
4313
4314 #ifdef CONFIG_INTEL_IOMMU_SVM
4315 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
4316 ret = intel_svm_enable_prq(iommu);
4317 if (ret)
4318 goto disable_iommu;
4319 }
4320 #endif
4321 ret = dmar_set_interrupt(iommu);
4322 if (ret)
4323 goto disable_iommu;
4324
4325 iommu_set_root_entry(iommu);
4326 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4327 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4328 iommu_enable_translation(iommu);
4329
4330 iommu_disable_protect_mem_regions(iommu);
4331 return 0;
4332
4333 disable_iommu:
4334 disable_dmar_iommu(iommu);
4335 out:
4336 free_dmar_iommu(iommu);
4337 return ret;
4338 }
4339
4340 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4341 {
4342 int ret = 0;
4343 struct intel_iommu *iommu = dmaru->iommu;
4344
4345 if (!intel_iommu_enabled)
4346 return 0;
4347 if (iommu == NULL)
4348 return -EINVAL;
4349
4350 if (insert) {
4351 ret = intel_iommu_add(dmaru);
4352 } else {
4353 disable_dmar_iommu(iommu);
4354 free_dmar_iommu(iommu);
4355 }
4356
4357 return ret;
4358 }
4359
4360 static void intel_iommu_free_dmars(void)
4361 {
4362 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4363 struct dmar_atsr_unit *atsru, *atsr_n;
4364
4365 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4366 list_del(&rmrru->list);
4367 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4368 kfree(rmrru);
4369 }
4370
4371 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4372 list_del(&atsru->list);
4373 intel_iommu_free_atsr(atsru);
4374 }
4375 }
4376
4377 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4378 {
4379 int i, ret = 1;
4380 struct pci_bus *bus;
4381 struct pci_dev *bridge = NULL;
4382 struct device *tmp;
4383 struct acpi_dmar_atsr *atsr;
4384 struct dmar_atsr_unit *atsru;
4385
4386 dev = pci_physfn(dev);
4387 for (bus = dev->bus; bus; bus = bus->parent) {
4388 bridge = bus->self;
4389 /* If it's an integrated device, allow ATS */
4390 if (!bridge)
4391 return 1;
4392 /* Connected via non-PCIe: no ATS */
4393 if (!pci_is_pcie(bridge) ||
4394 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4395 return 0;
4396 /* If we found the root port, look it up in the ATSR */
4397 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4398 break;
4399 }
4400
4401 rcu_read_lock();
4402 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4403 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4404 if (atsr->segment != pci_domain_nr(dev->bus))
4405 continue;
4406
4407 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4408 if (tmp == &bridge->dev)
4409 goto out;
4410
4411 if (atsru->include_all)
4412 goto out;
4413 }
4414 ret = 0;
4415 out:
4416 rcu_read_unlock();
4417
4418 return ret;
4419 }
4420
4421 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4422 {
4423 int ret = 0;
4424 struct dmar_rmrr_unit *rmrru;
4425 struct dmar_atsr_unit *atsru;
4426 struct acpi_dmar_atsr *atsr;
4427 struct acpi_dmar_reserved_memory *rmrr;
4428
4429 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
4430 return 0;
4431
4432 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4433 rmrr = container_of(rmrru->hdr,
4434 struct acpi_dmar_reserved_memory, header);
4435 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4436 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4437 ((void *)rmrr) + rmrr->header.length,
4438 rmrr->segment, rmrru->devices,
4439 rmrru->devices_cnt);
4440 if(ret < 0)
4441 return ret;
4442 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4443 dmar_remove_dev_scope(info, rmrr->segment,
4444 rmrru->devices, rmrru->devices_cnt);
4445 }
4446 }
4447
4448 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4449 if (atsru->include_all)
4450 continue;
4451
4452 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4453 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4454 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4455 (void *)atsr + atsr->header.length,
4456 atsr->segment, atsru->devices,
4457 atsru->devices_cnt);
4458 if (ret > 0)
4459 break;
4460 else if(ret < 0)
4461 return ret;
4462 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4463 if (dmar_remove_dev_scope(info, atsr->segment,
4464 atsru->devices, atsru->devices_cnt))
4465 break;
4466 }
4467 }
4468
4469 return 0;
4470 }
4471
4472 /*
4473 * Here we only respond to action of unbound device from driver.
4474 *
4475 * Added device is not attached to its DMAR domain here yet. That will happen
4476 * when mapping the device to iova.
4477 */
4478 static int device_notifier(struct notifier_block *nb,
4479 unsigned long action, void *data)
4480 {
4481 struct device *dev = data;
4482 struct dmar_domain *domain;
4483
4484 if (iommu_dummy(dev))
4485 return 0;
4486
4487 if (action != BUS_NOTIFY_REMOVED_DEVICE)
4488 return 0;
4489
4490 domain = find_domain(dev);
4491 if (!domain)
4492 return 0;
4493
4494 dmar_remove_one_dev_info(domain, dev);
4495 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4496 domain_exit(domain);
4497
4498 return 0;
4499 }
4500
4501 static struct notifier_block device_nb = {
4502 .notifier_call = device_notifier,
4503 };
4504
4505 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4506 unsigned long val, void *v)
4507 {
4508 struct memory_notify *mhp = v;
4509 unsigned long long start, end;
4510 unsigned long start_vpfn, last_vpfn;
4511
4512 switch (val) {
4513 case MEM_GOING_ONLINE:
4514 start = mhp->start_pfn << PAGE_SHIFT;
4515 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4516 if (iommu_domain_identity_map(si_domain, start, end)) {
4517 pr_warn("Failed to build identity map for [%llx-%llx]\n",
4518 start, end);
4519 return NOTIFY_BAD;
4520 }
4521 break;
4522
4523 case MEM_OFFLINE:
4524 case MEM_CANCEL_ONLINE:
4525 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4526 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4527 while (start_vpfn <= last_vpfn) {
4528 struct iova *iova;
4529 struct dmar_drhd_unit *drhd;
4530 struct intel_iommu *iommu;
4531 struct page *freelist;
4532
4533 iova = find_iova(&si_domain->iovad, start_vpfn);
4534 if (iova == NULL) {
4535 pr_debug("Failed get IOVA for PFN %lx\n",
4536 start_vpfn);
4537 break;
4538 }
4539
4540 iova = split_and_remove_iova(&si_domain->iovad, iova,
4541 start_vpfn, last_vpfn);
4542 if (iova == NULL) {
4543 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4544 start_vpfn, last_vpfn);
4545 return NOTIFY_BAD;
4546 }
4547
4548 freelist = domain_unmap(si_domain, iova->pfn_lo,
4549 iova->pfn_hi);
4550
4551 rcu_read_lock();
4552 for_each_active_iommu(iommu, drhd)
4553 iommu_flush_iotlb_psi(iommu, si_domain,
4554 iova->pfn_lo, iova_size(iova),
4555 !freelist, 0);
4556 rcu_read_unlock();
4557 dma_free_pagelist(freelist);
4558
4559 start_vpfn = iova->pfn_hi + 1;
4560 free_iova_mem(iova);
4561 }
4562 break;
4563 }
4564
4565 return NOTIFY_OK;
4566 }
4567
4568 static struct notifier_block intel_iommu_memory_nb = {
4569 .notifier_call = intel_iommu_memory_notifier,
4570 .priority = 0
4571 };
4572
4573
4574 static ssize_t intel_iommu_show_version(struct device *dev,
4575 struct device_attribute *attr,
4576 char *buf)
4577 {
4578 struct intel_iommu *iommu = dev_get_drvdata(dev);
4579 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4580 return sprintf(buf, "%d:%d\n",
4581 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4582 }
4583 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4584
4585 static ssize_t intel_iommu_show_address(struct device *dev,
4586 struct device_attribute *attr,
4587 char *buf)
4588 {
4589 struct intel_iommu *iommu = dev_get_drvdata(dev);
4590 return sprintf(buf, "%llx\n", iommu->reg_phys);
4591 }
4592 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4593
4594 static ssize_t intel_iommu_show_cap(struct device *dev,
4595 struct device_attribute *attr,
4596 char *buf)
4597 {
4598 struct intel_iommu *iommu = dev_get_drvdata(dev);
4599 return sprintf(buf, "%llx\n", iommu->cap);
4600 }
4601 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4602
4603 static ssize_t intel_iommu_show_ecap(struct device *dev,
4604 struct device_attribute *attr,
4605 char *buf)
4606 {
4607 struct intel_iommu *iommu = dev_get_drvdata(dev);
4608 return sprintf(buf, "%llx\n", iommu->ecap);
4609 }
4610 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4611
4612 static ssize_t intel_iommu_show_ndoms(struct device *dev,
4613 struct device_attribute *attr,
4614 char *buf)
4615 {
4616 struct intel_iommu *iommu = dev_get_drvdata(dev);
4617 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4618 }
4619 static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4620
4621 static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4622 struct device_attribute *attr,
4623 char *buf)
4624 {
4625 struct intel_iommu *iommu = dev_get_drvdata(dev);
4626 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4627 cap_ndoms(iommu->cap)));
4628 }
4629 static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4630
4631 static struct attribute *intel_iommu_attrs[] = {
4632 &dev_attr_version.attr,
4633 &dev_attr_address.attr,
4634 &dev_attr_cap.attr,
4635 &dev_attr_ecap.attr,
4636 &dev_attr_domains_supported.attr,
4637 &dev_attr_domains_used.attr,
4638 NULL,
4639 };
4640
4641 static struct attribute_group intel_iommu_group = {
4642 .name = "intel-iommu",
4643 .attrs = intel_iommu_attrs,
4644 };
4645
4646 const struct attribute_group *intel_iommu_groups[] = {
4647 &intel_iommu_group,
4648 NULL,
4649 };
4650
4651 int __init intel_iommu_init(void)
4652 {
4653 int ret = -ENODEV;
4654 struct dmar_drhd_unit *drhd;
4655 struct intel_iommu *iommu;
4656
4657 /* VT-d is required for a TXT/tboot launch, so enforce that */
4658 force_on = tboot_force_iommu();
4659
4660 if (iommu_init_mempool()) {
4661 if (force_on)
4662 panic("tboot: Failed to initialize iommu memory\n");
4663 return -ENOMEM;
4664 }
4665
4666 down_write(&dmar_global_lock);
4667 if (dmar_table_init()) {
4668 if (force_on)
4669 panic("tboot: Failed to initialize DMAR table\n");
4670 goto out_free_dmar;
4671 }
4672
4673 if (dmar_dev_scope_init() < 0) {
4674 if (force_on)
4675 panic("tboot: Failed to initialize DMAR device scope\n");
4676 goto out_free_dmar;
4677 }
4678
4679 if (no_iommu || dmar_disabled)
4680 goto out_free_dmar;
4681
4682 if (list_empty(&dmar_rmrr_units))
4683 pr_info("No RMRR found\n");
4684
4685 if (list_empty(&dmar_atsr_units))
4686 pr_info("No ATSR found\n");
4687
4688 if (dmar_init_reserved_ranges()) {
4689 if (force_on)
4690 panic("tboot: Failed to reserve iommu ranges\n");
4691 goto out_free_reserved_range;
4692 }
4693
4694 init_no_remapping_devices();
4695
4696 ret = init_dmars();
4697 if (ret) {
4698 if (force_on)
4699 panic("tboot: Failed to initialize DMARs\n");
4700 pr_err("Initialization failed\n");
4701 goto out_free_reserved_range;
4702 }
4703 up_write(&dmar_global_lock);
4704 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4705
4706 init_timer(&unmap_timer);
4707 #ifdef CONFIG_SWIOTLB
4708 swiotlb = 0;
4709 #endif
4710 dma_ops = &intel_dma_ops;
4711
4712 init_iommu_pm_ops();
4713
4714 for_each_active_iommu(iommu, drhd)
4715 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4716 intel_iommu_groups,
4717 "%s", iommu->name);
4718
4719 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4720 bus_register_notifier(&pci_bus_type, &device_nb);
4721 if (si_domain && !hw_pass_through)
4722 register_memory_notifier(&intel_iommu_memory_nb);
4723
4724 intel_iommu_enabled = 1;
4725
4726 return 0;
4727
4728 out_free_reserved_range:
4729 put_iova_domain(&reserved_iova_list);
4730 out_free_dmar:
4731 intel_iommu_free_dmars();
4732 up_write(&dmar_global_lock);
4733 iommu_exit_mempool();
4734 return ret;
4735 }
4736
4737 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4738 {
4739 struct intel_iommu *iommu = opaque;
4740
4741 domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4742 return 0;
4743 }
4744
4745 /*
4746 * NB - intel-iommu lacks any sort of reference counting for the users of
4747 * dependent devices. If multiple endpoints have intersecting dependent
4748 * devices, unbinding the driver from any one of them will possibly leave
4749 * the others unable to operate.
4750 */
4751 static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
4752 {
4753 if (!iommu || !dev || !dev_is_pci(dev))
4754 return;
4755
4756 pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
4757 }
4758
4759 static void __dmar_remove_one_dev_info(struct device_domain_info *info)
4760 {
4761 struct intel_iommu *iommu;
4762 unsigned long flags;
4763
4764 assert_spin_locked(&device_domain_lock);
4765
4766 if (WARN_ON(!info))
4767 return;
4768
4769 iommu = info->iommu;
4770
4771 if (info->dev) {
4772 iommu_disable_dev_iotlb(info);
4773 domain_context_clear(iommu, info->dev);
4774 }
4775
4776 unlink_domain_info(info);
4777
4778 spin_lock_irqsave(&iommu->lock, flags);
4779 domain_detach_iommu(info->domain, iommu);
4780 spin_unlock_irqrestore(&iommu->lock, flags);
4781
4782 free_devinfo_mem(info);
4783 }
4784
4785 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4786 struct device *dev)
4787 {
4788 struct device_domain_info *info;
4789 unsigned long flags;
4790
4791 spin_lock_irqsave(&device_domain_lock, flags);
4792 info = dev->archdata.iommu;
4793 __dmar_remove_one_dev_info(info);
4794 spin_unlock_irqrestore(&device_domain_lock, flags);
4795 }
4796
4797 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4798 {
4799 int adjust_width;
4800
4801 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4802 DMA_32BIT_PFN);
4803 domain_reserve_special_ranges(domain);
4804
4805 /* calculate AGAW */
4806 domain->gaw = guest_width;
4807 adjust_width = guestwidth_to_adjustwidth(guest_width);
4808 domain->agaw = width_to_agaw(adjust_width);
4809
4810 domain->iommu_coherency = 0;
4811 domain->iommu_snooping = 0;
4812 domain->iommu_superpage = 0;
4813 domain->max_addr = 0;
4814
4815 /* always allocate the top pgd */
4816 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
4817 if (!domain->pgd)
4818 return -ENOMEM;
4819 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4820 return 0;
4821 }
4822
4823 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
4824 {
4825 struct dmar_domain *dmar_domain;
4826 struct iommu_domain *domain;
4827
4828 if (type != IOMMU_DOMAIN_UNMANAGED)
4829 return NULL;
4830
4831 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
4832 if (!dmar_domain) {
4833 pr_err("Can't allocate dmar_domain\n");
4834 return NULL;
4835 }
4836 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4837 pr_err("Domain initialization failed\n");
4838 domain_exit(dmar_domain);
4839 return NULL;
4840 }
4841 domain_update_iommu_cap(dmar_domain);
4842
4843 domain = &dmar_domain->domain;
4844 domain->geometry.aperture_start = 0;
4845 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4846 domain->geometry.force_aperture = true;
4847
4848 return domain;
4849 }
4850
4851 static void intel_iommu_domain_free(struct iommu_domain *domain)
4852 {
4853 domain_exit(to_dmar_domain(domain));
4854 }
4855
4856 static int intel_iommu_attach_device(struct iommu_domain *domain,
4857 struct device *dev)
4858 {
4859 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4860 struct intel_iommu *iommu;
4861 int addr_width;
4862 u8 bus, devfn;
4863
4864 if (device_is_rmrr_locked(dev)) {
4865 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
4866 return -EPERM;
4867 }
4868
4869 /* normally dev is not mapped */
4870 if (unlikely(domain_context_mapped(dev))) {
4871 struct dmar_domain *old_domain;
4872
4873 old_domain = find_domain(dev);
4874 if (old_domain) {
4875 rcu_read_lock();
4876 dmar_remove_one_dev_info(old_domain, dev);
4877 rcu_read_unlock();
4878
4879 if (!domain_type_is_vm_or_si(old_domain) &&
4880 list_empty(&old_domain->devices))
4881 domain_exit(old_domain);
4882 }
4883 }
4884
4885 iommu = device_to_iommu(dev, &bus, &devfn);
4886 if (!iommu)
4887 return -ENODEV;
4888
4889 /* check if this iommu agaw is sufficient for max mapped address */
4890 addr_width = agaw_to_width(iommu->agaw);
4891 if (addr_width > cap_mgaw(iommu->cap))
4892 addr_width = cap_mgaw(iommu->cap);
4893
4894 if (dmar_domain->max_addr > (1LL << addr_width)) {
4895 pr_err("%s: iommu width (%d) is not "
4896 "sufficient for the mapped address (%llx)\n",
4897 __func__, addr_width, dmar_domain->max_addr);
4898 return -EFAULT;
4899 }
4900 dmar_domain->gaw = addr_width;
4901
4902 /*
4903 * Knock out extra levels of page tables if necessary
4904 */
4905 while (iommu->agaw < dmar_domain->agaw) {
4906 struct dma_pte *pte;
4907
4908 pte = dmar_domain->pgd;
4909 if (dma_pte_present(pte)) {
4910 dmar_domain->pgd = (struct dma_pte *)
4911 phys_to_virt(dma_pte_addr(pte));
4912 free_pgtable_page(pte);
4913 }
4914 dmar_domain->agaw--;
4915 }
4916
4917 return domain_add_dev_info(dmar_domain, dev);
4918 }
4919
4920 static void intel_iommu_detach_device(struct iommu_domain *domain,
4921 struct device *dev)
4922 {
4923 dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
4924 }
4925
4926 static int intel_iommu_map(struct iommu_domain *domain,
4927 unsigned long iova, phys_addr_t hpa,
4928 size_t size, int iommu_prot)
4929 {
4930 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4931 u64 max_addr;
4932 int prot = 0;
4933 int ret;
4934
4935 if (iommu_prot & IOMMU_READ)
4936 prot |= DMA_PTE_READ;
4937 if (iommu_prot & IOMMU_WRITE)
4938 prot |= DMA_PTE_WRITE;
4939 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4940 prot |= DMA_PTE_SNP;
4941
4942 max_addr = iova + size;
4943 if (dmar_domain->max_addr < max_addr) {
4944 u64 end;
4945
4946 /* check if minimum agaw is sufficient for mapped address */
4947 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
4948 if (end < max_addr) {
4949 pr_err("%s: iommu width (%d) is not "
4950 "sufficient for the mapped address (%llx)\n",
4951 __func__, dmar_domain->gaw, max_addr);
4952 return -EFAULT;
4953 }
4954 dmar_domain->max_addr = max_addr;
4955 }
4956 /* Round up size to next multiple of PAGE_SIZE, if it and
4957 the low bits of hpa would take us onto the next page */
4958 size = aligned_nrpages(hpa, size);
4959 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4960 hpa >> VTD_PAGE_SHIFT, size, prot);
4961 return ret;
4962 }
4963
4964 static size_t intel_iommu_unmap(struct iommu_domain *domain,
4965 unsigned long iova, size_t size)
4966 {
4967 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4968 struct page *freelist = NULL;
4969 struct intel_iommu *iommu;
4970 unsigned long start_pfn, last_pfn;
4971 unsigned int npages;
4972 int iommu_id, level = 0;
4973
4974 /* Cope with horrid API which requires us to unmap more than the
4975 size argument if it happens to be a large-page mapping. */
4976 BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
4977
4978 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4979 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4980
4981 start_pfn = iova >> VTD_PAGE_SHIFT;
4982 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4983
4984 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4985
4986 npages = last_pfn - start_pfn + 1;
4987
4988 for_each_domain_iommu(iommu_id, dmar_domain) {
4989 iommu = g_iommus[iommu_id];
4990
4991 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
4992 start_pfn, npages, !freelist, 0);
4993 }
4994
4995 dma_free_pagelist(freelist);
4996
4997 if (dmar_domain->max_addr == iova + size)
4998 dmar_domain->max_addr = iova;
4999
5000 return size;
5001 }
5002
5003 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
5004 dma_addr_t iova)
5005 {
5006 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5007 struct dma_pte *pte;
5008 int level = 0;
5009 u64 phys = 0;
5010
5011 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
5012 if (pte)
5013 phys = dma_pte_addr(pte);
5014
5015 return phys;
5016 }
5017
5018 static bool intel_iommu_capable(enum iommu_cap cap)
5019 {
5020 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5021 return domain_update_iommu_snooping(NULL) == 1;
5022 if (cap == IOMMU_CAP_INTR_REMAP)
5023 return irq_remapping_enabled == 1;
5024
5025 return false;
5026 }
5027
5028 static int intel_iommu_add_device(struct device *dev)
5029 {
5030 struct intel_iommu *iommu;
5031 struct iommu_group *group;
5032 u8 bus, devfn;
5033
5034 iommu = device_to_iommu(dev, &bus, &devfn);
5035 if (!iommu)
5036 return -ENODEV;
5037
5038 iommu_device_link(iommu->iommu_dev, dev);
5039
5040 group = iommu_group_get_for_dev(dev);
5041
5042 if (IS_ERR(group))
5043 return PTR_ERR(group);
5044
5045 iommu_group_put(group);
5046 return 0;
5047 }
5048
5049 static void intel_iommu_remove_device(struct device *dev)
5050 {
5051 struct intel_iommu *iommu;
5052 u8 bus, devfn;
5053
5054 iommu = device_to_iommu(dev, &bus, &devfn);
5055 if (!iommu)
5056 return;
5057
5058 iommu_group_remove_device(dev);
5059
5060 iommu_device_unlink(iommu->iommu_dev, dev);
5061 }
5062
5063 #ifdef CONFIG_INTEL_IOMMU_SVM
5064 #define MAX_NR_PASID_BITS (20)
5065 static inline unsigned long intel_iommu_get_pts(struct intel_iommu *iommu)
5066 {
5067 /*
5068 * Convert ecap_pss to extend context entry pts encoding, also
5069 * respect the soft pasid_max value set by the iommu.
5070 * - number of PASID bits = ecap_pss + 1
5071 * - number of PASID table entries = 2^(pts + 5)
5072 * Therefore, pts = ecap_pss - 4
5073 * e.g. KBL ecap_pss = 0x13, PASID has 20 bits, pts = 15
5074 */
5075 if (ecap_pss(iommu->ecap) < 5)
5076 return 0;
5077
5078 /* pasid_max is encoded as actual number of entries not the bits */
5079 return find_first_bit((unsigned long *)&iommu->pasid_max,
5080 MAX_NR_PASID_BITS) - 5;
5081 }
5082
5083 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5084 {
5085 struct device_domain_info *info;
5086 struct context_entry *context;
5087 struct dmar_domain *domain;
5088 unsigned long flags;
5089 u64 ctx_lo;
5090 int ret;
5091
5092 domain = get_valid_domain_for_dev(sdev->dev);
5093 if (!domain)
5094 return -EINVAL;
5095
5096 spin_lock_irqsave(&device_domain_lock, flags);
5097 spin_lock(&iommu->lock);
5098
5099 ret = -EINVAL;
5100 info = sdev->dev->archdata.iommu;
5101 if (!info || !info->pasid_supported)
5102 goto out;
5103
5104 context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5105 if (WARN_ON(!context))
5106 goto out;
5107
5108 ctx_lo = context[0].lo;
5109
5110 sdev->did = domain->iommu_did[iommu->seq_id];
5111 sdev->sid = PCI_DEVID(info->bus, info->devfn);
5112
5113 if (!(ctx_lo & CONTEXT_PASIDE)) {
5114 context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
5115 context[1].lo = (u64)virt_to_phys(iommu->pasid_table) |
5116 intel_iommu_get_pts(iommu);
5117
5118 wmb();
5119 /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both
5120 * extended to permit requests-with-PASID if the PASIDE bit
5121 * is set. which makes sense. For CONTEXT_TT_PASS_THROUGH,
5122 * however, the PASIDE bit is ignored and requests-with-PASID
5123 * are unconditionally blocked. Which makes less sense.
5124 * So convert from CONTEXT_TT_PASS_THROUGH to one of the new
5125 * "guest mode" translation types depending on whether ATS
5126 * is available or not. Annoyingly, we can't use the new
5127 * modes *unless* PASIDE is set. */
5128 if ((ctx_lo & CONTEXT_TT_MASK) == (CONTEXT_TT_PASS_THROUGH << 2)) {
5129 ctx_lo &= ~CONTEXT_TT_MASK;
5130 if (info->ats_supported)
5131 ctx_lo |= CONTEXT_TT_PT_PASID_DEV_IOTLB << 2;
5132 else
5133 ctx_lo |= CONTEXT_TT_PT_PASID << 2;
5134 }
5135 ctx_lo |= CONTEXT_PASIDE;
5136 if (iommu->pasid_state_table)
5137 ctx_lo |= CONTEXT_DINVE;
5138 if (info->pri_supported)
5139 ctx_lo |= CONTEXT_PRS;
5140 context[0].lo = ctx_lo;
5141 wmb();
5142 iommu->flush.flush_context(iommu, sdev->did, sdev->sid,
5143 DMA_CCMD_MASK_NOBIT,
5144 DMA_CCMD_DEVICE_INVL);
5145 }
5146
5147 /* Enable PASID support in the device, if it wasn't already */
5148 if (!info->pasid_enabled)
5149 iommu_enable_dev_iotlb(info);
5150
5151 if (info->ats_enabled) {
5152 sdev->dev_iotlb = 1;
5153 sdev->qdep = info->ats_qdep;
5154 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5155 sdev->qdep = 0;
5156 }
5157 ret = 0;
5158
5159 out:
5160 spin_unlock(&iommu->lock);
5161 spin_unlock_irqrestore(&device_domain_lock, flags);
5162
5163 return ret;
5164 }
5165
5166 struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5167 {
5168 struct intel_iommu *iommu;
5169 u8 bus, devfn;
5170
5171 if (iommu_dummy(dev)) {
5172 dev_warn(dev,
5173 "No IOMMU translation for device; cannot enable SVM\n");
5174 return NULL;
5175 }
5176
5177 iommu = device_to_iommu(dev, &bus, &devfn);
5178 if ((!iommu)) {
5179 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
5180 return NULL;
5181 }
5182
5183 if (!iommu->pasid_table) {
5184 dev_err(dev, "PASID not enabled on IOMMU; cannot enable SVM\n");
5185 return NULL;
5186 }
5187
5188 return iommu;
5189 }
5190 #endif /* CONFIG_INTEL_IOMMU_SVM */
5191
5192 static const struct iommu_ops intel_iommu_ops = {
5193 .capable = intel_iommu_capable,
5194 .domain_alloc = intel_iommu_domain_alloc,
5195 .domain_free = intel_iommu_domain_free,
5196 .attach_dev = intel_iommu_attach_device,
5197 .detach_dev = intel_iommu_detach_device,
5198 .map = intel_iommu_map,
5199 .unmap = intel_iommu_unmap,
5200 .map_sg = default_iommu_map_sg,
5201 .iova_to_phys = intel_iommu_iova_to_phys,
5202 .add_device = intel_iommu_add_device,
5203 .remove_device = intel_iommu_remove_device,
5204 .device_group = pci_device_group,
5205 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
5206 };
5207
5208 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5209 {
5210 /* G4x/GM45 integrated gfx dmar support is totally busted. */
5211 pr_info("Disabling IOMMU for graphics on this chipset\n");
5212 dmar_map_gfx = 0;
5213 }
5214
5215 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5216 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5217 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5218 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5219 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5220 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5221 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5222
5223 static void quirk_iommu_rwbf(struct pci_dev *dev)
5224 {
5225 /*
5226 * Mobile 4 Series Chipset neglects to set RWBF capability,
5227 * but needs it. Same seems to hold for the desktop versions.
5228 */
5229 pr_info("Forcing write-buffer flush capability\n");
5230 rwbf_quirk = 1;
5231 }
5232
5233 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
5234 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5235 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5236 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5237 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5238 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5239 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
5240
5241 #define GGC 0x52
5242 #define GGC_MEMORY_SIZE_MASK (0xf << 8)
5243 #define GGC_MEMORY_SIZE_NONE (0x0 << 8)
5244 #define GGC_MEMORY_SIZE_1M (0x1 << 8)
5245 #define GGC_MEMORY_SIZE_2M (0x3 << 8)
5246 #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
5247 #define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
5248 #define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
5249 #define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
5250
5251 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
5252 {
5253 unsigned short ggc;
5254
5255 if (pci_read_config_word(dev, GGC, &ggc))
5256 return;
5257
5258 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
5259 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
5260 dmar_map_gfx = 0;
5261 } else if (dmar_map_gfx) {
5262 /* we have to ensure the gfx device is idle before we flush */
5263 pr_info("Disabling batched IOTLB flush on Ironlake\n");
5264 intel_iommu_strict = 1;
5265 }
5266 }
5267 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5268 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5269 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5270 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5271
5272 /* On Tylersburg chipsets, some BIOSes have been known to enable the
5273 ISOCH DMAR unit for the Azalia sound device, but not give it any
5274 TLB entries, which causes it to deadlock. Check for that. We do
5275 this in a function called from init_dmars(), instead of in a PCI
5276 quirk, because we don't want to print the obnoxious "BIOS broken"
5277 message if VT-d is actually disabled.
5278 */
5279 static void __init check_tylersburg_isoch(void)
5280 {
5281 struct pci_dev *pdev;
5282 uint32_t vtisochctrl;
5283
5284 /* If there's no Azalia in the system anyway, forget it. */
5285 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5286 if (!pdev)
5287 return;
5288 pci_dev_put(pdev);
5289
5290 /* System Management Registers. Might be hidden, in which case
5291 we can't do the sanity check. But that's OK, because the
5292 known-broken BIOSes _don't_ actually hide it, so far. */
5293 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5294 if (!pdev)
5295 return;
5296
5297 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5298 pci_dev_put(pdev);
5299 return;
5300 }
5301
5302 pci_dev_put(pdev);
5303
5304 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5305 if (vtisochctrl & 1)
5306 return;
5307
5308 /* Drop all bits other than the number of TLB entries */
5309 vtisochctrl &= 0x1c;
5310
5311 /* If we have the recommended number of TLB entries (16), fine. */
5312 if (vtisochctrl == 0x10)
5313 return;
5314
5315 /* Zero TLB entries? You get to ride the short bus to school. */
5316 if (!vtisochctrl) {
5317 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5318 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5319 dmi_get_system_info(DMI_BIOS_VENDOR),
5320 dmi_get_system_info(DMI_BIOS_VERSION),
5321 dmi_get_system_info(DMI_PRODUCT_VERSION));
5322 iommu_identity_mapping |= IDENTMAP_AZALIA;
5323 return;
5324 }
5325
5326 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
5327 vtisochctrl);
5328 }
Linux with added FPC-III support