?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
kernel/locking: Compute 'current' directly
[cris-mirror.git] / drivers / iommu / intel-iommu.c
blob8a185250ae5a5923d8ab9f34d811caa0f5e09b79
1 /*
2 * Copyright © 2006-2014 Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
18 * Joerg Roedel <jroedel@suse.de>
19 */
20
21 #define pr_fmt(fmt) "DMAR: " fmt
22
23 #include <linux/init.h>
24 #include <linux/bitmap.h>
25 #include <linux/debugfs.h>
26 #include <linux/export.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/memory.h>
36 #include <linux/cpu.h>
37 #include <linux/timer.h>
38 #include <linux/io.h>
39 #include <linux/iova.h>
40 #include <linux/iommu.h>
41 #include <linux/intel-iommu.h>
42 #include <linux/syscore_ops.h>
43 #include <linux/tboot.h>
44 #include <linux/dmi.h>
45 #include <linux/pci-ats.h>
46 #include <linux/memblock.h>
47 #include <linux/dma-contiguous.h>
48 #include <linux/crash_dump.h>
49 #include <asm/irq_remapping.h>
50 #include <asm/cacheflush.h>
51 #include <asm/iommu.h>
52
53 #include "irq_remapping.h"
54
55 #define ROOT_SIZE VTD_PAGE_SIZE
56 #define CONTEXT_SIZE VTD_PAGE_SIZE
57
58 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
59 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
60 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
61 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
62
63 #define IOAPIC_RANGE_START (0xfee00000)
64 #define IOAPIC_RANGE_END (0xfeefffff)
65 #define IOVA_START_ADDR (0x1000)
66
67 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
68
69 #define MAX_AGAW_WIDTH 64
70 #define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
71
72 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
73 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
74
75 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
76 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
77 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
78 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
79 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
80
81 /* IO virtual address start page frame number */
82 #define IOVA_START_PFN (1)
83
84 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
85 #define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
86 #define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
87
88 /* page table handling */
89 #define LEVEL_STRIDE (9)
90 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
91
92 /*
93 * This bitmap is used to advertise the page sizes our hardware support
94 * to the IOMMU core, which will then use this information to split
95 * physically contiguous memory regions it is mapping into page sizes
96 * that we support.
97 *
98 * Traditionally the IOMMU core just handed us the mappings directly,
99 * after making sure the size is an order of a 4KiB page and that the
100 * mapping has natural alignment.
101 *
102 * To retain this behavior, we currently advertise that we support
103 * all page sizes that are an order of 4KiB.
104 *
105 * If at some point we'd like to utilize the IOMMU core's new behavior,
106 * we could change this to advertise the real page sizes we support.
107 */
108 #define INTEL_IOMMU_PGSIZES (~0xFFFUL)
109
110 static inline int agaw_to_level(int agaw)
111 {
112 return agaw + 2;
113 }
114
115 static inline int agaw_to_width(int agaw)
116 {
117 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
118 }
119
120 static inline int width_to_agaw(int width)
121 {
122 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
123 }
124
125 static inline unsigned int level_to_offset_bits(int level)
126 {
127 return (level - 1) * LEVEL_STRIDE;
128 }
129
130 static inline int pfn_level_offset(unsigned long pfn, int level)
131 {
132 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
133 }
134
135 static inline unsigned long level_mask(int level)
136 {
137 return -1UL << level_to_offset_bits(level);
138 }
139
140 static inline unsigned long level_size(int level)
141 {
142 return 1UL << level_to_offset_bits(level);
143 }
144
145 static inline unsigned long align_to_level(unsigned long pfn, int level)
146 {
147 return (pfn + level_size(level) - 1) & level_mask(level);
148 }
149
150 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
151 {
152 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
153 }
154
155 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
156 are never going to work. */
157 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
158 {
159 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
160 }
161
162 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
163 {
164 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
165 }
166 static inline unsigned long page_to_dma_pfn(struct page *pg)
167 {
168 return mm_to_dma_pfn(page_to_pfn(pg));
169 }
170 static inline unsigned long virt_to_dma_pfn(void *p)
171 {
172 return page_to_dma_pfn(virt_to_page(p));
173 }
174
175 /* global iommu list, set NULL for ignored DMAR units */
176 static struct intel_iommu **g_iommus;
177
178 static void __init check_tylersburg_isoch(void);
179 static int rwbf_quirk;
180
181 /*
182 * set to 1 to panic kernel if can't successfully enable VT-d
183 * (used when kernel is launched w/ TXT)
184 */
185 static int force_on = 0;
186
187 /*
188 * 0: Present
189 * 1-11: Reserved
190 * 12-63: Context Ptr (12 - (haw-1))
191 * 64-127: Reserved
192 */
193 struct root_entry {
194 u64 lo;
195 u64 hi;
196 };
197 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
198
199 /*
200 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
201 * if marked present.
202 */
203 static phys_addr_t root_entry_lctp(struct root_entry *re)
204 {
205 if (!(re->lo & 1))
206 return 0;
207
208 return re->lo & VTD_PAGE_MASK;
209 }
210
211 /*
212 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
213 * if marked present.
214 */
215 static phys_addr_t root_entry_uctp(struct root_entry *re)
216 {
217 if (!(re->hi & 1))
218 return 0;
219
220 return re->hi & VTD_PAGE_MASK;
221 }
222 /*
223 * low 64 bits:
224 * 0: present
225 * 1: fault processing disable
226 * 2-3: translation type
227 * 12-63: address space root
228 * high 64 bits:
229 * 0-2: address width
230 * 3-6: aval
231 * 8-23: domain id
232 */
233 struct context_entry {
234 u64 lo;
235 u64 hi;
236 };
237
238 static inline void context_clear_pasid_enable(struct context_entry *context)
239 {
240 context->lo &= ~(1ULL << 11);
241 }
242
243 static inline bool context_pasid_enabled(struct context_entry *context)
244 {
245 return !!(context->lo & (1ULL << 11));
246 }
247
248 static inline void context_set_copied(struct context_entry *context)
249 {
250 context->hi |= (1ull << 3);
251 }
252
253 static inline bool context_copied(struct context_entry *context)
254 {
255 return !!(context->hi & (1ULL << 3));
256 }
257
258 static inline bool __context_present(struct context_entry *context)
259 {
260 return (context->lo & 1);
261 }
262
263 static inline bool context_present(struct context_entry *context)
264 {
265 return context_pasid_enabled(context) ?
266 __context_present(context) :
267 __context_present(context) && !context_copied(context);
268 }
269
270 static inline void context_set_present(struct context_entry *context)
271 {
272 context->lo |= 1;
273 }
274
275 static inline void context_set_fault_enable(struct context_entry *context)
276 {
277 context->lo &= (((u64)-1) << 2) | 1;
278 }
279
280 static inline void context_set_translation_type(struct context_entry *context,
281 unsigned long value)
282 {
283 context->lo &= (((u64)-1) << 4) | 3;
284 context->lo |= (value & 3) << 2;
285 }
286
287 static inline void context_set_address_root(struct context_entry *context,
288 unsigned long value)
289 {
290 context->lo &= ~VTD_PAGE_MASK;
291 context->lo |= value & VTD_PAGE_MASK;
292 }
293
294 static inline void context_set_address_width(struct context_entry *context,
295 unsigned long value)
296 {
297 context->hi |= value & 7;
298 }
299
300 static inline void context_set_domain_id(struct context_entry *context,
301 unsigned long value)
302 {
303 context->hi |= (value & ((1 << 16) - 1)) << 8;
304 }
305
306 static inline int context_domain_id(struct context_entry *c)
307 {
308 return((c->hi >> 8) & 0xffff);
309 }
310
311 static inline void context_clear_entry(struct context_entry *context)
312 {
313 context->lo = 0;
314 context->hi = 0;
315 }
316
317 /*
318 * 0: readable
319 * 1: writable
320 * 2-6: reserved
321 * 7: super page
322 * 8-10: available
323 * 11: snoop behavior
324 * 12-63: Host physcial address
325 */
326 struct dma_pte {
327 u64 val;
328 };
329
330 static inline void dma_clear_pte(struct dma_pte *pte)
331 {
332 pte->val = 0;
333 }
334
335 static inline u64 dma_pte_addr(struct dma_pte *pte)
336 {
337 #ifdef CONFIG_64BIT
338 return pte->val & VTD_PAGE_MASK;
339 #else
340 /* Must have a full atomic 64-bit read */
341 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
342 #endif
343 }
344
345 static inline bool dma_pte_present(struct dma_pte *pte)
346 {
347 return (pte->val & 3) != 0;
348 }
349
350 static inline bool dma_pte_superpage(struct dma_pte *pte)
351 {
352 return (pte->val & DMA_PTE_LARGE_PAGE);
353 }
354
355 static inline int first_pte_in_page(struct dma_pte *pte)
356 {
357 return !((unsigned long)pte & ~VTD_PAGE_MASK);
358 }
359
360 /*
361 * This domain is a statically identity mapping domain.
362 * 1. This domain creats a static 1:1 mapping to all usable memory.
363 * 2. It maps to each iommu if successful.
364 * 3. Each iommu mapps to this domain if successful.
365 */
366 static struct dmar_domain *si_domain;
367 static int hw_pass_through = 1;
368
369 /*
370 * Domain represents a virtual machine, more than one devices
371 * across iommus may be owned in one domain, e.g. kvm guest.
372 */
373 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
374
375 /* si_domain contains mulitple devices */
376 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
377
378 #define for_each_domain_iommu(idx, domain) \
379 for (idx = 0; idx < g_num_of_iommus; idx++) \
380 if (domain->iommu_refcnt[idx])
381
382 struct dmar_domain {
383 int nid; /* node id */
384
385 unsigned iommu_refcnt[DMAR_UNITS_SUPPORTED];
386 /* Refcount of devices per iommu */
387
388
389 u16 iommu_did[DMAR_UNITS_SUPPORTED];
390 /* Domain ids per IOMMU. Use u16 since
391 * domain ids are 16 bit wide according
392 * to VT-d spec, section 9.3 */
393
394 bool has_iotlb_device;
395 struct list_head devices; /* all devices' list */
396 struct iova_domain iovad; /* iova's that belong to this domain */
397
398 struct dma_pte *pgd; /* virtual address */
399 int gaw; /* max guest address width */
400
401 /* adjusted guest address width, 0 is level 2 30-bit */
402 int agaw;
403
404 int flags; /* flags to find out type of domain */
405
406 int iommu_coherency;/* indicate coherency of iommu access */
407 int iommu_snooping; /* indicate snooping control feature*/
408 int iommu_count; /* reference count of iommu */
409 int iommu_superpage;/* Level of superpages supported:
410 0 == 4KiB (no superpages), 1 == 2MiB,
411 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
412 u64 max_addr; /* maximum mapped address */
413
414 struct iommu_domain domain; /* generic domain data structure for
415 iommu core */
416 };
417
418 /* PCI domain-device relationship */
419 struct device_domain_info {
420 struct list_head link; /* link to domain siblings */
421 struct list_head global; /* link to global list */
422 u8 bus; /* PCI bus number */
423 u8 devfn; /* PCI devfn number */
424 u8 pasid_supported:3;
425 u8 pasid_enabled:1;
426 u8 pri_supported:1;
427 u8 pri_enabled:1;
428 u8 ats_supported:1;
429 u8 ats_enabled:1;
430 u8 ats_qdep;
431 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
432 struct intel_iommu *iommu; /* IOMMU used by this device */
433 struct dmar_domain *domain; /* pointer to domain */
434 };
435
436 struct dmar_rmrr_unit {
437 struct list_head list; /* list of rmrr units */
438 struct acpi_dmar_header *hdr; /* ACPI header */
439 u64 base_address; /* reserved base address*/
440 u64 end_address; /* reserved end address */
441 struct dmar_dev_scope *devices; /* target devices */
442 int devices_cnt; /* target device count */
443 };
444
445 struct dmar_atsr_unit {
446 struct list_head list; /* list of ATSR units */
447 struct acpi_dmar_header *hdr; /* ACPI header */
448 struct dmar_dev_scope *devices; /* target devices */
449 int devices_cnt; /* target device count */
450 u8 include_all:1; /* include all ports */
451 };
452
453 static LIST_HEAD(dmar_atsr_units);
454 static LIST_HEAD(dmar_rmrr_units);
455
456 #define for_each_rmrr_units(rmrr) \
457 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
458
459 static void flush_unmaps_timeout(unsigned long data);
460
461 struct deferred_flush_entry {
462 unsigned long iova_pfn;
463 unsigned long nrpages;
464 struct dmar_domain *domain;
465 struct page *freelist;
466 };
467
468 #define HIGH_WATER_MARK 250
469 struct deferred_flush_table {
470 int next;
471 struct deferred_flush_entry entries[HIGH_WATER_MARK];
472 };
473
474 struct deferred_flush_data {
475 spinlock_t lock;
476 int timer_on;
477 struct timer_list timer;
478 long size;
479 struct deferred_flush_table *tables;
480 };
481
482 DEFINE_PER_CPU(struct deferred_flush_data, deferred_flush);
483
484 /* bitmap for indexing intel_iommus */
485 static int g_num_of_iommus;
486
487 static void domain_exit(struct dmar_domain *domain);
488 static void domain_remove_dev_info(struct dmar_domain *domain);
489 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
490 struct device *dev);
491 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
492 static void domain_context_clear(struct intel_iommu *iommu,
493 struct device *dev);
494 static int domain_detach_iommu(struct dmar_domain *domain,
495 struct intel_iommu *iommu);
496
497 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
498 int dmar_disabled = 0;
499 #else
500 int dmar_disabled = 1;
501 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
502
503 int intel_iommu_enabled = 0;
504 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
505
506 static int dmar_map_gfx = 1;
507 static int dmar_forcedac;
508 static int intel_iommu_strict;
509 static int intel_iommu_superpage = 1;
510 static int intel_iommu_ecs = 1;
511 static int intel_iommu_pasid28;
512 static int iommu_identity_mapping;
513
514 #define IDENTMAP_ALL 1
515 #define IDENTMAP_GFX 2
516 #define IDENTMAP_AZALIA 4
517
518 /* Broadwell and Skylake have broken ECS support — normal so-called "second
519 * level" translation of DMA requests-without-PASID doesn't actually happen
520 * unless you also set the NESTE bit in an extended context-entry. Which of
521 * course means that SVM doesn't work because it's trying to do nested
522 * translation of the physical addresses it finds in the process page tables,
523 * through the IOVA->phys mapping found in the "second level" page tables.
524 *
525 * The VT-d specification was retroactively changed to change the definition
526 * of the capability bits and pretend that Broadwell/Skylake never happened...
527 * but unfortunately the wrong bit was changed. It's ECS which is broken, but
528 * for some reason it was the PASID capability bit which was redefined (from
529 * bit 28 on BDW/SKL to bit 40 in future).
530 *
531 * So our test for ECS needs to eschew those implementations which set the old
532 * PASID capabiity bit 28, since those are the ones on which ECS is broken.
533 * Unless we are working around the 'pasid28' limitations, that is, by putting
534 * the device into passthrough mode for normal DMA and thus masking the bug.
535 */
536 #define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
537 (intel_iommu_pasid28 || !ecap_broken_pasid(iommu->ecap)))
538 /* PASID support is thus enabled if ECS is enabled and *either* of the old
539 * or new capability bits are set. */
540 #define pasid_enabled(iommu) (ecs_enabled(iommu) && \
541 (ecap_pasid(iommu->ecap) || ecap_broken_pasid(iommu->ecap)))
542
543 int intel_iommu_gfx_mapped;
544 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
545
546 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
547 static DEFINE_SPINLOCK(device_domain_lock);
548 static LIST_HEAD(device_domain_list);
549
550 static const struct iommu_ops intel_iommu_ops;
551
552 static bool translation_pre_enabled(struct intel_iommu *iommu)
553 {
554 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
555 }
556
557 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
558 {
559 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
560 }
561
562 static void init_translation_status(struct intel_iommu *iommu)
563 {
564 u32 gsts;
565
566 gsts = readl(iommu->reg + DMAR_GSTS_REG);
567 if (gsts & DMA_GSTS_TES)
568 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
569 }
570
571 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
572 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
573 {
574 return container_of(dom, struct dmar_domain, domain);
575 }
576
577 static int __init intel_iommu_setup(char *str)
578 {
579 if (!str)
580 return -EINVAL;
581 while (*str) {
582 if (!strncmp(str, "on", 2)) {
583 dmar_disabled = 0;
584 pr_info("IOMMU enabled\n");
585 } else if (!strncmp(str, "off", 3)) {
586 dmar_disabled = 1;
587 pr_info("IOMMU disabled\n");
588 } else if (!strncmp(str, "igfx_off", 8)) {
589 dmar_map_gfx = 0;
590 pr_info("Disable GFX device mapping\n");
591 } else if (!strncmp(str, "forcedac", 8)) {
592 pr_info("Forcing DAC for PCI devices\n");
593 dmar_forcedac = 1;
594 } else if (!strncmp(str, "strict", 6)) {
595 pr_info("Disable batched IOTLB flush\n");
596 intel_iommu_strict = 1;
597 } else if (!strncmp(str, "sp_off", 6)) {
598 pr_info("Disable supported super page\n");
599 intel_iommu_superpage = 0;
600 } else if (!strncmp(str, "ecs_off", 7)) {
601 printk(KERN_INFO
602 "Intel-IOMMU: disable extended context table support\n");
603 intel_iommu_ecs = 0;
604 } else if (!strncmp(str, "pasid28", 7)) {
605 printk(KERN_INFO
606 "Intel-IOMMU: enable pre-production PASID support\n");
607 intel_iommu_pasid28 = 1;
608 iommu_identity_mapping |= IDENTMAP_GFX;
609 }
610
611 str += strcspn(str, ",");
612 while (*str == ',')
613 str++;
614 }
615 return 0;
616 }
617 __setup("intel_iommu=", intel_iommu_setup);
618
619 static struct kmem_cache *iommu_domain_cache;
620 static struct kmem_cache *iommu_devinfo_cache;
621
622 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
623 {
624 struct dmar_domain **domains;
625 int idx = did >> 8;
626
627 domains = iommu->domains[idx];
628 if (!domains)
629 return NULL;
630
631 return domains[did & 0xff];
632 }
633
634 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
635 struct dmar_domain *domain)
636 {
637 struct dmar_domain **domains;
638 int idx = did >> 8;
639
640 if (!iommu->domains[idx]) {
641 size_t size = 256 * sizeof(struct dmar_domain *);
642 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
643 }
644
645 domains = iommu->domains[idx];
646 if (WARN_ON(!domains))
647 return;
648 else
649 domains[did & 0xff] = domain;
650 }
651
652 static inline void *alloc_pgtable_page(int node)
653 {
654 struct page *page;
655 void *vaddr = NULL;
656
657 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
658 if (page)
659 vaddr = page_address(page);
660 return vaddr;
661 }
662
663 static inline void free_pgtable_page(void *vaddr)
664 {
665 free_page((unsigned long)vaddr);
666 }
667
668 static inline void *alloc_domain_mem(void)
669 {
670 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
671 }
672
673 static void free_domain_mem(void *vaddr)
674 {
675 kmem_cache_free(iommu_domain_cache, vaddr);
676 }
677
678 static inline void * alloc_devinfo_mem(void)
679 {
680 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
681 }
682
683 static inline void free_devinfo_mem(void *vaddr)
684 {
685 kmem_cache_free(iommu_devinfo_cache, vaddr);
686 }
687
688 static inline int domain_type_is_vm(struct dmar_domain *domain)
689 {
690 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
691 }
692
693 static inline int domain_type_is_si(struct dmar_domain *domain)
694 {
695 return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
696 }
697
698 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
699 {
700 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
701 DOMAIN_FLAG_STATIC_IDENTITY);
702 }
703
704 static inline int domain_pfn_supported(struct dmar_domain *domain,
705 unsigned long pfn)
706 {
707 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
708
709 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
710 }
711
712 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
713 {
714 unsigned long sagaw;
715 int agaw = -1;
716
717 sagaw = cap_sagaw(iommu->cap);
718 for (agaw = width_to_agaw(max_gaw);
719 agaw >= 0; agaw--) {
720 if (test_bit(agaw, &sagaw))
721 break;
722 }
723
724 return agaw;
725 }
726
727 /*
728 * Calculate max SAGAW for each iommu.
729 */
730 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
731 {
732 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
733 }
734
735 /*
736 * calculate agaw for each iommu.
737 * "SAGAW" may be different across iommus, use a default agaw, and
738 * get a supported less agaw for iommus that don't support the default agaw.
739 */
740 int iommu_calculate_agaw(struct intel_iommu *iommu)
741 {
742 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
743 }
744
745 /* This functionin only returns single iommu in a domain */
746 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
747 {
748 int iommu_id;
749
750 /* si_domain and vm domain should not get here. */
751 BUG_ON(domain_type_is_vm_or_si(domain));
752 for_each_domain_iommu(iommu_id, domain)
753 break;
754
755 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
756 return NULL;
757
758 return g_iommus[iommu_id];
759 }
760
761 static void domain_update_iommu_coherency(struct dmar_domain *domain)
762 {
763 struct dmar_drhd_unit *drhd;
764 struct intel_iommu *iommu;
765 bool found = false;
766 int i;
767
768 domain->iommu_coherency = 1;
769
770 for_each_domain_iommu(i, domain) {
771 found = true;
772 if (!ecap_coherent(g_iommus[i]->ecap)) {
773 domain->iommu_coherency = 0;
774 break;
775 }
776 }
777 if (found)
778 return;
779
780 /* No hardware attached; use lowest common denominator */
781 rcu_read_lock();
782 for_each_active_iommu(iommu, drhd) {
783 if (!ecap_coherent(iommu->ecap)) {
784 domain->iommu_coherency = 0;
785 break;
786 }
787 }
788 rcu_read_unlock();
789 }
790
791 static int domain_update_iommu_snooping(struct intel_iommu *skip)
792 {
793 struct dmar_drhd_unit *drhd;
794 struct intel_iommu *iommu;
795 int ret = 1;
796
797 rcu_read_lock();
798 for_each_active_iommu(iommu, drhd) {
799 if (iommu != skip) {
800 if (!ecap_sc_support(iommu->ecap)) {
801 ret = 0;
802 break;
803 }
804 }
805 }
806 rcu_read_unlock();
807
808 return ret;
809 }
810
811 static int domain_update_iommu_superpage(struct intel_iommu *skip)
812 {
813 struct dmar_drhd_unit *drhd;
814 struct intel_iommu *iommu;
815 int mask = 0xf;
816
817 if (!intel_iommu_superpage) {
818 return 0;
819 }
820
821 /* set iommu_superpage to the smallest common denominator */
822 rcu_read_lock();
823 for_each_active_iommu(iommu, drhd) {
824 if (iommu != skip) {
825 mask &= cap_super_page_val(iommu->cap);
826 if (!mask)
827 break;
828 }
829 }
830 rcu_read_unlock();
831
832 return fls(mask);
833 }
834
835 /* Some capabilities may be different across iommus */
836 static void domain_update_iommu_cap(struct dmar_domain *domain)
837 {
838 domain_update_iommu_coherency(domain);
839 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
840 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
841 }
842
843 static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
844 u8 bus, u8 devfn, int alloc)
845 {
846 struct root_entry *root = &iommu->root_entry[bus];
847 struct context_entry *context;
848 u64 *entry;
849
850 entry = &root->lo;
851 if (ecs_enabled(iommu)) {
852 if (devfn >= 0x80) {
853 devfn -= 0x80;
854 entry = &root->hi;
855 }
856 devfn *= 2;
857 }
858 if (*entry & 1)
859 context = phys_to_virt(*entry & VTD_PAGE_MASK);
860 else {
861 unsigned long phy_addr;
862 if (!alloc)
863 return NULL;
864
865 context = alloc_pgtable_page(iommu->node);
866 if (!context)
867 return NULL;
868
869 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
870 phy_addr = virt_to_phys((void *)context);
871 *entry = phy_addr | 1;
872 __iommu_flush_cache(iommu, entry, sizeof(*entry));
873 }
874 return &context[devfn];
875 }
876
877 static int iommu_dummy(struct device *dev)
878 {
879 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
880 }
881
882 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
883 {
884 struct dmar_drhd_unit *drhd = NULL;
885 struct intel_iommu *iommu;
886 struct device *tmp;
887 struct pci_dev *ptmp, *pdev = NULL;
888 u16 segment = 0;
889 int i;
890
891 if (iommu_dummy(dev))
892 return NULL;
893
894 if (dev_is_pci(dev)) {
895 struct pci_dev *pf_pdev;
896
897 pdev = to_pci_dev(dev);
898 /* VFs aren't listed in scope tables; we need to look up
899 * the PF instead to find the IOMMU. */
900 pf_pdev = pci_physfn(pdev);
901 dev = &pf_pdev->dev;
902 segment = pci_domain_nr(pdev->bus);
903 } else if (has_acpi_companion(dev))
904 dev = &ACPI_COMPANION(dev)->dev;
905
906 rcu_read_lock();
907 for_each_active_iommu(iommu, drhd) {
908 if (pdev && segment != drhd->segment)
909 continue;
910
911 for_each_active_dev_scope(drhd->devices,
912 drhd->devices_cnt, i, tmp) {
913 if (tmp == dev) {
914 /* For a VF use its original BDF# not that of the PF
915 * which we used for the IOMMU lookup. Strictly speaking
916 * we could do this for all PCI devices; we only need to
917 * get the BDF# from the scope table for ACPI matches. */
918 if (pdev->is_virtfn)
919 goto got_pdev;
920
921 *bus = drhd->devices[i].bus;
922 *devfn = drhd->devices[i].devfn;
923 goto out;
924 }
925
926 if (!pdev || !dev_is_pci(tmp))
927 continue;
928
929 ptmp = to_pci_dev(tmp);
930 if (ptmp->subordinate &&
931 ptmp->subordinate->number <= pdev->bus->number &&
932 ptmp->subordinate->busn_res.end >= pdev->bus->number)
933 goto got_pdev;
934 }
935
936 if (pdev && drhd->include_all) {
937 got_pdev:
938 *bus = pdev->bus->number;
939 *devfn = pdev->devfn;
940 goto out;
941 }
942 }
943 iommu = NULL;
944 out:
945 rcu_read_unlock();
946
947 return iommu;
948 }
949
950 static void domain_flush_cache(struct dmar_domain *domain,
951 void *addr, int size)
952 {
953 if (!domain->iommu_coherency)
954 clflush_cache_range(addr, size);
955 }
956
957 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
958 {
959 struct context_entry *context;
960 int ret = 0;
961 unsigned long flags;
962
963 spin_lock_irqsave(&iommu->lock, flags);
964 context = iommu_context_addr(iommu, bus, devfn, 0);
965 if (context)
966 ret = context_present(context);
967 spin_unlock_irqrestore(&iommu->lock, flags);
968 return ret;
969 }
970
971 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
972 {
973 struct context_entry *context;
974 unsigned long flags;
975
976 spin_lock_irqsave(&iommu->lock, flags);
977 context = iommu_context_addr(iommu, bus, devfn, 0);
978 if (context) {
979 context_clear_entry(context);
980 __iommu_flush_cache(iommu, context, sizeof(*context));
981 }
982 spin_unlock_irqrestore(&iommu->lock, flags);
983 }
984
985 static void free_context_table(struct intel_iommu *iommu)
986 {
987 int i;
988 unsigned long flags;
989 struct context_entry *context;
990
991 spin_lock_irqsave(&iommu->lock, flags);
992 if (!iommu->root_entry) {
993 goto out;
994 }
995 for (i = 0; i < ROOT_ENTRY_NR; i++) {
996 context = iommu_context_addr(iommu, i, 0, 0);
997 if (context)
998 free_pgtable_page(context);
999
1000 if (!ecs_enabled(iommu))
1001 continue;
1002
1003 context = iommu_context_addr(iommu, i, 0x80, 0);
1004 if (context)
1005 free_pgtable_page(context);
1006
1007 }
1008 free_pgtable_page(iommu->root_entry);
1009 iommu->root_entry = NULL;
1010 out:
1011 spin_unlock_irqrestore(&iommu->lock, flags);
1012 }
1013
1014 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
1015 unsigned long pfn, int *target_level)
1016 {
1017 struct dma_pte *parent, *pte = NULL;
1018 int level = agaw_to_level(domain->agaw);
1019 int offset;
1020
1021 BUG_ON(!domain->pgd);
1022
1023 if (!domain_pfn_supported(domain, pfn))
1024 /* Address beyond IOMMU's addressing capabilities. */
1025 return NULL;
1026
1027 parent = domain->pgd;
1028
1029 while (1) {
1030 void *tmp_page;
1031
1032 offset = pfn_level_offset(pfn, level);
1033 pte = &parent[offset];
1034 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
1035 break;
1036 if (level == *target_level)
1037 break;
1038
1039 if (!dma_pte_present(pte)) {
1040 uint64_t pteval;
1041
1042 tmp_page = alloc_pgtable_page(domain->nid);
1043
1044 if (!tmp_page)
1045 return NULL;
1046
1047 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
1048 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
1049 if (cmpxchg64(&pte->val, 0ULL, pteval))
1050 /* Someone else set it while we were thinking; use theirs. */
1051 free_pgtable_page(tmp_page);
1052 else
1053 domain_flush_cache(domain, pte, sizeof(*pte));
1054 }
1055 if (level == 1)
1056 break;
1057
1058 parent = phys_to_virt(dma_pte_addr(pte));
1059 level--;
1060 }
1061
1062 if (!*target_level)
1063 *target_level = level;
1064
1065 return pte;
1066 }
1067
1068
1069 /* return address's pte at specific level */
1070 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1071 unsigned long pfn,
1072 int level, int *large_page)
1073 {
1074 struct dma_pte *parent, *pte = NULL;
1075 int total = agaw_to_level(domain->agaw);
1076 int offset;
1077
1078 parent = domain->pgd;
1079 while (level <= total) {
1080 offset = pfn_level_offset(pfn, total);
1081 pte = &parent[offset];
1082 if (level == total)
1083 return pte;
1084
1085 if (!dma_pte_present(pte)) {
1086 *large_page = total;
1087 break;
1088 }
1089
1090 if (dma_pte_superpage(pte)) {
1091 *large_page = total;
1092 return pte;
1093 }
1094
1095 parent = phys_to_virt(dma_pte_addr(pte));
1096 total--;
1097 }
1098 return NULL;
1099 }
1100
1101 /* clear last level pte, a tlb flush should be followed */
1102 static void dma_pte_clear_range(struct dmar_domain *domain,
1103 unsigned long start_pfn,
1104 unsigned long last_pfn)
1105 {
1106 unsigned int large_page = 1;
1107 struct dma_pte *first_pte, *pte;
1108
1109 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1110 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1111 BUG_ON(start_pfn > last_pfn);
1112
1113 /* we don't need lock here; nobody else touches the iova range */
1114 do {
1115 large_page = 1;
1116 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
1117 if (!pte) {
1118 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
1119 continue;
1120 }
1121 do {
1122 dma_clear_pte(pte);
1123 start_pfn += lvl_to_nr_pages(large_page);
1124 pte++;
1125 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1126
1127 domain_flush_cache(domain, first_pte,
1128 (void *)pte - (void *)first_pte);
1129
1130 } while (start_pfn && start_pfn <= last_pfn);
1131 }
1132
1133 static void dma_pte_free_level(struct dmar_domain *domain, int level,
1134 struct dma_pte *pte, unsigned long pfn,
1135 unsigned long start_pfn, unsigned long last_pfn)
1136 {
1137 pfn = max(start_pfn, pfn);
1138 pte = &pte[pfn_level_offset(pfn, level)];
1139
1140 do {
1141 unsigned long level_pfn;
1142 struct dma_pte *level_pte;
1143
1144 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1145 goto next;
1146
1147 level_pfn = pfn & level_mask(level - 1);
1148 level_pte = phys_to_virt(dma_pte_addr(pte));
1149
1150 if (level > 2)
1151 dma_pte_free_level(domain, level - 1, level_pte,
1152 level_pfn, start_pfn, last_pfn);
1153
1154 /* If range covers entire pagetable, free it */
1155 if (!(start_pfn > level_pfn ||
1156 last_pfn < level_pfn + level_size(level) - 1)) {
1157 dma_clear_pte(pte);
1158 domain_flush_cache(domain, pte, sizeof(*pte));
1159 free_pgtable_page(level_pte);
1160 }
1161 next:
1162 pfn += level_size(level);
1163 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1164 }
1165
1166 /* clear last level (leaf) ptes and free page table pages. */
1167 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1168 unsigned long start_pfn,
1169 unsigned long last_pfn)
1170 {
1171 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1172 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1173 BUG_ON(start_pfn > last_pfn);
1174
1175 dma_pte_clear_range(domain, start_pfn, last_pfn);
1176
1177 /* We don't need lock here; nobody else touches the iova range */
1178 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1179 domain->pgd, 0, start_pfn, last_pfn);
1180
1181 /* free pgd */
1182 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1183 free_pgtable_page(domain->pgd);
1184 domain->pgd = NULL;
1185 }
1186 }
1187
1188 /* When a page at a given level is being unlinked from its parent, we don't
1189 need to *modify* it at all. All we need to do is make a list of all the
1190 pages which can be freed just as soon as we've flushed the IOTLB and we
1191 know the hardware page-walk will no longer touch them.
1192 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1193 be freed. */
1194 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1195 int level, struct dma_pte *pte,
1196 struct page *freelist)
1197 {
1198 struct page *pg;
1199
1200 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1201 pg->freelist = freelist;
1202 freelist = pg;
1203
1204 if (level == 1)
1205 return freelist;
1206
1207 pte = page_address(pg);
1208 do {
1209 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1210 freelist = dma_pte_list_pagetables(domain, level - 1,
1211 pte, freelist);
1212 pte++;
1213 } while (!first_pte_in_page(pte));
1214
1215 return freelist;
1216 }
1217
1218 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1219 struct dma_pte *pte, unsigned long pfn,
1220 unsigned long start_pfn,
1221 unsigned long last_pfn,
1222 struct page *freelist)
1223 {
1224 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1225
1226 pfn = max(start_pfn, pfn);
1227 pte = &pte[pfn_level_offset(pfn, level)];
1228
1229 do {
1230 unsigned long level_pfn;
1231
1232 if (!dma_pte_present(pte))
1233 goto next;
1234
1235 level_pfn = pfn & level_mask(level);
1236
1237 /* If range covers entire pagetable, free it */
1238 if (start_pfn <= level_pfn &&
1239 last_pfn >= level_pfn + level_size(level) - 1) {
1240 /* These suborbinate page tables are going away entirely. Don't
1241 bother to clear them; we're just going to *free* them. */
1242 if (level > 1 && !dma_pte_superpage(pte))
1243 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1244
1245 dma_clear_pte(pte);
1246 if (!first_pte)
1247 first_pte = pte;
1248 last_pte = pte;
1249 } else if (level > 1) {
1250 /* Recurse down into a level that isn't *entirely* obsolete */
1251 freelist = dma_pte_clear_level(domain, level - 1,
1252 phys_to_virt(dma_pte_addr(pte)),
1253 level_pfn, start_pfn, last_pfn,
1254 freelist);
1255 }
1256 next:
1257 pfn += level_size(level);
1258 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1259
1260 if (first_pte)
1261 domain_flush_cache(domain, first_pte,
1262 (void *)++last_pte - (void *)first_pte);
1263
1264 return freelist;
1265 }
1266
1267 /* We can't just free the pages because the IOMMU may still be walking
1268 the page tables, and may have cached the intermediate levels. The
1269 pages can only be freed after the IOTLB flush has been done. */
1270 static struct page *domain_unmap(struct dmar_domain *domain,
1271 unsigned long start_pfn,
1272 unsigned long last_pfn)
1273 {
1274 struct page *freelist = NULL;
1275
1276 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1277 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1278 BUG_ON(start_pfn > last_pfn);
1279
1280 /* we don't need lock here; nobody else touches the iova range */
1281 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1282 domain->pgd, 0, start_pfn, last_pfn, NULL);
1283
1284 /* free pgd */
1285 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1286 struct page *pgd_page = virt_to_page(domain->pgd);
1287 pgd_page->freelist = freelist;
1288 freelist = pgd_page;
1289
1290 domain->pgd = NULL;
1291 }
1292
1293 return freelist;
1294 }
1295
1296 static void dma_free_pagelist(struct page *freelist)
1297 {
1298 struct page *pg;
1299
1300 while ((pg = freelist)) {
1301 freelist = pg->freelist;
1302 free_pgtable_page(page_address(pg));
1303 }
1304 }
1305
1306 /* iommu handling */
1307 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1308 {
1309 struct root_entry *root;
1310 unsigned long flags;
1311
1312 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1313 if (!root) {
1314 pr_err("Allocating root entry for %s failed\n",
1315 iommu->name);
1316 return -ENOMEM;
1317 }
1318
1319 __iommu_flush_cache(iommu, root, ROOT_SIZE);
1320
1321 spin_lock_irqsave(&iommu->lock, flags);
1322 iommu->root_entry = root;
1323 spin_unlock_irqrestore(&iommu->lock, flags);
1324
1325 return 0;
1326 }
1327
1328 static void iommu_set_root_entry(struct intel_iommu *iommu)
1329 {
1330 u64 addr;
1331 u32 sts;
1332 unsigned long flag;
1333
1334 addr = virt_to_phys(iommu->root_entry);
1335 if (ecs_enabled(iommu))
1336 addr |= DMA_RTADDR_RTT;
1337
1338 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1339 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1340
1341 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1342
1343 /* Make sure hardware complete it */
1344 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1345 readl, (sts & DMA_GSTS_RTPS), sts);
1346
1347 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1348 }
1349
1350 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1351 {
1352 u32 val;
1353 unsigned long flag;
1354
1355 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1356 return;
1357
1358 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1359 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1360
1361 /* Make sure hardware complete it */
1362 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1363 readl, (!(val & DMA_GSTS_WBFS)), val);
1364
1365 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1366 }
1367
1368 /* return value determine if we need a write buffer flush */
1369 static void __iommu_flush_context(struct intel_iommu *iommu,
1370 u16 did, u16 source_id, u8 function_mask,
1371 u64 type)
1372 {
1373 u64 val = 0;
1374 unsigned long flag;
1375
1376 switch (type) {
1377 case DMA_CCMD_GLOBAL_INVL:
1378 val = DMA_CCMD_GLOBAL_INVL;
1379 break;
1380 case DMA_CCMD_DOMAIN_INVL:
1381 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1382 break;
1383 case DMA_CCMD_DEVICE_INVL:
1384 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1385 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1386 break;
1387 default:
1388 BUG();
1389 }
1390 val |= DMA_CCMD_ICC;
1391
1392 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1393 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1394
1395 /* Make sure hardware complete it */
1396 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1397 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1398
1399 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1400 }
1401
1402 /* return value determine if we need a write buffer flush */
1403 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1404 u64 addr, unsigned int size_order, u64 type)
1405 {
1406 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1407 u64 val = 0, val_iva = 0;
1408 unsigned long flag;
1409
1410 switch (type) {
1411 case DMA_TLB_GLOBAL_FLUSH:
1412 /* global flush doesn't need set IVA_REG */
1413 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1414 break;
1415 case DMA_TLB_DSI_FLUSH:
1416 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1417 break;
1418 case DMA_TLB_PSI_FLUSH:
1419 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1420 /* IH bit is passed in as part of address */
1421 val_iva = size_order | addr;
1422 break;
1423 default:
1424 BUG();
1425 }
1426 /* Note: set drain read/write */
1427 #if 0
1428 /*
1429 * This is probably to be super secure.. Looks like we can
1430 * ignore it without any impact.
1431 */
1432 if (cap_read_drain(iommu->cap))
1433 val |= DMA_TLB_READ_DRAIN;
1434 #endif
1435 if (cap_write_drain(iommu->cap))
1436 val |= DMA_TLB_WRITE_DRAIN;
1437
1438 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1439 /* Note: Only uses first TLB reg currently */
1440 if (val_iva)
1441 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1442 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1443
1444 /* Make sure hardware complete it */
1445 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1446 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1447
1448 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1449
1450 /* check IOTLB invalidation granularity */
1451 if (DMA_TLB_IAIG(val) == 0)
1452 pr_err("Flush IOTLB failed\n");
1453 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1454 pr_debug("TLB flush request %Lx, actual %Lx\n",
1455 (unsigned long long)DMA_TLB_IIRG(type),
1456 (unsigned long long)DMA_TLB_IAIG(val));
1457 }
1458
1459 static struct device_domain_info *
1460 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1461 u8 bus, u8 devfn)
1462 {
1463 struct device_domain_info *info;
1464
1465 assert_spin_locked(&device_domain_lock);
1466
1467 if (!iommu->qi)
1468 return NULL;
1469
1470 list_for_each_entry(info, &domain->devices, link)
1471 if (info->iommu == iommu && info->bus == bus &&
1472 info->devfn == devfn) {
1473 if (info->ats_supported && info->dev)
1474 return info;
1475 break;
1476 }
1477
1478 return NULL;
1479 }
1480
1481 static void domain_update_iotlb(struct dmar_domain *domain)
1482 {
1483 struct device_domain_info *info;
1484 bool has_iotlb_device = false;
1485
1486 assert_spin_locked(&device_domain_lock);
1487
1488 list_for_each_entry(info, &domain->devices, link) {
1489 struct pci_dev *pdev;
1490
1491 if (!info->dev || !dev_is_pci(info->dev))
1492 continue;
1493
1494 pdev = to_pci_dev(info->dev);
1495 if (pdev->ats_enabled) {
1496 has_iotlb_device = true;
1497 break;
1498 }
1499 }
1500
1501 domain->has_iotlb_device = has_iotlb_device;
1502 }
1503
1504 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1505 {
1506 struct pci_dev *pdev;
1507
1508 assert_spin_locked(&device_domain_lock);
1509
1510 if (!info || !dev_is_pci(info->dev))
1511 return;
1512
1513 pdev = to_pci_dev(info->dev);
1514
1515 #ifdef CONFIG_INTEL_IOMMU_SVM
1516 /* The PCIe spec, in its wisdom, declares that the behaviour of
1517 the device if you enable PASID support after ATS support is
1518 undefined. So always enable PASID support on devices which
1519 have it, even if we can't yet know if we're ever going to
1520 use it. */
1521 if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1522 info->pasid_enabled = 1;
1523
1524 if (info->pri_supported && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1525 info->pri_enabled = 1;
1526 #endif
1527 if (info->ats_supported && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1528 info->ats_enabled = 1;
1529 domain_update_iotlb(info->domain);
1530 info->ats_qdep = pci_ats_queue_depth(pdev);
1531 }
1532 }
1533
1534 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1535 {
1536 struct pci_dev *pdev;
1537
1538 assert_spin_locked(&device_domain_lock);
1539
1540 if (!dev_is_pci(info->dev))
1541 return;
1542
1543 pdev = to_pci_dev(info->dev);
1544
1545 if (info->ats_enabled) {
1546 pci_disable_ats(pdev);
1547 info->ats_enabled = 0;
1548 domain_update_iotlb(info->domain);
1549 }
1550 #ifdef CONFIG_INTEL_IOMMU_SVM
1551 if (info->pri_enabled) {
1552 pci_disable_pri(pdev);
1553 info->pri_enabled = 0;
1554 }
1555 if (info->pasid_enabled) {
1556 pci_disable_pasid(pdev);
1557 info->pasid_enabled = 0;
1558 }
1559 #endif
1560 }
1561
1562 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1563 u64 addr, unsigned mask)
1564 {
1565 u16 sid, qdep;
1566 unsigned long flags;
1567 struct device_domain_info *info;
1568
1569 if (!domain->has_iotlb_device)
1570 return;
1571
1572 spin_lock_irqsave(&device_domain_lock, flags);
1573 list_for_each_entry(info, &domain->devices, link) {
1574 if (!info->ats_enabled)
1575 continue;
1576
1577 sid = info->bus << 8 | info->devfn;
1578 qdep = info->ats_qdep;
1579 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1580 }
1581 spin_unlock_irqrestore(&device_domain_lock, flags);
1582 }
1583
1584 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1585 struct dmar_domain *domain,
1586 unsigned long pfn, unsigned int pages,
1587 int ih, int map)
1588 {
1589 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1590 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1591 u16 did = domain->iommu_did[iommu->seq_id];
1592
1593 BUG_ON(pages == 0);
1594
1595 if (ih)
1596 ih = 1 << 6;
1597 /*
1598 * Fallback to domain selective flush if no PSI support or the size is
1599 * too big.
1600 * PSI requires page size to be 2 ^ x, and the base address is naturally
1601 * aligned to the size
1602 */
1603 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1604 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1605 DMA_TLB_DSI_FLUSH);
1606 else
1607 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1608 DMA_TLB_PSI_FLUSH);
1609
1610 /*
1611 * In caching mode, changes of pages from non-present to present require
1612 * flush. However, device IOTLB doesn't need to be flushed in this case.
1613 */
1614 if (!cap_caching_mode(iommu->cap) || !map)
1615 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1616 addr, mask);
1617 }
1618
1619 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1620 {
1621 u32 pmen;
1622 unsigned long flags;
1623
1624 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1625 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1626 pmen &= ~DMA_PMEN_EPM;
1627 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1628
1629 /* wait for the protected region status bit to clear */
1630 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1631 readl, !(pmen & DMA_PMEN_PRS), pmen);
1632
1633 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1634 }
1635
1636 static void iommu_enable_translation(struct intel_iommu *iommu)
1637 {
1638 u32 sts;
1639 unsigned long flags;
1640
1641 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1642 iommu->gcmd |= DMA_GCMD_TE;
1643 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1644
1645 /* Make sure hardware complete it */
1646 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1647 readl, (sts & DMA_GSTS_TES), sts);
1648
1649 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1650 }
1651
1652 static void iommu_disable_translation(struct intel_iommu *iommu)
1653 {
1654 u32 sts;
1655 unsigned long flag;
1656
1657 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1658 iommu->gcmd &= ~DMA_GCMD_TE;
1659 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1660
1661 /* Make sure hardware complete it */
1662 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1663 readl, (!(sts & DMA_GSTS_TES)), sts);
1664
1665 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1666 }
1667
1668
1669 static int iommu_init_domains(struct intel_iommu *iommu)
1670 {
1671 u32 ndomains, nlongs;
1672 size_t size;
1673
1674 ndomains = cap_ndoms(iommu->cap);
1675 pr_debug("%s: Number of Domains supported <%d>\n",
1676 iommu->name, ndomains);
1677 nlongs = BITS_TO_LONGS(ndomains);
1678
1679 spin_lock_init(&iommu->lock);
1680
1681 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1682 if (!iommu->domain_ids) {
1683 pr_err("%s: Allocating domain id array failed\n",
1684 iommu->name);
1685 return -ENOMEM;
1686 }
1687
1688 size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
1689 iommu->domains = kzalloc(size, GFP_KERNEL);
1690
1691 if (iommu->domains) {
1692 size = 256 * sizeof(struct dmar_domain *);
1693 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1694 }
1695
1696 if (!iommu->domains || !iommu->domains[0]) {
1697 pr_err("%s: Allocating domain array failed\n",
1698 iommu->name);
1699 kfree(iommu->domain_ids);
1700 kfree(iommu->domains);
1701 iommu->domain_ids = NULL;
1702 iommu->domains = NULL;
1703 return -ENOMEM;
1704 }
1705
1706
1707
1708 /*
1709 * If Caching mode is set, then invalid translations are tagged
1710 * with domain-id 0, hence we need to pre-allocate it. We also
1711 * use domain-id 0 as a marker for non-allocated domain-id, so
1712 * make sure it is not used for a real domain.
1713 */
1714 set_bit(0, iommu->domain_ids);
1715
1716 return 0;
1717 }
1718
1719 static void disable_dmar_iommu(struct intel_iommu *iommu)
1720 {
1721 struct device_domain_info *info, *tmp;
1722 unsigned long flags;
1723
1724 if (!iommu->domains || !iommu->domain_ids)
1725 return;
1726
1727 again:
1728 spin_lock_irqsave(&device_domain_lock, flags);
1729 list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1730 struct dmar_domain *domain;
1731
1732 if (info->iommu != iommu)
1733 continue;
1734
1735 if (!info->dev || !info->domain)
1736 continue;
1737
1738 domain = info->domain;
1739
1740 __dmar_remove_one_dev_info(info);
1741
1742 if (!domain_type_is_vm_or_si(domain)) {
1743 /*
1744 * The domain_exit() function can't be called under
1745 * device_domain_lock, as it takes this lock itself.
1746 * So release the lock here and re-run the loop
1747 * afterwards.
1748 */
1749 spin_unlock_irqrestore(&device_domain_lock, flags);
1750 domain_exit(domain);
1751 goto again;
1752 }
1753 }
1754 spin_unlock_irqrestore(&device_domain_lock, flags);
1755
1756 if (iommu->gcmd & DMA_GCMD_TE)
1757 iommu_disable_translation(iommu);
1758 }
1759
1760 static void free_dmar_iommu(struct intel_iommu *iommu)
1761 {
1762 if ((iommu->domains) && (iommu->domain_ids)) {
1763 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
1764 int i;
1765
1766 for (i = 0; i < elems; i++)
1767 kfree(iommu->domains[i]);
1768 kfree(iommu->domains);
1769 kfree(iommu->domain_ids);
1770 iommu->domains = NULL;
1771 iommu->domain_ids = NULL;
1772 }
1773
1774 g_iommus[iommu->seq_id] = NULL;
1775
1776 /* free context mapping */
1777 free_context_table(iommu);
1778
1779 #ifdef CONFIG_INTEL_IOMMU_SVM
1780 if (pasid_enabled(iommu)) {
1781 if (ecap_prs(iommu->ecap))
1782 intel_svm_finish_prq(iommu);
1783 intel_svm_free_pasid_tables(iommu);
1784 }
1785 #endif
1786 }
1787
1788 static struct dmar_domain *alloc_domain(int flags)
1789 {
1790 struct dmar_domain *domain;
1791
1792 domain = alloc_domain_mem();
1793 if (!domain)
1794 return NULL;
1795
1796 memset(domain, 0, sizeof(*domain));
1797 domain->nid = -1;
1798 domain->flags = flags;
1799 domain->has_iotlb_device = false;
1800 INIT_LIST_HEAD(&domain->devices);
1801
1802 return domain;
1803 }
1804
1805 /* Must be called with iommu->lock */
1806 static int domain_attach_iommu(struct dmar_domain *domain,
1807 struct intel_iommu *iommu)
1808 {
1809 unsigned long ndomains;
1810 int num;
1811
1812 assert_spin_locked(&device_domain_lock);
1813 assert_spin_locked(&iommu->lock);
1814
1815 domain->iommu_refcnt[iommu->seq_id] += 1;
1816 domain->iommu_count += 1;
1817 if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1818 ndomains = cap_ndoms(iommu->cap);
1819 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1820
1821 if (num >= ndomains) {
1822 pr_err("%s: No free domain ids\n", iommu->name);
1823 domain->iommu_refcnt[iommu->seq_id] -= 1;
1824 domain->iommu_count -= 1;
1825 return -ENOSPC;
1826 }
1827
1828 set_bit(num, iommu->domain_ids);
1829 set_iommu_domain(iommu, num, domain);
1830
1831 domain->iommu_did[iommu->seq_id] = num;
1832 domain->nid = iommu->node;
1833
1834 domain_update_iommu_cap(domain);
1835 }
1836
1837 return 0;
1838 }
1839
1840 static int domain_detach_iommu(struct dmar_domain *domain,
1841 struct intel_iommu *iommu)
1842 {
1843 int num, count = INT_MAX;
1844
1845 assert_spin_locked(&device_domain_lock);
1846 assert_spin_locked(&iommu->lock);
1847
1848 domain->iommu_refcnt[iommu->seq_id] -= 1;
1849 count = --domain->iommu_count;
1850 if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1851 num = domain->iommu_did[iommu->seq_id];
1852 clear_bit(num, iommu->domain_ids);
1853 set_iommu_domain(iommu, num, NULL);
1854
1855 domain_update_iommu_cap(domain);
1856 domain->iommu_did[iommu->seq_id] = 0;
1857 }
1858
1859 return count;
1860 }
1861
1862 static struct iova_domain reserved_iova_list;
1863 static struct lock_class_key reserved_rbtree_key;
1864
1865 static int dmar_init_reserved_ranges(void)
1866 {
1867 struct pci_dev *pdev = NULL;
1868 struct iova *iova;
1869 int i;
1870
1871 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1872 DMA_32BIT_PFN);
1873
1874 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1875 &reserved_rbtree_key);
1876
1877 /* IOAPIC ranges shouldn't be accessed by DMA */
1878 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1879 IOVA_PFN(IOAPIC_RANGE_END));
1880 if (!iova) {
1881 pr_err("Reserve IOAPIC range failed\n");
1882 return -ENODEV;
1883 }
1884
1885 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1886 for_each_pci_dev(pdev) {
1887 struct resource *r;
1888
1889 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1890 r = &pdev->resource[i];
1891 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1892 continue;
1893 iova = reserve_iova(&reserved_iova_list,
1894 IOVA_PFN(r->start),
1895 IOVA_PFN(r->end));
1896 if (!iova) {
1897 pr_err("Reserve iova failed\n");
1898 return -ENODEV;
1899 }
1900 }
1901 }
1902 return 0;
1903 }
1904
1905 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1906 {
1907 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1908 }
1909
1910 static inline int guestwidth_to_adjustwidth(int gaw)
1911 {
1912 int agaw;
1913 int r = (gaw - 12) % 9;
1914
1915 if (r == 0)
1916 agaw = gaw;
1917 else
1918 agaw = gaw + 9 - r;
1919 if (agaw > 64)
1920 agaw = 64;
1921 return agaw;
1922 }
1923
1924 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1925 int guest_width)
1926 {
1927 int adjust_width, agaw;
1928 unsigned long sagaw;
1929
1930 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1931 DMA_32BIT_PFN);
1932 domain_reserve_special_ranges(domain);
1933
1934 /* calculate AGAW */
1935 if (guest_width > cap_mgaw(iommu->cap))
1936 guest_width = cap_mgaw(iommu->cap);
1937 domain->gaw = guest_width;
1938 adjust_width = guestwidth_to_adjustwidth(guest_width);
1939 agaw = width_to_agaw(adjust_width);
1940 sagaw = cap_sagaw(iommu->cap);
1941 if (!test_bit(agaw, &sagaw)) {
1942 /* hardware doesn't support it, choose a bigger one */
1943 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1944 agaw = find_next_bit(&sagaw, 5, agaw);
1945 if (agaw >= 5)
1946 return -ENODEV;
1947 }
1948 domain->agaw = agaw;
1949
1950 if (ecap_coherent(iommu->ecap))
1951 domain->iommu_coherency = 1;
1952 else
1953 domain->iommu_coherency = 0;
1954
1955 if (ecap_sc_support(iommu->ecap))
1956 domain->iommu_snooping = 1;
1957 else
1958 domain->iommu_snooping = 0;
1959
1960 if (intel_iommu_superpage)
1961 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1962 else
1963 domain->iommu_superpage = 0;
1964
1965 domain->nid = iommu->node;
1966
1967 /* always allocate the top pgd */
1968 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1969 if (!domain->pgd)
1970 return -ENOMEM;
1971 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1972 return 0;
1973 }
1974
1975 static void domain_exit(struct dmar_domain *domain)
1976 {
1977 struct page *freelist = NULL;
1978
1979 /* Domain 0 is reserved, so dont process it */
1980 if (!domain)
1981 return;
1982
1983 /* Flush any lazy unmaps that may reference this domain */
1984 if (!intel_iommu_strict) {
1985 int cpu;
1986
1987 for_each_possible_cpu(cpu)
1988 flush_unmaps_timeout(cpu);
1989 }
1990
1991 /* Remove associated devices and clear attached or cached domains */
1992 rcu_read_lock();
1993 domain_remove_dev_info(domain);
1994 rcu_read_unlock();
1995
1996 /* destroy iovas */
1997 put_iova_domain(&domain->iovad);
1998
1999 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
2000
2001 dma_free_pagelist(freelist);
2002
2003 free_domain_mem(domain);
2004 }
2005
2006 static int domain_context_mapping_one(struct dmar_domain *domain,
2007 struct intel_iommu *iommu,
2008 u8 bus, u8 devfn)
2009 {
2010 u16 did = domain->iommu_did[iommu->seq_id];
2011 int translation = CONTEXT_TT_MULTI_LEVEL;
2012 struct device_domain_info *info = NULL;
2013 struct context_entry *context;
2014 unsigned long flags;
2015 struct dma_pte *pgd;
2016 int ret, agaw;
2017
2018 WARN_ON(did == 0);
2019
2020 if (hw_pass_through && domain_type_is_si(domain))
2021 translation = CONTEXT_TT_PASS_THROUGH;
2022
2023 pr_debug("Set context mapping for %02x:%02x.%d\n",
2024 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
2025
2026 BUG_ON(!domain->pgd);
2027
2028 spin_lock_irqsave(&device_domain_lock, flags);
2029 spin_lock(&iommu->lock);
2030
2031 ret = -ENOMEM;
2032 context = iommu_context_addr(iommu, bus, devfn, 1);
2033 if (!context)
2034 goto out_unlock;
2035
2036 ret = 0;
2037 if (context_present(context))
2038 goto out_unlock;
2039
2040 /*
2041 * For kdump cases, old valid entries may be cached due to the
2042 * in-flight DMA and copied pgtable, but there is no unmapping
2043 * behaviour for them, thus we need an explicit cache flush for
2044 * the newly-mapped device. For kdump, at this point, the device
2045 * is supposed to finish reset at its driver probe stage, so no
2046 * in-flight DMA will exist, and we don't need to worry anymore
2047 * hereafter.
2048 */
2049 if (context_copied(context)) {
2050 u16 did_old = context_domain_id(context);
2051
2052 if (did_old >= 0 && did_old < cap_ndoms(iommu->cap))
2053 iommu->flush.flush_context(iommu, did_old,
2054 (((u16)bus) << 8) | devfn,
2055 DMA_CCMD_MASK_NOBIT,
2056 DMA_CCMD_DEVICE_INVL);
2057 }
2058
2059 pgd = domain->pgd;
2060
2061 context_clear_entry(context);
2062 context_set_domain_id(context, did);
2063
2064 /*
2065 * Skip top levels of page tables for iommu which has less agaw
2066 * than default. Unnecessary for PT mode.
2067 */
2068 if (translation != CONTEXT_TT_PASS_THROUGH) {
2069 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
2070 ret = -ENOMEM;
2071 pgd = phys_to_virt(dma_pte_addr(pgd));
2072 if (!dma_pte_present(pgd))
2073 goto out_unlock;
2074 }
2075
2076 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2077 if (info && info->ats_supported)
2078 translation = CONTEXT_TT_DEV_IOTLB;
2079 else
2080 translation = CONTEXT_TT_MULTI_LEVEL;
2081
2082 context_set_address_root(context, virt_to_phys(pgd));
2083 context_set_address_width(context, iommu->agaw);
2084 } else {
2085 /*
2086 * In pass through mode, AW must be programmed to
2087 * indicate the largest AGAW value supported by
2088 * hardware. And ASR is ignored by hardware.
2089 */
2090 context_set_address_width(context, iommu->msagaw);
2091 }
2092
2093 context_set_translation_type(context, translation);
2094 context_set_fault_enable(context);
2095 context_set_present(context);
2096 domain_flush_cache(domain, context, sizeof(*context));
2097
2098 /*
2099 * It's a non-present to present mapping. If hardware doesn't cache
2100 * non-present entry we only need to flush the write-buffer. If the
2101 * _does_ cache non-present entries, then it does so in the special
2102 * domain #0, which we have to flush:
2103 */
2104 if (cap_caching_mode(iommu->cap)) {
2105 iommu->flush.flush_context(iommu, 0,
2106 (((u16)bus) << 8) | devfn,
2107 DMA_CCMD_MASK_NOBIT,
2108 DMA_CCMD_DEVICE_INVL);
2109 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2110 } else {
2111 iommu_flush_write_buffer(iommu);
2112 }
2113 iommu_enable_dev_iotlb(info);
2114
2115 ret = 0;
2116
2117 out_unlock:
2118 spin_unlock(&iommu->lock);
2119 spin_unlock_irqrestore(&device_domain_lock, flags);
2120
2121 return ret;
2122 }
2123
2124 struct domain_context_mapping_data {
2125 struct dmar_domain *domain;
2126 struct intel_iommu *iommu;
2127 };
2128
2129 static int domain_context_mapping_cb(struct pci_dev *pdev,
2130 u16 alias, void *opaque)
2131 {
2132 struct domain_context_mapping_data *data = opaque;
2133
2134 return domain_context_mapping_one(data->domain, data->iommu,
2135 PCI_BUS_NUM(alias), alias & 0xff);
2136 }
2137
2138 static int
2139 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2140 {
2141 struct intel_iommu *iommu;
2142 u8 bus, devfn;
2143 struct domain_context_mapping_data data;
2144
2145 iommu = device_to_iommu(dev, &bus, &devfn);
2146 if (!iommu)
2147 return -ENODEV;
2148
2149 if (!dev_is_pci(dev))
2150 return domain_context_mapping_one(domain, iommu, bus, devfn);
2151
2152 data.domain = domain;
2153 data.iommu = iommu;
2154
2155 return pci_for_each_dma_alias(to_pci_dev(dev),
2156 &domain_context_mapping_cb, &data);
2157 }
2158
2159 static int domain_context_mapped_cb(struct pci_dev *pdev,
2160 u16 alias, void *opaque)
2161 {
2162 struct intel_iommu *iommu = opaque;
2163
2164 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2165 }
2166
2167 static int domain_context_mapped(struct device *dev)
2168 {
2169 struct intel_iommu *iommu;
2170 u8 bus, devfn;
2171
2172 iommu = device_to_iommu(dev, &bus, &devfn);
2173 if (!iommu)
2174 return -ENODEV;
2175
2176 if (!dev_is_pci(dev))
2177 return device_context_mapped(iommu, bus, devfn);
2178
2179 return !pci_for_each_dma_alias(to_pci_dev(dev),
2180 domain_context_mapped_cb, iommu);
2181 }
2182
2183 /* Returns a number of VTD pages, but aligned to MM page size */
2184 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2185 size_t size)
2186 {
2187 host_addr &= ~PAGE_MASK;
2188 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2189 }
2190
2191 /* Return largest possible superpage level for a given mapping */
2192 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2193 unsigned long iov_pfn,
2194 unsigned long phy_pfn,
2195 unsigned long pages)
2196 {
2197 int support, level = 1;
2198 unsigned long pfnmerge;
2199
2200 support = domain->iommu_superpage;
2201
2202 /* To use a large page, the virtual *and* physical addresses
2203 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2204 of them will mean we have to use smaller pages. So just
2205 merge them and check both at once. */
2206 pfnmerge = iov_pfn | phy_pfn;
2207
2208 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2209 pages >>= VTD_STRIDE_SHIFT;
2210 if (!pages)
2211 break;
2212 pfnmerge >>= VTD_STRIDE_SHIFT;
2213 level++;
2214 support--;
2215 }
2216 return level;
2217 }
2218
2219 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2220 struct scatterlist *sg, unsigned long phys_pfn,
2221 unsigned long nr_pages, int prot)
2222 {
2223 struct dma_pte *first_pte = NULL, *pte = NULL;
2224 phys_addr_t uninitialized_var(pteval);
2225 unsigned long sg_res = 0;
2226 unsigned int largepage_lvl = 0;
2227 unsigned long lvl_pages = 0;
2228
2229 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2230
2231 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2232 return -EINVAL;
2233
2234 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2235
2236 if (!sg) {
2237 sg_res = nr_pages;
2238 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2239 }
2240
2241 while (nr_pages > 0) {
2242 uint64_t tmp;
2243
2244 if (!sg_res) {
2245 sg_res = aligned_nrpages(sg->offset, sg->length);
2246 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2247 sg->dma_length = sg->length;
2248 pteval = page_to_phys(sg_page(sg)) | prot;
2249 phys_pfn = pteval >> VTD_PAGE_SHIFT;
2250 }
2251
2252 if (!pte) {
2253 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2254
2255 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2256 if (!pte)
2257 return -ENOMEM;
2258 /* It is large page*/
2259 if (largepage_lvl > 1) {
2260 unsigned long nr_superpages, end_pfn;
2261
2262 pteval |= DMA_PTE_LARGE_PAGE;
2263 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2264
2265 nr_superpages = sg_res / lvl_pages;
2266 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2267
2268 /*
2269 * Ensure that old small page tables are
2270 * removed to make room for superpage(s).
2271 */
2272 dma_pte_free_pagetable(domain, iov_pfn, end_pfn);
2273 } else {
2274 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2275 }
2276
2277 }
2278 /* We don't need lock here, nobody else
2279 * touches the iova range
2280 */
2281 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2282 if (tmp) {
2283 static int dumps = 5;
2284 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2285 iov_pfn, tmp, (unsigned long long)pteval);
2286 if (dumps) {
2287 dumps--;
2288 debug_dma_dump_mappings(NULL);
2289 }
2290 WARN_ON(1);
2291 }
2292
2293 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2294
2295 BUG_ON(nr_pages < lvl_pages);
2296 BUG_ON(sg_res < lvl_pages);
2297
2298 nr_pages -= lvl_pages;
2299 iov_pfn += lvl_pages;
2300 phys_pfn += lvl_pages;
2301 pteval += lvl_pages * VTD_PAGE_SIZE;
2302 sg_res -= lvl_pages;
2303
2304 /* If the next PTE would be the first in a new page, then we
2305 need to flush the cache on the entries we've just written.
2306 And then we'll need to recalculate 'pte', so clear it and
2307 let it get set again in the if (!pte) block above.
2308
2309 If we're done (!nr_pages) we need to flush the cache too.
2310
2311 Also if we've been setting superpages, we may need to
2312 recalculate 'pte' and switch back to smaller pages for the
2313 end of the mapping, if the trailing size is not enough to
2314 use another superpage (i.e. sg_res < lvl_pages). */
2315 pte++;
2316 if (!nr_pages || first_pte_in_page(pte) ||
2317 (largepage_lvl > 1 && sg_res < lvl_pages)) {
2318 domain_flush_cache(domain, first_pte,
2319 (void *)pte - (void *)first_pte);
2320 pte = NULL;
2321 }
2322
2323 if (!sg_res && nr_pages)
2324 sg = sg_next(sg);
2325 }
2326 return 0;
2327 }
2328
2329 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2330 struct scatterlist *sg, unsigned long nr_pages,
2331 int prot)
2332 {
2333 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2334 }
2335
2336 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2337 unsigned long phys_pfn, unsigned long nr_pages,
2338 int prot)
2339 {
2340 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2341 }
2342
2343 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2344 {
2345 if (!iommu)
2346 return;
2347
2348 clear_context_table(iommu, bus, devfn);
2349 iommu->flush.flush_context(iommu, 0, 0, 0,
2350 DMA_CCMD_GLOBAL_INVL);
2351 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2352 }
2353
2354 static inline void unlink_domain_info(struct device_domain_info *info)
2355 {
2356 assert_spin_locked(&device_domain_lock);
2357 list_del(&info->link);
2358 list_del(&info->global);
2359 if (info->dev)
2360 info->dev->archdata.iommu = NULL;
2361 }
2362
2363 static void domain_remove_dev_info(struct dmar_domain *domain)
2364 {
2365 struct device_domain_info *info, *tmp;
2366 unsigned long flags;
2367
2368 spin_lock_irqsave(&device_domain_lock, flags);
2369 list_for_each_entry_safe(info, tmp, &domain->devices, link)
2370 __dmar_remove_one_dev_info(info);
2371 spin_unlock_irqrestore(&device_domain_lock, flags);
2372 }
2373
2374 /*
2375 * find_domain
2376 * Note: we use struct device->archdata.iommu stores the info
2377 */
2378 static struct dmar_domain *find_domain(struct device *dev)
2379 {
2380 struct device_domain_info *info;
2381
2382 /* No lock here, assumes no domain exit in normal case */
2383 info = dev->archdata.iommu;
2384 if (info)
2385 return info->domain;
2386 return NULL;
2387 }
2388
2389 static inline struct device_domain_info *
2390 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2391 {
2392 struct device_domain_info *info;
2393
2394 list_for_each_entry(info, &device_domain_list, global)
2395 if (info->iommu->segment == segment && info->bus == bus &&
2396 info->devfn == devfn)
2397 return info;
2398
2399 return NULL;
2400 }
2401
2402 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2403 int bus, int devfn,
2404 struct device *dev,
2405 struct dmar_domain *domain)
2406 {
2407 struct dmar_domain *found = NULL;
2408 struct device_domain_info *info;
2409 unsigned long flags;
2410 int ret;
2411
2412 info = alloc_devinfo_mem();
2413 if (!info)
2414 return NULL;
2415
2416 info->bus = bus;
2417 info->devfn = devfn;
2418 info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2419 info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2420 info->ats_qdep = 0;
2421 info->dev = dev;
2422 info->domain = domain;
2423 info->iommu = iommu;
2424
2425 if (dev && dev_is_pci(dev)) {
2426 struct pci_dev *pdev = to_pci_dev(info->dev);
2427
2428 if (ecap_dev_iotlb_support(iommu->ecap) &&
2429 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2430 dmar_find_matched_atsr_unit(pdev))
2431 info->ats_supported = 1;
2432
2433 if (ecs_enabled(iommu)) {
2434 if (pasid_enabled(iommu)) {
2435 int features = pci_pasid_features(pdev);
2436 if (features >= 0)
2437 info->pasid_supported = features | 1;
2438 }
2439
2440 if (info->ats_supported && ecap_prs(iommu->ecap) &&
2441 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2442 info->pri_supported = 1;
2443 }
2444 }
2445
2446 spin_lock_irqsave(&device_domain_lock, flags);
2447 if (dev)
2448 found = find_domain(dev);
2449
2450 if (!found) {
2451 struct device_domain_info *info2;
2452 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2453 if (info2) {
2454 found = info2->domain;
2455 info2->dev = dev;
2456 }
2457 }
2458
2459 if (found) {
2460 spin_unlock_irqrestore(&device_domain_lock, flags);
2461 free_devinfo_mem(info);
2462 /* Caller must free the original domain */
2463 return found;
2464 }
2465
2466 spin_lock(&iommu->lock);
2467 ret = domain_attach_iommu(domain, iommu);
2468 spin_unlock(&iommu->lock);
2469
2470 if (ret) {
2471 spin_unlock_irqrestore(&device_domain_lock, flags);
2472 free_devinfo_mem(info);
2473 return NULL;
2474 }
2475
2476 list_add(&info->link, &domain->devices);
2477 list_add(&info->global, &device_domain_list);
2478 if (dev)
2479 dev->archdata.iommu = info;
2480 spin_unlock_irqrestore(&device_domain_lock, flags);
2481
2482 if (dev && domain_context_mapping(domain, dev)) {
2483 pr_err("Domain context map for %s failed\n", dev_name(dev));
2484 dmar_remove_one_dev_info(domain, dev);
2485 return NULL;
2486 }
2487
2488 return domain;
2489 }
2490
2491 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2492 {
2493 *(u16 *)opaque = alias;
2494 return 0;
2495 }
2496
2497 static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
2498 {
2499 struct device_domain_info *info = NULL;
2500 struct dmar_domain *domain = NULL;
2501 struct intel_iommu *iommu;
2502 u16 req_id, dma_alias;
2503 unsigned long flags;
2504 u8 bus, devfn;
2505
2506 iommu = device_to_iommu(dev, &bus, &devfn);
2507 if (!iommu)
2508 return NULL;
2509
2510 req_id = ((u16)bus << 8) | devfn;
2511
2512 if (dev_is_pci(dev)) {
2513 struct pci_dev *pdev = to_pci_dev(dev);
2514
2515 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2516
2517 spin_lock_irqsave(&device_domain_lock, flags);
2518 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2519 PCI_BUS_NUM(dma_alias),
2520 dma_alias & 0xff);
2521 if (info) {
2522 iommu = info->iommu;
2523 domain = info->domain;
2524 }
2525 spin_unlock_irqrestore(&device_domain_lock, flags);
2526
2527 /* DMA alias already has a domain, use it */
2528 if (info)
2529 goto out;
2530 }
2531
2532 /* Allocate and initialize new domain for the device */
2533 domain = alloc_domain(0);
2534 if (!domain)
2535 return NULL;
2536 if (domain_init(domain, iommu, gaw)) {
2537 domain_exit(domain);
2538 return NULL;
2539 }
2540
2541 out:
2542
2543 return domain;
2544 }
2545
2546 static struct dmar_domain *set_domain_for_dev(struct device *dev,
2547 struct dmar_domain *domain)
2548 {
2549 struct intel_iommu *iommu;
2550 struct dmar_domain *tmp;
2551 u16 req_id, dma_alias;
2552 u8 bus, devfn;
2553
2554 iommu = device_to_iommu(dev, &bus, &devfn);
2555 if (!iommu)
2556 return NULL;
2557
2558 req_id = ((u16)bus << 8) | devfn;
2559
2560 if (dev_is_pci(dev)) {
2561 struct pci_dev *pdev = to_pci_dev(dev);
2562
2563 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2564
2565 /* register PCI DMA alias device */
2566 if (req_id != dma_alias) {
2567 tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2568 dma_alias & 0xff, NULL, domain);
2569
2570 if (!tmp || tmp != domain)
2571 return tmp;
2572 }
2573 }
2574
2575 tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2576 if (!tmp || tmp != domain)
2577 return tmp;
2578
2579 return domain;
2580 }
2581
2582 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2583 {
2584 struct dmar_domain *domain, *tmp;
2585
2586 domain = find_domain(dev);
2587 if (domain)
2588 goto out;
2589
2590 domain = find_or_alloc_domain(dev, gaw);
2591 if (!domain)
2592 goto out;
2593
2594 tmp = set_domain_for_dev(dev, domain);
2595 if (!tmp || domain != tmp) {
2596 domain_exit(domain);
2597 domain = tmp;
2598 }
2599
2600 out:
2601
2602 return domain;
2603 }
2604
2605 static int iommu_domain_identity_map(struct dmar_domain *domain,
2606 unsigned long long start,
2607 unsigned long long end)
2608 {
2609 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2610 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2611
2612 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2613 dma_to_mm_pfn(last_vpfn))) {
2614 pr_err("Reserving iova failed\n");
2615 return -ENOMEM;
2616 }
2617
2618 pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2619 /*
2620 * RMRR range might have overlap with physical memory range,
2621 * clear it first
2622 */
2623 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2624
2625 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2626 last_vpfn - first_vpfn + 1,
2627 DMA_PTE_READ|DMA_PTE_WRITE);
2628 }
2629
2630 static int domain_prepare_identity_map(struct device *dev,
2631 struct dmar_domain *domain,
2632 unsigned long long start,
2633 unsigned long long end)
2634 {
2635 /* For _hardware_ passthrough, don't bother. But for software
2636 passthrough, we do it anyway -- it may indicate a memory
2637 range which is reserved in E820, so which didn't get set
2638 up to start with in si_domain */
2639 if (domain == si_domain && hw_pass_through) {
2640 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2641 dev_name(dev), start, end);
2642 return 0;
2643 }
2644
2645 pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2646 dev_name(dev), start, end);
2647
2648 if (end < start) {
2649 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2650 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2651 dmi_get_system_info(DMI_BIOS_VENDOR),
2652 dmi_get_system_info(DMI_BIOS_VERSION),
2653 dmi_get_system_info(DMI_PRODUCT_VERSION));
2654 return -EIO;
2655 }
2656
2657 if (end >> agaw_to_width(domain->agaw)) {
2658 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2659 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2660 agaw_to_width(domain->agaw),
2661 dmi_get_system_info(DMI_BIOS_VENDOR),
2662 dmi_get_system_info(DMI_BIOS_VERSION),
2663 dmi_get_system_info(DMI_PRODUCT_VERSION));
2664 return -EIO;
2665 }
2666
2667 return iommu_domain_identity_map(domain, start, end);
2668 }
2669
2670 static int iommu_prepare_identity_map(struct device *dev,
2671 unsigned long long start,
2672 unsigned long long end)
2673 {
2674 struct dmar_domain *domain;
2675 int ret;
2676
2677 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2678 if (!domain)
2679 return -ENOMEM;
2680
2681 ret = domain_prepare_identity_map(dev, domain, start, end);
2682 if (ret)
2683 domain_exit(domain);
2684
2685 return ret;
2686 }
2687
2688 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2689 struct device *dev)
2690 {
2691 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2692 return 0;
2693 return iommu_prepare_identity_map(dev, rmrr->base_address,
2694 rmrr->end_address);
2695 }
2696
2697 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2698 static inline void iommu_prepare_isa(void)
2699 {
2700 struct pci_dev *pdev;
2701 int ret;
2702
2703 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2704 if (!pdev)
2705 return;
2706
2707 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2708 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2709
2710 if (ret)
2711 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2712
2713 pci_dev_put(pdev);
2714 }
2715 #else
2716 static inline void iommu_prepare_isa(void)
2717 {
2718 return;
2719 }
2720 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2721
2722 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2723
2724 static int __init si_domain_init(int hw)
2725 {
2726 int nid, ret = 0;
2727
2728 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2729 if (!si_domain)
2730 return -EFAULT;
2731
2732 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2733 domain_exit(si_domain);
2734 return -EFAULT;
2735 }
2736
2737 pr_debug("Identity mapping domain allocated\n");
2738
2739 if (hw)
2740 return 0;
2741
2742 for_each_online_node(nid) {
2743 unsigned long start_pfn, end_pfn;
2744 int i;
2745
2746 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2747 ret = iommu_domain_identity_map(si_domain,
2748 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2749 if (ret)
2750 return ret;
2751 }
2752 }
2753
2754 return 0;
2755 }
2756
2757 static int identity_mapping(struct device *dev)
2758 {
2759 struct device_domain_info *info;
2760
2761 if (likely(!iommu_identity_mapping))
2762 return 0;
2763
2764 info = dev->archdata.iommu;
2765 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2766 return (info->domain == si_domain);
2767
2768 return 0;
2769 }
2770
2771 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2772 {
2773 struct dmar_domain *ndomain;
2774 struct intel_iommu *iommu;
2775 u8 bus, devfn;
2776
2777 iommu = device_to_iommu(dev, &bus, &devfn);
2778 if (!iommu)
2779 return -ENODEV;
2780
2781 ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2782 if (ndomain != domain)
2783 return -EBUSY;
2784
2785 return 0;
2786 }
2787
2788 static bool device_has_rmrr(struct device *dev)
2789 {
2790 struct dmar_rmrr_unit *rmrr;
2791 struct device *tmp;
2792 int i;
2793
2794 rcu_read_lock();
2795 for_each_rmrr_units(rmrr) {
2796 /*
2797 * Return TRUE if this RMRR contains the device that
2798 * is passed in.
2799 */
2800 for_each_active_dev_scope(rmrr->devices,
2801 rmrr->devices_cnt, i, tmp)
2802 if (tmp == dev) {
2803 rcu_read_unlock();
2804 return true;
2805 }
2806 }
2807 rcu_read_unlock();
2808 return false;
2809 }
2810
2811 /*
2812 * There are a couple cases where we need to restrict the functionality of
2813 * devices associated with RMRRs. The first is when evaluating a device for
2814 * identity mapping because problems exist when devices are moved in and out
2815 * of domains and their respective RMRR information is lost. This means that
2816 * a device with associated RMRRs will never be in a "passthrough" domain.
2817 * The second is use of the device through the IOMMU API. This interface
2818 * expects to have full control of the IOVA space for the device. We cannot
2819 * satisfy both the requirement that RMRR access is maintained and have an
2820 * unencumbered IOVA space. We also have no ability to quiesce the device's
2821 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2822 * We therefore prevent devices associated with an RMRR from participating in
2823 * the IOMMU API, which eliminates them from device assignment.
2824 *
2825 * In both cases we assume that PCI USB devices with RMRRs have them largely
2826 * for historical reasons and that the RMRR space is not actively used post
2827 * boot. This exclusion may change if vendors begin to abuse it.
2828 *
2829 * The same exception is made for graphics devices, with the requirement that
2830 * any use of the RMRR regions will be torn down before assigning the device
2831 * to a guest.
2832 */
2833 static bool device_is_rmrr_locked(struct device *dev)
2834 {
2835 if (!device_has_rmrr(dev))
2836 return false;
2837
2838 if (dev_is_pci(dev)) {
2839 struct pci_dev *pdev = to_pci_dev(dev);
2840
2841 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2842 return false;
2843 }
2844
2845 return true;
2846 }
2847
2848 static int iommu_should_identity_map(struct device *dev, int startup)
2849 {
2850
2851 if (dev_is_pci(dev)) {
2852 struct pci_dev *pdev = to_pci_dev(dev);
2853
2854 if (device_is_rmrr_locked(dev))
2855 return 0;
2856
2857 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2858 return 1;
2859
2860 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2861 return 1;
2862
2863 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2864 return 0;
2865
2866 /*
2867 * We want to start off with all devices in the 1:1 domain, and
2868 * take them out later if we find they can't access all of memory.
2869 *
2870 * However, we can't do this for PCI devices behind bridges,
2871 * because all PCI devices behind the same bridge will end up
2872 * with the same source-id on their transactions.
2873 *
2874 * Practically speaking, we can't change things around for these
2875 * devices at run-time, because we can't be sure there'll be no
2876 * DMA transactions in flight for any of their siblings.
2877 *
2878 * So PCI devices (unless they're on the root bus) as well as
2879 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2880 * the 1:1 domain, just in _case_ one of their siblings turns out
2881 * not to be able to map all of memory.
2882 */
2883 if (!pci_is_pcie(pdev)) {
2884 if (!pci_is_root_bus(pdev->bus))
2885 return 0;
2886 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2887 return 0;
2888 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2889 return 0;
2890 } else {
2891 if (device_has_rmrr(dev))
2892 return 0;
2893 }
2894
2895 /*
2896 * At boot time, we don't yet know if devices will be 64-bit capable.
2897 * Assume that they will — if they turn out not to be, then we can
2898 * take them out of the 1:1 domain later.
2899 */
2900 if (!startup) {
2901 /*
2902 * If the device's dma_mask is less than the system's memory
2903 * size then this is not a candidate for identity mapping.
2904 */
2905 u64 dma_mask = *dev->dma_mask;
2906
2907 if (dev->coherent_dma_mask &&
2908 dev->coherent_dma_mask < dma_mask)
2909 dma_mask = dev->coherent_dma_mask;
2910
2911 return dma_mask >= dma_get_required_mask(dev);
2912 }
2913
2914 return 1;
2915 }
2916
2917 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2918 {
2919 int ret;
2920
2921 if (!iommu_should_identity_map(dev, 1))
2922 return 0;
2923
2924 ret = domain_add_dev_info(si_domain, dev);
2925 if (!ret)
2926 pr_info("%s identity mapping for device %s\n",
2927 hw ? "Hardware" : "Software", dev_name(dev));
2928 else if (ret == -ENODEV)
2929 /* device not associated with an iommu */
2930 ret = 0;
2931
2932 return ret;
2933 }
2934
2935
2936 static int __init iommu_prepare_static_identity_mapping(int hw)
2937 {
2938 struct pci_dev *pdev = NULL;
2939 struct dmar_drhd_unit *drhd;
2940 struct intel_iommu *iommu;
2941 struct device *dev;
2942 int i;
2943 int ret = 0;
2944
2945 for_each_pci_dev(pdev) {
2946 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2947 if (ret)
2948 return ret;
2949 }
2950
2951 for_each_active_iommu(iommu, drhd)
2952 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2953 struct acpi_device_physical_node *pn;
2954 struct acpi_device *adev;
2955
2956 if (dev->bus != &acpi_bus_type)
2957 continue;
2958
2959 adev= to_acpi_device(dev);
2960 mutex_lock(&adev->physical_node_lock);
2961 list_for_each_entry(pn, &adev->physical_node_list, node) {
2962 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2963 if (ret)
2964 break;
2965 }
2966 mutex_unlock(&adev->physical_node_lock);
2967 if (ret)
2968 return ret;
2969 }
2970
2971 return 0;
2972 }
2973
2974 static void intel_iommu_init_qi(struct intel_iommu *iommu)
2975 {
2976 /*
2977 * Start from the sane iommu hardware state.
2978 * If the queued invalidation is already initialized by us
2979 * (for example, while enabling interrupt-remapping) then
2980 * we got the things already rolling from a sane state.
2981 */
2982 if (!iommu->qi) {
2983 /*
2984 * Clear any previous faults.
2985 */
2986 dmar_fault(-1, iommu);
2987 /*
2988 * Disable queued invalidation if supported and already enabled
2989 * before OS handover.
2990 */
2991 dmar_disable_qi(iommu);
2992 }
2993
2994 if (dmar_enable_qi(iommu)) {
2995 /*
2996 * Queued Invalidate not enabled, use Register Based Invalidate
2997 */
2998 iommu->flush.flush_context = __iommu_flush_context;
2999 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
3000 pr_info("%s: Using Register based invalidation\n",
3001 iommu->name);
3002 } else {
3003 iommu->flush.flush_context = qi_flush_context;
3004 iommu->flush.flush_iotlb = qi_flush_iotlb;
3005 pr_info("%s: Using Queued invalidation\n", iommu->name);
3006 }
3007 }
3008
3009 static int copy_context_table(struct intel_iommu *iommu,
3010 struct root_entry *old_re,
3011 struct context_entry **tbl,
3012 int bus, bool ext)
3013 {
3014 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
3015 struct context_entry *new_ce = NULL, ce;
3016 struct context_entry *old_ce = NULL;
3017 struct root_entry re;
3018 phys_addr_t old_ce_phys;
3019
3020 tbl_idx = ext ? bus * 2 : bus;
3021 memcpy(&re, old_re, sizeof(re));
3022
3023 for (devfn = 0; devfn < 256; devfn++) {
3024 /* First calculate the correct index */
3025 idx = (ext ? devfn * 2 : devfn) % 256;
3026
3027 if (idx == 0) {
3028 /* First save what we may have and clean up */
3029 if (new_ce) {
3030 tbl[tbl_idx] = new_ce;
3031 __iommu_flush_cache(iommu, new_ce,
3032 VTD_PAGE_SIZE);
3033 pos = 1;
3034 }
3035
3036 if (old_ce)
3037 iounmap(old_ce);
3038
3039 ret = 0;
3040 if (devfn < 0x80)
3041 old_ce_phys = root_entry_lctp(&re);
3042 else
3043 old_ce_phys = root_entry_uctp(&re);
3044
3045 if (!old_ce_phys) {
3046 if (ext && devfn == 0) {
3047 /* No LCTP, try UCTP */
3048 devfn = 0x7f;
3049 continue;
3050 } else {
3051 goto out;
3052 }
3053 }
3054
3055 ret = -ENOMEM;
3056 old_ce = memremap(old_ce_phys, PAGE_SIZE,
3057 MEMREMAP_WB);
3058 if (!old_ce)
3059 goto out;
3060
3061 new_ce = alloc_pgtable_page(iommu->node);
3062 if (!new_ce)
3063 goto out_unmap;
3064
3065 ret = 0;
3066 }
3067
3068 /* Now copy the context entry */
3069 memcpy(&ce, old_ce + idx, sizeof(ce));
3070
3071 if (!__context_present(&ce))
3072 continue;
3073
3074 did = context_domain_id(&ce);
3075 if (did >= 0 && did < cap_ndoms(iommu->cap))
3076 set_bit(did, iommu->domain_ids);
3077
3078 /*
3079 * We need a marker for copied context entries. This
3080 * marker needs to work for the old format as well as
3081 * for extended context entries.
3082 *
3083 * Bit 67 of the context entry is used. In the old
3084 * format this bit is available to software, in the
3085 * extended format it is the PGE bit, but PGE is ignored
3086 * by HW if PASIDs are disabled (and thus still
3087 * available).
3088 *
3089 * So disable PASIDs first and then mark the entry
3090 * copied. This means that we don't copy PASID
3091 * translations from the old kernel, but this is fine as
3092 * faults there are not fatal.
3093 */
3094 context_clear_pasid_enable(&ce);
3095 context_set_copied(&ce);
3096
3097 new_ce[idx] = ce;
3098 }
3099
3100 tbl[tbl_idx + pos] = new_ce;
3101
3102 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3103
3104 out_unmap:
3105 memunmap(old_ce);
3106
3107 out:
3108 return ret;
3109 }
3110
3111 static int copy_translation_tables(struct intel_iommu *iommu)
3112 {
3113 struct context_entry **ctxt_tbls;
3114 struct root_entry *old_rt;
3115 phys_addr_t old_rt_phys;
3116 int ctxt_table_entries;
3117 unsigned long flags;
3118 u64 rtaddr_reg;
3119 int bus, ret;
3120 bool new_ext, ext;
3121
3122 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3123 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
3124 new_ext = !!ecap_ecs(iommu->ecap);
3125
3126 /*
3127 * The RTT bit can only be changed when translation is disabled,
3128 * but disabling translation means to open a window for data
3129 * corruption. So bail out and don't copy anything if we would
3130 * have to change the bit.
3131 */
3132 if (new_ext != ext)
3133 return -EINVAL;
3134
3135 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3136 if (!old_rt_phys)
3137 return -EINVAL;
3138
3139 old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
3140 if (!old_rt)
3141 return -ENOMEM;
3142
3143 /* This is too big for the stack - allocate it from slab */
3144 ctxt_table_entries = ext ? 512 : 256;
3145 ret = -ENOMEM;
3146 ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
3147 if (!ctxt_tbls)
3148 goto out_unmap;
3149
3150 for (bus = 0; bus < 256; bus++) {
3151 ret = copy_context_table(iommu, &old_rt[bus],
3152 ctxt_tbls, bus, ext);
3153 if (ret) {
3154 pr_err("%s: Failed to copy context table for bus %d\n",
3155 iommu->name, bus);
3156 continue;
3157 }
3158 }
3159
3160 spin_lock_irqsave(&iommu->lock, flags);
3161
3162 /* Context tables are copied, now write them to the root_entry table */
3163 for (bus = 0; bus < 256; bus++) {
3164 int idx = ext ? bus * 2 : bus;
3165 u64 val;
3166
3167 if (ctxt_tbls[idx]) {
3168 val = virt_to_phys(ctxt_tbls[idx]) | 1;
3169 iommu->root_entry[bus].lo = val;
3170 }
3171
3172 if (!ext || !ctxt_tbls[idx + 1])
3173 continue;
3174
3175 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3176 iommu->root_entry[bus].hi = val;
3177 }
3178
3179 spin_unlock_irqrestore(&iommu->lock, flags);
3180
3181 kfree(ctxt_tbls);
3182
3183 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3184
3185 ret = 0;
3186
3187 out_unmap:
3188 memunmap(old_rt);
3189
3190 return ret;
3191 }
3192
3193 static int __init init_dmars(void)
3194 {
3195 struct dmar_drhd_unit *drhd;
3196 struct dmar_rmrr_unit *rmrr;
3197 bool copied_tables = false;
3198 struct device *dev;
3199 struct intel_iommu *iommu;
3200 int i, ret, cpu;
3201
3202 /*
3203 * for each drhd
3204 * allocate root
3205 * initialize and program root entry to not present
3206 * endfor
3207 */
3208 for_each_drhd_unit(drhd) {
3209 /*
3210 * lock not needed as this is only incremented in the single
3211 * threaded kernel __init code path all other access are read
3212 * only
3213 */
3214 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3215 g_num_of_iommus++;
3216 continue;
3217 }
3218 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3219 }
3220
3221 /* Preallocate enough resources for IOMMU hot-addition */
3222 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3223 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3224
3225 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3226 GFP_KERNEL);
3227 if (!g_iommus) {
3228 pr_err("Allocating global iommu array failed\n");
3229 ret = -ENOMEM;
3230 goto error;
3231 }
3232
3233 for_each_possible_cpu(cpu) {
3234 struct deferred_flush_data *dfd = per_cpu_ptr(&deferred_flush,
3235 cpu);
3236
3237 dfd->tables = kzalloc(g_num_of_iommus *
3238 sizeof(struct deferred_flush_table),
3239 GFP_KERNEL);
3240 if (!dfd->tables) {
3241 ret = -ENOMEM;
3242 goto free_g_iommus;
3243 }
3244
3245 spin_lock_init(&dfd->lock);
3246 setup_timer(&dfd->timer, flush_unmaps_timeout, cpu);
3247 }
3248
3249 for_each_active_iommu(iommu, drhd) {
3250 g_iommus[iommu->seq_id] = iommu;
3251
3252 intel_iommu_init_qi(iommu);
3253
3254 ret = iommu_init_domains(iommu);
3255 if (ret)
3256 goto free_iommu;
3257
3258 init_translation_status(iommu);
3259
3260 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3261 iommu_disable_translation(iommu);
3262 clear_translation_pre_enabled(iommu);
3263 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3264 iommu->name);
3265 }
3266
3267 /*
3268 * TBD:
3269 * we could share the same root & context tables
3270 * among all IOMMU's. Need to Split it later.
3271 */
3272 ret = iommu_alloc_root_entry(iommu);
3273 if (ret)
3274 goto free_iommu;
3275
3276 if (translation_pre_enabled(iommu)) {
3277 pr_info("Translation already enabled - trying to copy translation structures\n");
3278
3279 ret = copy_translation_tables(iommu);
3280 if (ret) {
3281 /*
3282 * We found the IOMMU with translation
3283 * enabled - but failed to copy over the
3284 * old root-entry table. Try to proceed
3285 * by disabling translation now and
3286 * allocating a clean root-entry table.
3287 * This might cause DMAR faults, but
3288 * probably the dump will still succeed.
3289 */
3290 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3291 iommu->name);
3292 iommu_disable_translation(iommu);
3293 clear_translation_pre_enabled(iommu);
3294 } else {
3295 pr_info("Copied translation tables from previous kernel for %s\n",
3296 iommu->name);
3297 copied_tables = true;
3298 }
3299 }
3300
3301 if (!ecap_pass_through(iommu->ecap))
3302 hw_pass_through = 0;
3303 #ifdef CONFIG_INTEL_IOMMU_SVM
3304 if (pasid_enabled(iommu))
3305 intel_svm_alloc_pasid_tables(iommu);
3306 #endif
3307 }
3308
3309 /*
3310 * Now that qi is enabled on all iommus, set the root entry and flush
3311 * caches. This is required on some Intel X58 chipsets, otherwise the
3312 * flush_context function will loop forever and the boot hangs.
3313 */
3314 for_each_active_iommu(iommu, drhd) {
3315 iommu_flush_write_buffer(iommu);
3316 iommu_set_root_entry(iommu);
3317 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3318 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3319 }
3320
3321 if (iommu_pass_through)
3322 iommu_identity_mapping |= IDENTMAP_ALL;
3323
3324 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3325 iommu_identity_mapping |= IDENTMAP_GFX;
3326 #endif
3327
3328 if (iommu_identity_mapping) {
3329 ret = si_domain_init(hw_pass_through);
3330 if (ret)
3331 goto free_iommu;
3332 }
3333
3334 check_tylersburg_isoch();
3335
3336 /*
3337 * If we copied translations from a previous kernel in the kdump
3338 * case, we can not assign the devices to domains now, as that
3339 * would eliminate the old mappings. So skip this part and defer
3340 * the assignment to device driver initialization time.
3341 */
3342 if (copied_tables)
3343 goto domains_done;
3344
3345 /*
3346 * If pass through is not set or not enabled, setup context entries for
3347 * identity mappings for rmrr, gfx, and isa and may fall back to static
3348 * identity mapping if iommu_identity_mapping is set.
3349 */
3350 if (iommu_identity_mapping) {
3351 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3352 if (ret) {
3353 pr_crit("Failed to setup IOMMU pass-through\n");
3354 goto free_iommu;
3355 }
3356 }
3357 /*
3358 * For each rmrr
3359 * for each dev attached to rmrr
3360 * do
3361 * locate drhd for dev, alloc domain for dev
3362 * allocate free domain
3363 * allocate page table entries for rmrr
3364 * if context not allocated for bus
3365 * allocate and init context
3366 * set present in root table for this bus
3367 * init context with domain, translation etc
3368 * endfor
3369 * endfor
3370 */
3371 pr_info("Setting RMRR:\n");
3372 for_each_rmrr_units(rmrr) {
3373 /* some BIOS lists non-exist devices in DMAR table. */
3374 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3375 i, dev) {
3376 ret = iommu_prepare_rmrr_dev(rmrr, dev);
3377 if (ret)
3378 pr_err("Mapping reserved region failed\n");
3379 }
3380 }
3381
3382 iommu_prepare_isa();
3383
3384 domains_done:
3385
3386 /*
3387 * for each drhd
3388 * enable fault log
3389 * global invalidate context cache
3390 * global invalidate iotlb
3391 * enable translation
3392 */
3393 for_each_iommu(iommu, drhd) {
3394 if (drhd->ignored) {
3395 /*
3396 * we always have to disable PMRs or DMA may fail on
3397 * this device
3398 */
3399 if (force_on)
3400 iommu_disable_protect_mem_regions(iommu);
3401 continue;
3402 }
3403
3404 iommu_flush_write_buffer(iommu);
3405
3406 #ifdef CONFIG_INTEL_IOMMU_SVM
3407 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
3408 ret = intel_svm_enable_prq(iommu);
3409 if (ret)
3410 goto free_iommu;
3411 }
3412 #endif
3413 ret = dmar_set_interrupt(iommu);
3414 if (ret)
3415 goto free_iommu;
3416
3417 if (!translation_pre_enabled(iommu))
3418 iommu_enable_translation(iommu);
3419
3420 iommu_disable_protect_mem_regions(iommu);
3421 }
3422
3423 return 0;
3424
3425 free_iommu:
3426 for_each_active_iommu(iommu, drhd) {
3427 disable_dmar_iommu(iommu);
3428 free_dmar_iommu(iommu);
3429 }
3430 free_g_iommus:
3431 for_each_possible_cpu(cpu)
3432 kfree(per_cpu_ptr(&deferred_flush, cpu)->tables);
3433 kfree(g_iommus);
3434 error:
3435 return ret;
3436 }
3437
3438 /* This takes a number of _MM_ pages, not VTD pages */
3439 static unsigned long intel_alloc_iova(struct device *dev,
3440 struct dmar_domain *domain,
3441 unsigned long nrpages, uint64_t dma_mask)
3442 {
3443 unsigned long iova_pfn = 0;
3444
3445 /* Restrict dma_mask to the width that the iommu can handle */
3446 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3447 /* Ensure we reserve the whole size-aligned region */
3448 nrpages = __roundup_pow_of_two(nrpages);
3449
3450 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3451 /*
3452 * First try to allocate an io virtual address in
3453 * DMA_BIT_MASK(32) and if that fails then try allocating
3454 * from higher range
3455 */
3456 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3457 IOVA_PFN(DMA_BIT_MASK(32)));
3458 if (iova_pfn)
3459 return iova_pfn;
3460 }
3461 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages, IOVA_PFN(dma_mask));
3462 if (unlikely(!iova_pfn)) {
3463 pr_err("Allocating %ld-page iova for %s failed",
3464 nrpages, dev_name(dev));
3465 return 0;
3466 }
3467
3468 return iova_pfn;
3469 }
3470
3471 static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
3472 {
3473 struct dmar_domain *domain, *tmp;
3474 struct dmar_rmrr_unit *rmrr;
3475 struct device *i_dev;
3476 int i, ret;
3477
3478 domain = find_domain(dev);
3479 if (domain)
3480 goto out;
3481
3482 domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3483 if (!domain)
3484 goto out;
3485
3486 /* We have a new domain - setup possible RMRRs for the device */
3487 rcu_read_lock();
3488 for_each_rmrr_units(rmrr) {
3489 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3490 i, i_dev) {
3491 if (i_dev != dev)
3492 continue;
3493
3494 ret = domain_prepare_identity_map(dev, domain,
3495 rmrr->base_address,
3496 rmrr->end_address);
3497 if (ret)
3498 dev_err(dev, "Mapping reserved region failed\n");
3499 }
3500 }
3501 rcu_read_unlock();
3502
3503 tmp = set_domain_for_dev(dev, domain);
3504 if (!tmp || domain != tmp) {
3505 domain_exit(domain);
3506 domain = tmp;
3507 }
3508
3509 out:
3510
3511 if (!domain)
3512 pr_err("Allocating domain for %s failed\n", dev_name(dev));
3513
3514
3515 return domain;
3516 }
3517
3518 static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3519 {
3520 struct device_domain_info *info;
3521
3522 /* No lock here, assumes no domain exit in normal case */
3523 info = dev->archdata.iommu;
3524 if (likely(info))
3525 return info->domain;
3526
3527 return __get_valid_domain_for_dev(dev);
3528 }
3529
3530 /* Check if the dev needs to go through non-identity map and unmap process.*/
3531 static int iommu_no_mapping(struct device *dev)
3532 {
3533 int found;
3534
3535 if (iommu_dummy(dev))
3536 return 1;
3537
3538 if (!iommu_identity_mapping)
3539 return 0;
3540
3541 found = identity_mapping(dev);
3542 if (found) {
3543 if (iommu_should_identity_map(dev, 0))
3544 return 1;
3545 else {
3546 /*
3547 * 32 bit DMA is removed from si_domain and fall back
3548 * to non-identity mapping.
3549 */
3550 dmar_remove_one_dev_info(si_domain, dev);
3551 pr_info("32bit %s uses non-identity mapping\n",
3552 dev_name(dev));
3553 return 0;
3554 }
3555 } else {
3556 /*
3557 * In case of a detached 64 bit DMA device from vm, the device
3558 * is put into si_domain for identity mapping.
3559 */
3560 if (iommu_should_identity_map(dev, 0)) {
3561 int ret;
3562 ret = domain_add_dev_info(si_domain, dev);
3563 if (!ret) {
3564 pr_info("64bit %s uses identity mapping\n",
3565 dev_name(dev));
3566 return 1;
3567 }
3568 }
3569 }
3570
3571 return 0;
3572 }
3573
3574 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3575 size_t size, int dir, u64 dma_mask)
3576 {
3577 struct dmar_domain *domain;
3578 phys_addr_t start_paddr;
3579 unsigned long iova_pfn;
3580 int prot = 0;
3581 int ret;
3582 struct intel_iommu *iommu;
3583 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3584
3585 BUG_ON(dir == DMA_NONE);
3586
3587 if (iommu_no_mapping(dev))
3588 return paddr;
3589
3590 domain = get_valid_domain_for_dev(dev);
3591 if (!domain)
3592 return 0;
3593
3594 iommu = domain_get_iommu(domain);
3595 size = aligned_nrpages(paddr, size);
3596
3597 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3598 if (!iova_pfn)
3599 goto error;
3600
3601 /*
3602 * Check if DMAR supports zero-length reads on write only
3603 * mappings..
3604 */
3605 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3606 !cap_zlr(iommu->cap))
3607 prot |= DMA_PTE_READ;
3608 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3609 prot |= DMA_PTE_WRITE;
3610 /*
3611 * paddr - (paddr + size) might be partial page, we should map the whole
3612 * page. Note: if two part of one page are separately mapped, we
3613 * might have two guest_addr mapping to the same host paddr, but this
3614 * is not a big problem
3615 */
3616 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
3617 mm_to_dma_pfn(paddr_pfn), size, prot);
3618 if (ret)
3619 goto error;
3620
3621 /* it's a non-present to present mapping. Only flush if caching mode */
3622 if (cap_caching_mode(iommu->cap))
3623 iommu_flush_iotlb_psi(iommu, domain,
3624 mm_to_dma_pfn(iova_pfn),
3625 size, 0, 1);
3626 else
3627 iommu_flush_write_buffer(iommu);
3628
3629 start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
3630 start_paddr += paddr & ~PAGE_MASK;
3631 return start_paddr;
3632
3633 error:
3634 if (iova_pfn)
3635 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3636 pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
3637 dev_name(dev), size, (unsigned long long)paddr, dir);
3638 return 0;
3639 }
3640
3641 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3642 unsigned long offset, size_t size,
3643 enum dma_data_direction dir,
3644 unsigned long attrs)
3645 {
3646 return __intel_map_single(dev, page_to_phys(page) + offset, size,
3647 dir, *dev->dma_mask);
3648 }
3649
3650 static void flush_unmaps(struct deferred_flush_data *flush_data)
3651 {
3652 int i, j;
3653
3654 flush_data->timer_on = 0;
3655
3656 /* just flush them all */
3657 for (i = 0; i < g_num_of_iommus; i++) {
3658 struct intel_iommu *iommu = g_iommus[i];
3659 struct deferred_flush_table *flush_table =
3660 &flush_data->tables[i];
3661 if (!iommu)
3662 continue;
3663
3664 if (!flush_table->next)
3665 continue;
3666
3667 /* In caching mode, global flushes turn emulation expensive */
3668 if (!cap_caching_mode(iommu->cap))
3669 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3670 DMA_TLB_GLOBAL_FLUSH);
3671 for (j = 0; j < flush_table->next; j++) {
3672 unsigned long mask;
3673 struct deferred_flush_entry *entry =
3674 &flush_table->entries[j];
3675 unsigned long iova_pfn = entry->iova_pfn;
3676 unsigned long nrpages = entry->nrpages;
3677 struct dmar_domain *domain = entry->domain;
3678 struct page *freelist = entry->freelist;
3679
3680 /* On real hardware multiple invalidations are expensive */
3681 if (cap_caching_mode(iommu->cap))
3682 iommu_flush_iotlb_psi(iommu, domain,
3683 mm_to_dma_pfn(iova_pfn),
3684 nrpages, !freelist, 0);
3685 else {
3686 mask = ilog2(nrpages);
3687 iommu_flush_dev_iotlb(domain,
3688 (uint64_t)iova_pfn << PAGE_SHIFT, mask);
3689 }
3690 free_iova_fast(&domain->iovad, iova_pfn, nrpages);
3691 if (freelist)
3692 dma_free_pagelist(freelist);
3693 }
3694 flush_table->next = 0;
3695 }
3696
3697 flush_data->size = 0;
3698 }
3699
3700 static void flush_unmaps_timeout(unsigned long cpuid)
3701 {
3702 struct deferred_flush_data *flush_data = per_cpu_ptr(&deferred_flush, cpuid);
3703 unsigned long flags;
3704
3705 spin_lock_irqsave(&flush_data->lock, flags);
3706 flush_unmaps(flush_data);
3707 spin_unlock_irqrestore(&flush_data->lock, flags);
3708 }
3709
3710 static void add_unmap(struct dmar_domain *dom, unsigned long iova_pfn,
3711 unsigned long nrpages, struct page *freelist)
3712 {
3713 unsigned long flags;
3714 int entry_id, iommu_id;
3715 struct intel_iommu *iommu;
3716 struct deferred_flush_entry *entry;
3717 struct deferred_flush_data *flush_data;
3718 unsigned int cpuid;
3719
3720 cpuid = get_cpu();
3721 flush_data = per_cpu_ptr(&deferred_flush, cpuid);
3722
3723 /* Flush all CPUs' entries to avoid deferring too much. If
3724 * this becomes a bottleneck, can just flush us, and rely on
3725 * flush timer for the rest.
3726 */
3727 if (flush_data->size == HIGH_WATER_MARK) {
3728 int cpu;
3729
3730 for_each_online_cpu(cpu)
3731 flush_unmaps_timeout(cpu);
3732 }
3733
3734 spin_lock_irqsave(&flush_data->lock, flags);
3735
3736 iommu = domain_get_iommu(dom);
3737 iommu_id = iommu->seq_id;
3738
3739 entry_id = flush_data->tables[iommu_id].next;
3740 ++(flush_data->tables[iommu_id].next);
3741
3742 entry = &flush_data->tables[iommu_id].entries[entry_id];
3743 entry->domain = dom;
3744 entry->iova_pfn = iova_pfn;
3745 entry->nrpages = nrpages;
3746 entry->freelist = freelist;
3747
3748 if (!flush_data->timer_on) {
3749 mod_timer(&flush_data->timer, jiffies + msecs_to_jiffies(10));
3750 flush_data->timer_on = 1;
3751 }
3752 flush_data->size++;
3753 spin_unlock_irqrestore(&flush_data->lock, flags);
3754
3755 put_cpu();
3756 }
3757
3758 static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
3759 {
3760 struct dmar_domain *domain;
3761 unsigned long start_pfn, last_pfn;
3762 unsigned long nrpages;
3763 unsigned long iova_pfn;
3764 struct intel_iommu *iommu;
3765 struct page *freelist;
3766
3767 if (iommu_no_mapping(dev))
3768 return;
3769
3770 domain = find_domain(dev);
3771 BUG_ON(!domain);
3772
3773 iommu = domain_get_iommu(domain);
3774
3775 iova_pfn = IOVA_PFN(dev_addr);
3776
3777 nrpages = aligned_nrpages(dev_addr, size);
3778 start_pfn = mm_to_dma_pfn(iova_pfn);
3779 last_pfn = start_pfn + nrpages - 1;
3780
3781 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3782 dev_name(dev), start_pfn, last_pfn);
3783
3784 freelist = domain_unmap(domain, start_pfn, last_pfn);
3785
3786 if (intel_iommu_strict) {
3787 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3788 nrpages, !freelist, 0);
3789 /* free iova */
3790 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
3791 dma_free_pagelist(freelist);
3792 } else {
3793 add_unmap(domain, iova_pfn, nrpages, freelist);
3794 /*
3795 * queue up the release of the unmap to save the 1/6th of the
3796 * cpu used up by the iotlb flush operation...
3797 */
3798 }
3799 }
3800
3801 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3802 size_t size, enum dma_data_direction dir,
3803 unsigned long attrs)
3804 {
3805 intel_unmap(dev, dev_addr, size);
3806 }
3807
3808 static void *intel_alloc_coherent(struct device *dev, size_t size,
3809 dma_addr_t *dma_handle, gfp_t flags,
3810 unsigned long attrs)
3811 {
3812 struct page *page = NULL;
3813 int order;
3814
3815 size = PAGE_ALIGN(size);
3816 order = get_order(size);
3817
3818 if (!iommu_no_mapping(dev))
3819 flags &= ~(GFP_DMA | GFP_DMA32);
3820 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3821 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3822 flags |= GFP_DMA;
3823 else
3824 flags |= GFP_DMA32;
3825 }
3826
3827 if (gfpflags_allow_blocking(flags)) {
3828 unsigned int count = size >> PAGE_SHIFT;
3829
3830 page = dma_alloc_from_contiguous(dev, count, order);
3831 if (page && iommu_no_mapping(dev) &&
3832 page_to_phys(page) + size > dev->coherent_dma_mask) {
3833 dma_release_from_contiguous(dev, page, count);
3834 page = NULL;
3835 }
3836 }
3837
3838 if (!page)
3839 page = alloc_pages(flags, order);
3840 if (!page)
3841 return NULL;
3842 memset(page_address(page), 0, size);
3843
3844 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3845 DMA_BIDIRECTIONAL,
3846 dev->coherent_dma_mask);
3847 if (*dma_handle)
3848 return page_address(page);
3849 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3850 __free_pages(page, order);
3851
3852 return NULL;
3853 }
3854
3855 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3856 dma_addr_t dma_handle, unsigned long attrs)
3857 {
3858 int order;
3859 struct page *page = virt_to_page(vaddr);
3860
3861 size = PAGE_ALIGN(size);
3862 order = get_order(size);
3863
3864 intel_unmap(dev, dma_handle, size);
3865 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3866 __free_pages(page, order);
3867 }
3868
3869 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3870 int nelems, enum dma_data_direction dir,
3871 unsigned long attrs)
3872 {
3873 dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
3874 unsigned long nrpages = 0;
3875 struct scatterlist *sg;
3876 int i;
3877
3878 for_each_sg(sglist, sg, nelems, i) {
3879 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3880 }
3881
3882 intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
3883 }
3884
3885 static int intel_nontranslate_map_sg(struct device *hddev,
3886 struct scatterlist *sglist, int nelems, int dir)
3887 {
3888 int i;
3889 struct scatterlist *sg;
3890
3891 for_each_sg(sglist, sg, nelems, i) {
3892 BUG_ON(!sg_page(sg));
3893 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
3894 sg->dma_length = sg->length;
3895 }
3896 return nelems;
3897 }
3898
3899 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3900 enum dma_data_direction dir, unsigned long attrs)
3901 {
3902 int i;
3903 struct dmar_domain *domain;
3904 size_t size = 0;
3905 int prot = 0;
3906 unsigned long iova_pfn;
3907 int ret;
3908 struct scatterlist *sg;
3909 unsigned long start_vpfn;
3910 struct intel_iommu *iommu;
3911
3912 BUG_ON(dir == DMA_NONE);
3913 if (iommu_no_mapping(dev))
3914 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3915
3916 domain = get_valid_domain_for_dev(dev);
3917 if (!domain)
3918 return 0;
3919
3920 iommu = domain_get_iommu(domain);
3921
3922 for_each_sg(sglist, sg, nelems, i)
3923 size += aligned_nrpages(sg->offset, sg->length);
3924
3925 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3926 *dev->dma_mask);
3927 if (!iova_pfn) {
3928 sglist->dma_length = 0;
3929 return 0;
3930 }
3931
3932 /*
3933 * Check if DMAR supports zero-length reads on write only
3934 * mappings..
3935 */
3936 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3937 !cap_zlr(iommu->cap))
3938 prot |= DMA_PTE_READ;
3939 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3940 prot |= DMA_PTE_WRITE;
3941
3942 start_vpfn = mm_to_dma_pfn(iova_pfn);
3943
3944 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3945 if (unlikely(ret)) {
3946 dma_pte_free_pagetable(domain, start_vpfn,
3947 start_vpfn + size - 1);
3948 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3949 return 0;
3950 }
3951
3952 /* it's a non-present to present mapping. Only flush if caching mode */
3953 if (cap_caching_mode(iommu->cap))
3954 iommu_flush_iotlb_psi(iommu, domain, start_vpfn, size, 0, 1);
3955 else
3956 iommu_flush_write_buffer(iommu);
3957
3958 return nelems;
3959 }
3960
3961 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3962 {
3963 return !dma_addr;
3964 }
3965
3966 struct dma_map_ops intel_dma_ops = {
3967 .alloc = intel_alloc_coherent,
3968 .free = intel_free_coherent,
3969 .map_sg = intel_map_sg,
3970 .unmap_sg = intel_unmap_sg,
3971 .map_page = intel_map_page,
3972 .unmap_page = intel_unmap_page,
3973 .mapping_error = intel_mapping_error,
3974 };
3975
3976 static inline int iommu_domain_cache_init(void)
3977 {
3978 int ret = 0;
3979
3980 iommu_domain_cache = kmem_cache_create("iommu_domain",
3981 sizeof(struct dmar_domain),
3982 0,
3983 SLAB_HWCACHE_ALIGN,
3984
3985 NULL);
3986 if (!iommu_domain_cache) {
3987 pr_err("Couldn't create iommu_domain cache\n");
3988 ret = -ENOMEM;
3989 }
3990
3991 return ret;
3992 }
3993
3994 static inline int iommu_devinfo_cache_init(void)
3995 {
3996 int ret = 0;
3997
3998 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3999 sizeof(struct device_domain_info),
4000 0,
4001 SLAB_HWCACHE_ALIGN,
4002 NULL);
4003 if (!iommu_devinfo_cache) {
4004 pr_err("Couldn't create devinfo cache\n");
4005 ret = -ENOMEM;
4006 }
4007
4008 return ret;
4009 }
4010
4011 static int __init iommu_init_mempool(void)
4012 {
4013 int ret;
4014 ret = iova_cache_get();
4015 if (ret)
4016 return ret;
4017
4018 ret = iommu_domain_cache_init();
4019 if (ret)
4020 goto domain_error;
4021
4022 ret = iommu_devinfo_cache_init();
4023 if (!ret)
4024 return ret;
4025
4026 kmem_cache_destroy(iommu_domain_cache);
4027 domain_error:
4028 iova_cache_put();
4029
4030 return -ENOMEM;
4031 }
4032
4033 static void __init iommu_exit_mempool(void)
4034 {
4035 kmem_cache_destroy(iommu_devinfo_cache);
4036 kmem_cache_destroy(iommu_domain_cache);
4037 iova_cache_put();
4038 }
4039
4040 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
4041 {
4042 struct dmar_drhd_unit *drhd;
4043 u32 vtbar;
4044 int rc;
4045
4046 /* We know that this device on this chipset has its own IOMMU.
4047 * If we find it under a different IOMMU, then the BIOS is lying
4048 * to us. Hope that the IOMMU for this device is actually
4049 * disabled, and it needs no translation...
4050 */
4051 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
4052 if (rc) {
4053 /* "can't" happen */
4054 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
4055 return;
4056 }
4057 vtbar &= 0xffff0000;
4058
4059 /* we know that the this iommu should be at offset 0xa000 from vtbar */
4060 drhd = dmar_find_matched_drhd_unit(pdev);
4061 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
4062 TAINT_FIRMWARE_WORKAROUND,
4063 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
4064 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4065 }
4066 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
4067
4068 static void __init init_no_remapping_devices(void)
4069 {
4070 struct dmar_drhd_unit *drhd;
4071 struct device *dev;
4072 int i;
4073
4074 for_each_drhd_unit(drhd) {
4075 if (!drhd->include_all) {
4076 for_each_active_dev_scope(drhd->devices,
4077 drhd->devices_cnt, i, dev)
4078 break;
4079 /* ignore DMAR unit if no devices exist */
4080 if (i == drhd->devices_cnt)
4081 drhd->ignored = 1;
4082 }
4083 }
4084
4085 for_each_active_drhd_unit(drhd) {
4086 if (drhd->include_all)
4087 continue;
4088
4089 for_each_active_dev_scope(drhd->devices,
4090 drhd->devices_cnt, i, dev)
4091 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
4092 break;
4093 if (i < drhd->devices_cnt)
4094 continue;
4095
4096 /* This IOMMU has *only* gfx devices. Either bypass it or
4097 set the gfx_mapped flag, as appropriate */
4098 if (dmar_map_gfx) {
4099 intel_iommu_gfx_mapped = 1;
4100 } else {
4101 drhd->ignored = 1;
4102 for_each_active_dev_scope(drhd->devices,
4103 drhd->devices_cnt, i, dev)
4104 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4105 }
4106 }
4107 }
4108
4109 #ifdef CONFIG_SUSPEND
4110 static int init_iommu_hw(void)
4111 {
4112 struct dmar_drhd_unit *drhd;
4113 struct intel_iommu *iommu = NULL;
4114
4115 for_each_active_iommu(iommu, drhd)
4116 if (iommu->qi)
4117 dmar_reenable_qi(iommu);
4118
4119 for_each_iommu(iommu, drhd) {
4120 if (drhd->ignored) {
4121 /*
4122 * we always have to disable PMRs or DMA may fail on
4123 * this device
4124 */
4125 if (force_on)
4126 iommu_disable_protect_mem_regions(iommu);
4127 continue;
4128 }
4129
4130 iommu_flush_write_buffer(iommu);
4131
4132 iommu_set_root_entry(iommu);
4133
4134 iommu->flush.flush_context(iommu, 0, 0, 0,
4135 DMA_CCMD_GLOBAL_INVL);
4136 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4137 iommu_enable_translation(iommu);
4138 iommu_disable_protect_mem_regions(iommu);
4139 }
4140
4141 return 0;
4142 }
4143
4144 static void iommu_flush_all(void)
4145 {
4146 struct dmar_drhd_unit *drhd;
4147 struct intel_iommu *iommu;
4148
4149 for_each_active_iommu(iommu, drhd) {
4150 iommu->flush.flush_context(iommu, 0, 0, 0,
4151 DMA_CCMD_GLOBAL_INVL);
4152 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
4153 DMA_TLB_GLOBAL_FLUSH);
4154 }
4155 }
4156
4157 static int iommu_suspend(void)
4158 {
4159 struct dmar_drhd_unit *drhd;
4160 struct intel_iommu *iommu = NULL;
4161 unsigned long flag;
4162
4163 for_each_active_iommu(iommu, drhd) {
4164 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
4165 GFP_ATOMIC);
4166 if (!iommu->iommu_state)
4167 goto nomem;
4168 }
4169
4170 iommu_flush_all();
4171
4172 for_each_active_iommu(iommu, drhd) {
4173 iommu_disable_translation(iommu);
4174
4175 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4176
4177 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4178 readl(iommu->reg + DMAR_FECTL_REG);
4179 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4180 readl(iommu->reg + DMAR_FEDATA_REG);
4181 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4182 readl(iommu->reg + DMAR_FEADDR_REG);
4183 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4184 readl(iommu->reg + DMAR_FEUADDR_REG);
4185
4186 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4187 }
4188 return 0;
4189
4190 nomem:
4191 for_each_active_iommu(iommu, drhd)
4192 kfree(iommu->iommu_state);
4193
4194 return -ENOMEM;
4195 }
4196
4197 static void iommu_resume(void)
4198 {
4199 struct dmar_drhd_unit *drhd;
4200 struct intel_iommu *iommu = NULL;
4201 unsigned long flag;
4202
4203 if (init_iommu_hw()) {
4204 if (force_on)
4205 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4206 else
4207 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
4208 return;
4209 }
4210
4211 for_each_active_iommu(iommu, drhd) {
4212
4213 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4214
4215 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4216 iommu->reg + DMAR_FECTL_REG);
4217 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4218 iommu->reg + DMAR_FEDATA_REG);
4219 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4220 iommu->reg + DMAR_FEADDR_REG);
4221 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4222 iommu->reg + DMAR_FEUADDR_REG);
4223
4224 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4225 }
4226
4227 for_each_active_iommu(iommu, drhd)
4228 kfree(iommu->iommu_state);
4229 }
4230
4231 static struct syscore_ops iommu_syscore_ops = {
4232 .resume = iommu_resume,
4233 .suspend = iommu_suspend,
4234 };
4235
4236 static void __init init_iommu_pm_ops(void)
4237 {
4238 register_syscore_ops(&iommu_syscore_ops);
4239 }
4240
4241 #else
4242 static inline void init_iommu_pm_ops(void) {}
4243 #endif /* CONFIG_PM */
4244
4245
4246 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
4247 {
4248 struct acpi_dmar_reserved_memory *rmrr;
4249 struct dmar_rmrr_unit *rmrru;
4250
4251 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4252 if (!rmrru)
4253 return -ENOMEM;
4254
4255 rmrru->hdr = header;
4256 rmrr = (struct acpi_dmar_reserved_memory *)header;
4257 rmrru->base_address = rmrr->base_address;
4258 rmrru->end_address = rmrr->end_address;
4259 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4260 ((void *)rmrr) + rmrr->header.length,
4261 &rmrru->devices_cnt);
4262 if (rmrru->devices_cnt && rmrru->devices == NULL) {
4263 kfree(rmrru);
4264 return -ENOMEM;
4265 }
4266
4267 list_add(&rmrru->list, &dmar_rmrr_units);
4268
4269 return 0;
4270 }
4271
4272 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4273 {
4274 struct dmar_atsr_unit *atsru;
4275 struct acpi_dmar_atsr *tmp;
4276
4277 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4278 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4279 if (atsr->segment != tmp->segment)
4280 continue;
4281 if (atsr->header.length != tmp->header.length)
4282 continue;
4283 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4284 return atsru;
4285 }
4286
4287 return NULL;
4288 }
4289
4290 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4291 {
4292 struct acpi_dmar_atsr *atsr;
4293 struct dmar_atsr_unit *atsru;
4294
4295 if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
4296 return 0;
4297
4298 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4299 atsru = dmar_find_atsr(atsr);
4300 if (atsru)
4301 return 0;
4302
4303 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4304 if (!atsru)
4305 return -ENOMEM;
4306
4307 /*
4308 * If memory is allocated from slab by ACPI _DSM method, we need to
4309 * copy the memory content because the memory buffer will be freed
4310 * on return.
4311 */
4312 atsru->hdr = (void *)(atsru + 1);
4313 memcpy(atsru->hdr, hdr, hdr->length);
4314 atsru->include_all = atsr->flags & 0x1;
4315 if (!atsru->include_all) {
4316 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4317 (void *)atsr + atsr->header.length,
4318 &atsru->devices_cnt);
4319 if (atsru->devices_cnt && atsru->devices == NULL) {
4320 kfree(atsru);
4321 return -ENOMEM;
4322 }
4323 }
4324
4325 list_add_rcu(&atsru->list, &dmar_atsr_units);
4326
4327 return 0;
4328 }
4329
4330 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4331 {
4332 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4333 kfree(atsru);
4334 }
4335
4336 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4337 {
4338 struct acpi_dmar_atsr *atsr;
4339 struct dmar_atsr_unit *atsru;
4340
4341 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4342 atsru = dmar_find_atsr(atsr);
4343 if (atsru) {
4344 list_del_rcu(&atsru->list);
4345 synchronize_rcu();
4346 intel_iommu_free_atsr(atsru);
4347 }
4348
4349 return 0;
4350 }
4351
4352 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4353 {
4354 int i;
4355 struct device *dev;
4356 struct acpi_dmar_atsr *atsr;
4357 struct dmar_atsr_unit *atsru;
4358
4359 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4360 atsru = dmar_find_atsr(atsr);
4361 if (!atsru)
4362 return 0;
4363
4364 if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
4365 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4366 i, dev)
4367 return -EBUSY;
4368 }
4369
4370 return 0;
4371 }
4372
4373 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4374 {
4375 int sp, ret = 0;
4376 struct intel_iommu *iommu = dmaru->iommu;
4377
4378 if (g_iommus[iommu->seq_id])
4379 return 0;
4380
4381 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4382 pr_warn("%s: Doesn't support hardware pass through.\n",
4383 iommu->name);
4384 return -ENXIO;
4385 }
4386 if (!ecap_sc_support(iommu->ecap) &&
4387 domain_update_iommu_snooping(iommu)) {
4388 pr_warn("%s: Doesn't support snooping.\n",
4389 iommu->name);
4390 return -ENXIO;
4391 }
4392 sp = domain_update_iommu_superpage(iommu) - 1;
4393 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4394 pr_warn("%s: Doesn't support large page.\n",
4395 iommu->name);
4396 return -ENXIO;
4397 }
4398
4399 /*
4400 * Disable translation if already enabled prior to OS handover.
4401 */
4402 if (iommu->gcmd & DMA_GCMD_TE)
4403 iommu_disable_translation(iommu);
4404
4405 g_iommus[iommu->seq_id] = iommu;
4406 ret = iommu_init_domains(iommu);
4407 if (ret == 0)
4408 ret = iommu_alloc_root_entry(iommu);
4409 if (ret)
4410 goto out;
4411
4412 #ifdef CONFIG_INTEL_IOMMU_SVM
4413 if (pasid_enabled(iommu))
4414 intel_svm_alloc_pasid_tables(iommu);
4415 #endif
4416
4417 if (dmaru->ignored) {
4418 /*
4419 * we always have to disable PMRs or DMA may fail on this device
4420 */
4421 if (force_on)
4422 iommu_disable_protect_mem_regions(iommu);
4423 return 0;
4424 }
4425
4426 intel_iommu_init_qi(iommu);
4427 iommu_flush_write_buffer(iommu);
4428
4429 #ifdef CONFIG_INTEL_IOMMU_SVM
4430 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
4431 ret = intel_svm_enable_prq(iommu);
4432 if (ret)
4433 goto disable_iommu;
4434 }
4435 #endif
4436 ret = dmar_set_interrupt(iommu);
4437 if (ret)
4438 goto disable_iommu;
4439
4440 iommu_set_root_entry(iommu);
4441 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4442 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4443 iommu_enable_translation(iommu);
4444
4445 iommu_disable_protect_mem_regions(iommu);
4446 return 0;
4447
4448 disable_iommu:
4449 disable_dmar_iommu(iommu);
4450 out:
4451 free_dmar_iommu(iommu);
4452 return ret;
4453 }
4454
4455 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4456 {
4457 int ret = 0;
4458 struct intel_iommu *iommu = dmaru->iommu;
4459
4460 if (!intel_iommu_enabled)
4461 return 0;
4462 if (iommu == NULL)
4463 return -EINVAL;
4464
4465 if (insert) {
4466 ret = intel_iommu_add(dmaru);
4467 } else {
4468 disable_dmar_iommu(iommu);
4469 free_dmar_iommu(iommu);
4470 }
4471
4472 return ret;
4473 }
4474
4475 static void intel_iommu_free_dmars(void)
4476 {
4477 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4478 struct dmar_atsr_unit *atsru, *atsr_n;
4479
4480 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4481 list_del(&rmrru->list);
4482 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4483 kfree(rmrru);
4484 }
4485
4486 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4487 list_del(&atsru->list);
4488 intel_iommu_free_atsr(atsru);
4489 }
4490 }
4491
4492 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4493 {
4494 int i, ret = 1;
4495 struct pci_bus *bus;
4496 struct pci_dev *bridge = NULL;
4497 struct device *tmp;
4498 struct acpi_dmar_atsr *atsr;
4499 struct dmar_atsr_unit *atsru;
4500
4501 dev = pci_physfn(dev);
4502 for (bus = dev->bus; bus; bus = bus->parent) {
4503 bridge = bus->self;
4504 /* If it's an integrated device, allow ATS */
4505 if (!bridge)
4506 return 1;
4507 /* Connected via non-PCIe: no ATS */
4508 if (!pci_is_pcie(bridge) ||
4509 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4510 return 0;
4511 /* If we found the root port, look it up in the ATSR */
4512 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4513 break;
4514 }
4515
4516 rcu_read_lock();
4517 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4518 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4519 if (atsr->segment != pci_domain_nr(dev->bus))
4520 continue;
4521
4522 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4523 if (tmp == &bridge->dev)
4524 goto out;
4525
4526 if (atsru->include_all)
4527 goto out;
4528 }
4529 ret = 0;
4530 out:
4531 rcu_read_unlock();
4532
4533 return ret;
4534 }
4535
4536 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4537 {
4538 int ret = 0;
4539 struct dmar_rmrr_unit *rmrru;
4540 struct dmar_atsr_unit *atsru;
4541 struct acpi_dmar_atsr *atsr;
4542 struct acpi_dmar_reserved_memory *rmrr;
4543
4544 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
4545 return 0;
4546
4547 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4548 rmrr = container_of(rmrru->hdr,
4549 struct acpi_dmar_reserved_memory, header);
4550 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4551 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4552 ((void *)rmrr) + rmrr->header.length,
4553 rmrr->segment, rmrru->devices,
4554 rmrru->devices_cnt);
4555 if(ret < 0)
4556 return ret;
4557 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4558 dmar_remove_dev_scope(info, rmrr->segment,
4559 rmrru->devices, rmrru->devices_cnt);
4560 }
4561 }
4562
4563 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4564 if (atsru->include_all)
4565 continue;
4566
4567 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4568 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4569 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4570 (void *)atsr + atsr->header.length,
4571 atsr->segment, atsru->devices,
4572 atsru->devices_cnt);
4573 if (ret > 0)
4574 break;
4575 else if(ret < 0)
4576 return ret;
4577 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4578 if (dmar_remove_dev_scope(info, atsr->segment,
4579 atsru->devices, atsru->devices_cnt))
4580 break;
4581 }
4582 }
4583
4584 return 0;
4585 }
4586
4587 /*
4588 * Here we only respond to action of unbound device from driver.
4589 *
4590 * Added device is not attached to its DMAR domain here yet. That will happen
4591 * when mapping the device to iova.
4592 */
4593 static int device_notifier(struct notifier_block *nb,
4594 unsigned long action, void *data)
4595 {
4596 struct device *dev = data;
4597 struct dmar_domain *domain;
4598
4599 if (iommu_dummy(dev))
4600 return 0;
4601
4602 if (action != BUS_NOTIFY_REMOVED_DEVICE)
4603 return 0;
4604
4605 domain = find_domain(dev);
4606 if (!domain)
4607 return 0;
4608
4609 dmar_remove_one_dev_info(domain, dev);
4610 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4611 domain_exit(domain);
4612
4613 return 0;
4614 }
4615
4616 static struct notifier_block device_nb = {
4617 .notifier_call = device_notifier,
4618 };
4619
4620 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4621 unsigned long val, void *v)
4622 {
4623 struct memory_notify *mhp = v;
4624 unsigned long long start, end;
4625 unsigned long start_vpfn, last_vpfn;
4626
4627 switch (val) {
4628 case MEM_GOING_ONLINE:
4629 start = mhp->start_pfn << PAGE_SHIFT;
4630 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4631 if (iommu_domain_identity_map(si_domain, start, end)) {
4632 pr_warn("Failed to build identity map for [%llx-%llx]\n",
4633 start, end);
4634 return NOTIFY_BAD;
4635 }
4636 break;
4637
4638 case MEM_OFFLINE:
4639 case MEM_CANCEL_ONLINE:
4640 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4641 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4642 while (start_vpfn <= last_vpfn) {
4643 struct iova *iova;
4644 struct dmar_drhd_unit *drhd;
4645 struct intel_iommu *iommu;
4646 struct page *freelist;
4647
4648 iova = find_iova(&si_domain->iovad, start_vpfn);
4649 if (iova == NULL) {
4650 pr_debug("Failed get IOVA for PFN %lx\n",
4651 start_vpfn);
4652 break;
4653 }
4654
4655 iova = split_and_remove_iova(&si_domain->iovad, iova,
4656 start_vpfn, last_vpfn);
4657 if (iova == NULL) {
4658 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4659 start_vpfn, last_vpfn);
4660 return NOTIFY_BAD;
4661 }
4662
4663 freelist = domain_unmap(si_domain, iova->pfn_lo,
4664 iova->pfn_hi);
4665
4666 rcu_read_lock();
4667 for_each_active_iommu(iommu, drhd)
4668 iommu_flush_iotlb_psi(iommu, si_domain,
4669 iova->pfn_lo, iova_size(iova),
4670 !freelist, 0);
4671 rcu_read_unlock();
4672 dma_free_pagelist(freelist);
4673
4674 start_vpfn = iova->pfn_hi + 1;
4675 free_iova_mem(iova);
4676 }
4677 break;
4678 }
4679
4680 return NOTIFY_OK;
4681 }
4682
4683 static struct notifier_block intel_iommu_memory_nb = {
4684 .notifier_call = intel_iommu_memory_notifier,
4685 .priority = 0
4686 };
4687
4688 static void free_all_cpu_cached_iovas(unsigned int cpu)
4689 {
4690 int i;
4691
4692 for (i = 0; i < g_num_of_iommus; i++) {
4693 struct intel_iommu *iommu = g_iommus[i];
4694 struct dmar_domain *domain;
4695 int did;
4696
4697 if (!iommu)
4698 continue;
4699
4700 for (did = 0; did < cap_ndoms(iommu->cap); did++) {
4701 domain = get_iommu_domain(iommu, (u16)did);
4702
4703 if (!domain)
4704 continue;
4705 free_cpu_cached_iovas(cpu, &domain->iovad);
4706 }
4707 }
4708 }
4709
4710 static int intel_iommu_cpu_dead(unsigned int cpu)
4711 {
4712 free_all_cpu_cached_iovas(cpu);
4713 flush_unmaps_timeout(cpu);
4714 return 0;
4715 }
4716
4717 static ssize_t intel_iommu_show_version(struct device *dev,
4718 struct device_attribute *attr,
4719 char *buf)
4720 {
4721 struct intel_iommu *iommu = dev_get_drvdata(dev);
4722 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4723 return sprintf(buf, "%d:%d\n",
4724 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4725 }
4726 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4727
4728 static ssize_t intel_iommu_show_address(struct device *dev,
4729 struct device_attribute *attr,
4730 char *buf)
4731 {
4732 struct intel_iommu *iommu = dev_get_drvdata(dev);
4733 return sprintf(buf, "%llx\n", iommu->reg_phys);
4734 }
4735 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4736
4737 static ssize_t intel_iommu_show_cap(struct device *dev,
4738 struct device_attribute *attr,
4739 char *buf)
4740 {
4741 struct intel_iommu *iommu = dev_get_drvdata(dev);
4742 return sprintf(buf, "%llx\n", iommu->cap);
4743 }
4744 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4745
4746 static ssize_t intel_iommu_show_ecap(struct device *dev,
4747 struct device_attribute *attr,
4748 char *buf)
4749 {
4750 struct intel_iommu *iommu = dev_get_drvdata(dev);
4751 return sprintf(buf, "%llx\n", iommu->ecap);
4752 }
4753 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4754
4755 static ssize_t intel_iommu_show_ndoms(struct device *dev,
4756 struct device_attribute *attr,
4757 char *buf)
4758 {
4759 struct intel_iommu *iommu = dev_get_drvdata(dev);
4760 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4761 }
4762 static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4763
4764 static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4765 struct device_attribute *attr,
4766 char *buf)
4767 {
4768 struct intel_iommu *iommu = dev_get_drvdata(dev);
4769 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4770 cap_ndoms(iommu->cap)));
4771 }
4772 static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4773
4774 static struct attribute *intel_iommu_attrs[] = {
4775 &dev_attr_version.attr,
4776 &dev_attr_address.attr,
4777 &dev_attr_cap.attr,
4778 &dev_attr_ecap.attr,
4779 &dev_attr_domains_supported.attr,
4780 &dev_attr_domains_used.attr,
4781 NULL,
4782 };
4783
4784 static struct attribute_group intel_iommu_group = {
4785 .name = "intel-iommu",
4786 .attrs = intel_iommu_attrs,
4787 };
4788
4789 const struct attribute_group *intel_iommu_groups[] = {
4790 &intel_iommu_group,
4791 NULL,
4792 };
4793
4794 int __init intel_iommu_init(void)
4795 {
4796 int ret = -ENODEV;
4797 struct dmar_drhd_unit *drhd;
4798 struct intel_iommu *iommu;
4799
4800 /* VT-d is required for a TXT/tboot launch, so enforce that */
4801 force_on = tboot_force_iommu();
4802
4803 if (iommu_init_mempool()) {
4804 if (force_on)
4805 panic("tboot: Failed to initialize iommu memory\n");
4806 return -ENOMEM;
4807 }
4808
4809 down_write(&dmar_global_lock);
4810 if (dmar_table_init()) {
4811 if (force_on)
4812 panic("tboot: Failed to initialize DMAR table\n");
4813 goto out_free_dmar;
4814 }
4815
4816 if (dmar_dev_scope_init() < 0) {
4817 if (force_on)
4818 panic("tboot: Failed to initialize DMAR device scope\n");
4819 goto out_free_dmar;
4820 }
4821
4822 if (no_iommu || dmar_disabled)
4823 goto out_free_dmar;
4824
4825 if (list_empty(&dmar_rmrr_units))
4826 pr_info("No RMRR found\n");
4827
4828 if (list_empty(&dmar_atsr_units))
4829 pr_info("No ATSR found\n");
4830
4831 if (dmar_init_reserved_ranges()) {
4832 if (force_on)
4833 panic("tboot: Failed to reserve iommu ranges\n");
4834 goto out_free_reserved_range;
4835 }
4836
4837 init_no_remapping_devices();
4838
4839 ret = init_dmars();
4840 if (ret) {
4841 if (force_on)
4842 panic("tboot: Failed to initialize DMARs\n");
4843 pr_err("Initialization failed\n");
4844 goto out_free_reserved_range;
4845 }
4846 up_write(&dmar_global_lock);
4847 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4848
4849 #ifdef CONFIG_SWIOTLB
4850 swiotlb = 0;
4851 #endif
4852 dma_ops = &intel_dma_ops;
4853
4854 init_iommu_pm_ops();
4855
4856 for_each_active_iommu(iommu, drhd)
4857 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4858 intel_iommu_groups,
4859 "%s", iommu->name);
4860
4861 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4862 bus_register_notifier(&pci_bus_type, &device_nb);
4863 if (si_domain && !hw_pass_through)
4864 register_memory_notifier(&intel_iommu_memory_nb);
4865 cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL,
4866 intel_iommu_cpu_dead);
4867 intel_iommu_enabled = 1;
4868
4869 return 0;
4870
4871 out_free_reserved_range:
4872 put_iova_domain(&reserved_iova_list);
4873 out_free_dmar:
4874 intel_iommu_free_dmars();
4875 up_write(&dmar_global_lock);
4876 iommu_exit_mempool();
4877 return ret;
4878 }
4879
4880 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4881 {
4882 struct intel_iommu *iommu = opaque;
4883
4884 domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4885 return 0;
4886 }
4887
4888 /*
4889 * NB - intel-iommu lacks any sort of reference counting for the users of
4890 * dependent devices. If multiple endpoints have intersecting dependent
4891 * devices, unbinding the driver from any one of them will possibly leave
4892 * the others unable to operate.
4893 */
4894 static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
4895 {
4896 if (!iommu || !dev || !dev_is_pci(dev))
4897 return;
4898
4899 pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
4900 }
4901
4902 static void __dmar_remove_one_dev_info(struct device_domain_info *info)
4903 {
4904 struct intel_iommu *iommu;
4905 unsigned long flags;
4906
4907 assert_spin_locked(&device_domain_lock);
4908
4909 if (WARN_ON(!info))
4910 return;
4911
4912 iommu = info->iommu;
4913
4914 if (info->dev) {
4915 iommu_disable_dev_iotlb(info);
4916 domain_context_clear(iommu, info->dev);
4917 }
4918
4919 unlink_domain_info(info);
4920
4921 spin_lock_irqsave(&iommu->lock, flags);
4922 domain_detach_iommu(info->domain, iommu);
4923 spin_unlock_irqrestore(&iommu->lock, flags);
4924
4925 free_devinfo_mem(info);
4926 }
4927
4928 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4929 struct device *dev)
4930 {
4931 struct device_domain_info *info;
4932 unsigned long flags;
4933
4934 spin_lock_irqsave(&device_domain_lock, flags);
4935 info = dev->archdata.iommu;
4936 __dmar_remove_one_dev_info(info);
4937 spin_unlock_irqrestore(&device_domain_lock, flags);
4938 }
4939
4940 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4941 {
4942 int adjust_width;
4943
4944 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4945 DMA_32BIT_PFN);
4946 domain_reserve_special_ranges(domain);
4947
4948 /* calculate AGAW */
4949 domain->gaw = guest_width;
4950 adjust_width = guestwidth_to_adjustwidth(guest_width);
4951 domain->agaw = width_to_agaw(adjust_width);
4952
4953 domain->iommu_coherency = 0;
4954 domain->iommu_snooping = 0;
4955 domain->iommu_superpage = 0;
4956 domain->max_addr = 0;
4957
4958 /* always allocate the top pgd */
4959 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
4960 if (!domain->pgd)
4961 return -ENOMEM;
4962 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4963 return 0;
4964 }
4965
4966 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
4967 {
4968 struct dmar_domain *dmar_domain;
4969 struct iommu_domain *domain;
4970
4971 if (type != IOMMU_DOMAIN_UNMANAGED)
4972 return NULL;
4973
4974 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
4975 if (!dmar_domain) {
4976 pr_err("Can't allocate dmar_domain\n");
4977 return NULL;
4978 }
4979 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4980 pr_err("Domain initialization failed\n");
4981 domain_exit(dmar_domain);
4982 return NULL;
4983 }
4984 domain_update_iommu_cap(dmar_domain);
4985
4986 domain = &dmar_domain->domain;
4987 domain->geometry.aperture_start = 0;
4988 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4989 domain->geometry.force_aperture = true;
4990
4991 return domain;
4992 }
4993
4994 static void intel_iommu_domain_free(struct iommu_domain *domain)
4995 {
4996 domain_exit(to_dmar_domain(domain));
4997 }
4998
4999 static int intel_iommu_attach_device(struct iommu_domain *domain,
5000 struct device *dev)
5001 {
5002 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5003 struct intel_iommu *iommu;
5004 int addr_width;
5005 u8 bus, devfn;
5006
5007 if (device_is_rmrr_locked(dev)) {
5008 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
5009 return -EPERM;
5010 }
5011
5012 /* normally dev is not mapped */
5013 if (unlikely(domain_context_mapped(dev))) {
5014 struct dmar_domain *old_domain;
5015
5016 old_domain = find_domain(dev);
5017 if (old_domain) {
5018 rcu_read_lock();
5019 dmar_remove_one_dev_info(old_domain, dev);
5020 rcu_read_unlock();
5021
5022 if (!domain_type_is_vm_or_si(old_domain) &&
5023 list_empty(&old_domain->devices))
5024 domain_exit(old_domain);
5025 }
5026 }
5027
5028 iommu = device_to_iommu(dev, &bus, &devfn);
5029 if (!iommu)
5030 return -ENODEV;
5031
5032 /* check if this iommu agaw is sufficient for max mapped address */
5033 addr_width = agaw_to_width(iommu->agaw);
5034 if (addr_width > cap_mgaw(iommu->cap))
5035 addr_width = cap_mgaw(iommu->cap);
5036
5037 if (dmar_domain->max_addr > (1LL << addr_width)) {
5038 pr_err("%s: iommu width (%d) is not "
5039 "sufficient for the mapped address (%llx)\n",
5040 __func__, addr_width, dmar_domain->max_addr);
5041 return -EFAULT;
5042 }
5043 dmar_domain->gaw = addr_width;
5044
5045 /*
5046 * Knock out extra levels of page tables if necessary
5047 */
5048 while (iommu->agaw < dmar_domain->agaw) {
5049 struct dma_pte *pte;
5050
5051 pte = dmar_domain->pgd;
5052 if (dma_pte_present(pte)) {
5053 dmar_domain->pgd = (struct dma_pte *)
5054 phys_to_virt(dma_pte_addr(pte));
5055 free_pgtable_page(pte);
5056 }
5057 dmar_domain->agaw--;
5058 }
5059
5060 return domain_add_dev_info(dmar_domain, dev);
5061 }
5062
5063 static void intel_iommu_detach_device(struct iommu_domain *domain,
5064 struct device *dev)
5065 {
5066 dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
5067 }
5068
5069 static int intel_iommu_map(struct iommu_domain *domain,
5070 unsigned long iova, phys_addr_t hpa,
5071 size_t size, int iommu_prot)
5072 {
5073 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5074 u64 max_addr;
5075 int prot = 0;
5076 int ret;
5077
5078 if (iommu_prot & IOMMU_READ)
5079 prot |= DMA_PTE_READ;
5080 if (iommu_prot & IOMMU_WRITE)
5081 prot |= DMA_PTE_WRITE;
5082 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
5083 prot |= DMA_PTE_SNP;
5084
5085 max_addr = iova + size;
5086 if (dmar_domain->max_addr < max_addr) {
5087 u64 end;
5088
5089 /* check if minimum agaw is sufficient for mapped address */
5090 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
5091 if (end < max_addr) {
5092 pr_err("%s: iommu width (%d) is not "
5093 "sufficient for the mapped address (%llx)\n",
5094 __func__, dmar_domain->gaw, max_addr);
5095 return -EFAULT;
5096 }
5097 dmar_domain->max_addr = max_addr;
5098 }
5099 /* Round up size to next multiple of PAGE_SIZE, if it and
5100 the low bits of hpa would take us onto the next page */
5101 size = aligned_nrpages(hpa, size);
5102 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
5103 hpa >> VTD_PAGE_SHIFT, size, prot);
5104 return ret;
5105 }
5106
5107 static size_t intel_iommu_unmap(struct iommu_domain *domain,
5108 unsigned long iova, size_t size)
5109 {
5110 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5111 struct page *freelist = NULL;
5112 struct intel_iommu *iommu;
5113 unsigned long start_pfn, last_pfn;
5114 unsigned int npages;
5115 int iommu_id, level = 0;
5116
5117 /* Cope with horrid API which requires us to unmap more than the
5118 size argument if it happens to be a large-page mapping. */
5119 BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
5120
5121 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
5122 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
5123
5124 start_pfn = iova >> VTD_PAGE_SHIFT;
5125 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5126
5127 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
5128
5129 npages = last_pfn - start_pfn + 1;
5130
5131 for_each_domain_iommu(iommu_id, dmar_domain) {
5132 iommu = g_iommus[iommu_id];
5133
5134 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5135 start_pfn, npages, !freelist, 0);
5136 }
5137
5138 dma_free_pagelist(freelist);
5139
5140 if (dmar_domain->max_addr == iova + size)
5141 dmar_domain->max_addr = iova;
5142
5143 return size;
5144 }
5145
5146 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
5147 dma_addr_t iova)
5148 {
5149 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5150 struct dma_pte *pte;
5151 int level = 0;
5152 u64 phys = 0;
5153
5154 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
5155 if (pte)
5156 phys = dma_pte_addr(pte);
5157
5158 return phys;
5159 }
5160
5161 static bool intel_iommu_capable(enum iommu_cap cap)
5162 {
5163 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5164 return domain_update_iommu_snooping(NULL) == 1;
5165 if (cap == IOMMU_CAP_INTR_REMAP)
5166 return irq_remapping_enabled == 1;
5167
5168 return false;
5169 }
5170
5171 static int intel_iommu_add_device(struct device *dev)
5172 {
5173 struct intel_iommu *iommu;
5174 struct iommu_group *group;
5175 u8 bus, devfn;
5176
5177 iommu = device_to_iommu(dev, &bus, &devfn);
5178 if (!iommu)
5179 return -ENODEV;
5180
5181 iommu_device_link(iommu->iommu_dev, dev);
5182
5183 group = iommu_group_get_for_dev(dev);
5184
5185 if (IS_ERR(group))
5186 return PTR_ERR(group);
5187
5188 iommu_group_put(group);
5189 return 0;
5190 }
5191
5192 static void intel_iommu_remove_device(struct device *dev)
5193 {
5194 struct intel_iommu *iommu;
5195 u8 bus, devfn;
5196
5197 iommu = device_to_iommu(dev, &bus, &devfn);
5198 if (!iommu)
5199 return;
5200
5201 iommu_group_remove_device(dev);
5202
5203 iommu_device_unlink(iommu->iommu_dev, dev);
5204 }
5205
5206 #ifdef CONFIG_INTEL_IOMMU_SVM
5207 #define MAX_NR_PASID_BITS (20)
5208 static inline unsigned long intel_iommu_get_pts(struct intel_iommu *iommu)
5209 {
5210 /*
5211 * Convert ecap_pss to extend context entry pts encoding, also
5212 * respect the soft pasid_max value set by the iommu.
5213 * - number of PASID bits = ecap_pss + 1
5214 * - number of PASID table entries = 2^(pts + 5)
5215 * Therefore, pts = ecap_pss - 4
5216 * e.g. KBL ecap_pss = 0x13, PASID has 20 bits, pts = 15
5217 */
5218 if (ecap_pss(iommu->ecap) < 5)
5219 return 0;
5220
5221 /* pasid_max is encoded as actual number of entries not the bits */
5222 return find_first_bit((unsigned long *)&iommu->pasid_max,
5223 MAX_NR_PASID_BITS) - 5;
5224 }
5225
5226 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5227 {
5228 struct device_domain_info *info;
5229 struct context_entry *context;
5230 struct dmar_domain *domain;
5231 unsigned long flags;
5232 u64 ctx_lo;
5233 int ret;
5234
5235 domain = get_valid_domain_for_dev(sdev->dev);
5236 if (!domain)
5237 return -EINVAL;
5238
5239 spin_lock_irqsave(&device_domain_lock, flags);
5240 spin_lock(&iommu->lock);
5241
5242 ret = -EINVAL;
5243 info = sdev->dev->archdata.iommu;
5244 if (!info || !info->pasid_supported)
5245 goto out;
5246
5247 context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5248 if (WARN_ON(!context))
5249 goto out;
5250
5251 ctx_lo = context[0].lo;
5252
5253 sdev->did = domain->iommu_did[iommu->seq_id];
5254 sdev->sid = PCI_DEVID(info->bus, info->devfn);
5255
5256 if (!(ctx_lo & CONTEXT_PASIDE)) {
5257 context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
5258 context[1].lo = (u64)virt_to_phys(iommu->pasid_table) |
5259 intel_iommu_get_pts(iommu);
5260
5261 wmb();
5262 /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both
5263 * extended to permit requests-with-PASID if the PASIDE bit
5264 * is set. which makes sense. For CONTEXT_TT_PASS_THROUGH,
5265 * however, the PASIDE bit is ignored and requests-with-PASID
5266 * are unconditionally blocked. Which makes less sense.
5267 * So convert from CONTEXT_TT_PASS_THROUGH to one of the new
5268 * "guest mode" translation types depending on whether ATS
5269 * is available or not. Annoyingly, we can't use the new
5270 * modes *unless* PASIDE is set. */
5271 if ((ctx_lo & CONTEXT_TT_MASK) == (CONTEXT_TT_PASS_THROUGH << 2)) {
5272 ctx_lo &= ~CONTEXT_TT_MASK;
5273 if (info->ats_supported)
5274 ctx_lo |= CONTEXT_TT_PT_PASID_DEV_IOTLB << 2;
5275 else
5276 ctx_lo |= CONTEXT_TT_PT_PASID << 2;
5277 }
5278 ctx_lo |= CONTEXT_PASIDE;
5279 if (iommu->pasid_state_table)
5280 ctx_lo |= CONTEXT_DINVE;
5281 if (info->pri_supported)
5282 ctx_lo |= CONTEXT_PRS;
5283 context[0].lo = ctx_lo;
5284 wmb();
5285 iommu->flush.flush_context(iommu, sdev->did, sdev->sid,
5286 DMA_CCMD_MASK_NOBIT,
5287 DMA_CCMD_DEVICE_INVL);
5288 }
5289
5290 /* Enable PASID support in the device, if it wasn't already */
5291 if (!info->pasid_enabled)
5292 iommu_enable_dev_iotlb(info);
5293
5294 if (info->ats_enabled) {
5295 sdev->dev_iotlb = 1;
5296 sdev->qdep = info->ats_qdep;
5297 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5298 sdev->qdep = 0;
5299 }
5300 ret = 0;
5301
5302 out:
5303 spin_unlock(&iommu->lock);
5304 spin_unlock_irqrestore(&device_domain_lock, flags);
5305
5306 return ret;
5307 }
5308
5309 struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5310 {
5311 struct intel_iommu *iommu;
5312 u8 bus, devfn;
5313
5314 if (iommu_dummy(dev)) {
5315 dev_warn(dev,
5316 "No IOMMU translation for device; cannot enable SVM\n");
5317 return NULL;
5318 }
5319
5320 iommu = device_to_iommu(dev, &bus, &devfn);
5321 if ((!iommu)) {
5322 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
5323 return NULL;
5324 }
5325
5326 if (!iommu->pasid_table) {
5327 dev_err(dev, "PASID not enabled on IOMMU; cannot enable SVM\n");
5328 return NULL;
5329 }
5330
5331 return iommu;
5332 }
5333 #endif /* CONFIG_INTEL_IOMMU_SVM */
5334
5335 static const struct iommu_ops intel_iommu_ops = {
5336 .capable = intel_iommu_capable,
5337 .domain_alloc = intel_iommu_domain_alloc,
5338 .domain_free = intel_iommu_domain_free,
5339 .attach_dev = intel_iommu_attach_device,
5340 .detach_dev = intel_iommu_detach_device,
5341 .map = intel_iommu_map,
5342 .unmap = intel_iommu_unmap,
5343 .map_sg = default_iommu_map_sg,
5344 .iova_to_phys = intel_iommu_iova_to_phys,
5345 .add_device = intel_iommu_add_device,
5346 .remove_device = intel_iommu_remove_device,
5347 .device_group = pci_device_group,
5348 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
5349 };
5350
5351 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5352 {
5353 /* G4x/GM45 integrated gfx dmar support is totally busted. */
5354 pr_info("Disabling IOMMU for graphics on this chipset\n");
5355 dmar_map_gfx = 0;
5356 }
5357
5358 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5359 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5360 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5361 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5362 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5363 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5364 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5365
5366 static void quirk_iommu_rwbf(struct pci_dev *dev)
5367 {
5368 /*
5369 * Mobile 4 Series Chipset neglects to set RWBF capability,
5370 * but needs it. Same seems to hold for the desktop versions.
5371 */
5372 pr_info("Forcing write-buffer flush capability\n");
5373 rwbf_quirk = 1;
5374 }
5375
5376 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
5377 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5378 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5379 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5380 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5381 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5382 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
5383
5384 #define GGC 0x52
5385 #define GGC_MEMORY_SIZE_MASK (0xf << 8)
5386 #define GGC_MEMORY_SIZE_NONE (0x0 << 8)
5387 #define GGC_MEMORY_SIZE_1M (0x1 << 8)
5388 #define GGC_MEMORY_SIZE_2M (0x3 << 8)
5389 #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
5390 #define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
5391 #define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
5392 #define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
5393
5394 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
5395 {
5396 unsigned short ggc;
5397
5398 if (pci_read_config_word(dev, GGC, &ggc))
5399 return;
5400
5401 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
5402 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
5403 dmar_map_gfx = 0;
5404 } else if (dmar_map_gfx) {
5405 /* we have to ensure the gfx device is idle before we flush */
5406 pr_info("Disabling batched IOTLB flush on Ironlake\n");
5407 intel_iommu_strict = 1;
5408 }
5409 }
5410 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5411 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5412 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5413 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5414
5415 /* On Tylersburg chipsets, some BIOSes have been known to enable the
5416 ISOCH DMAR unit for the Azalia sound device, but not give it any
5417 TLB entries, which causes it to deadlock. Check for that. We do
5418 this in a function called from init_dmars(), instead of in a PCI
5419 quirk, because we don't want to print the obnoxious "BIOS broken"
5420 message if VT-d is actually disabled.
5421 */
5422 static void __init check_tylersburg_isoch(void)
5423 {
5424 struct pci_dev *pdev;
5425 uint32_t vtisochctrl;
5426
5427 /* If there's no Azalia in the system anyway, forget it. */
5428 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5429 if (!pdev)
5430 return;
5431 pci_dev_put(pdev);
5432
5433 /* System Management Registers. Might be hidden, in which case
5434 we can't do the sanity check. But that's OK, because the
5435 known-broken BIOSes _don't_ actually hide it, so far. */
5436 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5437 if (!pdev)
5438 return;
5439
5440 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5441 pci_dev_put(pdev);
5442 return;
5443 }
5444
5445 pci_dev_put(pdev);
5446
5447 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5448 if (vtisochctrl & 1)
5449 return;
5450
5451 /* Drop all bits other than the number of TLB entries */
5452 vtisochctrl &= 0x1c;
5453
5454 /* If we have the recommended number of TLB entries (16), fine. */
5455 if (vtisochctrl == 0x10)
5456 return;
5457
5458 /* Zero TLB entries? You get to ride the short bus to school. */
5459 if (!vtisochctrl) {
5460 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5461 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5462 dmi_get_system_info(DMI_BIOS_VENDOR),
5463 dmi_get_system_info(DMI_BIOS_VERSION),
5464 dmi_get_system_info(DMI_PRODUCT_VERSION));
5465 iommu_identity_mapping |= IDENTMAP_AZALIA;
5466 return;
5467 }
5468
5469 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
5470 vtisochctrl);
5471 }
CRIS linux kernel tree