?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 4.16.11
[linux/fpc-iii.git] / drivers / iommu / intel-iommu.c
blob582fd01cb7d1b5ab9a0ece0d08e4bbdc0aac6188
1 /*
2 * Copyright © 2006-2014 Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
18 * Joerg Roedel <jroedel@suse.de>
19 */
20
21 #define pr_fmt(fmt) "DMAR: " fmt
22
23 #include <linux/init.h>
24 #include <linux/bitmap.h>
25 #include <linux/debugfs.h>
26 #include <linux/export.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/memory.h>
36 #include <linux/cpu.h>
37 #include <linux/timer.h>
38 #include <linux/io.h>
39 #include <linux/iova.h>
40 #include <linux/iommu.h>
41 #include <linux/intel-iommu.h>
42 #include <linux/syscore_ops.h>
43 #include <linux/tboot.h>
44 #include <linux/dmi.h>
45 #include <linux/pci-ats.h>
46 #include <linux/memblock.h>
47 #include <linux/dma-contiguous.h>
48 #include <linux/crash_dump.h>
49 #include <asm/irq_remapping.h>
50 #include <asm/cacheflush.h>
51 #include <asm/iommu.h>
52
53 #include "irq_remapping.h"
54
55 #define ROOT_SIZE VTD_PAGE_SIZE
56 #define CONTEXT_SIZE VTD_PAGE_SIZE
57
58 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
59 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
60 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
61 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
62
63 #define IOAPIC_RANGE_START (0xfee00000)
64 #define IOAPIC_RANGE_END (0xfeefffff)
65 #define IOVA_START_ADDR (0x1000)
66
67 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 57
68
69 #define MAX_AGAW_WIDTH 64
70 #define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
71
72 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
73 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
74
75 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
76 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
77 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
78 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
79 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
80
81 /* IO virtual address start page frame number */
82 #define IOVA_START_PFN (1)
83
84 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
85
86 /* page table handling */
87 #define LEVEL_STRIDE (9)
88 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
89
90 /*
91 * This bitmap is used to advertise the page sizes our hardware support
92 * to the IOMMU core, which will then use this information to split
93 * physically contiguous memory regions it is mapping into page sizes
94 * that we support.
95 *
96 * Traditionally the IOMMU core just handed us the mappings directly,
97 * after making sure the size is an order of a 4KiB page and that the
98 * mapping has natural alignment.
99 *
100 * To retain this behavior, we currently advertise that we support
101 * all page sizes that are an order of 4KiB.
102 *
103 * If at some point we'd like to utilize the IOMMU core's new behavior,
104 * we could change this to advertise the real page sizes we support.
105 */
106 #define INTEL_IOMMU_PGSIZES (~0xFFFUL)
107
108 static inline int agaw_to_level(int agaw)
109 {
110 return agaw + 2;
111 }
112
113 static inline int agaw_to_width(int agaw)
114 {
115 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
116 }
117
118 static inline int width_to_agaw(int width)
119 {
120 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
121 }
122
123 static inline unsigned int level_to_offset_bits(int level)
124 {
125 return (level - 1) * LEVEL_STRIDE;
126 }
127
128 static inline int pfn_level_offset(unsigned long pfn, int level)
129 {
130 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
131 }
132
133 static inline unsigned long level_mask(int level)
134 {
135 return -1UL << level_to_offset_bits(level);
136 }
137
138 static inline unsigned long level_size(int level)
139 {
140 return 1UL << level_to_offset_bits(level);
141 }
142
143 static inline unsigned long align_to_level(unsigned long pfn, int level)
144 {
145 return (pfn + level_size(level) - 1) & level_mask(level);
146 }
147
148 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
149 {
150 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
151 }
152
153 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
154 are never going to work. */
155 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
156 {
157 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
158 }
159
160 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
161 {
162 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
163 }
164 static inline unsigned long page_to_dma_pfn(struct page *pg)
165 {
166 return mm_to_dma_pfn(page_to_pfn(pg));
167 }
168 static inline unsigned long virt_to_dma_pfn(void *p)
169 {
170 return page_to_dma_pfn(virt_to_page(p));
171 }
172
173 /* global iommu list, set NULL for ignored DMAR units */
174 static struct intel_iommu **g_iommus;
175
176 static void __init check_tylersburg_isoch(void);
177 static int rwbf_quirk;
178
179 /*
180 * set to 1 to panic kernel if can't successfully enable VT-d
181 * (used when kernel is launched w/ TXT)
182 */
183 static int force_on = 0;
184 int intel_iommu_tboot_noforce;
185
186 /*
187 * 0: Present
188 * 1-11: Reserved
189 * 12-63: Context Ptr (12 - (haw-1))
190 * 64-127: Reserved
191 */
192 struct root_entry {
193 u64 lo;
194 u64 hi;
195 };
196 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
197
198 /*
199 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
200 * if marked present.
201 */
202 static phys_addr_t root_entry_lctp(struct root_entry *re)
203 {
204 if (!(re->lo & 1))
205 return 0;
206
207 return re->lo & VTD_PAGE_MASK;
208 }
209
210 /*
211 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
212 * if marked present.
213 */
214 static phys_addr_t root_entry_uctp(struct root_entry *re)
215 {
216 if (!(re->hi & 1))
217 return 0;
218
219 return re->hi & VTD_PAGE_MASK;
220 }
221 /*
222 * low 64 bits:
223 * 0: present
224 * 1: fault processing disable
225 * 2-3: translation type
226 * 12-63: address space root
227 * high 64 bits:
228 * 0-2: address width
229 * 3-6: aval
230 * 8-23: domain id
231 */
232 struct context_entry {
233 u64 lo;
234 u64 hi;
235 };
236
237 static inline void context_clear_pasid_enable(struct context_entry *context)
238 {
239 context->lo &= ~(1ULL << 11);
240 }
241
242 static inline bool context_pasid_enabled(struct context_entry *context)
243 {
244 return !!(context->lo & (1ULL << 11));
245 }
246
247 static inline void context_set_copied(struct context_entry *context)
248 {
249 context->hi |= (1ull << 3);
250 }
251
252 static inline bool context_copied(struct context_entry *context)
253 {
254 return !!(context->hi & (1ULL << 3));
255 }
256
257 static inline bool __context_present(struct context_entry *context)
258 {
259 return (context->lo & 1);
260 }
261
262 static inline bool context_present(struct context_entry *context)
263 {
264 return context_pasid_enabled(context) ?
265 __context_present(context) :
266 __context_present(context) && !context_copied(context);
267 }
268
269 static inline void context_set_present(struct context_entry *context)
270 {
271 context->lo |= 1;
272 }
273
274 static inline void context_set_fault_enable(struct context_entry *context)
275 {
276 context->lo &= (((u64)-1) << 2) | 1;
277 }
278
279 static inline void context_set_translation_type(struct context_entry *context,
280 unsigned long value)
281 {
282 context->lo &= (((u64)-1) << 4) | 3;
283 context->lo |= (value & 3) << 2;
284 }
285
286 static inline void context_set_address_root(struct context_entry *context,
287 unsigned long value)
288 {
289 context->lo &= ~VTD_PAGE_MASK;
290 context->lo |= value & VTD_PAGE_MASK;
291 }
292
293 static inline void context_set_address_width(struct context_entry *context,
294 unsigned long value)
295 {
296 context->hi |= value & 7;
297 }
298
299 static inline void context_set_domain_id(struct context_entry *context,
300 unsigned long value)
301 {
302 context->hi |= (value & ((1 << 16) - 1)) << 8;
303 }
304
305 static inline int context_domain_id(struct context_entry *c)
306 {
307 return((c->hi >> 8) & 0xffff);
308 }
309
310 static inline void context_clear_entry(struct context_entry *context)
311 {
312 context->lo = 0;
313 context->hi = 0;
314 }
315
316 /*
317 * 0: readable
318 * 1: writable
319 * 2-6: reserved
320 * 7: super page
321 * 8-10: available
322 * 11: snoop behavior
323 * 12-63: Host physcial address
324 */
325 struct dma_pte {
326 u64 val;
327 };
328
329 static inline void dma_clear_pte(struct dma_pte *pte)
330 {
331 pte->val = 0;
332 }
333
334 static inline u64 dma_pte_addr(struct dma_pte *pte)
335 {
336 #ifdef CONFIG_64BIT
337 return pte->val & VTD_PAGE_MASK;
338 #else
339 /* Must have a full atomic 64-bit read */
340 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
341 #endif
342 }
343
344 static inline bool dma_pte_present(struct dma_pte *pte)
345 {
346 return (pte->val & 3) != 0;
347 }
348
349 static inline bool dma_pte_superpage(struct dma_pte *pte)
350 {
351 return (pte->val & DMA_PTE_LARGE_PAGE);
352 }
353
354 static inline int first_pte_in_page(struct dma_pte *pte)
355 {
356 return !((unsigned long)pte & ~VTD_PAGE_MASK);
357 }
358
359 /*
360 * This domain is a statically identity mapping domain.
361 * 1. This domain creats a static 1:1 mapping to all usable memory.
362 * 2. It maps to each iommu if successful.
363 * 3. Each iommu mapps to this domain if successful.
364 */
365 static struct dmar_domain *si_domain;
366 static int hw_pass_through = 1;
367
368 /*
369 * Domain represents a virtual machine, more than one devices
370 * across iommus may be owned in one domain, e.g. kvm guest.
371 */
372 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
373
374 /* si_domain contains mulitple devices */
375 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
376
377 #define for_each_domain_iommu(idx, domain) \
378 for (idx = 0; idx < g_num_of_iommus; idx++) \
379 if (domain->iommu_refcnt[idx])
380
381 struct dmar_domain {
382 int nid; /* node id */
383
384 unsigned iommu_refcnt[DMAR_UNITS_SUPPORTED];
385 /* Refcount of devices per iommu */
386
387
388 u16 iommu_did[DMAR_UNITS_SUPPORTED];
389 /* Domain ids per IOMMU. Use u16 since
390 * domain ids are 16 bit wide according
391 * to VT-d spec, section 9.3 */
392
393 bool has_iotlb_device;
394 struct list_head devices; /* all devices' list */
395 struct iova_domain iovad; /* iova's that belong to this domain */
396
397 struct dma_pte *pgd; /* virtual address */
398 int gaw; /* max guest address width */
399
400 /* adjusted guest address width, 0 is level 2 30-bit */
401 int agaw;
402
403 int flags; /* flags to find out type of domain */
404
405 int iommu_coherency;/* indicate coherency of iommu access */
406 int iommu_snooping; /* indicate snooping control feature*/
407 int iommu_count; /* reference count of iommu */
408 int iommu_superpage;/* Level of superpages supported:
409 0 == 4KiB (no superpages), 1 == 2MiB,
410 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
411 u64 max_addr; /* maximum mapped address */
412
413 struct iommu_domain domain; /* generic domain data structure for
414 iommu core */
415 };
416
417 /* PCI domain-device relationship */
418 struct device_domain_info {
419 struct list_head link; /* link to domain siblings */
420 struct list_head global; /* link to global list */
421 u8 bus; /* PCI bus number */
422 u8 devfn; /* PCI devfn number */
423 u8 pasid_supported:3;
424 u8 pasid_enabled:1;
425 u8 pri_supported:1;
426 u8 pri_enabled:1;
427 u8 ats_supported:1;
428 u8 ats_enabled:1;
429 u8 ats_qdep;
430 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
431 struct intel_iommu *iommu; /* IOMMU used by this device */
432 struct dmar_domain *domain; /* pointer to domain */
433 };
434
435 struct dmar_rmrr_unit {
436 struct list_head list; /* list of rmrr units */
437 struct acpi_dmar_header *hdr; /* ACPI header */
438 u64 base_address; /* reserved base address*/
439 u64 end_address; /* reserved end address */
440 struct dmar_dev_scope *devices; /* target devices */
441 int devices_cnt; /* target device count */
442 struct iommu_resv_region *resv; /* reserved region handle */
443 };
444
445 struct dmar_atsr_unit {
446 struct list_head list; /* list of ATSR units */
447 struct acpi_dmar_header *hdr; /* ACPI header */
448 struct dmar_dev_scope *devices; /* target devices */
449 int devices_cnt; /* target device count */
450 u8 include_all:1; /* include all ports */
451 };
452
453 static LIST_HEAD(dmar_atsr_units);
454 static LIST_HEAD(dmar_rmrr_units);
455
456 #define for_each_rmrr_units(rmrr) \
457 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
458
459 /* bitmap for indexing intel_iommus */
460 static int g_num_of_iommus;
461
462 static void domain_exit(struct dmar_domain *domain);
463 static void domain_remove_dev_info(struct dmar_domain *domain);
464 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
465 struct device *dev);
466 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
467 static void domain_context_clear(struct intel_iommu *iommu,
468 struct device *dev);
469 static int domain_detach_iommu(struct dmar_domain *domain,
470 struct intel_iommu *iommu);
471
472 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
473 int dmar_disabled = 0;
474 #else
475 int dmar_disabled = 1;
476 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
477
478 int intel_iommu_enabled = 0;
479 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
480
481 static int dmar_map_gfx = 1;
482 static int dmar_forcedac;
483 static int intel_iommu_strict;
484 static int intel_iommu_superpage = 1;
485 static int intel_iommu_ecs = 1;
486 static int intel_iommu_pasid28;
487 static int iommu_identity_mapping;
488
489 #define IDENTMAP_ALL 1
490 #define IDENTMAP_GFX 2
491 #define IDENTMAP_AZALIA 4
492
493 /* Broadwell and Skylake have broken ECS support — normal so-called "second
494 * level" translation of DMA requests-without-PASID doesn't actually happen
495 * unless you also set the NESTE bit in an extended context-entry. Which of
496 * course means that SVM doesn't work because it's trying to do nested
497 * translation of the physical addresses it finds in the process page tables,
498 * through the IOVA->phys mapping found in the "second level" page tables.
499 *
500 * The VT-d specification was retroactively changed to change the definition
501 * of the capability bits and pretend that Broadwell/Skylake never happened...
502 * but unfortunately the wrong bit was changed. It's ECS which is broken, but
503 * for some reason it was the PASID capability bit which was redefined (from
504 * bit 28 on BDW/SKL to bit 40 in future).
505 *
506 * So our test for ECS needs to eschew those implementations which set the old
507 * PASID capabiity bit 28, since those are the ones on which ECS is broken.
508 * Unless we are working around the 'pasid28' limitations, that is, by putting
509 * the device into passthrough mode for normal DMA and thus masking the bug.
510 */
511 #define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
512 (intel_iommu_pasid28 || !ecap_broken_pasid(iommu->ecap)))
513 /* PASID support is thus enabled if ECS is enabled and *either* of the old
514 * or new capability bits are set. */
515 #define pasid_enabled(iommu) (ecs_enabled(iommu) && \
516 (ecap_pasid(iommu->ecap) || ecap_broken_pasid(iommu->ecap)))
517
518 int intel_iommu_gfx_mapped;
519 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
520
521 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
522 static DEFINE_SPINLOCK(device_domain_lock);
523 static LIST_HEAD(device_domain_list);
524
525 const struct iommu_ops intel_iommu_ops;
526
527 static bool translation_pre_enabled(struct intel_iommu *iommu)
528 {
529 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
530 }
531
532 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
533 {
534 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
535 }
536
537 static void init_translation_status(struct intel_iommu *iommu)
538 {
539 u32 gsts;
540
541 gsts = readl(iommu->reg + DMAR_GSTS_REG);
542 if (gsts & DMA_GSTS_TES)
543 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
544 }
545
546 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
547 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
548 {
549 return container_of(dom, struct dmar_domain, domain);
550 }
551
552 static int __init intel_iommu_setup(char *str)
553 {
554 if (!str)
555 return -EINVAL;
556 while (*str) {
557 if (!strncmp(str, "on", 2)) {
558 dmar_disabled = 0;
559 pr_info("IOMMU enabled\n");
560 } else if (!strncmp(str, "off", 3)) {
561 dmar_disabled = 1;
562 pr_info("IOMMU disabled\n");
563 } else if (!strncmp(str, "igfx_off", 8)) {
564 dmar_map_gfx = 0;
565 pr_info("Disable GFX device mapping\n");
566 } else if (!strncmp(str, "forcedac", 8)) {
567 pr_info("Forcing DAC for PCI devices\n");
568 dmar_forcedac = 1;
569 } else if (!strncmp(str, "strict", 6)) {
570 pr_info("Disable batched IOTLB flush\n");
571 intel_iommu_strict = 1;
572 } else if (!strncmp(str, "sp_off", 6)) {
573 pr_info("Disable supported super page\n");
574 intel_iommu_superpage = 0;
575 } else if (!strncmp(str, "ecs_off", 7)) {
576 printk(KERN_INFO
577 "Intel-IOMMU: disable extended context table support\n");
578 intel_iommu_ecs = 0;
579 } else if (!strncmp(str, "pasid28", 7)) {
580 printk(KERN_INFO
581 "Intel-IOMMU: enable pre-production PASID support\n");
582 intel_iommu_pasid28 = 1;
583 iommu_identity_mapping |= IDENTMAP_GFX;
584 } else if (!strncmp(str, "tboot_noforce", 13)) {
585 printk(KERN_INFO
586 "Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
587 intel_iommu_tboot_noforce = 1;
588 }
589
590 str += strcspn(str, ",");
591 while (*str == ',')
592 str++;
593 }
594 return 0;
595 }
596 __setup("intel_iommu=", intel_iommu_setup);
597
598 static struct kmem_cache *iommu_domain_cache;
599 static struct kmem_cache *iommu_devinfo_cache;
600
601 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
602 {
603 struct dmar_domain **domains;
604 int idx = did >> 8;
605
606 domains = iommu->domains[idx];
607 if (!domains)
608 return NULL;
609
610 return domains[did & 0xff];
611 }
612
613 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
614 struct dmar_domain *domain)
615 {
616 struct dmar_domain **domains;
617 int idx = did >> 8;
618
619 if (!iommu->domains[idx]) {
620 size_t size = 256 * sizeof(struct dmar_domain *);
621 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
622 }
623
624 domains = iommu->domains[idx];
625 if (WARN_ON(!domains))
626 return;
627 else
628 domains[did & 0xff] = domain;
629 }
630
631 static inline void *alloc_pgtable_page(int node)
632 {
633 struct page *page;
634 void *vaddr = NULL;
635
636 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
637 if (page)
638 vaddr = page_address(page);
639 return vaddr;
640 }
641
642 static inline void free_pgtable_page(void *vaddr)
643 {
644 free_page((unsigned long)vaddr);
645 }
646
647 static inline void *alloc_domain_mem(void)
648 {
649 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
650 }
651
652 static void free_domain_mem(void *vaddr)
653 {
654 kmem_cache_free(iommu_domain_cache, vaddr);
655 }
656
657 static inline void * alloc_devinfo_mem(void)
658 {
659 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
660 }
661
662 static inline void free_devinfo_mem(void *vaddr)
663 {
664 kmem_cache_free(iommu_devinfo_cache, vaddr);
665 }
666
667 static inline int domain_type_is_vm(struct dmar_domain *domain)
668 {
669 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
670 }
671
672 static inline int domain_type_is_si(struct dmar_domain *domain)
673 {
674 return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
675 }
676
677 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
678 {
679 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
680 DOMAIN_FLAG_STATIC_IDENTITY);
681 }
682
683 static inline int domain_pfn_supported(struct dmar_domain *domain,
684 unsigned long pfn)
685 {
686 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
687
688 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
689 }
690
691 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
692 {
693 unsigned long sagaw;
694 int agaw = -1;
695
696 sagaw = cap_sagaw(iommu->cap);
697 for (agaw = width_to_agaw(max_gaw);
698 agaw >= 0; agaw--) {
699 if (test_bit(agaw, &sagaw))
700 break;
701 }
702
703 return agaw;
704 }
705
706 /*
707 * Calculate max SAGAW for each iommu.
708 */
709 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
710 {
711 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
712 }
713
714 /*
715 * calculate agaw for each iommu.
716 * "SAGAW" may be different across iommus, use a default agaw, and
717 * get a supported less agaw for iommus that don't support the default agaw.
718 */
719 int iommu_calculate_agaw(struct intel_iommu *iommu)
720 {
721 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
722 }
723
724 /* This functionin only returns single iommu in a domain */
725 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
726 {
727 int iommu_id;
728
729 /* si_domain and vm domain should not get here. */
730 BUG_ON(domain_type_is_vm_or_si(domain));
731 for_each_domain_iommu(iommu_id, domain)
732 break;
733
734 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
735 return NULL;
736
737 return g_iommus[iommu_id];
738 }
739
740 static void domain_update_iommu_coherency(struct dmar_domain *domain)
741 {
742 struct dmar_drhd_unit *drhd;
743 struct intel_iommu *iommu;
744 bool found = false;
745 int i;
746
747 domain->iommu_coherency = 1;
748
749 for_each_domain_iommu(i, domain) {
750 found = true;
751 if (!ecap_coherent(g_iommus[i]->ecap)) {
752 domain->iommu_coherency = 0;
753 break;
754 }
755 }
756 if (found)
757 return;
758
759 /* No hardware attached; use lowest common denominator */
760 rcu_read_lock();
761 for_each_active_iommu(iommu, drhd) {
762 if (!ecap_coherent(iommu->ecap)) {
763 domain->iommu_coherency = 0;
764 break;
765 }
766 }
767 rcu_read_unlock();
768 }
769
770 static int domain_update_iommu_snooping(struct intel_iommu *skip)
771 {
772 struct dmar_drhd_unit *drhd;
773 struct intel_iommu *iommu;
774 int ret = 1;
775
776 rcu_read_lock();
777 for_each_active_iommu(iommu, drhd) {
778 if (iommu != skip) {
779 if (!ecap_sc_support(iommu->ecap)) {
780 ret = 0;
781 break;
782 }
783 }
784 }
785 rcu_read_unlock();
786
787 return ret;
788 }
789
790 static int domain_update_iommu_superpage(struct intel_iommu *skip)
791 {
792 struct dmar_drhd_unit *drhd;
793 struct intel_iommu *iommu;
794 int mask = 0xf;
795
796 if (!intel_iommu_superpage) {
797 return 0;
798 }
799
800 /* set iommu_superpage to the smallest common denominator */
801 rcu_read_lock();
802 for_each_active_iommu(iommu, drhd) {
803 if (iommu != skip) {
804 mask &= cap_super_page_val(iommu->cap);
805 if (!mask)
806 break;
807 }
808 }
809 rcu_read_unlock();
810
811 return fls(mask);
812 }
813
814 /* Some capabilities may be different across iommus */
815 static void domain_update_iommu_cap(struct dmar_domain *domain)
816 {
817 domain_update_iommu_coherency(domain);
818 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
819 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
820 }
821
822 static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
823 u8 bus, u8 devfn, int alloc)
824 {
825 struct root_entry *root = &iommu->root_entry[bus];
826 struct context_entry *context;
827 u64 *entry;
828
829 entry = &root->lo;
830 if (ecs_enabled(iommu)) {
831 if (devfn >= 0x80) {
832 devfn -= 0x80;
833 entry = &root->hi;
834 }
835 devfn *= 2;
836 }
837 if (*entry & 1)
838 context = phys_to_virt(*entry & VTD_PAGE_MASK);
839 else {
840 unsigned long phy_addr;
841 if (!alloc)
842 return NULL;
843
844 context = alloc_pgtable_page(iommu->node);
845 if (!context)
846 return NULL;
847
848 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
849 phy_addr = virt_to_phys((void *)context);
850 *entry = phy_addr | 1;
851 __iommu_flush_cache(iommu, entry, sizeof(*entry));
852 }
853 return &context[devfn];
854 }
855
856 static int iommu_dummy(struct device *dev)
857 {
858 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
859 }
860
861 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
862 {
863 struct dmar_drhd_unit *drhd = NULL;
864 struct intel_iommu *iommu;
865 struct device *tmp;
866 struct pci_dev *ptmp, *pdev = NULL;
867 u16 segment = 0;
868 int i;
869
870 if (iommu_dummy(dev))
871 return NULL;
872
873 if (dev_is_pci(dev)) {
874 struct pci_dev *pf_pdev;
875
876 pdev = to_pci_dev(dev);
877
878 #ifdef CONFIG_X86
879 /* VMD child devices currently cannot be handled individually */
880 if (is_vmd(pdev->bus))
881 return NULL;
882 #endif
883
884 /* VFs aren't listed in scope tables; we need to look up
885 * the PF instead to find the IOMMU. */
886 pf_pdev = pci_physfn(pdev);
887 dev = &pf_pdev->dev;
888 segment = pci_domain_nr(pdev->bus);
889 } else if (has_acpi_companion(dev))
890 dev = &ACPI_COMPANION(dev)->dev;
891
892 rcu_read_lock();
893 for_each_active_iommu(iommu, drhd) {
894 if (pdev && segment != drhd->segment)
895 continue;
896
897 for_each_active_dev_scope(drhd->devices,
898 drhd->devices_cnt, i, tmp) {
899 if (tmp == dev) {
900 /* For a VF use its original BDF# not that of the PF
901 * which we used for the IOMMU lookup. Strictly speaking
902 * we could do this for all PCI devices; we only need to
903 * get the BDF# from the scope table for ACPI matches. */
904 if (pdev && pdev->is_virtfn)
905 goto got_pdev;
906
907 *bus = drhd->devices[i].bus;
908 *devfn = drhd->devices[i].devfn;
909 goto out;
910 }
911
912 if (!pdev || !dev_is_pci(tmp))
913 continue;
914
915 ptmp = to_pci_dev(tmp);
916 if (ptmp->subordinate &&
917 ptmp->subordinate->number <= pdev->bus->number &&
918 ptmp->subordinate->busn_res.end >= pdev->bus->number)
919 goto got_pdev;
920 }
921
922 if (pdev && drhd->include_all) {
923 got_pdev:
924 *bus = pdev->bus->number;
925 *devfn = pdev->devfn;
926 goto out;
927 }
928 }
929 iommu = NULL;
930 out:
931 rcu_read_unlock();
932
933 return iommu;
934 }
935
936 static void domain_flush_cache(struct dmar_domain *domain,
937 void *addr, int size)
938 {
939 if (!domain->iommu_coherency)
940 clflush_cache_range(addr, size);
941 }
942
943 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
944 {
945 struct context_entry *context;
946 int ret = 0;
947 unsigned long flags;
948
949 spin_lock_irqsave(&iommu->lock, flags);
950 context = iommu_context_addr(iommu, bus, devfn, 0);
951 if (context)
952 ret = context_present(context);
953 spin_unlock_irqrestore(&iommu->lock, flags);
954 return ret;
955 }
956
957 static void free_context_table(struct intel_iommu *iommu)
958 {
959 int i;
960 unsigned long flags;
961 struct context_entry *context;
962
963 spin_lock_irqsave(&iommu->lock, flags);
964 if (!iommu->root_entry) {
965 goto out;
966 }
967 for (i = 0; i < ROOT_ENTRY_NR; i++) {
968 context = iommu_context_addr(iommu, i, 0, 0);
969 if (context)
970 free_pgtable_page(context);
971
972 if (!ecs_enabled(iommu))
973 continue;
974
975 context = iommu_context_addr(iommu, i, 0x80, 0);
976 if (context)
977 free_pgtable_page(context);
978
979 }
980 free_pgtable_page(iommu->root_entry);
981 iommu->root_entry = NULL;
982 out:
983 spin_unlock_irqrestore(&iommu->lock, flags);
984 }
985
986 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
987 unsigned long pfn, int *target_level)
988 {
989 struct dma_pte *parent, *pte = NULL;
990 int level = agaw_to_level(domain->agaw);
991 int offset;
992
993 BUG_ON(!domain->pgd);
994
995 if (!domain_pfn_supported(domain, pfn))
996 /* Address beyond IOMMU's addressing capabilities. */
997 return NULL;
998
999 parent = domain->pgd;
1000
1001 while (1) {
1002 void *tmp_page;
1003
1004 offset = pfn_level_offset(pfn, level);
1005 pte = &parent[offset];
1006 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
1007 break;
1008 if (level == *target_level)
1009 break;
1010
1011 if (!dma_pte_present(pte)) {
1012 uint64_t pteval;
1013
1014 tmp_page = alloc_pgtable_page(domain->nid);
1015
1016 if (!tmp_page)
1017 return NULL;
1018
1019 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
1020 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
1021 if (cmpxchg64(&pte->val, 0ULL, pteval))
1022 /* Someone else set it while we were thinking; use theirs. */
1023 free_pgtable_page(tmp_page);
1024 else
1025 domain_flush_cache(domain, pte, sizeof(*pte));
1026 }
1027 if (level == 1)
1028 break;
1029
1030 parent = phys_to_virt(dma_pte_addr(pte));
1031 level--;
1032 }
1033
1034 if (!*target_level)
1035 *target_level = level;
1036
1037 return pte;
1038 }
1039
1040
1041 /* return address's pte at specific level */
1042 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1043 unsigned long pfn,
1044 int level, int *large_page)
1045 {
1046 struct dma_pte *parent, *pte = NULL;
1047 int total = agaw_to_level(domain->agaw);
1048 int offset;
1049
1050 parent = domain->pgd;
1051 while (level <= total) {
1052 offset = pfn_level_offset(pfn, total);
1053 pte = &parent[offset];
1054 if (level == total)
1055 return pte;
1056
1057 if (!dma_pte_present(pte)) {
1058 *large_page = total;
1059 break;
1060 }
1061
1062 if (dma_pte_superpage(pte)) {
1063 *large_page = total;
1064 return pte;
1065 }
1066
1067 parent = phys_to_virt(dma_pte_addr(pte));
1068 total--;
1069 }
1070 return NULL;
1071 }
1072
1073 /* clear last level pte, a tlb flush should be followed */
1074 static void dma_pte_clear_range(struct dmar_domain *domain,
1075 unsigned long start_pfn,
1076 unsigned long last_pfn)
1077 {
1078 unsigned int large_page = 1;
1079 struct dma_pte *first_pte, *pte;
1080
1081 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1082 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1083 BUG_ON(start_pfn > last_pfn);
1084
1085 /* we don't need lock here; nobody else touches the iova range */
1086 do {
1087 large_page = 1;
1088 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
1089 if (!pte) {
1090 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
1091 continue;
1092 }
1093 do {
1094 dma_clear_pte(pte);
1095 start_pfn += lvl_to_nr_pages(large_page);
1096 pte++;
1097 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1098
1099 domain_flush_cache(domain, first_pte,
1100 (void *)pte - (void *)first_pte);
1101
1102 } while (start_pfn && start_pfn <= last_pfn);
1103 }
1104
1105 static void dma_pte_free_level(struct dmar_domain *domain, int level,
1106 int retain_level, struct dma_pte *pte,
1107 unsigned long pfn, unsigned long start_pfn,
1108 unsigned long last_pfn)
1109 {
1110 pfn = max(start_pfn, pfn);
1111 pte = &pte[pfn_level_offset(pfn, level)];
1112
1113 do {
1114 unsigned long level_pfn;
1115 struct dma_pte *level_pte;
1116
1117 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1118 goto next;
1119
1120 level_pfn = pfn & level_mask(level);
1121 level_pte = phys_to_virt(dma_pte_addr(pte));
1122
1123 if (level > 2) {
1124 dma_pte_free_level(domain, level - 1, retain_level,
1125 level_pte, level_pfn, start_pfn,
1126 last_pfn);
1127 }
1128
1129 /*
1130 * Free the page table if we're below the level we want to
1131 * retain and the range covers the entire table.
1132 */
1133 if (level < retain_level && !(start_pfn > level_pfn ||
1134 last_pfn < level_pfn + level_size(level) - 1)) {
1135 dma_clear_pte(pte);
1136 domain_flush_cache(domain, pte, sizeof(*pte));
1137 free_pgtable_page(level_pte);
1138 }
1139 next:
1140 pfn += level_size(level);
1141 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1142 }
1143
1144 /*
1145 * clear last level (leaf) ptes and free page table pages below the
1146 * level we wish to keep intact.
1147 */
1148 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1149 unsigned long start_pfn,
1150 unsigned long last_pfn,
1151 int retain_level)
1152 {
1153 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1154 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1155 BUG_ON(start_pfn > last_pfn);
1156
1157 dma_pte_clear_range(domain, start_pfn, last_pfn);
1158
1159 /* We don't need lock here; nobody else touches the iova range */
1160 dma_pte_free_level(domain, agaw_to_level(domain->agaw), retain_level,
1161 domain->pgd, 0, start_pfn, last_pfn);
1162
1163 /* free pgd */
1164 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1165 free_pgtable_page(domain->pgd);
1166 domain->pgd = NULL;
1167 }
1168 }
1169
1170 /* When a page at a given level is being unlinked from its parent, we don't
1171 need to *modify* it at all. All we need to do is make a list of all the
1172 pages which can be freed just as soon as we've flushed the IOTLB and we
1173 know the hardware page-walk will no longer touch them.
1174 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1175 be freed. */
1176 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1177 int level, struct dma_pte *pte,
1178 struct page *freelist)
1179 {
1180 struct page *pg;
1181
1182 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1183 pg->freelist = freelist;
1184 freelist = pg;
1185
1186 if (level == 1)
1187 return freelist;
1188
1189 pte = page_address(pg);
1190 do {
1191 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1192 freelist = dma_pte_list_pagetables(domain, level - 1,
1193 pte, freelist);
1194 pte++;
1195 } while (!first_pte_in_page(pte));
1196
1197 return freelist;
1198 }
1199
1200 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1201 struct dma_pte *pte, unsigned long pfn,
1202 unsigned long start_pfn,
1203 unsigned long last_pfn,
1204 struct page *freelist)
1205 {
1206 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1207
1208 pfn = max(start_pfn, pfn);
1209 pte = &pte[pfn_level_offset(pfn, level)];
1210
1211 do {
1212 unsigned long level_pfn;
1213
1214 if (!dma_pte_present(pte))
1215 goto next;
1216
1217 level_pfn = pfn & level_mask(level);
1218
1219 /* If range covers entire pagetable, free it */
1220 if (start_pfn <= level_pfn &&
1221 last_pfn >= level_pfn + level_size(level) - 1) {
1222 /* These suborbinate page tables are going away entirely. Don't
1223 bother to clear them; we're just going to *free* them. */
1224 if (level > 1 && !dma_pte_superpage(pte))
1225 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1226
1227 dma_clear_pte(pte);
1228 if (!first_pte)
1229 first_pte = pte;
1230 last_pte = pte;
1231 } else if (level > 1) {
1232 /* Recurse down into a level that isn't *entirely* obsolete */
1233 freelist = dma_pte_clear_level(domain, level - 1,
1234 phys_to_virt(dma_pte_addr(pte)),
1235 level_pfn, start_pfn, last_pfn,
1236 freelist);
1237 }
1238 next:
1239 pfn += level_size(level);
1240 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1241
1242 if (first_pte)
1243 domain_flush_cache(domain, first_pte,
1244 (void *)++last_pte - (void *)first_pte);
1245
1246 return freelist;
1247 }
1248
1249 /* We can't just free the pages because the IOMMU may still be walking
1250 the page tables, and may have cached the intermediate levels. The
1251 pages can only be freed after the IOTLB flush has been done. */
1252 static struct page *domain_unmap(struct dmar_domain *domain,
1253 unsigned long start_pfn,
1254 unsigned long last_pfn)
1255 {
1256 struct page *freelist = NULL;
1257
1258 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1259 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1260 BUG_ON(start_pfn > last_pfn);
1261
1262 /* we don't need lock here; nobody else touches the iova range */
1263 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1264 domain->pgd, 0, start_pfn, last_pfn, NULL);
1265
1266 /* free pgd */
1267 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1268 struct page *pgd_page = virt_to_page(domain->pgd);
1269 pgd_page->freelist = freelist;
1270 freelist = pgd_page;
1271
1272 domain->pgd = NULL;
1273 }
1274
1275 return freelist;
1276 }
1277
1278 static void dma_free_pagelist(struct page *freelist)
1279 {
1280 struct page *pg;
1281
1282 while ((pg = freelist)) {
1283 freelist = pg->freelist;
1284 free_pgtable_page(page_address(pg));
1285 }
1286 }
1287
1288 static void iova_entry_free(unsigned long data)
1289 {
1290 struct page *freelist = (struct page *)data;
1291
1292 dma_free_pagelist(freelist);
1293 }
1294
1295 /* iommu handling */
1296 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1297 {
1298 struct root_entry *root;
1299 unsigned long flags;
1300
1301 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1302 if (!root) {
1303 pr_err("Allocating root entry for %s failed\n",
1304 iommu->name);
1305 return -ENOMEM;
1306 }
1307
1308 __iommu_flush_cache(iommu, root, ROOT_SIZE);
1309
1310 spin_lock_irqsave(&iommu->lock, flags);
1311 iommu->root_entry = root;
1312 spin_unlock_irqrestore(&iommu->lock, flags);
1313
1314 return 0;
1315 }
1316
1317 static void iommu_set_root_entry(struct intel_iommu *iommu)
1318 {
1319 u64 addr;
1320 u32 sts;
1321 unsigned long flag;
1322
1323 addr = virt_to_phys(iommu->root_entry);
1324 if (ecs_enabled(iommu))
1325 addr |= DMA_RTADDR_RTT;
1326
1327 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1328 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1329
1330 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1331
1332 /* Make sure hardware complete it */
1333 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1334 readl, (sts & DMA_GSTS_RTPS), sts);
1335
1336 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1337 }
1338
1339 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1340 {
1341 u32 val;
1342 unsigned long flag;
1343
1344 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1345 return;
1346
1347 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1348 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1349
1350 /* Make sure hardware complete it */
1351 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1352 readl, (!(val & DMA_GSTS_WBFS)), val);
1353
1354 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1355 }
1356
1357 /* return value determine if we need a write buffer flush */
1358 static void __iommu_flush_context(struct intel_iommu *iommu,
1359 u16 did, u16 source_id, u8 function_mask,
1360 u64 type)
1361 {
1362 u64 val = 0;
1363 unsigned long flag;
1364
1365 switch (type) {
1366 case DMA_CCMD_GLOBAL_INVL:
1367 val = DMA_CCMD_GLOBAL_INVL;
1368 break;
1369 case DMA_CCMD_DOMAIN_INVL:
1370 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1371 break;
1372 case DMA_CCMD_DEVICE_INVL:
1373 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1374 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1375 break;
1376 default:
1377 BUG();
1378 }
1379 val |= DMA_CCMD_ICC;
1380
1381 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1382 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1383
1384 /* Make sure hardware complete it */
1385 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1386 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1387
1388 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1389 }
1390
1391 /* return value determine if we need a write buffer flush */
1392 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1393 u64 addr, unsigned int size_order, u64 type)
1394 {
1395 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1396 u64 val = 0, val_iva = 0;
1397 unsigned long flag;
1398
1399 switch (type) {
1400 case DMA_TLB_GLOBAL_FLUSH:
1401 /* global flush doesn't need set IVA_REG */
1402 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1403 break;
1404 case DMA_TLB_DSI_FLUSH:
1405 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1406 break;
1407 case DMA_TLB_PSI_FLUSH:
1408 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1409 /* IH bit is passed in as part of address */
1410 val_iva = size_order | addr;
1411 break;
1412 default:
1413 BUG();
1414 }
1415 /* Note: set drain read/write */
1416 #if 0
1417 /*
1418 * This is probably to be super secure.. Looks like we can
1419 * ignore it without any impact.
1420 */
1421 if (cap_read_drain(iommu->cap))
1422 val |= DMA_TLB_READ_DRAIN;
1423 #endif
1424 if (cap_write_drain(iommu->cap))
1425 val |= DMA_TLB_WRITE_DRAIN;
1426
1427 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1428 /* Note: Only uses first TLB reg currently */
1429 if (val_iva)
1430 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1431 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1432
1433 /* Make sure hardware complete it */
1434 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1435 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1436
1437 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1438
1439 /* check IOTLB invalidation granularity */
1440 if (DMA_TLB_IAIG(val) == 0)
1441 pr_err("Flush IOTLB failed\n");
1442 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1443 pr_debug("TLB flush request %Lx, actual %Lx\n",
1444 (unsigned long long)DMA_TLB_IIRG(type),
1445 (unsigned long long)DMA_TLB_IAIG(val));
1446 }
1447
1448 static struct device_domain_info *
1449 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1450 u8 bus, u8 devfn)
1451 {
1452 struct device_domain_info *info;
1453
1454 assert_spin_locked(&device_domain_lock);
1455
1456 if (!iommu->qi)
1457 return NULL;
1458
1459 list_for_each_entry(info, &domain->devices, link)
1460 if (info->iommu == iommu && info->bus == bus &&
1461 info->devfn == devfn) {
1462 if (info->ats_supported && info->dev)
1463 return info;
1464 break;
1465 }
1466
1467 return NULL;
1468 }
1469
1470 static void domain_update_iotlb(struct dmar_domain *domain)
1471 {
1472 struct device_domain_info *info;
1473 bool has_iotlb_device = false;
1474
1475 assert_spin_locked(&device_domain_lock);
1476
1477 list_for_each_entry(info, &domain->devices, link) {
1478 struct pci_dev *pdev;
1479
1480 if (!info->dev || !dev_is_pci(info->dev))
1481 continue;
1482
1483 pdev = to_pci_dev(info->dev);
1484 if (pdev->ats_enabled) {
1485 has_iotlb_device = true;
1486 break;
1487 }
1488 }
1489
1490 domain->has_iotlb_device = has_iotlb_device;
1491 }
1492
1493 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1494 {
1495 struct pci_dev *pdev;
1496
1497 assert_spin_locked(&device_domain_lock);
1498
1499 if (!info || !dev_is_pci(info->dev))
1500 return;
1501
1502 pdev = to_pci_dev(info->dev);
1503
1504 #ifdef CONFIG_INTEL_IOMMU_SVM
1505 /* The PCIe spec, in its wisdom, declares that the behaviour of
1506 the device if you enable PASID support after ATS support is
1507 undefined. So always enable PASID support on devices which
1508 have it, even if we can't yet know if we're ever going to
1509 use it. */
1510 if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1511 info->pasid_enabled = 1;
1512
1513 if (info->pri_supported && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1514 info->pri_enabled = 1;
1515 #endif
1516 if (info->ats_supported && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1517 info->ats_enabled = 1;
1518 domain_update_iotlb(info->domain);
1519 info->ats_qdep = pci_ats_queue_depth(pdev);
1520 }
1521 }
1522
1523 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1524 {
1525 struct pci_dev *pdev;
1526
1527 assert_spin_locked(&device_domain_lock);
1528
1529 if (!dev_is_pci(info->dev))
1530 return;
1531
1532 pdev = to_pci_dev(info->dev);
1533
1534 if (info->ats_enabled) {
1535 pci_disable_ats(pdev);
1536 info->ats_enabled = 0;
1537 domain_update_iotlb(info->domain);
1538 }
1539 #ifdef CONFIG_INTEL_IOMMU_SVM
1540 if (info->pri_enabled) {
1541 pci_disable_pri(pdev);
1542 info->pri_enabled = 0;
1543 }
1544 if (info->pasid_enabled) {
1545 pci_disable_pasid(pdev);
1546 info->pasid_enabled = 0;
1547 }
1548 #endif
1549 }
1550
1551 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1552 u64 addr, unsigned mask)
1553 {
1554 u16 sid, qdep;
1555 unsigned long flags;
1556 struct device_domain_info *info;
1557
1558 if (!domain->has_iotlb_device)
1559 return;
1560
1561 spin_lock_irqsave(&device_domain_lock, flags);
1562 list_for_each_entry(info, &domain->devices, link) {
1563 if (!info->ats_enabled)
1564 continue;
1565
1566 sid = info->bus << 8 | info->devfn;
1567 qdep = info->ats_qdep;
1568 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1569 }
1570 spin_unlock_irqrestore(&device_domain_lock, flags);
1571 }
1572
1573 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1574 struct dmar_domain *domain,
1575 unsigned long pfn, unsigned int pages,
1576 int ih, int map)
1577 {
1578 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1579 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1580 u16 did = domain->iommu_did[iommu->seq_id];
1581
1582 BUG_ON(pages == 0);
1583
1584 if (ih)
1585 ih = 1 << 6;
1586 /*
1587 * Fallback to domain selective flush if no PSI support or the size is
1588 * too big.
1589 * PSI requires page size to be 2 ^ x, and the base address is naturally
1590 * aligned to the size
1591 */
1592 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1593 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1594 DMA_TLB_DSI_FLUSH);
1595 else
1596 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1597 DMA_TLB_PSI_FLUSH);
1598
1599 /*
1600 * In caching mode, changes of pages from non-present to present require
1601 * flush. However, device IOTLB doesn't need to be flushed in this case.
1602 */
1603 if (!cap_caching_mode(iommu->cap) || !map)
1604 iommu_flush_dev_iotlb(domain, addr, mask);
1605 }
1606
1607 static void iommu_flush_iova(struct iova_domain *iovad)
1608 {
1609 struct dmar_domain *domain;
1610 int idx;
1611
1612 domain = container_of(iovad, struct dmar_domain, iovad);
1613
1614 for_each_domain_iommu(idx, domain) {
1615 struct intel_iommu *iommu = g_iommus[idx];
1616 u16 did = domain->iommu_did[iommu->seq_id];
1617
1618 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
1619
1620 if (!cap_caching_mode(iommu->cap))
1621 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1622 0, MAX_AGAW_PFN_WIDTH);
1623 }
1624 }
1625
1626 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1627 {
1628 u32 pmen;
1629 unsigned long flags;
1630
1631 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1632 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1633 pmen &= ~DMA_PMEN_EPM;
1634 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1635
1636 /* wait for the protected region status bit to clear */
1637 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1638 readl, !(pmen & DMA_PMEN_PRS), pmen);
1639
1640 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1641 }
1642
1643 static void iommu_enable_translation(struct intel_iommu *iommu)
1644 {
1645 u32 sts;
1646 unsigned long flags;
1647
1648 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1649 iommu->gcmd |= DMA_GCMD_TE;
1650 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1651
1652 /* Make sure hardware complete it */
1653 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1654 readl, (sts & DMA_GSTS_TES), sts);
1655
1656 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1657 }
1658
1659 static void iommu_disable_translation(struct intel_iommu *iommu)
1660 {
1661 u32 sts;
1662 unsigned long flag;
1663
1664 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1665 iommu->gcmd &= ~DMA_GCMD_TE;
1666 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1667
1668 /* Make sure hardware complete it */
1669 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1670 readl, (!(sts & DMA_GSTS_TES)), sts);
1671
1672 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1673 }
1674
1675
1676 static int iommu_init_domains(struct intel_iommu *iommu)
1677 {
1678 u32 ndomains, nlongs;
1679 size_t size;
1680
1681 ndomains = cap_ndoms(iommu->cap);
1682 pr_debug("%s: Number of Domains supported <%d>\n",
1683 iommu->name, ndomains);
1684 nlongs = BITS_TO_LONGS(ndomains);
1685
1686 spin_lock_init(&iommu->lock);
1687
1688 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1689 if (!iommu->domain_ids) {
1690 pr_err("%s: Allocating domain id array failed\n",
1691 iommu->name);
1692 return -ENOMEM;
1693 }
1694
1695 size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
1696 iommu->domains = kzalloc(size, GFP_KERNEL);
1697
1698 if (iommu->domains) {
1699 size = 256 * sizeof(struct dmar_domain *);
1700 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1701 }
1702
1703 if (!iommu->domains || !iommu->domains[0]) {
1704 pr_err("%s: Allocating domain array failed\n",
1705 iommu->name);
1706 kfree(iommu->domain_ids);
1707 kfree(iommu->domains);
1708 iommu->domain_ids = NULL;
1709 iommu->domains = NULL;
1710 return -ENOMEM;
1711 }
1712
1713
1714
1715 /*
1716 * If Caching mode is set, then invalid translations are tagged
1717 * with domain-id 0, hence we need to pre-allocate it. We also
1718 * use domain-id 0 as a marker for non-allocated domain-id, so
1719 * make sure it is not used for a real domain.
1720 */
1721 set_bit(0, iommu->domain_ids);
1722
1723 return 0;
1724 }
1725
1726 static void disable_dmar_iommu(struct intel_iommu *iommu)
1727 {
1728 struct device_domain_info *info, *tmp;
1729 unsigned long flags;
1730
1731 if (!iommu->domains || !iommu->domain_ids)
1732 return;
1733
1734 again:
1735 spin_lock_irqsave(&device_domain_lock, flags);
1736 list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1737 struct dmar_domain *domain;
1738
1739 if (info->iommu != iommu)
1740 continue;
1741
1742 if (!info->dev || !info->domain)
1743 continue;
1744
1745 domain = info->domain;
1746
1747 __dmar_remove_one_dev_info(info);
1748
1749 if (!domain_type_is_vm_or_si(domain)) {
1750 /*
1751 * The domain_exit() function can't be called under
1752 * device_domain_lock, as it takes this lock itself.
1753 * So release the lock here and re-run the loop
1754 * afterwards.
1755 */
1756 spin_unlock_irqrestore(&device_domain_lock, flags);
1757 domain_exit(domain);
1758 goto again;
1759 }
1760 }
1761 spin_unlock_irqrestore(&device_domain_lock, flags);
1762
1763 if (iommu->gcmd & DMA_GCMD_TE)
1764 iommu_disable_translation(iommu);
1765 }
1766
1767 static void free_dmar_iommu(struct intel_iommu *iommu)
1768 {
1769 if ((iommu->domains) && (iommu->domain_ids)) {
1770 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
1771 int i;
1772
1773 for (i = 0; i < elems; i++)
1774 kfree(iommu->domains[i]);
1775 kfree(iommu->domains);
1776 kfree(iommu->domain_ids);
1777 iommu->domains = NULL;
1778 iommu->domain_ids = NULL;
1779 }
1780
1781 g_iommus[iommu->seq_id] = NULL;
1782
1783 /* free context mapping */
1784 free_context_table(iommu);
1785
1786 #ifdef CONFIG_INTEL_IOMMU_SVM
1787 if (pasid_enabled(iommu)) {
1788 if (ecap_prs(iommu->ecap))
1789 intel_svm_finish_prq(iommu);
1790 intel_svm_free_pasid_tables(iommu);
1791 }
1792 #endif
1793 }
1794
1795 static struct dmar_domain *alloc_domain(int flags)
1796 {
1797 struct dmar_domain *domain;
1798
1799 domain = alloc_domain_mem();
1800 if (!domain)
1801 return NULL;
1802
1803 memset(domain, 0, sizeof(*domain));
1804 domain->nid = -1;
1805 domain->flags = flags;
1806 domain->has_iotlb_device = false;
1807 INIT_LIST_HEAD(&domain->devices);
1808
1809 return domain;
1810 }
1811
1812 /* Must be called with iommu->lock */
1813 static int domain_attach_iommu(struct dmar_domain *domain,
1814 struct intel_iommu *iommu)
1815 {
1816 unsigned long ndomains;
1817 int num;
1818
1819 assert_spin_locked(&device_domain_lock);
1820 assert_spin_locked(&iommu->lock);
1821
1822 domain->iommu_refcnt[iommu->seq_id] += 1;
1823 domain->iommu_count += 1;
1824 if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1825 ndomains = cap_ndoms(iommu->cap);
1826 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1827
1828 if (num >= ndomains) {
1829 pr_err("%s: No free domain ids\n", iommu->name);
1830 domain->iommu_refcnt[iommu->seq_id] -= 1;
1831 domain->iommu_count -= 1;
1832 return -ENOSPC;
1833 }
1834
1835 set_bit(num, iommu->domain_ids);
1836 set_iommu_domain(iommu, num, domain);
1837
1838 domain->iommu_did[iommu->seq_id] = num;
1839 domain->nid = iommu->node;
1840
1841 domain_update_iommu_cap(domain);
1842 }
1843
1844 return 0;
1845 }
1846
1847 static int domain_detach_iommu(struct dmar_domain *domain,
1848 struct intel_iommu *iommu)
1849 {
1850 int num, count = INT_MAX;
1851
1852 assert_spin_locked(&device_domain_lock);
1853 assert_spin_locked(&iommu->lock);
1854
1855 domain->iommu_refcnt[iommu->seq_id] -= 1;
1856 count = --domain->iommu_count;
1857 if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1858 num = domain->iommu_did[iommu->seq_id];
1859 clear_bit(num, iommu->domain_ids);
1860 set_iommu_domain(iommu, num, NULL);
1861
1862 domain_update_iommu_cap(domain);
1863 domain->iommu_did[iommu->seq_id] = 0;
1864 }
1865
1866 return count;
1867 }
1868
1869 static struct iova_domain reserved_iova_list;
1870 static struct lock_class_key reserved_rbtree_key;
1871
1872 static int dmar_init_reserved_ranges(void)
1873 {
1874 struct pci_dev *pdev = NULL;
1875 struct iova *iova;
1876 int i;
1877
1878 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN);
1879
1880 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1881 &reserved_rbtree_key);
1882
1883 /* IOAPIC ranges shouldn't be accessed by DMA */
1884 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1885 IOVA_PFN(IOAPIC_RANGE_END));
1886 if (!iova) {
1887 pr_err("Reserve IOAPIC range failed\n");
1888 return -ENODEV;
1889 }
1890
1891 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1892 for_each_pci_dev(pdev) {
1893 struct resource *r;
1894
1895 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1896 r = &pdev->resource[i];
1897 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1898 continue;
1899 iova = reserve_iova(&reserved_iova_list,
1900 IOVA_PFN(r->start),
1901 IOVA_PFN(r->end));
1902 if (!iova) {
1903 pr_err("Reserve iova failed\n");
1904 return -ENODEV;
1905 }
1906 }
1907 }
1908 return 0;
1909 }
1910
1911 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1912 {
1913 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1914 }
1915
1916 static inline int guestwidth_to_adjustwidth(int gaw)
1917 {
1918 int agaw;
1919 int r = (gaw - 12) % 9;
1920
1921 if (r == 0)
1922 agaw = gaw;
1923 else
1924 agaw = gaw + 9 - r;
1925 if (agaw > 64)
1926 agaw = 64;
1927 return agaw;
1928 }
1929
1930 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1931 int guest_width)
1932 {
1933 int adjust_width, agaw;
1934 unsigned long sagaw;
1935 int err;
1936
1937 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
1938
1939 err = init_iova_flush_queue(&domain->iovad,
1940 iommu_flush_iova, iova_entry_free);
1941 if (err)
1942 return err;
1943
1944 domain_reserve_special_ranges(domain);
1945
1946 /* calculate AGAW */
1947 if (guest_width > cap_mgaw(iommu->cap))
1948 guest_width = cap_mgaw(iommu->cap);
1949 domain->gaw = guest_width;
1950 adjust_width = guestwidth_to_adjustwidth(guest_width);
1951 agaw = width_to_agaw(adjust_width);
1952 sagaw = cap_sagaw(iommu->cap);
1953 if (!test_bit(agaw, &sagaw)) {
1954 /* hardware doesn't support it, choose a bigger one */
1955 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1956 agaw = find_next_bit(&sagaw, 5, agaw);
1957 if (agaw >= 5)
1958 return -ENODEV;
1959 }
1960 domain->agaw = agaw;
1961
1962 if (ecap_coherent(iommu->ecap))
1963 domain->iommu_coherency = 1;
1964 else
1965 domain->iommu_coherency = 0;
1966
1967 if (ecap_sc_support(iommu->ecap))
1968 domain->iommu_snooping = 1;
1969 else
1970 domain->iommu_snooping = 0;
1971
1972 if (intel_iommu_superpage)
1973 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1974 else
1975 domain->iommu_superpage = 0;
1976
1977 domain->nid = iommu->node;
1978
1979 /* always allocate the top pgd */
1980 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1981 if (!domain->pgd)
1982 return -ENOMEM;
1983 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1984 return 0;
1985 }
1986
1987 static void domain_exit(struct dmar_domain *domain)
1988 {
1989 struct page *freelist = NULL;
1990
1991 /* Domain 0 is reserved, so dont process it */
1992 if (!domain)
1993 return;
1994
1995 /* Remove associated devices and clear attached or cached domains */
1996 rcu_read_lock();
1997 domain_remove_dev_info(domain);
1998 rcu_read_unlock();
1999
2000 /* destroy iovas */
2001 put_iova_domain(&domain->iovad);
2002
2003 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
2004
2005 dma_free_pagelist(freelist);
2006
2007 free_domain_mem(domain);
2008 }
2009
2010 static int domain_context_mapping_one(struct dmar_domain *domain,
2011 struct intel_iommu *iommu,
2012 u8 bus, u8 devfn)
2013 {
2014 u16 did = domain->iommu_did[iommu->seq_id];
2015 int translation = CONTEXT_TT_MULTI_LEVEL;
2016 struct device_domain_info *info = NULL;
2017 struct context_entry *context;
2018 unsigned long flags;
2019 struct dma_pte *pgd;
2020 int ret, agaw;
2021
2022 WARN_ON(did == 0);
2023
2024 if (hw_pass_through && domain_type_is_si(domain))
2025 translation = CONTEXT_TT_PASS_THROUGH;
2026
2027 pr_debug("Set context mapping for %02x:%02x.%d\n",
2028 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
2029
2030 BUG_ON(!domain->pgd);
2031
2032 spin_lock_irqsave(&device_domain_lock, flags);
2033 spin_lock(&iommu->lock);
2034
2035 ret = -ENOMEM;
2036 context = iommu_context_addr(iommu, bus, devfn, 1);
2037 if (!context)
2038 goto out_unlock;
2039
2040 ret = 0;
2041 if (context_present(context))
2042 goto out_unlock;
2043
2044 /*
2045 * For kdump cases, old valid entries may be cached due to the
2046 * in-flight DMA and copied pgtable, but there is no unmapping
2047 * behaviour for them, thus we need an explicit cache flush for
2048 * the newly-mapped device. For kdump, at this point, the device
2049 * is supposed to finish reset at its driver probe stage, so no
2050 * in-flight DMA will exist, and we don't need to worry anymore
2051 * hereafter.
2052 */
2053 if (context_copied(context)) {
2054 u16 did_old = context_domain_id(context);
2055
2056 if (did_old < cap_ndoms(iommu->cap)) {
2057 iommu->flush.flush_context(iommu, did_old,
2058 (((u16)bus) << 8) | devfn,
2059 DMA_CCMD_MASK_NOBIT,
2060 DMA_CCMD_DEVICE_INVL);
2061 iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
2062 DMA_TLB_DSI_FLUSH);
2063 }
2064 }
2065
2066 pgd = domain->pgd;
2067
2068 context_clear_entry(context);
2069 context_set_domain_id(context, did);
2070
2071 /*
2072 * Skip top levels of page tables for iommu which has less agaw
2073 * than default. Unnecessary for PT mode.
2074 */
2075 if (translation != CONTEXT_TT_PASS_THROUGH) {
2076 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
2077 ret = -ENOMEM;
2078 pgd = phys_to_virt(dma_pte_addr(pgd));
2079 if (!dma_pte_present(pgd))
2080 goto out_unlock;
2081 }
2082
2083 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2084 if (info && info->ats_supported)
2085 translation = CONTEXT_TT_DEV_IOTLB;
2086 else
2087 translation = CONTEXT_TT_MULTI_LEVEL;
2088
2089 context_set_address_root(context, virt_to_phys(pgd));
2090 context_set_address_width(context, iommu->agaw);
2091 } else {
2092 /*
2093 * In pass through mode, AW must be programmed to
2094 * indicate the largest AGAW value supported by
2095 * hardware. And ASR is ignored by hardware.
2096 */
2097 context_set_address_width(context, iommu->msagaw);
2098 }
2099
2100 context_set_translation_type(context, translation);
2101 context_set_fault_enable(context);
2102 context_set_present(context);
2103 domain_flush_cache(domain, context, sizeof(*context));
2104
2105 /*
2106 * It's a non-present to present mapping. If hardware doesn't cache
2107 * non-present entry we only need to flush the write-buffer. If the
2108 * _does_ cache non-present entries, then it does so in the special
2109 * domain #0, which we have to flush:
2110 */
2111 if (cap_caching_mode(iommu->cap)) {
2112 iommu->flush.flush_context(iommu, 0,
2113 (((u16)bus) << 8) | devfn,
2114 DMA_CCMD_MASK_NOBIT,
2115 DMA_CCMD_DEVICE_INVL);
2116 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2117 } else {
2118 iommu_flush_write_buffer(iommu);
2119 }
2120 iommu_enable_dev_iotlb(info);
2121
2122 ret = 0;
2123
2124 out_unlock:
2125 spin_unlock(&iommu->lock);
2126 spin_unlock_irqrestore(&device_domain_lock, flags);
2127
2128 return ret;
2129 }
2130
2131 struct domain_context_mapping_data {
2132 struct dmar_domain *domain;
2133 struct intel_iommu *iommu;
2134 };
2135
2136 static int domain_context_mapping_cb(struct pci_dev *pdev,
2137 u16 alias, void *opaque)
2138 {
2139 struct domain_context_mapping_data *data = opaque;
2140
2141 return domain_context_mapping_one(data->domain, data->iommu,
2142 PCI_BUS_NUM(alias), alias & 0xff);
2143 }
2144
2145 static int
2146 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2147 {
2148 struct intel_iommu *iommu;
2149 u8 bus, devfn;
2150 struct domain_context_mapping_data data;
2151
2152 iommu = device_to_iommu(dev, &bus, &devfn);
2153 if (!iommu)
2154 return -ENODEV;
2155
2156 if (!dev_is_pci(dev))
2157 return domain_context_mapping_one(domain, iommu, bus, devfn);
2158
2159 data.domain = domain;
2160 data.iommu = iommu;
2161
2162 return pci_for_each_dma_alias(to_pci_dev(dev),
2163 &domain_context_mapping_cb, &data);
2164 }
2165
2166 static int domain_context_mapped_cb(struct pci_dev *pdev,
2167 u16 alias, void *opaque)
2168 {
2169 struct intel_iommu *iommu = opaque;
2170
2171 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2172 }
2173
2174 static int domain_context_mapped(struct device *dev)
2175 {
2176 struct intel_iommu *iommu;
2177 u8 bus, devfn;
2178
2179 iommu = device_to_iommu(dev, &bus, &devfn);
2180 if (!iommu)
2181 return -ENODEV;
2182
2183 if (!dev_is_pci(dev))
2184 return device_context_mapped(iommu, bus, devfn);
2185
2186 return !pci_for_each_dma_alias(to_pci_dev(dev),
2187 domain_context_mapped_cb, iommu);
2188 }
2189
2190 /* Returns a number of VTD pages, but aligned to MM page size */
2191 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2192 size_t size)
2193 {
2194 host_addr &= ~PAGE_MASK;
2195 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2196 }
2197
2198 /* Return largest possible superpage level for a given mapping */
2199 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2200 unsigned long iov_pfn,
2201 unsigned long phy_pfn,
2202 unsigned long pages)
2203 {
2204 int support, level = 1;
2205 unsigned long pfnmerge;
2206
2207 support = domain->iommu_superpage;
2208
2209 /* To use a large page, the virtual *and* physical addresses
2210 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2211 of them will mean we have to use smaller pages. So just
2212 merge them and check both at once. */
2213 pfnmerge = iov_pfn | phy_pfn;
2214
2215 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2216 pages >>= VTD_STRIDE_SHIFT;
2217 if (!pages)
2218 break;
2219 pfnmerge >>= VTD_STRIDE_SHIFT;
2220 level++;
2221 support--;
2222 }
2223 return level;
2224 }
2225
2226 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2227 struct scatterlist *sg, unsigned long phys_pfn,
2228 unsigned long nr_pages, int prot)
2229 {
2230 struct dma_pte *first_pte = NULL, *pte = NULL;
2231 phys_addr_t uninitialized_var(pteval);
2232 unsigned long sg_res = 0;
2233 unsigned int largepage_lvl = 0;
2234 unsigned long lvl_pages = 0;
2235
2236 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2237
2238 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2239 return -EINVAL;
2240
2241 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2242
2243 if (!sg) {
2244 sg_res = nr_pages;
2245 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2246 }
2247
2248 while (nr_pages > 0) {
2249 uint64_t tmp;
2250
2251 if (!sg_res) {
2252 unsigned int pgoff = sg->offset & ~PAGE_MASK;
2253
2254 sg_res = aligned_nrpages(sg->offset, sg->length);
2255 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + pgoff;
2256 sg->dma_length = sg->length;
2257 pteval = (sg_phys(sg) - pgoff) | prot;
2258 phys_pfn = pteval >> VTD_PAGE_SHIFT;
2259 }
2260
2261 if (!pte) {
2262 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2263
2264 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2265 if (!pte)
2266 return -ENOMEM;
2267 /* It is large page*/
2268 if (largepage_lvl > 1) {
2269 unsigned long nr_superpages, end_pfn;
2270
2271 pteval |= DMA_PTE_LARGE_PAGE;
2272 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2273
2274 nr_superpages = sg_res / lvl_pages;
2275 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2276
2277 /*
2278 * Ensure that old small page tables are
2279 * removed to make room for superpage(s).
2280 * We're adding new large pages, so make sure
2281 * we don't remove their parent tables.
2282 */
2283 dma_pte_free_pagetable(domain, iov_pfn, end_pfn,
2284 largepage_lvl + 1);
2285 } else {
2286 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2287 }
2288
2289 }
2290 /* We don't need lock here, nobody else
2291 * touches the iova range
2292 */
2293 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2294 if (tmp) {
2295 static int dumps = 5;
2296 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2297 iov_pfn, tmp, (unsigned long long)pteval);
2298 if (dumps) {
2299 dumps--;
2300 debug_dma_dump_mappings(NULL);
2301 }
2302 WARN_ON(1);
2303 }
2304
2305 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2306
2307 BUG_ON(nr_pages < lvl_pages);
2308 BUG_ON(sg_res < lvl_pages);
2309
2310 nr_pages -= lvl_pages;
2311 iov_pfn += lvl_pages;
2312 phys_pfn += lvl_pages;
2313 pteval += lvl_pages * VTD_PAGE_SIZE;
2314 sg_res -= lvl_pages;
2315
2316 /* If the next PTE would be the first in a new page, then we
2317 need to flush the cache on the entries we've just written.
2318 And then we'll need to recalculate 'pte', so clear it and
2319 let it get set again in the if (!pte) block above.
2320
2321 If we're done (!nr_pages) we need to flush the cache too.
2322
2323 Also if we've been setting superpages, we may need to
2324 recalculate 'pte' and switch back to smaller pages for the
2325 end of the mapping, if the trailing size is not enough to
2326 use another superpage (i.e. sg_res < lvl_pages). */
2327 pte++;
2328 if (!nr_pages || first_pte_in_page(pte) ||
2329 (largepage_lvl > 1 && sg_res < lvl_pages)) {
2330 domain_flush_cache(domain, first_pte,
2331 (void *)pte - (void *)first_pte);
2332 pte = NULL;
2333 }
2334
2335 if (!sg_res && nr_pages)
2336 sg = sg_next(sg);
2337 }
2338 return 0;
2339 }
2340
2341 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2342 struct scatterlist *sg, unsigned long nr_pages,
2343 int prot)
2344 {
2345 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2346 }
2347
2348 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2349 unsigned long phys_pfn, unsigned long nr_pages,
2350 int prot)
2351 {
2352 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2353 }
2354
2355 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2356 {
2357 unsigned long flags;
2358 struct context_entry *context;
2359 u16 did_old;
2360
2361 if (!iommu)
2362 return;
2363
2364 spin_lock_irqsave(&iommu->lock, flags);
2365 context = iommu_context_addr(iommu, bus, devfn, 0);
2366 if (!context) {
2367 spin_unlock_irqrestore(&iommu->lock, flags);
2368 return;
2369 }
2370 did_old = context_domain_id(context);
2371 context_clear_entry(context);
2372 __iommu_flush_cache(iommu, context, sizeof(*context));
2373 spin_unlock_irqrestore(&iommu->lock, flags);
2374 iommu->flush.flush_context(iommu,
2375 did_old,
2376 (((u16)bus) << 8) | devfn,
2377 DMA_CCMD_MASK_NOBIT,
2378 DMA_CCMD_DEVICE_INVL);
2379 iommu->flush.flush_iotlb(iommu,
2380 did_old,
2381 0,
2382 0,
2383 DMA_TLB_DSI_FLUSH);
2384 }
2385
2386 static inline void unlink_domain_info(struct device_domain_info *info)
2387 {
2388 assert_spin_locked(&device_domain_lock);
2389 list_del(&info->link);
2390 list_del(&info->global);
2391 if (info->dev)
2392 info->dev->archdata.iommu = NULL;
2393 }
2394
2395 static void domain_remove_dev_info(struct dmar_domain *domain)
2396 {
2397 struct device_domain_info *info, *tmp;
2398 unsigned long flags;
2399
2400 spin_lock_irqsave(&device_domain_lock, flags);
2401 list_for_each_entry_safe(info, tmp, &domain->devices, link)
2402 __dmar_remove_one_dev_info(info);
2403 spin_unlock_irqrestore(&device_domain_lock, flags);
2404 }
2405
2406 /*
2407 * find_domain
2408 * Note: we use struct device->archdata.iommu stores the info
2409 */
2410 static struct dmar_domain *find_domain(struct device *dev)
2411 {
2412 struct device_domain_info *info;
2413
2414 /* No lock here, assumes no domain exit in normal case */
2415 info = dev->archdata.iommu;
2416 if (likely(info))
2417 return info->domain;
2418 return NULL;
2419 }
2420
2421 static inline struct device_domain_info *
2422 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2423 {
2424 struct device_domain_info *info;
2425
2426 list_for_each_entry(info, &device_domain_list, global)
2427 if (info->iommu->segment == segment && info->bus == bus &&
2428 info->devfn == devfn)
2429 return info;
2430
2431 return NULL;
2432 }
2433
2434 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2435 int bus, int devfn,
2436 struct device *dev,
2437 struct dmar_domain *domain)
2438 {
2439 struct dmar_domain *found = NULL;
2440 struct device_domain_info *info;
2441 unsigned long flags;
2442 int ret;
2443
2444 info = alloc_devinfo_mem();
2445 if (!info)
2446 return NULL;
2447
2448 info->bus = bus;
2449 info->devfn = devfn;
2450 info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2451 info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2452 info->ats_qdep = 0;
2453 info->dev = dev;
2454 info->domain = domain;
2455 info->iommu = iommu;
2456
2457 if (dev && dev_is_pci(dev)) {
2458 struct pci_dev *pdev = to_pci_dev(info->dev);
2459
2460 if (ecap_dev_iotlb_support(iommu->ecap) &&
2461 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2462 dmar_find_matched_atsr_unit(pdev))
2463 info->ats_supported = 1;
2464
2465 if (ecs_enabled(iommu)) {
2466 if (pasid_enabled(iommu)) {
2467 int features = pci_pasid_features(pdev);
2468 if (features >= 0)
2469 info->pasid_supported = features | 1;
2470 }
2471
2472 if (info->ats_supported && ecap_prs(iommu->ecap) &&
2473 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2474 info->pri_supported = 1;
2475 }
2476 }
2477
2478 spin_lock_irqsave(&device_domain_lock, flags);
2479 if (dev)
2480 found = find_domain(dev);
2481
2482 if (!found) {
2483 struct device_domain_info *info2;
2484 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2485 if (info2) {
2486 found = info2->domain;
2487 info2->dev = dev;
2488 }
2489 }
2490
2491 if (found) {
2492 spin_unlock_irqrestore(&device_domain_lock, flags);
2493 free_devinfo_mem(info);
2494 /* Caller must free the original domain */
2495 return found;
2496 }
2497
2498 spin_lock(&iommu->lock);
2499 ret = domain_attach_iommu(domain, iommu);
2500 spin_unlock(&iommu->lock);
2501
2502 if (ret) {
2503 spin_unlock_irqrestore(&device_domain_lock, flags);
2504 free_devinfo_mem(info);
2505 return NULL;
2506 }
2507
2508 list_add(&info->link, &domain->devices);
2509 list_add(&info->global, &device_domain_list);
2510 if (dev)
2511 dev->archdata.iommu = info;
2512 spin_unlock_irqrestore(&device_domain_lock, flags);
2513
2514 if (dev && domain_context_mapping(domain, dev)) {
2515 pr_err("Domain context map for %s failed\n", dev_name(dev));
2516 dmar_remove_one_dev_info(domain, dev);
2517 return NULL;
2518 }
2519
2520 return domain;
2521 }
2522
2523 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2524 {
2525 *(u16 *)opaque = alias;
2526 return 0;
2527 }
2528
2529 static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
2530 {
2531 struct device_domain_info *info = NULL;
2532 struct dmar_domain *domain = NULL;
2533 struct intel_iommu *iommu;
2534 u16 req_id, dma_alias;
2535 unsigned long flags;
2536 u8 bus, devfn;
2537
2538 iommu = device_to_iommu(dev, &bus, &devfn);
2539 if (!iommu)
2540 return NULL;
2541
2542 req_id = ((u16)bus << 8) | devfn;
2543
2544 if (dev_is_pci(dev)) {
2545 struct pci_dev *pdev = to_pci_dev(dev);
2546
2547 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2548
2549 spin_lock_irqsave(&device_domain_lock, flags);
2550 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2551 PCI_BUS_NUM(dma_alias),
2552 dma_alias & 0xff);
2553 if (info) {
2554 iommu = info->iommu;
2555 domain = info->domain;
2556 }
2557 spin_unlock_irqrestore(&device_domain_lock, flags);
2558
2559 /* DMA alias already has a domain, use it */
2560 if (info)
2561 goto out;
2562 }
2563
2564 /* Allocate and initialize new domain for the device */
2565 domain = alloc_domain(0);
2566 if (!domain)
2567 return NULL;
2568 if (domain_init(domain, iommu, gaw)) {
2569 domain_exit(domain);
2570 return NULL;
2571 }
2572
2573 out:
2574
2575 return domain;
2576 }
2577
2578 static struct dmar_domain *set_domain_for_dev(struct device *dev,
2579 struct dmar_domain *domain)
2580 {
2581 struct intel_iommu *iommu;
2582 struct dmar_domain *tmp;
2583 u16 req_id, dma_alias;
2584 u8 bus, devfn;
2585
2586 iommu = device_to_iommu(dev, &bus, &devfn);
2587 if (!iommu)
2588 return NULL;
2589
2590 req_id = ((u16)bus << 8) | devfn;
2591
2592 if (dev_is_pci(dev)) {
2593 struct pci_dev *pdev = to_pci_dev(dev);
2594
2595 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2596
2597 /* register PCI DMA alias device */
2598 if (req_id != dma_alias) {
2599 tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2600 dma_alias & 0xff, NULL, domain);
2601
2602 if (!tmp || tmp != domain)
2603 return tmp;
2604 }
2605 }
2606
2607 tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2608 if (!tmp || tmp != domain)
2609 return tmp;
2610
2611 return domain;
2612 }
2613
2614 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2615 {
2616 struct dmar_domain *domain, *tmp;
2617
2618 domain = find_domain(dev);
2619 if (domain)
2620 goto out;
2621
2622 domain = find_or_alloc_domain(dev, gaw);
2623 if (!domain)
2624 goto out;
2625
2626 tmp = set_domain_for_dev(dev, domain);
2627 if (!tmp || domain != tmp) {
2628 domain_exit(domain);
2629 domain = tmp;
2630 }
2631
2632 out:
2633
2634 return domain;
2635 }
2636
2637 static int iommu_domain_identity_map(struct dmar_domain *domain,
2638 unsigned long long start,
2639 unsigned long long end)
2640 {
2641 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2642 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2643
2644 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2645 dma_to_mm_pfn(last_vpfn))) {
2646 pr_err("Reserving iova failed\n");
2647 return -ENOMEM;
2648 }
2649
2650 pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2651 /*
2652 * RMRR range might have overlap with physical memory range,
2653 * clear it first
2654 */
2655 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2656
2657 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2658 last_vpfn - first_vpfn + 1,
2659 DMA_PTE_READ|DMA_PTE_WRITE);
2660 }
2661
2662 static int domain_prepare_identity_map(struct device *dev,
2663 struct dmar_domain *domain,
2664 unsigned long long start,
2665 unsigned long long end)
2666 {
2667 /* For _hardware_ passthrough, don't bother. But for software
2668 passthrough, we do it anyway -- it may indicate a memory
2669 range which is reserved in E820, so which didn't get set
2670 up to start with in si_domain */
2671 if (domain == si_domain && hw_pass_through) {
2672 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2673 dev_name(dev), start, end);
2674 return 0;
2675 }
2676
2677 pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2678 dev_name(dev), start, end);
2679
2680 if (end < start) {
2681 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2682 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2683 dmi_get_system_info(DMI_BIOS_VENDOR),
2684 dmi_get_system_info(DMI_BIOS_VERSION),
2685 dmi_get_system_info(DMI_PRODUCT_VERSION));
2686 return -EIO;
2687 }
2688
2689 if (end >> agaw_to_width(domain->agaw)) {
2690 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2691 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2692 agaw_to_width(domain->agaw),
2693 dmi_get_system_info(DMI_BIOS_VENDOR),
2694 dmi_get_system_info(DMI_BIOS_VERSION),
2695 dmi_get_system_info(DMI_PRODUCT_VERSION));
2696 return -EIO;
2697 }
2698
2699 return iommu_domain_identity_map(domain, start, end);
2700 }
2701
2702 static int iommu_prepare_identity_map(struct device *dev,
2703 unsigned long long start,
2704 unsigned long long end)
2705 {
2706 struct dmar_domain *domain;
2707 int ret;
2708
2709 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2710 if (!domain)
2711 return -ENOMEM;
2712
2713 ret = domain_prepare_identity_map(dev, domain, start, end);
2714 if (ret)
2715 domain_exit(domain);
2716
2717 return ret;
2718 }
2719
2720 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2721 struct device *dev)
2722 {
2723 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2724 return 0;
2725 return iommu_prepare_identity_map(dev, rmrr->base_address,
2726 rmrr->end_address);
2727 }
2728
2729 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2730 static inline void iommu_prepare_isa(void)
2731 {
2732 struct pci_dev *pdev;
2733 int ret;
2734
2735 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2736 if (!pdev)
2737 return;
2738
2739 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2740 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2741
2742 if (ret)
2743 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2744
2745 pci_dev_put(pdev);
2746 }
2747 #else
2748 static inline void iommu_prepare_isa(void)
2749 {
2750 return;
2751 }
2752 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2753
2754 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2755
2756 static int __init si_domain_init(int hw)
2757 {
2758 int nid, ret = 0;
2759
2760 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2761 if (!si_domain)
2762 return -EFAULT;
2763
2764 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2765 domain_exit(si_domain);
2766 return -EFAULT;
2767 }
2768
2769 pr_debug("Identity mapping domain allocated\n");
2770
2771 if (hw)
2772 return 0;
2773
2774 for_each_online_node(nid) {
2775 unsigned long start_pfn, end_pfn;
2776 int i;
2777
2778 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2779 ret = iommu_domain_identity_map(si_domain,
2780 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2781 if (ret)
2782 return ret;
2783 }
2784 }
2785
2786 return 0;
2787 }
2788
2789 static int identity_mapping(struct device *dev)
2790 {
2791 struct device_domain_info *info;
2792
2793 if (likely(!iommu_identity_mapping))
2794 return 0;
2795
2796 info = dev->archdata.iommu;
2797 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2798 return (info->domain == si_domain);
2799
2800 return 0;
2801 }
2802
2803 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2804 {
2805 struct dmar_domain *ndomain;
2806 struct intel_iommu *iommu;
2807 u8 bus, devfn;
2808
2809 iommu = device_to_iommu(dev, &bus, &devfn);
2810 if (!iommu)
2811 return -ENODEV;
2812
2813 ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2814 if (ndomain != domain)
2815 return -EBUSY;
2816
2817 return 0;
2818 }
2819
2820 static bool device_has_rmrr(struct device *dev)
2821 {
2822 struct dmar_rmrr_unit *rmrr;
2823 struct device *tmp;
2824 int i;
2825
2826 rcu_read_lock();
2827 for_each_rmrr_units(rmrr) {
2828 /*
2829 * Return TRUE if this RMRR contains the device that
2830 * is passed in.
2831 */
2832 for_each_active_dev_scope(rmrr->devices,
2833 rmrr->devices_cnt, i, tmp)
2834 if (tmp == dev) {
2835 rcu_read_unlock();
2836 return true;
2837 }
2838 }
2839 rcu_read_unlock();
2840 return false;
2841 }
2842
2843 /*
2844 * There are a couple cases where we need to restrict the functionality of
2845 * devices associated with RMRRs. The first is when evaluating a device for
2846 * identity mapping because problems exist when devices are moved in and out
2847 * of domains and their respective RMRR information is lost. This means that
2848 * a device with associated RMRRs will never be in a "passthrough" domain.
2849 * The second is use of the device through the IOMMU API. This interface
2850 * expects to have full control of the IOVA space for the device. We cannot
2851 * satisfy both the requirement that RMRR access is maintained and have an
2852 * unencumbered IOVA space. We also have no ability to quiesce the device's
2853 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2854 * We therefore prevent devices associated with an RMRR from participating in
2855 * the IOMMU API, which eliminates them from device assignment.
2856 *
2857 * In both cases we assume that PCI USB devices with RMRRs have them largely
2858 * for historical reasons and that the RMRR space is not actively used post
2859 * boot. This exclusion may change if vendors begin to abuse it.
2860 *
2861 * The same exception is made for graphics devices, with the requirement that
2862 * any use of the RMRR regions will be torn down before assigning the device
2863 * to a guest.
2864 */
2865 static bool device_is_rmrr_locked(struct device *dev)
2866 {
2867 if (!device_has_rmrr(dev))
2868 return false;
2869
2870 if (dev_is_pci(dev)) {
2871 struct pci_dev *pdev = to_pci_dev(dev);
2872
2873 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2874 return false;
2875 }
2876
2877 return true;
2878 }
2879
2880 static int iommu_should_identity_map(struct device *dev, int startup)
2881 {
2882
2883 if (dev_is_pci(dev)) {
2884 struct pci_dev *pdev = to_pci_dev(dev);
2885
2886 if (device_is_rmrr_locked(dev))
2887 return 0;
2888
2889 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2890 return 1;
2891
2892 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2893 return 1;
2894
2895 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2896 return 0;
2897
2898 /*
2899 * We want to start off with all devices in the 1:1 domain, and
2900 * take them out later if we find they can't access all of memory.
2901 *
2902 * However, we can't do this for PCI devices behind bridges,
2903 * because all PCI devices behind the same bridge will end up
2904 * with the same source-id on their transactions.
2905 *
2906 * Practically speaking, we can't change things around for these
2907 * devices at run-time, because we can't be sure there'll be no
2908 * DMA transactions in flight for any of their siblings.
2909 *
2910 * So PCI devices (unless they're on the root bus) as well as
2911 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2912 * the 1:1 domain, just in _case_ one of their siblings turns out
2913 * not to be able to map all of memory.
2914 */
2915 if (!pci_is_pcie(pdev)) {
2916 if (!pci_is_root_bus(pdev->bus))
2917 return 0;
2918 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2919 return 0;
2920 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2921 return 0;
2922 } else {
2923 if (device_has_rmrr(dev))
2924 return 0;
2925 }
2926
2927 /*
2928 * At boot time, we don't yet know if devices will be 64-bit capable.
2929 * Assume that they will — if they turn out not to be, then we can
2930 * take them out of the 1:1 domain later.
2931 */
2932 if (!startup) {
2933 /*
2934 * If the device's dma_mask is less than the system's memory
2935 * size then this is not a candidate for identity mapping.
2936 */
2937 u64 dma_mask = *dev->dma_mask;
2938
2939 if (dev->coherent_dma_mask &&
2940 dev->coherent_dma_mask < dma_mask)
2941 dma_mask = dev->coherent_dma_mask;
2942
2943 return dma_mask >= dma_get_required_mask(dev);
2944 }
2945
2946 return 1;
2947 }
2948
2949 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2950 {
2951 int ret;
2952
2953 if (!iommu_should_identity_map(dev, 1))
2954 return 0;
2955
2956 ret = domain_add_dev_info(si_domain, dev);
2957 if (!ret)
2958 pr_info("%s identity mapping for device %s\n",
2959 hw ? "Hardware" : "Software", dev_name(dev));
2960 else if (ret == -ENODEV)
2961 /* device not associated with an iommu */
2962 ret = 0;
2963
2964 return ret;
2965 }
2966
2967
2968 static int __init iommu_prepare_static_identity_mapping(int hw)
2969 {
2970 struct pci_dev *pdev = NULL;
2971 struct dmar_drhd_unit *drhd;
2972 struct intel_iommu *iommu;
2973 struct device *dev;
2974 int i;
2975 int ret = 0;
2976
2977 for_each_pci_dev(pdev) {
2978 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2979 if (ret)
2980 return ret;
2981 }
2982
2983 for_each_active_iommu(iommu, drhd)
2984 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2985 struct acpi_device_physical_node *pn;
2986 struct acpi_device *adev;
2987
2988 if (dev->bus != &acpi_bus_type)
2989 continue;
2990
2991 adev= to_acpi_device(dev);
2992 mutex_lock(&adev->physical_node_lock);
2993 list_for_each_entry(pn, &adev->physical_node_list, node) {
2994 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2995 if (ret)
2996 break;
2997 }
2998 mutex_unlock(&adev->physical_node_lock);
2999 if (ret)
3000 return ret;
3001 }
3002
3003 return 0;
3004 }
3005
3006 static void intel_iommu_init_qi(struct intel_iommu *iommu)
3007 {
3008 /*
3009 * Start from the sane iommu hardware state.
3010 * If the queued invalidation is already initialized by us
3011 * (for example, while enabling interrupt-remapping) then
3012 * we got the things already rolling from a sane state.
3013 */
3014 if (!iommu->qi) {
3015 /*
3016 * Clear any previous faults.
3017 */
3018 dmar_fault(-1, iommu);
3019 /*
3020 * Disable queued invalidation if supported and already enabled
3021 * before OS handover.
3022 */
3023 dmar_disable_qi(iommu);
3024 }
3025
3026 if (dmar_enable_qi(iommu)) {
3027 /*
3028 * Queued Invalidate not enabled, use Register Based Invalidate
3029 */
3030 iommu->flush.flush_context = __iommu_flush_context;
3031 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
3032 pr_info("%s: Using Register based invalidation\n",
3033 iommu->name);
3034 } else {
3035 iommu->flush.flush_context = qi_flush_context;
3036 iommu->flush.flush_iotlb = qi_flush_iotlb;
3037 pr_info("%s: Using Queued invalidation\n", iommu->name);
3038 }
3039 }
3040
3041 static int copy_context_table(struct intel_iommu *iommu,
3042 struct root_entry *old_re,
3043 struct context_entry **tbl,
3044 int bus, bool ext)
3045 {
3046 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
3047 struct context_entry *new_ce = NULL, ce;
3048 struct context_entry *old_ce = NULL;
3049 struct root_entry re;
3050 phys_addr_t old_ce_phys;
3051
3052 tbl_idx = ext ? bus * 2 : bus;
3053 memcpy(&re, old_re, sizeof(re));
3054
3055 for (devfn = 0; devfn < 256; devfn++) {
3056 /* First calculate the correct index */
3057 idx = (ext ? devfn * 2 : devfn) % 256;
3058
3059 if (idx == 0) {
3060 /* First save what we may have and clean up */
3061 if (new_ce) {
3062 tbl[tbl_idx] = new_ce;
3063 __iommu_flush_cache(iommu, new_ce,
3064 VTD_PAGE_SIZE);
3065 pos = 1;
3066 }
3067
3068 if (old_ce)
3069 iounmap(old_ce);
3070
3071 ret = 0;
3072 if (devfn < 0x80)
3073 old_ce_phys = root_entry_lctp(&re);
3074 else
3075 old_ce_phys = root_entry_uctp(&re);
3076
3077 if (!old_ce_phys) {
3078 if (ext && devfn == 0) {
3079 /* No LCTP, try UCTP */
3080 devfn = 0x7f;
3081 continue;
3082 } else {
3083 goto out;
3084 }
3085 }
3086
3087 ret = -ENOMEM;
3088 old_ce = memremap(old_ce_phys, PAGE_SIZE,
3089 MEMREMAP_WB);
3090 if (!old_ce)
3091 goto out;
3092
3093 new_ce = alloc_pgtable_page(iommu->node);
3094 if (!new_ce)
3095 goto out_unmap;
3096
3097 ret = 0;
3098 }
3099
3100 /* Now copy the context entry */
3101 memcpy(&ce, old_ce + idx, sizeof(ce));
3102
3103 if (!__context_present(&ce))
3104 continue;
3105
3106 did = context_domain_id(&ce);
3107 if (did >= 0 && did < cap_ndoms(iommu->cap))
3108 set_bit(did, iommu->domain_ids);
3109
3110 /*
3111 * We need a marker for copied context entries. This
3112 * marker needs to work for the old format as well as
3113 * for extended context entries.
3114 *
3115 * Bit 67 of the context entry is used. In the old
3116 * format this bit is available to software, in the
3117 * extended format it is the PGE bit, but PGE is ignored
3118 * by HW if PASIDs are disabled (and thus still
3119 * available).
3120 *
3121 * So disable PASIDs first and then mark the entry
3122 * copied. This means that we don't copy PASID
3123 * translations from the old kernel, but this is fine as
3124 * faults there are not fatal.
3125 */
3126 context_clear_pasid_enable(&ce);
3127 context_set_copied(&ce);
3128
3129 new_ce[idx] = ce;
3130 }
3131
3132 tbl[tbl_idx + pos] = new_ce;
3133
3134 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3135
3136 out_unmap:
3137 memunmap(old_ce);
3138
3139 out:
3140 return ret;
3141 }
3142
3143 static int copy_translation_tables(struct intel_iommu *iommu)
3144 {
3145 struct context_entry **ctxt_tbls;
3146 struct root_entry *old_rt;
3147 phys_addr_t old_rt_phys;
3148 int ctxt_table_entries;
3149 unsigned long flags;
3150 u64 rtaddr_reg;
3151 int bus, ret;
3152 bool new_ext, ext;
3153
3154 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3155 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
3156 new_ext = !!ecap_ecs(iommu->ecap);
3157
3158 /*
3159 * The RTT bit can only be changed when translation is disabled,
3160 * but disabling translation means to open a window for data
3161 * corruption. So bail out and don't copy anything if we would
3162 * have to change the bit.
3163 */
3164 if (new_ext != ext)
3165 return -EINVAL;
3166
3167 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3168 if (!old_rt_phys)
3169 return -EINVAL;
3170
3171 old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
3172 if (!old_rt)
3173 return -ENOMEM;
3174
3175 /* This is too big for the stack - allocate it from slab */
3176 ctxt_table_entries = ext ? 512 : 256;
3177 ret = -ENOMEM;
3178 ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
3179 if (!ctxt_tbls)
3180 goto out_unmap;
3181
3182 for (bus = 0; bus < 256; bus++) {
3183 ret = copy_context_table(iommu, &old_rt[bus],
3184 ctxt_tbls, bus, ext);
3185 if (ret) {
3186 pr_err("%s: Failed to copy context table for bus %d\n",
3187 iommu->name, bus);
3188 continue;
3189 }
3190 }
3191
3192 spin_lock_irqsave(&iommu->lock, flags);
3193
3194 /* Context tables are copied, now write them to the root_entry table */
3195 for (bus = 0; bus < 256; bus++) {
3196 int idx = ext ? bus * 2 : bus;
3197 u64 val;
3198
3199 if (ctxt_tbls[idx]) {
3200 val = virt_to_phys(ctxt_tbls[idx]) | 1;
3201 iommu->root_entry[bus].lo = val;
3202 }
3203
3204 if (!ext || !ctxt_tbls[idx + 1])
3205 continue;
3206
3207 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3208 iommu->root_entry[bus].hi = val;
3209 }
3210
3211 spin_unlock_irqrestore(&iommu->lock, flags);
3212
3213 kfree(ctxt_tbls);
3214
3215 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3216
3217 ret = 0;
3218
3219 out_unmap:
3220 memunmap(old_rt);
3221
3222 return ret;
3223 }
3224
3225 static int __init init_dmars(void)
3226 {
3227 struct dmar_drhd_unit *drhd;
3228 struct dmar_rmrr_unit *rmrr;
3229 bool copied_tables = false;
3230 struct device *dev;
3231 struct intel_iommu *iommu;
3232 int i, ret;
3233
3234 /*
3235 * for each drhd
3236 * allocate root
3237 * initialize and program root entry to not present
3238 * endfor
3239 */
3240 for_each_drhd_unit(drhd) {
3241 /*
3242 * lock not needed as this is only incremented in the single
3243 * threaded kernel __init code path all other access are read
3244 * only
3245 */
3246 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3247 g_num_of_iommus++;
3248 continue;
3249 }
3250 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3251 }
3252
3253 /* Preallocate enough resources for IOMMU hot-addition */
3254 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3255 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3256
3257 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3258 GFP_KERNEL);
3259 if (!g_iommus) {
3260 pr_err("Allocating global iommu array failed\n");
3261 ret = -ENOMEM;
3262 goto error;
3263 }
3264
3265 for_each_active_iommu(iommu, drhd) {
3266 g_iommus[iommu->seq_id] = iommu;
3267
3268 intel_iommu_init_qi(iommu);
3269
3270 ret = iommu_init_domains(iommu);
3271 if (ret)
3272 goto free_iommu;
3273
3274 init_translation_status(iommu);
3275
3276 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3277 iommu_disable_translation(iommu);
3278 clear_translation_pre_enabled(iommu);
3279 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3280 iommu->name);
3281 }
3282
3283 /*
3284 * TBD:
3285 * we could share the same root & context tables
3286 * among all IOMMU's. Need to Split it later.
3287 */
3288 ret = iommu_alloc_root_entry(iommu);
3289 if (ret)
3290 goto free_iommu;
3291
3292 if (translation_pre_enabled(iommu)) {
3293 pr_info("Translation already enabled - trying to copy translation structures\n");
3294
3295 ret = copy_translation_tables(iommu);
3296 if (ret) {
3297 /*
3298 * We found the IOMMU with translation
3299 * enabled - but failed to copy over the
3300 * old root-entry table. Try to proceed
3301 * by disabling translation now and
3302 * allocating a clean root-entry table.
3303 * This might cause DMAR faults, but
3304 * probably the dump will still succeed.
3305 */
3306 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3307 iommu->name);
3308 iommu_disable_translation(iommu);
3309 clear_translation_pre_enabled(iommu);
3310 } else {
3311 pr_info("Copied translation tables from previous kernel for %s\n",
3312 iommu->name);
3313 copied_tables = true;
3314 }
3315 }
3316
3317 if (!ecap_pass_through(iommu->ecap))
3318 hw_pass_through = 0;
3319 #ifdef CONFIG_INTEL_IOMMU_SVM
3320 if (pasid_enabled(iommu))
3321 intel_svm_alloc_pasid_tables(iommu);
3322 #endif
3323 }
3324
3325 /*
3326 * Now that qi is enabled on all iommus, set the root entry and flush
3327 * caches. This is required on some Intel X58 chipsets, otherwise the
3328 * flush_context function will loop forever and the boot hangs.
3329 */
3330 for_each_active_iommu(iommu, drhd) {
3331 iommu_flush_write_buffer(iommu);
3332 iommu_set_root_entry(iommu);
3333 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3334 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3335 }
3336
3337 if (iommu_pass_through)
3338 iommu_identity_mapping |= IDENTMAP_ALL;
3339
3340 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3341 iommu_identity_mapping |= IDENTMAP_GFX;
3342 #endif
3343
3344 check_tylersburg_isoch();
3345
3346 if (iommu_identity_mapping) {
3347 ret = si_domain_init(hw_pass_through);
3348 if (ret)
3349 goto free_iommu;
3350 }
3351
3352
3353 /*
3354 * If we copied translations from a previous kernel in the kdump
3355 * case, we can not assign the devices to domains now, as that
3356 * would eliminate the old mappings. So skip this part and defer
3357 * the assignment to device driver initialization time.
3358 */
3359 if (copied_tables)
3360 goto domains_done;
3361
3362 /*
3363 * If pass through is not set or not enabled, setup context entries for
3364 * identity mappings for rmrr, gfx, and isa and may fall back to static
3365 * identity mapping if iommu_identity_mapping is set.
3366 */
3367 if (iommu_identity_mapping) {
3368 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3369 if (ret) {
3370 pr_crit("Failed to setup IOMMU pass-through\n");
3371 goto free_iommu;
3372 }
3373 }
3374 /*
3375 * For each rmrr
3376 * for each dev attached to rmrr
3377 * do
3378 * locate drhd for dev, alloc domain for dev
3379 * allocate free domain
3380 * allocate page table entries for rmrr
3381 * if context not allocated for bus
3382 * allocate and init context
3383 * set present in root table for this bus
3384 * init context with domain, translation etc
3385 * endfor
3386 * endfor
3387 */
3388 pr_info("Setting RMRR:\n");
3389 for_each_rmrr_units(rmrr) {
3390 /* some BIOS lists non-exist devices in DMAR table. */
3391 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3392 i, dev) {
3393 ret = iommu_prepare_rmrr_dev(rmrr, dev);
3394 if (ret)
3395 pr_err("Mapping reserved region failed\n");
3396 }
3397 }
3398
3399 iommu_prepare_isa();
3400
3401 domains_done:
3402
3403 /*
3404 * for each drhd
3405 * enable fault log
3406 * global invalidate context cache
3407 * global invalidate iotlb
3408 * enable translation
3409 */
3410 for_each_iommu(iommu, drhd) {
3411 if (drhd->ignored) {
3412 /*
3413 * we always have to disable PMRs or DMA may fail on
3414 * this device
3415 */
3416 if (force_on)
3417 iommu_disable_protect_mem_regions(iommu);
3418 continue;
3419 }
3420
3421 iommu_flush_write_buffer(iommu);
3422
3423 #ifdef CONFIG_INTEL_IOMMU_SVM
3424 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
3425 ret = intel_svm_enable_prq(iommu);
3426 if (ret)
3427 goto free_iommu;
3428 }
3429 #endif
3430 ret = dmar_set_interrupt(iommu);
3431 if (ret)
3432 goto free_iommu;
3433
3434 if (!translation_pre_enabled(iommu))
3435 iommu_enable_translation(iommu);
3436
3437 iommu_disable_protect_mem_regions(iommu);
3438 }
3439
3440 return 0;
3441
3442 free_iommu:
3443 for_each_active_iommu(iommu, drhd) {
3444 disable_dmar_iommu(iommu);
3445 free_dmar_iommu(iommu);
3446 }
3447
3448 kfree(g_iommus);
3449
3450 error:
3451 return ret;
3452 }
3453
3454 /* This takes a number of _MM_ pages, not VTD pages */
3455 static unsigned long intel_alloc_iova(struct device *dev,
3456 struct dmar_domain *domain,
3457 unsigned long nrpages, uint64_t dma_mask)
3458 {
3459 unsigned long iova_pfn = 0;
3460
3461 /* Restrict dma_mask to the width that the iommu can handle */
3462 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3463 /* Ensure we reserve the whole size-aligned region */
3464 nrpages = __roundup_pow_of_two(nrpages);
3465
3466 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3467 /*
3468 * First try to allocate an io virtual address in
3469 * DMA_BIT_MASK(32) and if that fails then try allocating
3470 * from higher range
3471 */
3472 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3473 IOVA_PFN(DMA_BIT_MASK(32)), false);
3474 if (iova_pfn)
3475 return iova_pfn;
3476 }
3477 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3478 IOVA_PFN(dma_mask), true);
3479 if (unlikely(!iova_pfn)) {
3480 pr_err("Allocating %ld-page iova for %s failed",
3481 nrpages, dev_name(dev));
3482 return 0;
3483 }
3484
3485 return iova_pfn;
3486 }
3487
3488 static struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3489 {
3490 struct dmar_domain *domain, *tmp;
3491 struct dmar_rmrr_unit *rmrr;
3492 struct device *i_dev;
3493 int i, ret;
3494
3495 domain = find_domain(dev);
3496 if (domain)
3497 goto out;
3498
3499 domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3500 if (!domain)
3501 goto out;
3502
3503 /* We have a new domain - setup possible RMRRs for the device */
3504 rcu_read_lock();
3505 for_each_rmrr_units(rmrr) {
3506 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3507 i, i_dev) {
3508 if (i_dev != dev)
3509 continue;
3510
3511 ret = domain_prepare_identity_map(dev, domain,
3512 rmrr->base_address,
3513 rmrr->end_address);
3514 if (ret)
3515 dev_err(dev, "Mapping reserved region failed\n");
3516 }
3517 }
3518 rcu_read_unlock();
3519
3520 tmp = set_domain_for_dev(dev, domain);
3521 if (!tmp || domain != tmp) {
3522 domain_exit(domain);
3523 domain = tmp;
3524 }
3525
3526 out:
3527
3528 if (!domain)
3529 pr_err("Allocating domain for %s failed\n", dev_name(dev));
3530
3531
3532 return domain;
3533 }
3534
3535 /* Check if the dev needs to go through non-identity map and unmap process.*/
3536 static int iommu_no_mapping(struct device *dev)
3537 {
3538 int found;
3539
3540 if (iommu_dummy(dev))
3541 return 1;
3542
3543 if (!iommu_identity_mapping)
3544 return 0;
3545
3546 found = identity_mapping(dev);
3547 if (found) {
3548 if (iommu_should_identity_map(dev, 0))
3549 return 1;
3550 else {
3551 /*
3552 * 32 bit DMA is removed from si_domain and fall back
3553 * to non-identity mapping.
3554 */
3555 dmar_remove_one_dev_info(si_domain, dev);
3556 pr_info("32bit %s uses non-identity mapping\n",
3557 dev_name(dev));
3558 return 0;
3559 }
3560 } else {
3561 /*
3562 * In case of a detached 64 bit DMA device from vm, the device
3563 * is put into si_domain for identity mapping.
3564 */
3565 if (iommu_should_identity_map(dev, 0)) {
3566 int ret;
3567 ret = domain_add_dev_info(si_domain, dev);
3568 if (!ret) {
3569 pr_info("64bit %s uses identity mapping\n",
3570 dev_name(dev));
3571 return 1;
3572 }
3573 }
3574 }
3575
3576 return 0;
3577 }
3578
3579 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3580 size_t size, int dir, u64 dma_mask)
3581 {
3582 struct dmar_domain *domain;
3583 phys_addr_t start_paddr;
3584 unsigned long iova_pfn;
3585 int prot = 0;
3586 int ret;
3587 struct intel_iommu *iommu;
3588 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3589
3590 BUG_ON(dir == DMA_NONE);
3591
3592 if (iommu_no_mapping(dev))
3593 return paddr;
3594
3595 domain = get_valid_domain_for_dev(dev);
3596 if (!domain)
3597 return 0;
3598
3599 iommu = domain_get_iommu(domain);
3600 size = aligned_nrpages(paddr, size);
3601
3602 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3603 if (!iova_pfn)
3604 goto error;
3605
3606 /*
3607 * Check if DMAR supports zero-length reads on write only
3608 * mappings..
3609 */
3610 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3611 !cap_zlr(iommu->cap))
3612 prot |= DMA_PTE_READ;
3613 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3614 prot |= DMA_PTE_WRITE;
3615 /*
3616 * paddr - (paddr + size) might be partial page, we should map the whole
3617 * page. Note: if two part of one page are separately mapped, we
3618 * might have two guest_addr mapping to the same host paddr, but this
3619 * is not a big problem
3620 */
3621 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
3622 mm_to_dma_pfn(paddr_pfn), size, prot);
3623 if (ret)
3624 goto error;
3625
3626 /* it's a non-present to present mapping. Only flush if caching mode */
3627 if (cap_caching_mode(iommu->cap))
3628 iommu_flush_iotlb_psi(iommu, domain,
3629 mm_to_dma_pfn(iova_pfn),
3630 size, 0, 1);
3631 else
3632 iommu_flush_write_buffer(iommu);
3633
3634 start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
3635 start_paddr += paddr & ~PAGE_MASK;
3636 return start_paddr;
3637
3638 error:
3639 if (iova_pfn)
3640 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3641 pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
3642 dev_name(dev), size, (unsigned long long)paddr, dir);
3643 return 0;
3644 }
3645
3646 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3647 unsigned long offset, size_t size,
3648 enum dma_data_direction dir,
3649 unsigned long attrs)
3650 {
3651 return __intel_map_single(dev, page_to_phys(page) + offset, size,
3652 dir, *dev->dma_mask);
3653 }
3654
3655 static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
3656 {
3657 struct dmar_domain *domain;
3658 unsigned long start_pfn, last_pfn;
3659 unsigned long nrpages;
3660 unsigned long iova_pfn;
3661 struct intel_iommu *iommu;
3662 struct page *freelist;
3663
3664 if (iommu_no_mapping(dev))
3665 return;
3666
3667 domain = find_domain(dev);
3668 BUG_ON(!domain);
3669
3670 iommu = domain_get_iommu(domain);
3671
3672 iova_pfn = IOVA_PFN(dev_addr);
3673
3674 nrpages = aligned_nrpages(dev_addr, size);
3675 start_pfn = mm_to_dma_pfn(iova_pfn);
3676 last_pfn = start_pfn + nrpages - 1;
3677
3678 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3679 dev_name(dev), start_pfn, last_pfn);
3680
3681 freelist = domain_unmap(domain, start_pfn, last_pfn);
3682
3683 if (intel_iommu_strict) {
3684 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3685 nrpages, !freelist, 0);
3686 /* free iova */
3687 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
3688 dma_free_pagelist(freelist);
3689 } else {
3690 queue_iova(&domain->iovad, iova_pfn, nrpages,
3691 (unsigned long)freelist);
3692 /*
3693 * queue up the release of the unmap to save the 1/6th of the
3694 * cpu used up by the iotlb flush operation...
3695 */
3696 }
3697 }
3698
3699 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3700 size_t size, enum dma_data_direction dir,
3701 unsigned long attrs)
3702 {
3703 intel_unmap(dev, dev_addr, size);
3704 }
3705
3706 static void *intel_alloc_coherent(struct device *dev, size_t size,
3707 dma_addr_t *dma_handle, gfp_t flags,
3708 unsigned long attrs)
3709 {
3710 struct page *page = NULL;
3711 int order;
3712
3713 size = PAGE_ALIGN(size);
3714 order = get_order(size);
3715
3716 if (!iommu_no_mapping(dev))
3717 flags &= ~(GFP_DMA | GFP_DMA32);
3718 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3719 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3720 flags |= GFP_DMA;
3721 else
3722 flags |= GFP_DMA32;
3723 }
3724
3725 if (gfpflags_allow_blocking(flags)) {
3726 unsigned int count = size >> PAGE_SHIFT;
3727
3728 page = dma_alloc_from_contiguous(dev, count, order, flags);
3729 if (page && iommu_no_mapping(dev) &&
3730 page_to_phys(page) + size > dev->coherent_dma_mask) {
3731 dma_release_from_contiguous(dev, page, count);
3732 page = NULL;
3733 }
3734 }
3735
3736 if (!page)
3737 page = alloc_pages(flags, order);
3738 if (!page)
3739 return NULL;
3740 memset(page_address(page), 0, size);
3741
3742 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3743 DMA_BIDIRECTIONAL,
3744 dev->coherent_dma_mask);
3745 if (*dma_handle)
3746 return page_address(page);
3747 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3748 __free_pages(page, order);
3749
3750 return NULL;
3751 }
3752
3753 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3754 dma_addr_t dma_handle, unsigned long attrs)
3755 {
3756 int order;
3757 struct page *page = virt_to_page(vaddr);
3758
3759 size = PAGE_ALIGN(size);
3760 order = get_order(size);
3761
3762 intel_unmap(dev, dma_handle, size);
3763 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3764 __free_pages(page, order);
3765 }
3766
3767 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3768 int nelems, enum dma_data_direction dir,
3769 unsigned long attrs)
3770 {
3771 dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
3772 unsigned long nrpages = 0;
3773 struct scatterlist *sg;
3774 int i;
3775
3776 for_each_sg(sglist, sg, nelems, i) {
3777 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3778 }
3779
3780 intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
3781 }
3782
3783 static int intel_nontranslate_map_sg(struct device *hddev,
3784 struct scatterlist *sglist, int nelems, int dir)
3785 {
3786 int i;
3787 struct scatterlist *sg;
3788
3789 for_each_sg(sglist, sg, nelems, i) {
3790 BUG_ON(!sg_page(sg));
3791 sg->dma_address = sg_phys(sg);
3792 sg->dma_length = sg->length;
3793 }
3794 return nelems;
3795 }
3796
3797 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3798 enum dma_data_direction dir, unsigned long attrs)
3799 {
3800 int i;
3801 struct dmar_domain *domain;
3802 size_t size = 0;
3803 int prot = 0;
3804 unsigned long iova_pfn;
3805 int ret;
3806 struct scatterlist *sg;
3807 unsigned long start_vpfn;
3808 struct intel_iommu *iommu;
3809
3810 BUG_ON(dir == DMA_NONE);
3811 if (iommu_no_mapping(dev))
3812 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3813
3814 domain = get_valid_domain_for_dev(dev);
3815 if (!domain)
3816 return 0;
3817
3818 iommu = domain_get_iommu(domain);
3819
3820 for_each_sg(sglist, sg, nelems, i)
3821 size += aligned_nrpages(sg->offset, sg->length);
3822
3823 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3824 *dev->dma_mask);
3825 if (!iova_pfn) {
3826 sglist->dma_length = 0;
3827 return 0;
3828 }
3829
3830 /*
3831 * Check if DMAR supports zero-length reads on write only
3832 * mappings..
3833 */
3834 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3835 !cap_zlr(iommu->cap))
3836 prot |= DMA_PTE_READ;
3837 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3838 prot |= DMA_PTE_WRITE;
3839
3840 start_vpfn = mm_to_dma_pfn(iova_pfn);
3841
3842 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3843 if (unlikely(ret)) {
3844 dma_pte_free_pagetable(domain, start_vpfn,
3845 start_vpfn + size - 1,
3846 agaw_to_level(domain->agaw) + 1);
3847 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3848 return 0;
3849 }
3850
3851 /* it's a non-present to present mapping. Only flush if caching mode */
3852 if (cap_caching_mode(iommu->cap))
3853 iommu_flush_iotlb_psi(iommu, domain, start_vpfn, size, 0, 1);
3854 else
3855 iommu_flush_write_buffer(iommu);
3856
3857 return nelems;
3858 }
3859
3860 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3861 {
3862 return !dma_addr;
3863 }
3864
3865 const struct dma_map_ops intel_dma_ops = {
3866 .alloc = intel_alloc_coherent,
3867 .free = intel_free_coherent,
3868 .map_sg = intel_map_sg,
3869 .unmap_sg = intel_unmap_sg,
3870 .map_page = intel_map_page,
3871 .unmap_page = intel_unmap_page,
3872 .mapping_error = intel_mapping_error,
3873 #ifdef CONFIG_X86
3874 .dma_supported = x86_dma_supported,
3875 #endif
3876 };
3877
3878 static inline int iommu_domain_cache_init(void)
3879 {
3880 int ret = 0;
3881
3882 iommu_domain_cache = kmem_cache_create("iommu_domain",
3883 sizeof(struct dmar_domain),
3884 0,
3885 SLAB_HWCACHE_ALIGN,
3886
3887 NULL);
3888 if (!iommu_domain_cache) {
3889 pr_err("Couldn't create iommu_domain cache\n");
3890 ret = -ENOMEM;
3891 }
3892
3893 return ret;
3894 }
3895
3896 static inline int iommu_devinfo_cache_init(void)
3897 {
3898 int ret = 0;
3899
3900 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3901 sizeof(struct device_domain_info),
3902 0,
3903 SLAB_HWCACHE_ALIGN,
3904 NULL);
3905 if (!iommu_devinfo_cache) {
3906 pr_err("Couldn't create devinfo cache\n");
3907 ret = -ENOMEM;
3908 }
3909
3910 return ret;
3911 }
3912
3913 static int __init iommu_init_mempool(void)
3914 {
3915 int ret;
3916 ret = iova_cache_get();
3917 if (ret)
3918 return ret;
3919
3920 ret = iommu_domain_cache_init();
3921 if (ret)
3922 goto domain_error;
3923
3924 ret = iommu_devinfo_cache_init();
3925 if (!ret)
3926 return ret;
3927
3928 kmem_cache_destroy(iommu_domain_cache);
3929 domain_error:
3930 iova_cache_put();
3931
3932 return -ENOMEM;
3933 }
3934
3935 static void __init iommu_exit_mempool(void)
3936 {
3937 kmem_cache_destroy(iommu_devinfo_cache);
3938 kmem_cache_destroy(iommu_domain_cache);
3939 iova_cache_put();
3940 }
3941
3942 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3943 {
3944 struct dmar_drhd_unit *drhd;
3945 u32 vtbar;
3946 int rc;
3947
3948 /* We know that this device on this chipset has its own IOMMU.
3949 * If we find it under a different IOMMU, then the BIOS is lying
3950 * to us. Hope that the IOMMU for this device is actually
3951 * disabled, and it needs no translation...
3952 */
3953 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3954 if (rc) {
3955 /* "can't" happen */
3956 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3957 return;
3958 }
3959 vtbar &= 0xffff0000;
3960
3961 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3962 drhd = dmar_find_matched_drhd_unit(pdev);
3963 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3964 TAINT_FIRMWARE_WORKAROUND,
3965 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3966 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3967 }
3968 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3969
3970 static void __init init_no_remapping_devices(void)
3971 {
3972 struct dmar_drhd_unit *drhd;
3973 struct device *dev;
3974 int i;
3975
3976 for_each_drhd_unit(drhd) {
3977 if (!drhd->include_all) {
3978 for_each_active_dev_scope(drhd->devices,
3979 drhd->devices_cnt, i, dev)
3980 break;
3981 /* ignore DMAR unit if no devices exist */
3982 if (i == drhd->devices_cnt)
3983 drhd->ignored = 1;
3984 }
3985 }
3986
3987 for_each_active_drhd_unit(drhd) {
3988 if (drhd->include_all)
3989 continue;
3990
3991 for_each_active_dev_scope(drhd->devices,
3992 drhd->devices_cnt, i, dev)
3993 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
3994 break;
3995 if (i < drhd->devices_cnt)
3996 continue;
3997
3998 /* This IOMMU has *only* gfx devices. Either bypass it or
3999 set the gfx_mapped flag, as appropriate */
4000 if (dmar_map_gfx) {
4001 intel_iommu_gfx_mapped = 1;
4002 } else {
4003 drhd->ignored = 1;
4004 for_each_active_dev_scope(drhd->devices,
4005 drhd->devices_cnt, i, dev)
4006 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4007 }
4008 }
4009 }
4010
4011 #ifdef CONFIG_SUSPEND
4012 static int init_iommu_hw(void)
4013 {
4014 struct dmar_drhd_unit *drhd;
4015 struct intel_iommu *iommu = NULL;
4016
4017 for_each_active_iommu(iommu, drhd)
4018 if (iommu->qi)
4019 dmar_reenable_qi(iommu);
4020
4021 for_each_iommu(iommu, drhd) {
4022 if (drhd->ignored) {
4023 /*
4024 * we always have to disable PMRs or DMA may fail on
4025 * this device
4026 */
4027 if (force_on)
4028 iommu_disable_protect_mem_regions(iommu);
4029 continue;
4030 }
4031
4032 iommu_flush_write_buffer(iommu);
4033
4034 iommu_set_root_entry(iommu);
4035
4036 iommu->flush.flush_context(iommu, 0, 0, 0,
4037 DMA_CCMD_GLOBAL_INVL);
4038 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4039 iommu_enable_translation(iommu);
4040 iommu_disable_protect_mem_regions(iommu);
4041 }
4042
4043 return 0;
4044 }
4045
4046 static void iommu_flush_all(void)
4047 {
4048 struct dmar_drhd_unit *drhd;
4049 struct intel_iommu *iommu;
4050
4051 for_each_active_iommu(iommu, drhd) {
4052 iommu->flush.flush_context(iommu, 0, 0, 0,
4053 DMA_CCMD_GLOBAL_INVL);
4054 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
4055 DMA_TLB_GLOBAL_FLUSH);
4056 }
4057 }
4058
4059 static int iommu_suspend(void)
4060 {
4061 struct dmar_drhd_unit *drhd;
4062 struct intel_iommu *iommu = NULL;
4063 unsigned long flag;
4064
4065 for_each_active_iommu(iommu, drhd) {
4066 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
4067 GFP_ATOMIC);
4068 if (!iommu->iommu_state)
4069 goto nomem;
4070 }
4071
4072 iommu_flush_all();
4073
4074 for_each_active_iommu(iommu, drhd) {
4075 iommu_disable_translation(iommu);
4076
4077 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4078
4079 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4080 readl(iommu->reg + DMAR_FECTL_REG);
4081 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4082 readl(iommu->reg + DMAR_FEDATA_REG);
4083 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4084 readl(iommu->reg + DMAR_FEADDR_REG);
4085 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4086 readl(iommu->reg + DMAR_FEUADDR_REG);
4087
4088 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4089 }
4090 return 0;
4091
4092 nomem:
4093 for_each_active_iommu(iommu, drhd)
4094 kfree(iommu->iommu_state);
4095
4096 return -ENOMEM;
4097 }
4098
4099 static void iommu_resume(void)
4100 {
4101 struct dmar_drhd_unit *drhd;
4102 struct intel_iommu *iommu = NULL;
4103 unsigned long flag;
4104
4105 if (init_iommu_hw()) {
4106 if (force_on)
4107 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4108 else
4109 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
4110 return;
4111 }
4112
4113 for_each_active_iommu(iommu, drhd) {
4114
4115 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4116
4117 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4118 iommu->reg + DMAR_FECTL_REG);
4119 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4120 iommu->reg + DMAR_FEDATA_REG);
4121 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4122 iommu->reg + DMAR_FEADDR_REG);
4123 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4124 iommu->reg + DMAR_FEUADDR_REG);
4125
4126 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4127 }
4128
4129 for_each_active_iommu(iommu, drhd)
4130 kfree(iommu->iommu_state);
4131 }
4132
4133 static struct syscore_ops iommu_syscore_ops = {
4134 .resume = iommu_resume,
4135 .suspend = iommu_suspend,
4136 };
4137
4138 static void __init init_iommu_pm_ops(void)
4139 {
4140 register_syscore_ops(&iommu_syscore_ops);
4141 }
4142
4143 #else
4144 static inline void init_iommu_pm_ops(void) {}
4145 #endif /* CONFIG_PM */
4146
4147
4148 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
4149 {
4150 struct acpi_dmar_reserved_memory *rmrr;
4151 int prot = DMA_PTE_READ|DMA_PTE_WRITE;
4152 struct dmar_rmrr_unit *rmrru;
4153 size_t length;
4154
4155 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4156 if (!rmrru)
4157 goto out;
4158
4159 rmrru->hdr = header;
4160 rmrr = (struct acpi_dmar_reserved_memory *)header;
4161 rmrru->base_address = rmrr->base_address;
4162 rmrru->end_address = rmrr->end_address;
4163
4164 length = rmrr->end_address - rmrr->base_address + 1;
4165 rmrru->resv = iommu_alloc_resv_region(rmrr->base_address, length, prot,
4166 IOMMU_RESV_DIRECT);
4167 if (!rmrru->resv)
4168 goto free_rmrru;
4169
4170 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4171 ((void *)rmrr) + rmrr->header.length,
4172 &rmrru->devices_cnt);
4173 if (rmrru->devices_cnt && rmrru->devices == NULL)
4174 goto free_all;
4175
4176 list_add(&rmrru->list, &dmar_rmrr_units);
4177
4178 return 0;
4179 free_all:
4180 kfree(rmrru->resv);
4181 free_rmrru:
4182 kfree(rmrru);
4183 out:
4184 return -ENOMEM;
4185 }
4186
4187 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4188 {
4189 struct dmar_atsr_unit *atsru;
4190 struct acpi_dmar_atsr *tmp;
4191
4192 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4193 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4194 if (atsr->segment != tmp->segment)
4195 continue;
4196 if (atsr->header.length != tmp->header.length)
4197 continue;
4198 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4199 return atsru;
4200 }
4201
4202 return NULL;
4203 }
4204
4205 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4206 {
4207 struct acpi_dmar_atsr *atsr;
4208 struct dmar_atsr_unit *atsru;
4209
4210 if (system_state >= SYSTEM_RUNNING && !intel_iommu_enabled)
4211 return 0;
4212
4213 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4214 atsru = dmar_find_atsr(atsr);
4215 if (atsru)
4216 return 0;
4217
4218 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4219 if (!atsru)
4220 return -ENOMEM;
4221
4222 /*
4223 * If memory is allocated from slab by ACPI _DSM method, we need to
4224 * copy the memory content because the memory buffer will be freed
4225 * on return.
4226 */
4227 atsru->hdr = (void *)(atsru + 1);
4228 memcpy(atsru->hdr, hdr, hdr->length);
4229 atsru->include_all = atsr->flags & 0x1;
4230 if (!atsru->include_all) {
4231 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4232 (void *)atsr + atsr->header.length,
4233 &atsru->devices_cnt);
4234 if (atsru->devices_cnt && atsru->devices == NULL) {
4235 kfree(atsru);
4236 return -ENOMEM;
4237 }
4238 }
4239
4240 list_add_rcu(&atsru->list, &dmar_atsr_units);
4241
4242 return 0;
4243 }
4244
4245 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4246 {
4247 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4248 kfree(atsru);
4249 }
4250
4251 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4252 {
4253 struct acpi_dmar_atsr *atsr;
4254 struct dmar_atsr_unit *atsru;
4255
4256 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4257 atsru = dmar_find_atsr(atsr);
4258 if (atsru) {
4259 list_del_rcu(&atsru->list);
4260 synchronize_rcu();
4261 intel_iommu_free_atsr(atsru);
4262 }
4263
4264 return 0;
4265 }
4266
4267 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4268 {
4269 int i;
4270 struct device *dev;
4271 struct acpi_dmar_atsr *atsr;
4272 struct dmar_atsr_unit *atsru;
4273
4274 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4275 atsru = dmar_find_atsr(atsr);
4276 if (!atsru)
4277 return 0;
4278
4279 if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
4280 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4281 i, dev)
4282 return -EBUSY;
4283 }
4284
4285 return 0;
4286 }
4287
4288 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4289 {
4290 int sp, ret = 0;
4291 struct intel_iommu *iommu = dmaru->iommu;
4292
4293 if (g_iommus[iommu->seq_id])
4294 return 0;
4295
4296 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4297 pr_warn("%s: Doesn't support hardware pass through.\n",
4298 iommu->name);
4299 return -ENXIO;
4300 }
4301 if (!ecap_sc_support(iommu->ecap) &&
4302 domain_update_iommu_snooping(iommu)) {
4303 pr_warn("%s: Doesn't support snooping.\n",
4304 iommu->name);
4305 return -ENXIO;
4306 }
4307 sp = domain_update_iommu_superpage(iommu) - 1;
4308 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4309 pr_warn("%s: Doesn't support large page.\n",
4310 iommu->name);
4311 return -ENXIO;
4312 }
4313
4314 /*
4315 * Disable translation if already enabled prior to OS handover.
4316 */
4317 if (iommu->gcmd & DMA_GCMD_TE)
4318 iommu_disable_translation(iommu);
4319
4320 g_iommus[iommu->seq_id] = iommu;
4321 ret = iommu_init_domains(iommu);
4322 if (ret == 0)
4323 ret = iommu_alloc_root_entry(iommu);
4324 if (ret)
4325 goto out;
4326
4327 #ifdef CONFIG_INTEL_IOMMU_SVM
4328 if (pasid_enabled(iommu))
4329 intel_svm_alloc_pasid_tables(iommu);
4330 #endif
4331
4332 if (dmaru->ignored) {
4333 /*
4334 * we always have to disable PMRs or DMA may fail on this device
4335 */
4336 if (force_on)
4337 iommu_disable_protect_mem_regions(iommu);
4338 return 0;
4339 }
4340
4341 intel_iommu_init_qi(iommu);
4342 iommu_flush_write_buffer(iommu);
4343
4344 #ifdef CONFIG_INTEL_IOMMU_SVM
4345 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
4346 ret = intel_svm_enable_prq(iommu);
4347 if (ret)
4348 goto disable_iommu;
4349 }
4350 #endif
4351 ret = dmar_set_interrupt(iommu);
4352 if (ret)
4353 goto disable_iommu;
4354
4355 iommu_set_root_entry(iommu);
4356 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4357 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4358 iommu_enable_translation(iommu);
4359
4360 iommu_disable_protect_mem_regions(iommu);
4361 return 0;
4362
4363 disable_iommu:
4364 disable_dmar_iommu(iommu);
4365 out:
4366 free_dmar_iommu(iommu);
4367 return ret;
4368 }
4369
4370 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4371 {
4372 int ret = 0;
4373 struct intel_iommu *iommu = dmaru->iommu;
4374
4375 if (!intel_iommu_enabled)
4376 return 0;
4377 if (iommu == NULL)
4378 return -EINVAL;
4379
4380 if (insert) {
4381 ret = intel_iommu_add(dmaru);
4382 } else {
4383 disable_dmar_iommu(iommu);
4384 free_dmar_iommu(iommu);
4385 }
4386
4387 return ret;
4388 }
4389
4390 static void intel_iommu_free_dmars(void)
4391 {
4392 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4393 struct dmar_atsr_unit *atsru, *atsr_n;
4394
4395 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4396 list_del(&rmrru->list);
4397 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4398 kfree(rmrru->resv);
4399 kfree(rmrru);
4400 }
4401
4402 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4403 list_del(&atsru->list);
4404 intel_iommu_free_atsr(atsru);
4405 }
4406 }
4407
4408 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4409 {
4410 int i, ret = 1;
4411 struct pci_bus *bus;
4412 struct pci_dev *bridge = NULL;
4413 struct device *tmp;
4414 struct acpi_dmar_atsr *atsr;
4415 struct dmar_atsr_unit *atsru;
4416
4417 dev = pci_physfn(dev);
4418 for (bus = dev->bus; bus; bus = bus->parent) {
4419 bridge = bus->self;
4420 /* If it's an integrated device, allow ATS */
4421 if (!bridge)
4422 return 1;
4423 /* Connected via non-PCIe: no ATS */
4424 if (!pci_is_pcie(bridge) ||
4425 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4426 return 0;
4427 /* If we found the root port, look it up in the ATSR */
4428 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4429 break;
4430 }
4431
4432 rcu_read_lock();
4433 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4434 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4435 if (atsr->segment != pci_domain_nr(dev->bus))
4436 continue;
4437
4438 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4439 if (tmp == &bridge->dev)
4440 goto out;
4441
4442 if (atsru->include_all)
4443 goto out;
4444 }
4445 ret = 0;
4446 out:
4447 rcu_read_unlock();
4448
4449 return ret;
4450 }
4451
4452 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4453 {
4454 int ret = 0;
4455 struct dmar_rmrr_unit *rmrru;
4456 struct dmar_atsr_unit *atsru;
4457 struct acpi_dmar_atsr *atsr;
4458 struct acpi_dmar_reserved_memory *rmrr;
4459
4460 if (!intel_iommu_enabled && system_state >= SYSTEM_RUNNING)
4461 return 0;
4462
4463 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4464 rmrr = container_of(rmrru->hdr,
4465 struct acpi_dmar_reserved_memory, header);
4466 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4467 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4468 ((void *)rmrr) + rmrr->header.length,
4469 rmrr->segment, rmrru->devices,
4470 rmrru->devices_cnt);
4471 if(ret < 0)
4472 return ret;
4473 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4474 dmar_remove_dev_scope(info, rmrr->segment,
4475 rmrru->devices, rmrru->devices_cnt);
4476 }
4477 }
4478
4479 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4480 if (atsru->include_all)
4481 continue;
4482
4483 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4484 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4485 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4486 (void *)atsr + atsr->header.length,
4487 atsr->segment, atsru->devices,
4488 atsru->devices_cnt);
4489 if (ret > 0)
4490 break;
4491 else if(ret < 0)
4492 return ret;
4493 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4494 if (dmar_remove_dev_scope(info, atsr->segment,
4495 atsru->devices, atsru->devices_cnt))
4496 break;
4497 }
4498 }
4499
4500 return 0;
4501 }
4502
4503 /*
4504 * Here we only respond to action of unbound device from driver.
4505 *
4506 * Added device is not attached to its DMAR domain here yet. That will happen
4507 * when mapping the device to iova.
4508 */
4509 static int device_notifier(struct notifier_block *nb,
4510 unsigned long action, void *data)
4511 {
4512 struct device *dev = data;
4513 struct dmar_domain *domain;
4514
4515 if (iommu_dummy(dev))
4516 return 0;
4517
4518 if (action != BUS_NOTIFY_REMOVED_DEVICE)
4519 return 0;
4520
4521 domain = find_domain(dev);
4522 if (!domain)
4523 return 0;
4524
4525 dmar_remove_one_dev_info(domain, dev);
4526 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4527 domain_exit(domain);
4528
4529 return 0;
4530 }
4531
4532 static struct notifier_block device_nb = {
4533 .notifier_call = device_notifier,
4534 };
4535
4536 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4537 unsigned long val, void *v)
4538 {
4539 struct memory_notify *mhp = v;
4540 unsigned long long start, end;
4541 unsigned long start_vpfn, last_vpfn;
4542
4543 switch (val) {
4544 case MEM_GOING_ONLINE:
4545 start = mhp->start_pfn << PAGE_SHIFT;
4546 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4547 if (iommu_domain_identity_map(si_domain, start, end)) {
4548 pr_warn("Failed to build identity map for [%llx-%llx]\n",
4549 start, end);
4550 return NOTIFY_BAD;
4551 }
4552 break;
4553
4554 case MEM_OFFLINE:
4555 case MEM_CANCEL_ONLINE:
4556 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4557 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4558 while (start_vpfn <= last_vpfn) {
4559 struct iova *iova;
4560 struct dmar_drhd_unit *drhd;
4561 struct intel_iommu *iommu;
4562 struct page *freelist;
4563
4564 iova = find_iova(&si_domain->iovad, start_vpfn);
4565 if (iova == NULL) {
4566 pr_debug("Failed get IOVA for PFN %lx\n",
4567 start_vpfn);
4568 break;
4569 }
4570
4571 iova = split_and_remove_iova(&si_domain->iovad, iova,
4572 start_vpfn, last_vpfn);
4573 if (iova == NULL) {
4574 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4575 start_vpfn, last_vpfn);
4576 return NOTIFY_BAD;
4577 }
4578
4579 freelist = domain_unmap(si_domain, iova->pfn_lo,
4580 iova->pfn_hi);
4581
4582 rcu_read_lock();
4583 for_each_active_iommu(iommu, drhd)
4584 iommu_flush_iotlb_psi(iommu, si_domain,
4585 iova->pfn_lo, iova_size(iova),
4586 !freelist, 0);
4587 rcu_read_unlock();
4588 dma_free_pagelist(freelist);
4589
4590 start_vpfn = iova->pfn_hi + 1;
4591 free_iova_mem(iova);
4592 }
4593 break;
4594 }
4595
4596 return NOTIFY_OK;
4597 }
4598
4599 static struct notifier_block intel_iommu_memory_nb = {
4600 .notifier_call = intel_iommu_memory_notifier,
4601 .priority = 0
4602 };
4603
4604 static void free_all_cpu_cached_iovas(unsigned int cpu)
4605 {
4606 int i;
4607
4608 for (i = 0; i < g_num_of_iommus; i++) {
4609 struct intel_iommu *iommu = g_iommus[i];
4610 struct dmar_domain *domain;
4611 int did;
4612
4613 if (!iommu)
4614 continue;
4615
4616 for (did = 0; did < cap_ndoms(iommu->cap); did++) {
4617 domain = get_iommu_domain(iommu, (u16)did);
4618
4619 if (!domain)
4620 continue;
4621 free_cpu_cached_iovas(cpu, &domain->iovad);
4622 }
4623 }
4624 }
4625
4626 static int intel_iommu_cpu_dead(unsigned int cpu)
4627 {
4628 free_all_cpu_cached_iovas(cpu);
4629 return 0;
4630 }
4631
4632 static void intel_disable_iommus(void)
4633 {
4634 struct intel_iommu *iommu = NULL;
4635 struct dmar_drhd_unit *drhd;
4636
4637 for_each_iommu(iommu, drhd)
4638 iommu_disable_translation(iommu);
4639 }
4640
4641 static inline struct intel_iommu *dev_to_intel_iommu(struct device *dev)
4642 {
4643 struct iommu_device *iommu_dev = dev_to_iommu_device(dev);
4644
4645 return container_of(iommu_dev, struct intel_iommu, iommu);
4646 }
4647
4648 static ssize_t intel_iommu_show_version(struct device *dev,
4649 struct device_attribute *attr,
4650 char *buf)
4651 {
4652 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4653 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4654 return sprintf(buf, "%d:%d\n",
4655 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4656 }
4657 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4658
4659 static ssize_t intel_iommu_show_address(struct device *dev,
4660 struct device_attribute *attr,
4661 char *buf)
4662 {
4663 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4664 return sprintf(buf, "%llx\n", iommu->reg_phys);
4665 }
4666 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4667
4668 static ssize_t intel_iommu_show_cap(struct device *dev,
4669 struct device_attribute *attr,
4670 char *buf)
4671 {
4672 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4673 return sprintf(buf, "%llx\n", iommu->cap);
4674 }
4675 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4676
4677 static ssize_t intel_iommu_show_ecap(struct device *dev,
4678 struct device_attribute *attr,
4679 char *buf)
4680 {
4681 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4682 return sprintf(buf, "%llx\n", iommu->ecap);
4683 }
4684 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4685
4686 static ssize_t intel_iommu_show_ndoms(struct device *dev,
4687 struct device_attribute *attr,
4688 char *buf)
4689 {
4690 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4691 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4692 }
4693 static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4694
4695 static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4696 struct device_attribute *attr,
4697 char *buf)
4698 {
4699 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4700 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4701 cap_ndoms(iommu->cap)));
4702 }
4703 static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4704
4705 static struct attribute *intel_iommu_attrs[] = {
4706 &dev_attr_version.attr,
4707 &dev_attr_address.attr,
4708 &dev_attr_cap.attr,
4709 &dev_attr_ecap.attr,
4710 &dev_attr_domains_supported.attr,
4711 &dev_attr_domains_used.attr,
4712 NULL,
4713 };
4714
4715 static struct attribute_group intel_iommu_group = {
4716 .name = "intel-iommu",
4717 .attrs = intel_iommu_attrs,
4718 };
4719
4720 const struct attribute_group *intel_iommu_groups[] = {
4721 &intel_iommu_group,
4722 NULL,
4723 };
4724
4725 int __init intel_iommu_init(void)
4726 {
4727 int ret = -ENODEV;
4728 struct dmar_drhd_unit *drhd;
4729 struct intel_iommu *iommu;
4730
4731 /* VT-d is required for a TXT/tboot launch, so enforce that */
4732 force_on = tboot_force_iommu();
4733
4734 if (iommu_init_mempool()) {
4735 if (force_on)
4736 panic("tboot: Failed to initialize iommu memory\n");
4737 return -ENOMEM;
4738 }
4739
4740 down_write(&dmar_global_lock);
4741 if (dmar_table_init()) {
4742 if (force_on)
4743 panic("tboot: Failed to initialize DMAR table\n");
4744 goto out_free_dmar;
4745 }
4746
4747 if (dmar_dev_scope_init() < 0) {
4748 if (force_on)
4749 panic("tboot: Failed to initialize DMAR device scope\n");
4750 goto out_free_dmar;
4751 }
4752
4753 up_write(&dmar_global_lock);
4754
4755 /*
4756 * The bus notifier takes the dmar_global_lock, so lockdep will
4757 * complain later when we register it under the lock.
4758 */
4759 dmar_register_bus_notifier();
4760
4761 down_write(&dmar_global_lock);
4762
4763 if (no_iommu || dmar_disabled) {
4764 /*
4765 * We exit the function here to ensure IOMMU's remapping and
4766 * mempool aren't setup, which means that the IOMMU's PMRs
4767 * won't be disabled via the call to init_dmars(). So disable
4768 * it explicitly here. The PMRs were setup by tboot prior to
4769 * calling SENTER, but the kernel is expected to reset/tear
4770 * down the PMRs.
4771 */
4772 if (intel_iommu_tboot_noforce) {
4773 for_each_iommu(iommu, drhd)
4774 iommu_disable_protect_mem_regions(iommu);
4775 }
4776
4777 /*
4778 * Make sure the IOMMUs are switched off, even when we
4779 * boot into a kexec kernel and the previous kernel left
4780 * them enabled
4781 */
4782 intel_disable_iommus();
4783 goto out_free_dmar;
4784 }
4785
4786 if (list_empty(&dmar_rmrr_units))
4787 pr_info("No RMRR found\n");
4788
4789 if (list_empty(&dmar_atsr_units))
4790 pr_info("No ATSR found\n");
4791
4792 if (dmar_init_reserved_ranges()) {
4793 if (force_on)
4794 panic("tboot: Failed to reserve iommu ranges\n");
4795 goto out_free_reserved_range;
4796 }
4797
4798 init_no_remapping_devices();
4799
4800 ret = init_dmars();
4801 if (ret) {
4802 if (force_on)
4803 panic("tboot: Failed to initialize DMARs\n");
4804 pr_err("Initialization failed\n");
4805 goto out_free_reserved_range;
4806 }
4807 up_write(&dmar_global_lock);
4808 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4809
4810 #if defined(CONFIG_X86) && defined(CONFIG_SWIOTLB)
4811 swiotlb = 0;
4812 #endif
4813 dma_ops = &intel_dma_ops;
4814
4815 init_iommu_pm_ops();
4816
4817 for_each_active_iommu(iommu, drhd) {
4818 iommu_device_sysfs_add(&iommu->iommu, NULL,
4819 intel_iommu_groups,
4820 "%s", iommu->name);
4821 iommu_device_set_ops(&iommu->iommu, &intel_iommu_ops);
4822 iommu_device_register(&iommu->iommu);
4823 }
4824
4825 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4826 bus_register_notifier(&pci_bus_type, &device_nb);
4827 if (si_domain && !hw_pass_through)
4828 register_memory_notifier(&intel_iommu_memory_nb);
4829 cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL,
4830 intel_iommu_cpu_dead);
4831 intel_iommu_enabled = 1;
4832
4833 return 0;
4834
4835 out_free_reserved_range:
4836 put_iova_domain(&reserved_iova_list);
4837 out_free_dmar:
4838 intel_iommu_free_dmars();
4839 up_write(&dmar_global_lock);
4840 iommu_exit_mempool();
4841 return ret;
4842 }
4843
4844 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4845 {
4846 struct intel_iommu *iommu = opaque;
4847
4848 domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4849 return 0;
4850 }
4851
4852 /*
4853 * NB - intel-iommu lacks any sort of reference counting for the users of
4854 * dependent devices. If multiple endpoints have intersecting dependent
4855 * devices, unbinding the driver from any one of them will possibly leave
4856 * the others unable to operate.
4857 */
4858 static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
4859 {
4860 if (!iommu || !dev || !dev_is_pci(dev))
4861 return;
4862
4863 pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
4864 }
4865
4866 static void __dmar_remove_one_dev_info(struct device_domain_info *info)
4867 {
4868 struct intel_iommu *iommu;
4869 unsigned long flags;
4870
4871 assert_spin_locked(&device_domain_lock);
4872
4873 if (WARN_ON(!info))
4874 return;
4875
4876 iommu = info->iommu;
4877
4878 if (info->dev) {
4879 iommu_disable_dev_iotlb(info);
4880 domain_context_clear(iommu, info->dev);
4881 }
4882
4883 unlink_domain_info(info);
4884
4885 spin_lock_irqsave(&iommu->lock, flags);
4886 domain_detach_iommu(info->domain, iommu);
4887 spin_unlock_irqrestore(&iommu->lock, flags);
4888
4889 free_devinfo_mem(info);
4890 }
4891
4892 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4893 struct device *dev)
4894 {
4895 struct device_domain_info *info;
4896 unsigned long flags;
4897
4898 spin_lock_irqsave(&device_domain_lock, flags);
4899 info = dev->archdata.iommu;
4900 __dmar_remove_one_dev_info(info);
4901 spin_unlock_irqrestore(&device_domain_lock, flags);
4902 }
4903
4904 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4905 {
4906 int adjust_width;
4907
4908 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
4909 domain_reserve_special_ranges(domain);
4910
4911 /* calculate AGAW */
4912 domain->gaw = guest_width;
4913 adjust_width = guestwidth_to_adjustwidth(guest_width);
4914 domain->agaw = width_to_agaw(adjust_width);
4915
4916 domain->iommu_coherency = 0;
4917 domain->iommu_snooping = 0;
4918 domain->iommu_superpage = 0;
4919 domain->max_addr = 0;
4920
4921 /* always allocate the top pgd */
4922 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
4923 if (!domain->pgd)
4924 return -ENOMEM;
4925 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4926 return 0;
4927 }
4928
4929 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
4930 {
4931 struct dmar_domain *dmar_domain;
4932 struct iommu_domain *domain;
4933
4934 if (type != IOMMU_DOMAIN_UNMANAGED)
4935 return NULL;
4936
4937 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
4938 if (!dmar_domain) {
4939 pr_err("Can't allocate dmar_domain\n");
4940 return NULL;
4941 }
4942 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4943 pr_err("Domain initialization failed\n");
4944 domain_exit(dmar_domain);
4945 return NULL;
4946 }
4947 domain_update_iommu_cap(dmar_domain);
4948
4949 domain = &dmar_domain->domain;
4950 domain->geometry.aperture_start = 0;
4951 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4952 domain->geometry.force_aperture = true;
4953
4954 return domain;
4955 }
4956
4957 static void intel_iommu_domain_free(struct iommu_domain *domain)
4958 {
4959 domain_exit(to_dmar_domain(domain));
4960 }
4961
4962 static int intel_iommu_attach_device(struct iommu_domain *domain,
4963 struct device *dev)
4964 {
4965 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4966 struct intel_iommu *iommu;
4967 int addr_width;
4968 u8 bus, devfn;
4969
4970 if (device_is_rmrr_locked(dev)) {
4971 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
4972 return -EPERM;
4973 }
4974
4975 /* normally dev is not mapped */
4976 if (unlikely(domain_context_mapped(dev))) {
4977 struct dmar_domain *old_domain;
4978
4979 old_domain = find_domain(dev);
4980 if (old_domain) {
4981 rcu_read_lock();
4982 dmar_remove_one_dev_info(old_domain, dev);
4983 rcu_read_unlock();
4984
4985 if (!domain_type_is_vm_or_si(old_domain) &&
4986 list_empty(&old_domain->devices))
4987 domain_exit(old_domain);
4988 }
4989 }
4990
4991 iommu = device_to_iommu(dev, &bus, &devfn);
4992 if (!iommu)
4993 return -ENODEV;
4994
4995 /* check if this iommu agaw is sufficient for max mapped address */
4996 addr_width = agaw_to_width(iommu->agaw);
4997 if (addr_width > cap_mgaw(iommu->cap))
4998 addr_width = cap_mgaw(iommu->cap);
4999
5000 if (dmar_domain->max_addr > (1LL << addr_width)) {
5001 pr_err("%s: iommu width (%d) is not "
5002 "sufficient for the mapped address (%llx)\n",
5003 __func__, addr_width, dmar_domain->max_addr);
5004 return -EFAULT;
5005 }
5006 dmar_domain->gaw = addr_width;
5007
5008 /*
5009 * Knock out extra levels of page tables if necessary
5010 */
5011 while (iommu->agaw < dmar_domain->agaw) {
5012 struct dma_pte *pte;
5013
5014 pte = dmar_domain->pgd;
5015 if (dma_pte_present(pte)) {
5016 dmar_domain->pgd = (struct dma_pte *)
5017 phys_to_virt(dma_pte_addr(pte));
5018 free_pgtable_page(pte);
5019 }
5020 dmar_domain->agaw--;
5021 }
5022
5023 return domain_add_dev_info(dmar_domain, dev);
5024 }
5025
5026 static void intel_iommu_detach_device(struct iommu_domain *domain,
5027 struct device *dev)
5028 {
5029 dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
5030 }
5031
5032 static int intel_iommu_map(struct iommu_domain *domain,
5033 unsigned long iova, phys_addr_t hpa,
5034 size_t size, int iommu_prot)
5035 {
5036 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5037 u64 max_addr;
5038 int prot = 0;
5039 int ret;
5040
5041 if (iommu_prot & IOMMU_READ)
5042 prot |= DMA_PTE_READ;
5043 if (iommu_prot & IOMMU_WRITE)
5044 prot |= DMA_PTE_WRITE;
5045 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
5046 prot |= DMA_PTE_SNP;
5047
5048 max_addr = iova + size;
5049 if (dmar_domain->max_addr < max_addr) {
5050 u64 end;
5051
5052 /* check if minimum agaw is sufficient for mapped address */
5053 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
5054 if (end < max_addr) {
5055 pr_err("%s: iommu width (%d) is not "
5056 "sufficient for the mapped address (%llx)\n",
5057 __func__, dmar_domain->gaw, max_addr);
5058 return -EFAULT;
5059 }
5060 dmar_domain->max_addr = max_addr;
5061 }
5062 /* Round up size to next multiple of PAGE_SIZE, if it and
5063 the low bits of hpa would take us onto the next page */
5064 size = aligned_nrpages(hpa, size);
5065 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
5066 hpa >> VTD_PAGE_SHIFT, size, prot);
5067 return ret;
5068 }
5069
5070 static size_t intel_iommu_unmap(struct iommu_domain *domain,
5071 unsigned long iova, size_t size)
5072 {
5073 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5074 struct page *freelist = NULL;
5075 struct intel_iommu *iommu;
5076 unsigned long start_pfn, last_pfn;
5077 unsigned int npages;
5078 int iommu_id, level = 0;
5079
5080 /* Cope with horrid API which requires us to unmap more than the
5081 size argument if it happens to be a large-page mapping. */
5082 BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
5083
5084 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
5085 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
5086
5087 start_pfn = iova >> VTD_PAGE_SHIFT;
5088 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5089
5090 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
5091
5092 npages = last_pfn - start_pfn + 1;
5093
5094 for_each_domain_iommu(iommu_id, dmar_domain) {
5095 iommu = g_iommus[iommu_id];
5096
5097 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5098 start_pfn, npages, !freelist, 0);
5099 }
5100
5101 dma_free_pagelist(freelist);
5102
5103 if (dmar_domain->max_addr == iova + size)
5104 dmar_domain->max_addr = iova;
5105
5106 return size;
5107 }
5108
5109 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
5110 dma_addr_t iova)
5111 {
5112 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5113 struct dma_pte *pte;
5114 int level = 0;
5115 u64 phys = 0;
5116
5117 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
5118 if (pte)
5119 phys = dma_pte_addr(pte);
5120
5121 return phys;
5122 }
5123
5124 static bool intel_iommu_capable(enum iommu_cap cap)
5125 {
5126 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5127 return domain_update_iommu_snooping(NULL) == 1;
5128 if (cap == IOMMU_CAP_INTR_REMAP)
5129 return irq_remapping_enabled == 1;
5130
5131 return false;
5132 }
5133
5134 static int intel_iommu_add_device(struct device *dev)
5135 {
5136 struct intel_iommu *iommu;
5137 struct iommu_group *group;
5138 u8 bus, devfn;
5139
5140 iommu = device_to_iommu(dev, &bus, &devfn);
5141 if (!iommu)
5142 return -ENODEV;
5143
5144 iommu_device_link(&iommu->iommu, dev);
5145
5146 group = iommu_group_get_for_dev(dev);
5147
5148 if (IS_ERR(group))
5149 return PTR_ERR(group);
5150
5151 iommu_group_put(group);
5152 return 0;
5153 }
5154
5155 static void intel_iommu_remove_device(struct device *dev)
5156 {
5157 struct intel_iommu *iommu;
5158 u8 bus, devfn;
5159
5160 iommu = device_to_iommu(dev, &bus, &devfn);
5161 if (!iommu)
5162 return;
5163
5164 iommu_group_remove_device(dev);
5165
5166 iommu_device_unlink(&iommu->iommu, dev);
5167 }
5168
5169 static void intel_iommu_get_resv_regions(struct device *device,
5170 struct list_head *head)
5171 {
5172 struct iommu_resv_region *reg;
5173 struct dmar_rmrr_unit *rmrr;
5174 struct device *i_dev;
5175 int i;
5176
5177 rcu_read_lock();
5178 for_each_rmrr_units(rmrr) {
5179 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
5180 i, i_dev) {
5181 if (i_dev != device)
5182 continue;
5183
5184 list_add_tail(&rmrr->resv->list, head);
5185 }
5186 }
5187 rcu_read_unlock();
5188
5189 reg = iommu_alloc_resv_region(IOAPIC_RANGE_START,
5190 IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1,
5191 0, IOMMU_RESV_MSI);
5192 if (!reg)
5193 return;
5194 list_add_tail(&reg->list, head);
5195 }
5196
5197 static void intel_iommu_put_resv_regions(struct device *dev,
5198 struct list_head *head)
5199 {
5200 struct iommu_resv_region *entry, *next;
5201
5202 list_for_each_entry_safe(entry, next, head, list) {
5203 if (entry->type == IOMMU_RESV_RESERVED)
5204 kfree(entry);
5205 }
5206 }
5207
5208 #ifdef CONFIG_INTEL_IOMMU_SVM
5209 #define MAX_NR_PASID_BITS (20)
5210 static inline unsigned long intel_iommu_get_pts(struct intel_iommu *iommu)
5211 {
5212 /*
5213 * Convert ecap_pss to extend context entry pts encoding, also
5214 * respect the soft pasid_max value set by the iommu.
5215 * - number of PASID bits = ecap_pss + 1
5216 * - number of PASID table entries = 2^(pts + 5)
5217 * Therefore, pts = ecap_pss - 4
5218 * e.g. KBL ecap_pss = 0x13, PASID has 20 bits, pts = 15
5219 */
5220 if (ecap_pss(iommu->ecap) < 5)
5221 return 0;
5222
5223 /* pasid_max is encoded as actual number of entries not the bits */
5224 return find_first_bit((unsigned long *)&iommu->pasid_max,
5225 MAX_NR_PASID_BITS) - 5;
5226 }
5227
5228 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5229 {
5230 struct device_domain_info *info;
5231 struct context_entry *context;
5232 struct dmar_domain *domain;
5233 unsigned long flags;
5234 u64 ctx_lo;
5235 int ret;
5236
5237 domain = get_valid_domain_for_dev(sdev->dev);
5238 if (!domain)
5239 return -EINVAL;
5240
5241 spin_lock_irqsave(&device_domain_lock, flags);
5242 spin_lock(&iommu->lock);
5243
5244 ret = -EINVAL;
5245 info = sdev->dev->archdata.iommu;
5246 if (!info || !info->pasid_supported)
5247 goto out;
5248
5249 context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5250 if (WARN_ON(!context))
5251 goto out;
5252
5253 ctx_lo = context[0].lo;
5254
5255 sdev->did = domain->iommu_did[iommu->seq_id];
5256 sdev->sid = PCI_DEVID(info->bus, info->devfn);
5257
5258 if (!(ctx_lo & CONTEXT_PASIDE)) {
5259 if (iommu->pasid_state_table)
5260 context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
5261 context[1].lo = (u64)virt_to_phys(iommu->pasid_table) |
5262 intel_iommu_get_pts(iommu);
5263
5264 wmb();
5265 /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both
5266 * extended to permit requests-with-PASID if the PASIDE bit
5267 * is set. which makes sense. For CONTEXT_TT_PASS_THROUGH,
5268 * however, the PASIDE bit is ignored and requests-with-PASID
5269 * are unconditionally blocked. Which makes less sense.
5270 * So convert from CONTEXT_TT_PASS_THROUGH to one of the new
5271 * "guest mode" translation types depending on whether ATS
5272 * is available or not. Annoyingly, we can't use the new
5273 * modes *unless* PASIDE is set. */
5274 if ((ctx_lo & CONTEXT_TT_MASK) == (CONTEXT_TT_PASS_THROUGH << 2)) {
5275 ctx_lo &= ~CONTEXT_TT_MASK;
5276 if (info->ats_supported)
5277 ctx_lo |= CONTEXT_TT_PT_PASID_DEV_IOTLB << 2;
5278 else
5279 ctx_lo |= CONTEXT_TT_PT_PASID << 2;
5280 }
5281 ctx_lo |= CONTEXT_PASIDE;
5282 if (iommu->pasid_state_table)
5283 ctx_lo |= CONTEXT_DINVE;
5284 if (info->pri_supported)
5285 ctx_lo |= CONTEXT_PRS;
5286 context[0].lo = ctx_lo;
5287 wmb();
5288 iommu->flush.flush_context(iommu, sdev->did, sdev->sid,
5289 DMA_CCMD_MASK_NOBIT,
5290 DMA_CCMD_DEVICE_INVL);
5291 }
5292
5293 /* Enable PASID support in the device, if it wasn't already */
5294 if (!info->pasid_enabled)
5295 iommu_enable_dev_iotlb(info);
5296
5297 if (info->ats_enabled) {
5298 sdev->dev_iotlb = 1;
5299 sdev->qdep = info->ats_qdep;
5300 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5301 sdev->qdep = 0;
5302 }
5303 ret = 0;
5304
5305 out:
5306 spin_unlock(&iommu->lock);
5307 spin_unlock_irqrestore(&device_domain_lock, flags);
5308
5309 return ret;
5310 }
5311
5312 struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5313 {
5314 struct intel_iommu *iommu;
5315 u8 bus, devfn;
5316
5317 if (iommu_dummy(dev)) {
5318 dev_warn(dev,
5319 "No IOMMU translation for device; cannot enable SVM\n");
5320 return NULL;
5321 }
5322
5323 iommu = device_to_iommu(dev, &bus, &devfn);
5324 if ((!iommu)) {
5325 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
5326 return NULL;
5327 }
5328
5329 if (!iommu->pasid_table) {
5330 dev_err(dev, "PASID not enabled on IOMMU; cannot enable SVM\n");
5331 return NULL;
5332 }
5333
5334 return iommu;
5335 }
5336 #endif /* CONFIG_INTEL_IOMMU_SVM */
5337
5338 const struct iommu_ops intel_iommu_ops = {
5339 .capable = intel_iommu_capable,
5340 .domain_alloc = intel_iommu_domain_alloc,
5341 .domain_free = intel_iommu_domain_free,
5342 .attach_dev = intel_iommu_attach_device,
5343 .detach_dev = intel_iommu_detach_device,
5344 .map = intel_iommu_map,
5345 .unmap = intel_iommu_unmap,
5346 .map_sg = default_iommu_map_sg,
5347 .iova_to_phys = intel_iommu_iova_to_phys,
5348 .add_device = intel_iommu_add_device,
5349 .remove_device = intel_iommu_remove_device,
5350 .get_resv_regions = intel_iommu_get_resv_regions,
5351 .put_resv_regions = intel_iommu_put_resv_regions,
5352 .device_group = pci_device_group,
5353 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
5354 };
5355
5356 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5357 {
5358 /* G4x/GM45 integrated gfx dmar support is totally busted. */
5359 pr_info("Disabling IOMMU for graphics on this chipset\n");
5360 dmar_map_gfx = 0;
5361 }
5362
5363 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5364 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5365 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5366 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5367 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5368 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5369 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5370
5371 static void quirk_iommu_rwbf(struct pci_dev *dev)
5372 {
5373 /*
5374 * Mobile 4 Series Chipset neglects to set RWBF capability,
5375 * but needs it. Same seems to hold for the desktop versions.
5376 */
5377 pr_info("Forcing write-buffer flush capability\n");
5378 rwbf_quirk = 1;
5379 }
5380
5381 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
5382 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5383 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5384 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5385 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5386 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5387 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
5388
5389 #define GGC 0x52
5390 #define GGC_MEMORY_SIZE_MASK (0xf << 8)
5391 #define GGC_MEMORY_SIZE_NONE (0x0 << 8)
5392 #define GGC_MEMORY_SIZE_1M (0x1 << 8)
5393 #define GGC_MEMORY_SIZE_2M (0x3 << 8)
5394 #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
5395 #define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
5396 #define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
5397 #define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
5398
5399 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
5400 {
5401 unsigned short ggc;
5402
5403 if (pci_read_config_word(dev, GGC, &ggc))
5404 return;
5405
5406 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
5407 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
5408 dmar_map_gfx = 0;
5409 } else if (dmar_map_gfx) {
5410 /* we have to ensure the gfx device is idle before we flush */
5411 pr_info("Disabling batched IOTLB flush on Ironlake\n");
5412 intel_iommu_strict = 1;
5413 }
5414 }
5415 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5416 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5417 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5418 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5419
5420 /* On Tylersburg chipsets, some BIOSes have been known to enable the
5421 ISOCH DMAR unit for the Azalia sound device, but not give it any
5422 TLB entries, which causes it to deadlock. Check for that. We do
5423 this in a function called from init_dmars(), instead of in a PCI
5424 quirk, because we don't want to print the obnoxious "BIOS broken"
5425 message if VT-d is actually disabled.
5426 */
5427 static void __init check_tylersburg_isoch(void)
5428 {
5429 struct pci_dev *pdev;
5430 uint32_t vtisochctrl;
5431
5432 /* If there's no Azalia in the system anyway, forget it. */
5433 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5434 if (!pdev)
5435 return;
5436 pci_dev_put(pdev);
5437
5438 /* System Management Registers. Might be hidden, in which case
5439 we can't do the sanity check. But that's OK, because the
5440 known-broken BIOSes _don't_ actually hide it, so far. */
5441 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5442 if (!pdev)
5443 return;
5444
5445 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5446 pci_dev_put(pdev);
5447 return;
5448 }
5449
5450 pci_dev_put(pdev);
5451
5452 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5453 if (vtisochctrl & 1)
5454 return;
5455
5456 /* Drop all bits other than the number of TLB entries */
5457 vtisochctrl &= 0x1c;
5458
5459 /* If we have the recommended number of TLB entries (16), fine. */
5460 if (vtisochctrl == 0x10)
5461 return;
5462
5463 /* Zero TLB entries? You get to ride the short bus to school. */
5464 if (!vtisochctrl) {
5465 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5466 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5467 dmi_get_system_info(DMI_BIOS_VENDOR),
5468 dmi_get_system_info(DMI_BIOS_VERSION),
5469 dmi_get_system_info(DMI_PRODUCT_VERSION));
5470 iommu_identity_mapping |= IDENTMAP_AZALIA;
5471 return;
5472 }
5473
5474 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
5475 vtisochctrl);
5476 }
Linux with added FPC-III support