?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 4.19.133
[linux/fpc-iii.git] / drivers / iommu / intel-iommu.c
blob2a83fc31dd47a217b7bcc4f966dfdf37847ce391
1 /*
2 * Copyright © 2006-2014 Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
18 * Joerg Roedel <jroedel@suse.de>
19 */
20
21 #define pr_fmt(fmt) "DMAR: " fmt
22
23 #include <linux/init.h>
24 #include <linux/bitmap.h>
25 #include <linux/debugfs.h>
26 #include <linux/export.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/memory.h>
36 #include <linux/cpu.h>
37 #include <linux/timer.h>
38 #include <linux/io.h>
39 #include <linux/iova.h>
40 #include <linux/iommu.h>
41 #include <linux/intel-iommu.h>
42 #include <linux/syscore_ops.h>
43 #include <linux/tboot.h>
44 #include <linux/dmi.h>
45 #include <linux/pci-ats.h>
46 #include <linux/memblock.h>
47 #include <linux/dma-contiguous.h>
48 #include <linux/dma-direct.h>
49 #include <linux/crash_dump.h>
50 #include <asm/irq_remapping.h>
51 #include <asm/cacheflush.h>
52 #include <asm/iommu.h>
53
54 #include "irq_remapping.h"
55 #include "intel-pasid.h"
56
57 #define ROOT_SIZE VTD_PAGE_SIZE
58 #define CONTEXT_SIZE VTD_PAGE_SIZE
59
60 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
61 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
62 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
63 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
64
65 #define IOAPIC_RANGE_START (0xfee00000)
66 #define IOAPIC_RANGE_END (0xfeefffff)
67 #define IOVA_START_ADDR (0x1000)
68
69 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 57
70
71 #define MAX_AGAW_WIDTH 64
72 #define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
73
74 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
75 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
76
77 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
78 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
79 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
80 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
81 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
82
83 /* IO virtual address start page frame number */
84 #define IOVA_START_PFN (1)
85
86 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
87
88 /* page table handling */
89 #define LEVEL_STRIDE (9)
90 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
91
92 /*
93 * This bitmap is used to advertise the page sizes our hardware support
94 * to the IOMMU core, which will then use this information to split
95 * physically contiguous memory regions it is mapping into page sizes
96 * that we support.
97 *
98 * Traditionally the IOMMU core just handed us the mappings directly,
99 * after making sure the size is an order of a 4KiB page and that the
100 * mapping has natural alignment.
101 *
102 * To retain this behavior, we currently advertise that we support
103 * all page sizes that are an order of 4KiB.
104 *
105 * If at some point we'd like to utilize the IOMMU core's new behavior,
106 * we could change this to advertise the real page sizes we support.
107 */
108 #define INTEL_IOMMU_PGSIZES (~0xFFFUL)
109
110 static inline int agaw_to_level(int agaw)
111 {
112 return agaw + 2;
113 }
114
115 static inline int agaw_to_width(int agaw)
116 {
117 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
118 }
119
120 static inline int width_to_agaw(int width)
121 {
122 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
123 }
124
125 static inline unsigned int level_to_offset_bits(int level)
126 {
127 return (level - 1) * LEVEL_STRIDE;
128 }
129
130 static inline int pfn_level_offset(unsigned long pfn, int level)
131 {
132 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
133 }
134
135 static inline unsigned long level_mask(int level)
136 {
137 return -1UL << level_to_offset_bits(level);
138 }
139
140 static inline unsigned long level_size(int level)
141 {
142 return 1UL << level_to_offset_bits(level);
143 }
144
145 static inline unsigned long align_to_level(unsigned long pfn, int level)
146 {
147 return (pfn + level_size(level) - 1) & level_mask(level);
148 }
149
150 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
151 {
152 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
153 }
154
155 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
156 are never going to work. */
157 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
158 {
159 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
160 }
161
162 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
163 {
164 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
165 }
166 static inline unsigned long page_to_dma_pfn(struct page *pg)
167 {
168 return mm_to_dma_pfn(page_to_pfn(pg));
169 }
170 static inline unsigned long virt_to_dma_pfn(void *p)
171 {
172 return page_to_dma_pfn(virt_to_page(p));
173 }
174
175 /* global iommu list, set NULL for ignored DMAR units */
176 static struct intel_iommu **g_iommus;
177
178 static void __init check_tylersburg_isoch(void);
179 static int rwbf_quirk;
180
181 /*
182 * set to 1 to panic kernel if can't successfully enable VT-d
183 * (used when kernel is launched w/ TXT)
184 */
185 static int force_on = 0;
186 int intel_iommu_tboot_noforce;
187
188 /*
189 * 0: Present
190 * 1-11: Reserved
191 * 12-63: Context Ptr (12 - (haw-1))
192 * 64-127: Reserved
193 */
194 struct root_entry {
195 u64 lo;
196 u64 hi;
197 };
198 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
199
200 /*
201 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
202 * if marked present.
203 */
204 static phys_addr_t root_entry_lctp(struct root_entry *re)
205 {
206 if (!(re->lo & 1))
207 return 0;
208
209 return re->lo & VTD_PAGE_MASK;
210 }
211
212 /*
213 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
214 * if marked present.
215 */
216 static phys_addr_t root_entry_uctp(struct root_entry *re)
217 {
218 if (!(re->hi & 1))
219 return 0;
220
221 return re->hi & VTD_PAGE_MASK;
222 }
223 /*
224 * low 64 bits:
225 * 0: present
226 * 1: fault processing disable
227 * 2-3: translation type
228 * 12-63: address space root
229 * high 64 bits:
230 * 0-2: address width
231 * 3-6: aval
232 * 8-23: domain id
233 */
234 struct context_entry {
235 u64 lo;
236 u64 hi;
237 };
238
239 static inline void context_clear_pasid_enable(struct context_entry *context)
240 {
241 context->lo &= ~(1ULL << 11);
242 }
243
244 static inline bool context_pasid_enabled(struct context_entry *context)
245 {
246 return !!(context->lo & (1ULL << 11));
247 }
248
249 static inline void context_set_copied(struct context_entry *context)
250 {
251 context->hi |= (1ull << 3);
252 }
253
254 static inline bool context_copied(struct context_entry *context)
255 {
256 return !!(context->hi & (1ULL << 3));
257 }
258
259 static inline bool __context_present(struct context_entry *context)
260 {
261 return (context->lo & 1);
262 }
263
264 static inline bool context_present(struct context_entry *context)
265 {
266 return context_pasid_enabled(context) ?
267 __context_present(context) :
268 __context_present(context) && !context_copied(context);
269 }
270
271 static inline void context_set_present(struct context_entry *context)
272 {
273 context->lo |= 1;
274 }
275
276 static inline void context_set_fault_enable(struct context_entry *context)
277 {
278 context->lo &= (((u64)-1) << 2) | 1;
279 }
280
281 static inline void context_set_translation_type(struct context_entry *context,
282 unsigned long value)
283 {
284 context->lo &= (((u64)-1) << 4) | 3;
285 context->lo |= (value & 3) << 2;
286 }
287
288 static inline void context_set_address_root(struct context_entry *context,
289 unsigned long value)
290 {
291 context->lo &= ~VTD_PAGE_MASK;
292 context->lo |= value & VTD_PAGE_MASK;
293 }
294
295 static inline void context_set_address_width(struct context_entry *context,
296 unsigned long value)
297 {
298 context->hi |= value & 7;
299 }
300
301 static inline void context_set_domain_id(struct context_entry *context,
302 unsigned long value)
303 {
304 context->hi |= (value & ((1 << 16) - 1)) << 8;
305 }
306
307 static inline int context_domain_id(struct context_entry *c)
308 {
309 return((c->hi >> 8) & 0xffff);
310 }
311
312 static inline void context_clear_entry(struct context_entry *context)
313 {
314 context->lo = 0;
315 context->hi = 0;
316 }
317
318 /*
319 * 0: readable
320 * 1: writable
321 * 2-6: reserved
322 * 7: super page
323 * 8-10: available
324 * 11: snoop behavior
325 * 12-63: Host physcial address
326 */
327 struct dma_pte {
328 u64 val;
329 };
330
331 static inline void dma_clear_pte(struct dma_pte *pte)
332 {
333 pte->val = 0;
334 }
335
336 static inline u64 dma_pte_addr(struct dma_pte *pte)
337 {
338 #ifdef CONFIG_64BIT
339 return pte->val & VTD_PAGE_MASK;
340 #else
341 /* Must have a full atomic 64-bit read */
342 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
343 #endif
344 }
345
346 static inline bool dma_pte_present(struct dma_pte *pte)
347 {
348 return (pte->val & 3) != 0;
349 }
350
351 static inline bool dma_pte_superpage(struct dma_pte *pte)
352 {
353 return (pte->val & DMA_PTE_LARGE_PAGE);
354 }
355
356 static inline int first_pte_in_page(struct dma_pte *pte)
357 {
358 return !((unsigned long)pte & ~VTD_PAGE_MASK);
359 }
360
361 /*
362 * This domain is a statically identity mapping domain.
363 * 1. This domain creats a static 1:1 mapping to all usable memory.
364 * 2. It maps to each iommu if successful.
365 * 3. Each iommu mapps to this domain if successful.
366 */
367 static struct dmar_domain *si_domain;
368 static int hw_pass_through = 1;
369
370 /*
371 * Domain represents a virtual machine, more than one devices
372 * across iommus may be owned in one domain, e.g. kvm guest.
373 */
374 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
375
376 /* si_domain contains mulitple devices */
377 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
378
379 #define for_each_domain_iommu(idx, domain) \
380 for (idx = 0; idx < g_num_of_iommus; idx++) \
381 if (domain->iommu_refcnt[idx])
382
383 struct dmar_rmrr_unit {
384 struct list_head list; /* list of rmrr units */
385 struct acpi_dmar_header *hdr; /* ACPI header */
386 u64 base_address; /* reserved base address*/
387 u64 end_address; /* reserved end address */
388 struct dmar_dev_scope *devices; /* target devices */
389 int devices_cnt; /* target device count */
390 };
391
392 struct dmar_atsr_unit {
393 struct list_head list; /* list of ATSR units */
394 struct acpi_dmar_header *hdr; /* ACPI header */
395 struct dmar_dev_scope *devices; /* target devices */
396 int devices_cnt; /* target device count */
397 u8 include_all:1; /* include all ports */
398 };
399
400 static LIST_HEAD(dmar_atsr_units);
401 static LIST_HEAD(dmar_rmrr_units);
402
403 #define for_each_rmrr_units(rmrr) \
404 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
405
406 /* bitmap for indexing intel_iommus */
407 static int g_num_of_iommus;
408
409 static void domain_exit(struct dmar_domain *domain);
410 static void domain_remove_dev_info(struct dmar_domain *domain);
411 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
412 struct device *dev);
413 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
414 static void domain_context_clear(struct intel_iommu *iommu,
415 struct device *dev);
416 static int domain_detach_iommu(struct dmar_domain *domain,
417 struct intel_iommu *iommu);
418
419 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
420 int dmar_disabled = 0;
421 #else
422 int dmar_disabled = 1;
423 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
424
425 int intel_iommu_enabled = 0;
426 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
427
428 static int dmar_map_gfx = 1;
429 static int dmar_forcedac;
430 static int intel_iommu_strict;
431 static int intel_iommu_superpage = 1;
432 static int intel_iommu_ecs = 1;
433 static int intel_iommu_pasid28;
434 static int iommu_identity_mapping;
435
436 #define IDENTMAP_ALL 1
437 #define IDENTMAP_GFX 2
438 #define IDENTMAP_AZALIA 4
439
440 /* Broadwell and Skylake have broken ECS support — normal so-called "second
441 * level" translation of DMA requests-without-PASID doesn't actually happen
442 * unless you also set the NESTE bit in an extended context-entry. Which of
443 * course means that SVM doesn't work because it's trying to do nested
444 * translation of the physical addresses it finds in the process page tables,
445 * through the IOVA->phys mapping found in the "second level" page tables.
446 *
447 * The VT-d specification was retroactively changed to change the definition
448 * of the capability bits and pretend that Broadwell/Skylake never happened...
449 * but unfortunately the wrong bit was changed. It's ECS which is broken, but
450 * for some reason it was the PASID capability bit which was redefined (from
451 * bit 28 on BDW/SKL to bit 40 in future).
452 *
453 * So our test for ECS needs to eschew those implementations which set the old
454 * PASID capabiity bit 28, since those are the ones on which ECS is broken.
455 * Unless we are working around the 'pasid28' limitations, that is, by putting
456 * the device into passthrough mode for normal DMA and thus masking the bug.
457 */
458 #define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
459 (intel_iommu_pasid28 || !ecap_broken_pasid(iommu->ecap)))
460 /* PASID support is thus enabled if ECS is enabled and *either* of the old
461 * or new capability bits are set. */
462 #define pasid_enabled(iommu) (ecs_enabled(iommu) && \
463 (ecap_pasid(iommu->ecap) || ecap_broken_pasid(iommu->ecap)))
464
465 int intel_iommu_gfx_mapped;
466 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
467
468 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
469 static DEFINE_SPINLOCK(device_domain_lock);
470 static LIST_HEAD(device_domain_list);
471
472 /*
473 * Iterate over elements in device_domain_list and call the specified
474 * callback @fn against each element. This helper should only be used
475 * in the context where the device_domain_lock has already been holden.
476 */
477 int for_each_device_domain(int (*fn)(struct device_domain_info *info,
478 void *data), void *data)
479 {
480 int ret = 0;
481 struct device_domain_info *info;
482
483 assert_spin_locked(&device_domain_lock);
484 list_for_each_entry(info, &device_domain_list, global) {
485 ret = fn(info, data);
486 if (ret)
487 return ret;
488 }
489
490 return 0;
491 }
492
493 const struct iommu_ops intel_iommu_ops;
494
495 static bool translation_pre_enabled(struct intel_iommu *iommu)
496 {
497 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
498 }
499
500 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
501 {
502 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
503 }
504
505 static void init_translation_status(struct intel_iommu *iommu)
506 {
507 u32 gsts;
508
509 gsts = readl(iommu->reg + DMAR_GSTS_REG);
510 if (gsts & DMA_GSTS_TES)
511 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
512 }
513
514 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
515 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
516 {
517 return container_of(dom, struct dmar_domain, domain);
518 }
519
520 static int __init intel_iommu_setup(char *str)
521 {
522 if (!str)
523 return -EINVAL;
524 while (*str) {
525 if (!strncmp(str, "on", 2)) {
526 dmar_disabled = 0;
527 pr_info("IOMMU enabled\n");
528 } else if (!strncmp(str, "off", 3)) {
529 dmar_disabled = 1;
530 pr_info("IOMMU disabled\n");
531 } else if (!strncmp(str, "igfx_off", 8)) {
532 dmar_map_gfx = 0;
533 pr_info("Disable GFX device mapping\n");
534 } else if (!strncmp(str, "forcedac", 8)) {
535 pr_info("Forcing DAC for PCI devices\n");
536 dmar_forcedac = 1;
537 } else if (!strncmp(str, "strict", 6)) {
538 pr_info("Disable batched IOTLB flush\n");
539 intel_iommu_strict = 1;
540 } else if (!strncmp(str, "sp_off", 6)) {
541 pr_info("Disable supported super page\n");
542 intel_iommu_superpage = 0;
543 } else if (!strncmp(str, "ecs_off", 7)) {
544 printk(KERN_INFO
545 "Intel-IOMMU: disable extended context table support\n");
546 intel_iommu_ecs = 0;
547 } else if (!strncmp(str, "pasid28", 7)) {
548 printk(KERN_INFO
549 "Intel-IOMMU: enable pre-production PASID support\n");
550 intel_iommu_pasid28 = 1;
551 iommu_identity_mapping |= IDENTMAP_GFX;
552 } else if (!strncmp(str, "tboot_noforce", 13)) {
553 printk(KERN_INFO
554 "Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
555 intel_iommu_tboot_noforce = 1;
556 }
557
558 str += strcspn(str, ",");
559 while (*str == ',')
560 str++;
561 }
562 return 0;
563 }
564 __setup("intel_iommu=", intel_iommu_setup);
565
566 static struct kmem_cache *iommu_domain_cache;
567 static struct kmem_cache *iommu_devinfo_cache;
568
569 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
570 {
571 struct dmar_domain **domains;
572 int idx = did >> 8;
573
574 domains = iommu->domains[idx];
575 if (!domains)
576 return NULL;
577
578 return domains[did & 0xff];
579 }
580
581 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
582 struct dmar_domain *domain)
583 {
584 struct dmar_domain **domains;
585 int idx = did >> 8;
586
587 if (!iommu->domains[idx]) {
588 size_t size = 256 * sizeof(struct dmar_domain *);
589 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
590 }
591
592 domains = iommu->domains[idx];
593 if (WARN_ON(!domains))
594 return;
595 else
596 domains[did & 0xff] = domain;
597 }
598
599 void *alloc_pgtable_page(int node)
600 {
601 struct page *page;
602 void *vaddr = NULL;
603
604 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
605 if (page)
606 vaddr = page_address(page);
607 return vaddr;
608 }
609
610 void free_pgtable_page(void *vaddr)
611 {
612 free_page((unsigned long)vaddr);
613 }
614
615 static inline void *alloc_domain_mem(void)
616 {
617 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
618 }
619
620 static void free_domain_mem(void *vaddr)
621 {
622 kmem_cache_free(iommu_domain_cache, vaddr);
623 }
624
625 static inline void * alloc_devinfo_mem(void)
626 {
627 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
628 }
629
630 static inline void free_devinfo_mem(void *vaddr)
631 {
632 kmem_cache_free(iommu_devinfo_cache, vaddr);
633 }
634
635 static inline int domain_type_is_vm(struct dmar_domain *domain)
636 {
637 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
638 }
639
640 static inline int domain_type_is_si(struct dmar_domain *domain)
641 {
642 return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
643 }
644
645 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
646 {
647 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
648 DOMAIN_FLAG_STATIC_IDENTITY);
649 }
650
651 static inline int domain_pfn_supported(struct dmar_domain *domain,
652 unsigned long pfn)
653 {
654 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
655
656 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
657 }
658
659 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
660 {
661 unsigned long sagaw;
662 int agaw = -1;
663
664 sagaw = cap_sagaw(iommu->cap);
665 for (agaw = width_to_agaw(max_gaw);
666 agaw >= 0; agaw--) {
667 if (test_bit(agaw, &sagaw))
668 break;
669 }
670
671 return agaw;
672 }
673
674 /*
675 * Calculate max SAGAW for each iommu.
676 */
677 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
678 {
679 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
680 }
681
682 /*
683 * calculate agaw for each iommu.
684 * "SAGAW" may be different across iommus, use a default agaw, and
685 * get a supported less agaw for iommus that don't support the default agaw.
686 */
687 int iommu_calculate_agaw(struct intel_iommu *iommu)
688 {
689 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
690 }
691
692 /* This functionin only returns single iommu in a domain */
693 struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
694 {
695 int iommu_id;
696
697 /* si_domain and vm domain should not get here. */
698 BUG_ON(domain_type_is_vm_or_si(domain));
699 for_each_domain_iommu(iommu_id, domain)
700 break;
701
702 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
703 return NULL;
704
705 return g_iommus[iommu_id];
706 }
707
708 static void domain_update_iommu_coherency(struct dmar_domain *domain)
709 {
710 struct dmar_drhd_unit *drhd;
711 struct intel_iommu *iommu;
712 bool found = false;
713 int i;
714
715 domain->iommu_coherency = 1;
716
717 for_each_domain_iommu(i, domain) {
718 found = true;
719 if (!ecap_coherent(g_iommus[i]->ecap)) {
720 domain->iommu_coherency = 0;
721 break;
722 }
723 }
724 if (found)
725 return;
726
727 /* No hardware attached; use lowest common denominator */
728 rcu_read_lock();
729 for_each_active_iommu(iommu, drhd) {
730 if (!ecap_coherent(iommu->ecap)) {
731 domain->iommu_coherency = 0;
732 break;
733 }
734 }
735 rcu_read_unlock();
736 }
737
738 static int domain_update_iommu_snooping(struct intel_iommu *skip)
739 {
740 struct dmar_drhd_unit *drhd;
741 struct intel_iommu *iommu;
742 int ret = 1;
743
744 rcu_read_lock();
745 for_each_active_iommu(iommu, drhd) {
746 if (iommu != skip) {
747 if (!ecap_sc_support(iommu->ecap)) {
748 ret = 0;
749 break;
750 }
751 }
752 }
753 rcu_read_unlock();
754
755 return ret;
756 }
757
758 static int domain_update_iommu_superpage(struct intel_iommu *skip)
759 {
760 struct dmar_drhd_unit *drhd;
761 struct intel_iommu *iommu;
762 int mask = 0xf;
763
764 if (!intel_iommu_superpage) {
765 return 0;
766 }
767
768 /* set iommu_superpage to the smallest common denominator */
769 rcu_read_lock();
770 for_each_active_iommu(iommu, drhd) {
771 if (iommu != skip) {
772 mask &= cap_super_page_val(iommu->cap);
773 if (!mask)
774 break;
775 }
776 }
777 rcu_read_unlock();
778
779 return fls(mask);
780 }
781
782 /* Some capabilities may be different across iommus */
783 static void domain_update_iommu_cap(struct dmar_domain *domain)
784 {
785 domain_update_iommu_coherency(domain);
786 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
787 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
788 }
789
790 static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
791 u8 bus, u8 devfn, int alloc)
792 {
793 struct root_entry *root = &iommu->root_entry[bus];
794 struct context_entry *context;
795 u64 *entry;
796
797 entry = &root->lo;
798 if (ecs_enabled(iommu)) {
799 if (devfn >= 0x80) {
800 devfn -= 0x80;
801 entry = &root->hi;
802 }
803 devfn *= 2;
804 }
805 if (*entry & 1)
806 context = phys_to_virt(*entry & VTD_PAGE_MASK);
807 else {
808 unsigned long phy_addr;
809 if (!alloc)
810 return NULL;
811
812 context = alloc_pgtable_page(iommu->node);
813 if (!context)
814 return NULL;
815
816 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
817 phy_addr = virt_to_phys((void *)context);
818 *entry = phy_addr | 1;
819 __iommu_flush_cache(iommu, entry, sizeof(*entry));
820 }
821 return &context[devfn];
822 }
823
824 static int iommu_dummy(struct device *dev)
825 {
826 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
827 }
828
829 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
830 {
831 struct dmar_drhd_unit *drhd = NULL;
832 struct intel_iommu *iommu;
833 struct device *tmp;
834 struct pci_dev *ptmp, *pdev = NULL;
835 u16 segment = 0;
836 int i;
837
838 if (iommu_dummy(dev))
839 return NULL;
840
841 if (dev_is_pci(dev)) {
842 struct pci_dev *pf_pdev;
843
844 pdev = to_pci_dev(dev);
845
846 #ifdef CONFIG_X86
847 /* VMD child devices currently cannot be handled individually */
848 if (is_vmd(pdev->bus))
849 return NULL;
850 #endif
851
852 /* VFs aren't listed in scope tables; we need to look up
853 * the PF instead to find the IOMMU. */
854 pf_pdev = pci_physfn(pdev);
855 dev = &pf_pdev->dev;
856 segment = pci_domain_nr(pdev->bus);
857 } else if (has_acpi_companion(dev))
858 dev = &ACPI_COMPANION(dev)->dev;
859
860 rcu_read_lock();
861 for_each_active_iommu(iommu, drhd) {
862 if (pdev && segment != drhd->segment)
863 continue;
864
865 for_each_active_dev_scope(drhd->devices,
866 drhd->devices_cnt, i, tmp) {
867 if (tmp == dev) {
868 /* For a VF use its original BDF# not that of the PF
869 * which we used for the IOMMU lookup. Strictly speaking
870 * we could do this for all PCI devices; we only need to
871 * get the BDF# from the scope table for ACPI matches. */
872 if (pdev && pdev->is_virtfn)
873 goto got_pdev;
874
875 *bus = drhd->devices[i].bus;
876 *devfn = drhd->devices[i].devfn;
877 goto out;
878 }
879
880 if (!pdev || !dev_is_pci(tmp))
881 continue;
882
883 ptmp = to_pci_dev(tmp);
884 if (ptmp->subordinate &&
885 ptmp->subordinate->number <= pdev->bus->number &&
886 ptmp->subordinate->busn_res.end >= pdev->bus->number)
887 goto got_pdev;
888 }
889
890 if (pdev && drhd->include_all) {
891 got_pdev:
892 *bus = pdev->bus->number;
893 *devfn = pdev->devfn;
894 goto out;
895 }
896 }
897 iommu = NULL;
898 out:
899 rcu_read_unlock();
900
901 return iommu;
902 }
903
904 static void domain_flush_cache(struct dmar_domain *domain,
905 void *addr, int size)
906 {
907 if (!domain->iommu_coherency)
908 clflush_cache_range(addr, size);
909 }
910
911 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
912 {
913 struct context_entry *context;
914 int ret = 0;
915 unsigned long flags;
916
917 spin_lock_irqsave(&iommu->lock, flags);
918 context = iommu_context_addr(iommu, bus, devfn, 0);
919 if (context)
920 ret = context_present(context);
921 spin_unlock_irqrestore(&iommu->lock, flags);
922 return ret;
923 }
924
925 static void free_context_table(struct intel_iommu *iommu)
926 {
927 int i;
928 unsigned long flags;
929 struct context_entry *context;
930
931 spin_lock_irqsave(&iommu->lock, flags);
932 if (!iommu->root_entry) {
933 goto out;
934 }
935 for (i = 0; i < ROOT_ENTRY_NR; i++) {
936 context = iommu_context_addr(iommu, i, 0, 0);
937 if (context)
938 free_pgtable_page(context);
939
940 if (!ecs_enabled(iommu))
941 continue;
942
943 context = iommu_context_addr(iommu, i, 0x80, 0);
944 if (context)
945 free_pgtable_page(context);
946
947 }
948 free_pgtable_page(iommu->root_entry);
949 iommu->root_entry = NULL;
950 out:
951 spin_unlock_irqrestore(&iommu->lock, flags);
952 }
953
954 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
955 unsigned long pfn, int *target_level)
956 {
957 struct dma_pte *parent, *pte = NULL;
958 int level = agaw_to_level(domain->agaw);
959 int offset;
960
961 BUG_ON(!domain->pgd);
962
963 if (!domain_pfn_supported(domain, pfn))
964 /* Address beyond IOMMU's addressing capabilities. */
965 return NULL;
966
967 parent = domain->pgd;
968
969 while (1) {
970 void *tmp_page;
971
972 offset = pfn_level_offset(pfn, level);
973 pte = &parent[offset];
974 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
975 break;
976 if (level == *target_level)
977 break;
978
979 if (!dma_pte_present(pte)) {
980 uint64_t pteval;
981
982 tmp_page = alloc_pgtable_page(domain->nid);
983
984 if (!tmp_page)
985 return NULL;
986
987 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
988 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
989 if (cmpxchg64(&pte->val, 0ULL, pteval))
990 /* Someone else set it while we were thinking; use theirs. */
991 free_pgtable_page(tmp_page);
992 else
993 domain_flush_cache(domain, pte, sizeof(*pte));
994 }
995 if (level == 1)
996 break;
997
998 parent = phys_to_virt(dma_pte_addr(pte));
999 level--;
1000 }
1001
1002 if (!*target_level)
1003 *target_level = level;
1004
1005 return pte;
1006 }
1007
1008
1009 /* return address's pte at specific level */
1010 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1011 unsigned long pfn,
1012 int level, int *large_page)
1013 {
1014 struct dma_pte *parent, *pte = NULL;
1015 int total = agaw_to_level(domain->agaw);
1016 int offset;
1017
1018 parent = domain->pgd;
1019 while (level <= total) {
1020 offset = pfn_level_offset(pfn, total);
1021 pte = &parent[offset];
1022 if (level == total)
1023 return pte;
1024
1025 if (!dma_pte_present(pte)) {
1026 *large_page = total;
1027 break;
1028 }
1029
1030 if (dma_pte_superpage(pte)) {
1031 *large_page = total;
1032 return pte;
1033 }
1034
1035 parent = phys_to_virt(dma_pte_addr(pte));
1036 total--;
1037 }
1038 return NULL;
1039 }
1040
1041 /* clear last level pte, a tlb flush should be followed */
1042 static void dma_pte_clear_range(struct dmar_domain *domain,
1043 unsigned long start_pfn,
1044 unsigned long last_pfn)
1045 {
1046 unsigned int large_page = 1;
1047 struct dma_pte *first_pte, *pte;
1048
1049 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1050 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1051 BUG_ON(start_pfn > last_pfn);
1052
1053 /* we don't need lock here; nobody else touches the iova range */
1054 do {
1055 large_page = 1;
1056 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
1057 if (!pte) {
1058 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
1059 continue;
1060 }
1061 do {
1062 dma_clear_pte(pte);
1063 start_pfn += lvl_to_nr_pages(large_page);
1064 pte++;
1065 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1066
1067 domain_flush_cache(domain, first_pte,
1068 (void *)pte - (void *)first_pte);
1069
1070 } while (start_pfn && start_pfn <= last_pfn);
1071 }
1072
1073 static void dma_pte_free_level(struct dmar_domain *domain, int level,
1074 int retain_level, struct dma_pte *pte,
1075 unsigned long pfn, unsigned long start_pfn,
1076 unsigned long last_pfn)
1077 {
1078 pfn = max(start_pfn, pfn);
1079 pte = &pte[pfn_level_offset(pfn, level)];
1080
1081 do {
1082 unsigned long level_pfn;
1083 struct dma_pte *level_pte;
1084
1085 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1086 goto next;
1087
1088 level_pfn = pfn & level_mask(level);
1089 level_pte = phys_to_virt(dma_pte_addr(pte));
1090
1091 if (level > 2) {
1092 dma_pte_free_level(domain, level - 1, retain_level,
1093 level_pte, level_pfn, start_pfn,
1094 last_pfn);
1095 }
1096
1097 /*
1098 * Free the page table if we're below the level we want to
1099 * retain and the range covers the entire table.
1100 */
1101 if (level < retain_level && !(start_pfn > level_pfn ||
1102 last_pfn < level_pfn + level_size(level) - 1)) {
1103 dma_clear_pte(pte);
1104 domain_flush_cache(domain, pte, sizeof(*pte));
1105 free_pgtable_page(level_pte);
1106 }
1107 next:
1108 pfn += level_size(level);
1109 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1110 }
1111
1112 /*
1113 * clear last level (leaf) ptes and free page table pages below the
1114 * level we wish to keep intact.
1115 */
1116 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1117 unsigned long start_pfn,
1118 unsigned long last_pfn,
1119 int retain_level)
1120 {
1121 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1122 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1123 BUG_ON(start_pfn > last_pfn);
1124
1125 dma_pte_clear_range(domain, start_pfn, last_pfn);
1126
1127 /* We don't need lock here; nobody else touches the iova range */
1128 dma_pte_free_level(domain, agaw_to_level(domain->agaw), retain_level,
1129 domain->pgd, 0, start_pfn, last_pfn);
1130
1131 /* free pgd */
1132 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1133 free_pgtable_page(domain->pgd);
1134 domain->pgd = NULL;
1135 }
1136 }
1137
1138 /* When a page at a given level is being unlinked from its parent, we don't
1139 need to *modify* it at all. All we need to do is make a list of all the
1140 pages which can be freed just as soon as we've flushed the IOTLB and we
1141 know the hardware page-walk will no longer touch them.
1142 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1143 be freed. */
1144 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1145 int level, struct dma_pte *pte,
1146 struct page *freelist)
1147 {
1148 struct page *pg;
1149
1150 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1151 pg->freelist = freelist;
1152 freelist = pg;
1153
1154 if (level == 1)
1155 return freelist;
1156
1157 pte = page_address(pg);
1158 do {
1159 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1160 freelist = dma_pte_list_pagetables(domain, level - 1,
1161 pte, freelist);
1162 pte++;
1163 } while (!first_pte_in_page(pte));
1164
1165 return freelist;
1166 }
1167
1168 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1169 struct dma_pte *pte, unsigned long pfn,
1170 unsigned long start_pfn,
1171 unsigned long last_pfn,
1172 struct page *freelist)
1173 {
1174 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1175
1176 pfn = max(start_pfn, pfn);
1177 pte = &pte[pfn_level_offset(pfn, level)];
1178
1179 do {
1180 unsigned long level_pfn;
1181
1182 if (!dma_pte_present(pte))
1183 goto next;
1184
1185 level_pfn = pfn & level_mask(level);
1186
1187 /* If range covers entire pagetable, free it */
1188 if (start_pfn <= level_pfn &&
1189 last_pfn >= level_pfn + level_size(level) - 1) {
1190 /* These suborbinate page tables are going away entirely. Don't
1191 bother to clear them; we're just going to *free* them. */
1192 if (level > 1 && !dma_pte_superpage(pte))
1193 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1194
1195 dma_clear_pte(pte);
1196 if (!first_pte)
1197 first_pte = pte;
1198 last_pte = pte;
1199 } else if (level > 1) {
1200 /* Recurse down into a level that isn't *entirely* obsolete */
1201 freelist = dma_pte_clear_level(domain, level - 1,
1202 phys_to_virt(dma_pte_addr(pte)),
1203 level_pfn, start_pfn, last_pfn,
1204 freelist);
1205 }
1206 next:
1207 pfn += level_size(level);
1208 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1209
1210 if (first_pte)
1211 domain_flush_cache(domain, first_pte,
1212 (void *)++last_pte - (void *)first_pte);
1213
1214 return freelist;
1215 }
1216
1217 /* We can't just free the pages because the IOMMU may still be walking
1218 the page tables, and may have cached the intermediate levels. The
1219 pages can only be freed after the IOTLB flush has been done. */
1220 static struct page *domain_unmap(struct dmar_domain *domain,
1221 unsigned long start_pfn,
1222 unsigned long last_pfn)
1223 {
1224 struct page *freelist = NULL;
1225
1226 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1227 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1228 BUG_ON(start_pfn > last_pfn);
1229
1230 /* we don't need lock here; nobody else touches the iova range */
1231 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1232 domain->pgd, 0, start_pfn, last_pfn, NULL);
1233
1234 /* free pgd */
1235 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1236 struct page *pgd_page = virt_to_page(domain->pgd);
1237 pgd_page->freelist = freelist;
1238 freelist = pgd_page;
1239
1240 domain->pgd = NULL;
1241 }
1242
1243 return freelist;
1244 }
1245
1246 static void dma_free_pagelist(struct page *freelist)
1247 {
1248 struct page *pg;
1249
1250 while ((pg = freelist)) {
1251 freelist = pg->freelist;
1252 free_pgtable_page(page_address(pg));
1253 }
1254 }
1255
1256 static void iova_entry_free(unsigned long data)
1257 {
1258 struct page *freelist = (struct page *)data;
1259
1260 dma_free_pagelist(freelist);
1261 }
1262
1263 /* iommu handling */
1264 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1265 {
1266 struct root_entry *root;
1267 unsigned long flags;
1268
1269 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1270 if (!root) {
1271 pr_err("Allocating root entry for %s failed\n",
1272 iommu->name);
1273 return -ENOMEM;
1274 }
1275
1276 __iommu_flush_cache(iommu, root, ROOT_SIZE);
1277
1278 spin_lock_irqsave(&iommu->lock, flags);
1279 iommu->root_entry = root;
1280 spin_unlock_irqrestore(&iommu->lock, flags);
1281
1282 return 0;
1283 }
1284
1285 static void iommu_set_root_entry(struct intel_iommu *iommu)
1286 {
1287 u64 addr;
1288 u32 sts;
1289 unsigned long flag;
1290
1291 addr = virt_to_phys(iommu->root_entry);
1292 if (ecs_enabled(iommu))
1293 addr |= DMA_RTADDR_RTT;
1294
1295 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1296 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1297
1298 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1299
1300 /* Make sure hardware complete it */
1301 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1302 readl, (sts & DMA_GSTS_RTPS), sts);
1303
1304 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1305 }
1306
1307 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1308 {
1309 u32 val;
1310 unsigned long flag;
1311
1312 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1313 return;
1314
1315 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1316 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1317
1318 /* Make sure hardware complete it */
1319 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1320 readl, (!(val & DMA_GSTS_WBFS)), val);
1321
1322 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1323 }
1324
1325 /* return value determine if we need a write buffer flush */
1326 static void __iommu_flush_context(struct intel_iommu *iommu,
1327 u16 did, u16 source_id, u8 function_mask,
1328 u64 type)
1329 {
1330 u64 val = 0;
1331 unsigned long flag;
1332
1333 switch (type) {
1334 case DMA_CCMD_GLOBAL_INVL:
1335 val = DMA_CCMD_GLOBAL_INVL;
1336 break;
1337 case DMA_CCMD_DOMAIN_INVL:
1338 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1339 break;
1340 case DMA_CCMD_DEVICE_INVL:
1341 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1342 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1343 break;
1344 default:
1345 BUG();
1346 }
1347 val |= DMA_CCMD_ICC;
1348
1349 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1350 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1351
1352 /* Make sure hardware complete it */
1353 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1354 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1355
1356 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1357 }
1358
1359 /* return value determine if we need a write buffer flush */
1360 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1361 u64 addr, unsigned int size_order, u64 type)
1362 {
1363 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1364 u64 val = 0, val_iva = 0;
1365 unsigned long flag;
1366
1367 switch (type) {
1368 case DMA_TLB_GLOBAL_FLUSH:
1369 /* global flush doesn't need set IVA_REG */
1370 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1371 break;
1372 case DMA_TLB_DSI_FLUSH:
1373 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1374 break;
1375 case DMA_TLB_PSI_FLUSH:
1376 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1377 /* IH bit is passed in as part of address */
1378 val_iva = size_order | addr;
1379 break;
1380 default:
1381 BUG();
1382 }
1383 /* Note: set drain read/write */
1384 #if 0
1385 /*
1386 * This is probably to be super secure.. Looks like we can
1387 * ignore it without any impact.
1388 */
1389 if (cap_read_drain(iommu->cap))
1390 val |= DMA_TLB_READ_DRAIN;
1391 #endif
1392 if (cap_write_drain(iommu->cap))
1393 val |= DMA_TLB_WRITE_DRAIN;
1394
1395 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1396 /* Note: Only uses first TLB reg currently */
1397 if (val_iva)
1398 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1399 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1400
1401 /* Make sure hardware complete it */
1402 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1403 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1404
1405 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1406
1407 /* check IOTLB invalidation granularity */
1408 if (DMA_TLB_IAIG(val) == 0)
1409 pr_err("Flush IOTLB failed\n");
1410 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1411 pr_debug("TLB flush request %Lx, actual %Lx\n",
1412 (unsigned long long)DMA_TLB_IIRG(type),
1413 (unsigned long long)DMA_TLB_IAIG(val));
1414 }
1415
1416 static struct device_domain_info *
1417 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1418 u8 bus, u8 devfn)
1419 {
1420 struct device_domain_info *info;
1421
1422 assert_spin_locked(&device_domain_lock);
1423
1424 if (!iommu->qi)
1425 return NULL;
1426
1427 list_for_each_entry(info, &domain->devices, link)
1428 if (info->iommu == iommu && info->bus == bus &&
1429 info->devfn == devfn) {
1430 if (info->ats_supported && info->dev)
1431 return info;
1432 break;
1433 }
1434
1435 return NULL;
1436 }
1437
1438 static void domain_update_iotlb(struct dmar_domain *domain)
1439 {
1440 struct device_domain_info *info;
1441 bool has_iotlb_device = false;
1442
1443 assert_spin_locked(&device_domain_lock);
1444
1445 list_for_each_entry(info, &domain->devices, link) {
1446 struct pci_dev *pdev;
1447
1448 if (!info->dev || !dev_is_pci(info->dev))
1449 continue;
1450
1451 pdev = to_pci_dev(info->dev);
1452 if (pdev->ats_enabled) {
1453 has_iotlb_device = true;
1454 break;
1455 }
1456 }
1457
1458 domain->has_iotlb_device = has_iotlb_device;
1459 }
1460
1461 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1462 {
1463 struct pci_dev *pdev;
1464
1465 assert_spin_locked(&device_domain_lock);
1466
1467 if (!info || !dev_is_pci(info->dev))
1468 return;
1469
1470 pdev = to_pci_dev(info->dev);
1471 /* For IOMMU that supports device IOTLB throttling (DIT), we assign
1472 * PFSID to the invalidation desc of a VF such that IOMMU HW can gauge
1473 * queue depth at PF level. If DIT is not set, PFSID will be treated as
1474 * reserved, which should be set to 0.
1475 */
1476 if (!ecap_dit(info->iommu->ecap))
1477 info->pfsid = 0;
1478 else {
1479 struct pci_dev *pf_pdev;
1480
1481 /* pdev will be returned if device is not a vf */
1482 pf_pdev = pci_physfn(pdev);
1483 info->pfsid = PCI_DEVID(pf_pdev->bus->number, pf_pdev->devfn);
1484 }
1485
1486 #ifdef CONFIG_INTEL_IOMMU_SVM
1487 /* The PCIe spec, in its wisdom, declares that the behaviour of
1488 the device if you enable PASID support after ATS support is
1489 undefined. So always enable PASID support on devices which
1490 have it, even if we can't yet know if we're ever going to
1491 use it. */
1492 if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1493 info->pasid_enabled = 1;
1494
1495 if (info->pri_supported && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1496 info->pri_enabled = 1;
1497 #endif
1498 if (info->ats_supported && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1499 info->ats_enabled = 1;
1500 domain_update_iotlb(info->domain);
1501 info->ats_qdep = pci_ats_queue_depth(pdev);
1502 }
1503 }
1504
1505 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1506 {
1507 struct pci_dev *pdev;
1508
1509 assert_spin_locked(&device_domain_lock);
1510
1511 if (!dev_is_pci(info->dev))
1512 return;
1513
1514 pdev = to_pci_dev(info->dev);
1515
1516 if (info->ats_enabled) {
1517 pci_disable_ats(pdev);
1518 info->ats_enabled = 0;
1519 domain_update_iotlb(info->domain);
1520 }
1521 #ifdef CONFIG_INTEL_IOMMU_SVM
1522 if (info->pri_enabled) {
1523 pci_disable_pri(pdev);
1524 info->pri_enabled = 0;
1525 }
1526 if (info->pasid_enabled) {
1527 pci_disable_pasid(pdev);
1528 info->pasid_enabled = 0;
1529 }
1530 #endif
1531 }
1532
1533 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1534 u64 addr, unsigned mask)
1535 {
1536 u16 sid, qdep;
1537 unsigned long flags;
1538 struct device_domain_info *info;
1539
1540 if (!domain->has_iotlb_device)
1541 return;
1542
1543 spin_lock_irqsave(&device_domain_lock, flags);
1544 list_for_each_entry(info, &domain->devices, link) {
1545 if (!info->ats_enabled)
1546 continue;
1547
1548 sid = info->bus << 8 | info->devfn;
1549 qdep = info->ats_qdep;
1550 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
1551 qdep, addr, mask);
1552 }
1553 spin_unlock_irqrestore(&device_domain_lock, flags);
1554 }
1555
1556 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1557 struct dmar_domain *domain,
1558 unsigned long pfn, unsigned int pages,
1559 int ih, int map)
1560 {
1561 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1562 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1563 u16 did = domain->iommu_did[iommu->seq_id];
1564
1565 BUG_ON(pages == 0);
1566
1567 if (ih)
1568 ih = 1 << 6;
1569 /*
1570 * Fallback to domain selective flush if no PSI support or the size is
1571 * too big.
1572 * PSI requires page size to be 2 ^ x, and the base address is naturally
1573 * aligned to the size
1574 */
1575 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1576 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1577 DMA_TLB_DSI_FLUSH);
1578 else
1579 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1580 DMA_TLB_PSI_FLUSH);
1581
1582 /*
1583 * In caching mode, changes of pages from non-present to present require
1584 * flush. However, device IOTLB doesn't need to be flushed in this case.
1585 */
1586 if (!cap_caching_mode(iommu->cap) || !map)
1587 iommu_flush_dev_iotlb(domain, addr, mask);
1588 }
1589
1590 /* Notification for newly created mappings */
1591 static inline void __mapping_notify_one(struct intel_iommu *iommu,
1592 struct dmar_domain *domain,
1593 unsigned long pfn, unsigned int pages)
1594 {
1595 /* It's a non-present to present mapping. Only flush if caching mode */
1596 if (cap_caching_mode(iommu->cap))
1597 iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1);
1598 else
1599 iommu_flush_write_buffer(iommu);
1600 }
1601
1602 static void iommu_flush_iova(struct iova_domain *iovad)
1603 {
1604 struct dmar_domain *domain;
1605 int idx;
1606
1607 domain = container_of(iovad, struct dmar_domain, iovad);
1608
1609 for_each_domain_iommu(idx, domain) {
1610 struct intel_iommu *iommu = g_iommus[idx];
1611 u16 did = domain->iommu_did[iommu->seq_id];
1612
1613 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
1614
1615 if (!cap_caching_mode(iommu->cap))
1616 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1617 0, MAX_AGAW_PFN_WIDTH);
1618 }
1619 }
1620
1621 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1622 {
1623 u32 pmen;
1624 unsigned long flags;
1625
1626 if (!cap_plmr(iommu->cap) && !cap_phmr(iommu->cap))
1627 return;
1628
1629 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1630 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1631 pmen &= ~DMA_PMEN_EPM;
1632 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1633
1634 /* wait for the protected region status bit to clear */
1635 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1636 readl, !(pmen & DMA_PMEN_PRS), pmen);
1637
1638 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1639 }
1640
1641 static void iommu_enable_translation(struct intel_iommu *iommu)
1642 {
1643 u32 sts;
1644 unsigned long flags;
1645
1646 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1647 iommu->gcmd |= DMA_GCMD_TE;
1648 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1649
1650 /* Make sure hardware complete it */
1651 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1652 readl, (sts & DMA_GSTS_TES), sts);
1653
1654 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1655 }
1656
1657 static void iommu_disable_translation(struct intel_iommu *iommu)
1658 {
1659 u32 sts;
1660 unsigned long flag;
1661
1662 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1663 iommu->gcmd &= ~DMA_GCMD_TE;
1664 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1665
1666 /* Make sure hardware complete it */
1667 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1668 readl, (!(sts & DMA_GSTS_TES)), sts);
1669
1670 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1671 }
1672
1673
1674 static int iommu_init_domains(struct intel_iommu *iommu)
1675 {
1676 u32 ndomains, nlongs;
1677 size_t size;
1678
1679 ndomains = cap_ndoms(iommu->cap);
1680 pr_debug("%s: Number of Domains supported <%d>\n",
1681 iommu->name, ndomains);
1682 nlongs = BITS_TO_LONGS(ndomains);
1683
1684 spin_lock_init(&iommu->lock);
1685
1686 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1687 if (!iommu->domain_ids) {
1688 pr_err("%s: Allocating domain id array failed\n",
1689 iommu->name);
1690 return -ENOMEM;
1691 }
1692
1693 size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
1694 iommu->domains = kzalloc(size, GFP_KERNEL);
1695
1696 if (iommu->domains) {
1697 size = 256 * sizeof(struct dmar_domain *);
1698 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1699 }
1700
1701 if (!iommu->domains || !iommu->domains[0]) {
1702 pr_err("%s: Allocating domain array failed\n",
1703 iommu->name);
1704 kfree(iommu->domain_ids);
1705 kfree(iommu->domains);
1706 iommu->domain_ids = NULL;
1707 iommu->domains = NULL;
1708 return -ENOMEM;
1709 }
1710
1711
1712
1713 /*
1714 * If Caching mode is set, then invalid translations are tagged
1715 * with domain-id 0, hence we need to pre-allocate it. We also
1716 * use domain-id 0 as a marker for non-allocated domain-id, so
1717 * make sure it is not used for a real domain.
1718 */
1719 set_bit(0, iommu->domain_ids);
1720
1721 return 0;
1722 }
1723
1724 static void disable_dmar_iommu(struct intel_iommu *iommu)
1725 {
1726 struct device_domain_info *info, *tmp;
1727 unsigned long flags;
1728
1729 if (!iommu->domains || !iommu->domain_ids)
1730 return;
1731
1732 again:
1733 spin_lock_irqsave(&device_domain_lock, flags);
1734 list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1735 struct dmar_domain *domain;
1736
1737 if (info->iommu != iommu)
1738 continue;
1739
1740 if (!info->dev || !info->domain)
1741 continue;
1742
1743 domain = info->domain;
1744
1745 __dmar_remove_one_dev_info(info);
1746
1747 if (!domain_type_is_vm_or_si(domain)) {
1748 /*
1749 * The domain_exit() function can't be called under
1750 * device_domain_lock, as it takes this lock itself.
1751 * So release the lock here and re-run the loop
1752 * afterwards.
1753 */
1754 spin_unlock_irqrestore(&device_domain_lock, flags);
1755 domain_exit(domain);
1756 goto again;
1757 }
1758 }
1759 spin_unlock_irqrestore(&device_domain_lock, flags);
1760
1761 if (iommu->gcmd & DMA_GCMD_TE)
1762 iommu_disable_translation(iommu);
1763 }
1764
1765 static void free_dmar_iommu(struct intel_iommu *iommu)
1766 {
1767 if ((iommu->domains) && (iommu->domain_ids)) {
1768 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
1769 int i;
1770
1771 for (i = 0; i < elems; i++)
1772 kfree(iommu->domains[i]);
1773 kfree(iommu->domains);
1774 kfree(iommu->domain_ids);
1775 iommu->domains = NULL;
1776 iommu->domain_ids = NULL;
1777 }
1778
1779 g_iommus[iommu->seq_id] = NULL;
1780
1781 /* free context mapping */
1782 free_context_table(iommu);
1783
1784 #ifdef CONFIG_INTEL_IOMMU_SVM
1785 if (pasid_enabled(iommu)) {
1786 if (ecap_prs(iommu->ecap))
1787 intel_svm_finish_prq(iommu);
1788 intel_svm_exit(iommu);
1789 }
1790 #endif
1791 }
1792
1793 static struct dmar_domain *alloc_domain(int flags)
1794 {
1795 struct dmar_domain *domain;
1796
1797 domain = alloc_domain_mem();
1798 if (!domain)
1799 return NULL;
1800
1801 memset(domain, 0, sizeof(*domain));
1802 domain->nid = -1;
1803 domain->flags = flags;
1804 domain->has_iotlb_device = false;
1805 INIT_LIST_HEAD(&domain->devices);
1806
1807 return domain;
1808 }
1809
1810 /* Must be called with iommu->lock */
1811 static int domain_attach_iommu(struct dmar_domain *domain,
1812 struct intel_iommu *iommu)
1813 {
1814 unsigned long ndomains;
1815 int num;
1816
1817 assert_spin_locked(&device_domain_lock);
1818 assert_spin_locked(&iommu->lock);
1819
1820 domain->iommu_refcnt[iommu->seq_id] += 1;
1821 domain->iommu_count += 1;
1822 if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1823 ndomains = cap_ndoms(iommu->cap);
1824 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1825
1826 if (num >= ndomains) {
1827 pr_err("%s: No free domain ids\n", iommu->name);
1828 domain->iommu_refcnt[iommu->seq_id] -= 1;
1829 domain->iommu_count -= 1;
1830 return -ENOSPC;
1831 }
1832
1833 set_bit(num, iommu->domain_ids);
1834 set_iommu_domain(iommu, num, domain);
1835
1836 domain->iommu_did[iommu->seq_id] = num;
1837 domain->nid = iommu->node;
1838
1839 domain_update_iommu_cap(domain);
1840 }
1841
1842 return 0;
1843 }
1844
1845 static int domain_detach_iommu(struct dmar_domain *domain,
1846 struct intel_iommu *iommu)
1847 {
1848 int num, count = INT_MAX;
1849
1850 assert_spin_locked(&device_domain_lock);
1851 assert_spin_locked(&iommu->lock);
1852
1853 domain->iommu_refcnt[iommu->seq_id] -= 1;
1854 count = --domain->iommu_count;
1855 if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1856 num = domain->iommu_did[iommu->seq_id];
1857 clear_bit(num, iommu->domain_ids);
1858 set_iommu_domain(iommu, num, NULL);
1859
1860 domain_update_iommu_cap(domain);
1861 domain->iommu_did[iommu->seq_id] = 0;
1862 }
1863
1864 return count;
1865 }
1866
1867 static struct iova_domain reserved_iova_list;
1868 static struct lock_class_key reserved_rbtree_key;
1869
1870 static int dmar_init_reserved_ranges(void)
1871 {
1872 struct pci_dev *pdev = NULL;
1873 struct iova *iova;
1874 int i;
1875
1876 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN);
1877
1878 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1879 &reserved_rbtree_key);
1880
1881 /* IOAPIC ranges shouldn't be accessed by DMA */
1882 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1883 IOVA_PFN(IOAPIC_RANGE_END));
1884 if (!iova) {
1885 pr_err("Reserve IOAPIC range failed\n");
1886 return -ENODEV;
1887 }
1888
1889 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1890 for_each_pci_dev(pdev) {
1891 struct resource *r;
1892
1893 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1894 r = &pdev->resource[i];
1895 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1896 continue;
1897 iova = reserve_iova(&reserved_iova_list,
1898 IOVA_PFN(r->start),
1899 IOVA_PFN(r->end));
1900 if (!iova) {
1901 pr_err("Reserve iova failed\n");
1902 return -ENODEV;
1903 }
1904 }
1905 }
1906 return 0;
1907 }
1908
1909 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1910 {
1911 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1912 }
1913
1914 static inline int guestwidth_to_adjustwidth(int gaw)
1915 {
1916 int agaw;
1917 int r = (gaw - 12) % 9;
1918
1919 if (r == 0)
1920 agaw = gaw;
1921 else
1922 agaw = gaw + 9 - r;
1923 if (agaw > 64)
1924 agaw = 64;
1925 return agaw;
1926 }
1927
1928 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1929 int guest_width)
1930 {
1931 int adjust_width, agaw;
1932 unsigned long sagaw;
1933 int err;
1934
1935 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
1936
1937 err = init_iova_flush_queue(&domain->iovad,
1938 iommu_flush_iova, iova_entry_free);
1939 if (err)
1940 return err;
1941
1942 domain_reserve_special_ranges(domain);
1943
1944 /* calculate AGAW */
1945 if (guest_width > cap_mgaw(iommu->cap))
1946 guest_width = cap_mgaw(iommu->cap);
1947 domain->gaw = guest_width;
1948 adjust_width = guestwidth_to_adjustwidth(guest_width);
1949 agaw = width_to_agaw(adjust_width);
1950 sagaw = cap_sagaw(iommu->cap);
1951 if (!test_bit(agaw, &sagaw)) {
1952 /* hardware doesn't support it, choose a bigger one */
1953 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1954 agaw = find_next_bit(&sagaw, 5, agaw);
1955 if (agaw >= 5)
1956 return -ENODEV;
1957 }
1958 domain->agaw = agaw;
1959
1960 if (ecap_coherent(iommu->ecap))
1961 domain->iommu_coherency = 1;
1962 else
1963 domain->iommu_coherency = 0;
1964
1965 if (ecap_sc_support(iommu->ecap))
1966 domain->iommu_snooping = 1;
1967 else
1968 domain->iommu_snooping = 0;
1969
1970 if (intel_iommu_superpage)
1971 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1972 else
1973 domain->iommu_superpage = 0;
1974
1975 domain->nid = iommu->node;
1976
1977 /* always allocate the top pgd */
1978 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1979 if (!domain->pgd)
1980 return -ENOMEM;
1981 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1982 return 0;
1983 }
1984
1985 static void domain_exit(struct dmar_domain *domain)
1986 {
1987 struct page *freelist = NULL;
1988
1989 /* Domain 0 is reserved, so dont process it */
1990 if (!domain)
1991 return;
1992
1993 /* Remove associated devices and clear attached or cached domains */
1994 rcu_read_lock();
1995 domain_remove_dev_info(domain);
1996 rcu_read_unlock();
1997
1998 /* destroy iovas */
1999 put_iova_domain(&domain->iovad);
2000
2001 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
2002
2003 dma_free_pagelist(freelist);
2004
2005 free_domain_mem(domain);
2006 }
2007
2008 static int domain_context_mapping_one(struct dmar_domain *domain,
2009 struct intel_iommu *iommu,
2010 u8 bus, u8 devfn)
2011 {
2012 u16 did = domain->iommu_did[iommu->seq_id];
2013 int translation = CONTEXT_TT_MULTI_LEVEL;
2014 struct device_domain_info *info = NULL;
2015 struct context_entry *context;
2016 unsigned long flags;
2017 struct dma_pte *pgd;
2018 int ret, agaw;
2019
2020 WARN_ON(did == 0);
2021
2022 if (hw_pass_through && domain_type_is_si(domain))
2023 translation = CONTEXT_TT_PASS_THROUGH;
2024
2025 pr_debug("Set context mapping for %02x:%02x.%d\n",
2026 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
2027
2028 BUG_ON(!domain->pgd);
2029
2030 spin_lock_irqsave(&device_domain_lock, flags);
2031 spin_lock(&iommu->lock);
2032
2033 ret = -ENOMEM;
2034 context = iommu_context_addr(iommu, bus, devfn, 1);
2035 if (!context)
2036 goto out_unlock;
2037
2038 ret = 0;
2039 if (context_present(context))
2040 goto out_unlock;
2041
2042 /*
2043 * For kdump cases, old valid entries may be cached due to the
2044 * in-flight DMA and copied pgtable, but there is no unmapping
2045 * behaviour for them, thus we need an explicit cache flush for
2046 * the newly-mapped device. For kdump, at this point, the device
2047 * is supposed to finish reset at its driver probe stage, so no
2048 * in-flight DMA will exist, and we don't need to worry anymore
2049 * hereafter.
2050 */
2051 if (context_copied(context)) {
2052 u16 did_old = context_domain_id(context);
2053
2054 if (did_old < cap_ndoms(iommu->cap)) {
2055 iommu->flush.flush_context(iommu, did_old,
2056 (((u16)bus) << 8) | devfn,
2057 DMA_CCMD_MASK_NOBIT,
2058 DMA_CCMD_DEVICE_INVL);
2059 iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
2060 DMA_TLB_DSI_FLUSH);
2061 }
2062 }
2063
2064 pgd = domain->pgd;
2065
2066 context_clear_entry(context);
2067 context_set_domain_id(context, did);
2068
2069 /*
2070 * Skip top levels of page tables for iommu which has less agaw
2071 * than default. Unnecessary for PT mode.
2072 */
2073 if (translation != CONTEXT_TT_PASS_THROUGH) {
2074 for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
2075 ret = -ENOMEM;
2076 pgd = phys_to_virt(dma_pte_addr(pgd));
2077 if (!dma_pte_present(pgd))
2078 goto out_unlock;
2079 }
2080
2081 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2082 if (info && info->ats_supported)
2083 translation = CONTEXT_TT_DEV_IOTLB;
2084 else
2085 translation = CONTEXT_TT_MULTI_LEVEL;
2086
2087 context_set_address_root(context, virt_to_phys(pgd));
2088 context_set_address_width(context, agaw);
2089 } else {
2090 /*
2091 * In pass through mode, AW must be programmed to
2092 * indicate the largest AGAW value supported by
2093 * hardware. And ASR is ignored by hardware.
2094 */
2095 context_set_address_width(context, iommu->msagaw);
2096 }
2097
2098 context_set_translation_type(context, translation);
2099 context_set_fault_enable(context);
2100 context_set_present(context);
2101 domain_flush_cache(domain, context, sizeof(*context));
2102
2103 /*
2104 * It's a non-present to present mapping. If hardware doesn't cache
2105 * non-present entry we only need to flush the write-buffer. If the
2106 * _does_ cache non-present entries, then it does so in the special
2107 * domain #0, which we have to flush:
2108 */
2109 if (cap_caching_mode(iommu->cap)) {
2110 iommu->flush.flush_context(iommu, 0,
2111 (((u16)bus) << 8) | devfn,
2112 DMA_CCMD_MASK_NOBIT,
2113 DMA_CCMD_DEVICE_INVL);
2114 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2115 } else {
2116 iommu_flush_write_buffer(iommu);
2117 }
2118 iommu_enable_dev_iotlb(info);
2119
2120 ret = 0;
2121
2122 out_unlock:
2123 spin_unlock(&iommu->lock);
2124 spin_unlock_irqrestore(&device_domain_lock, flags);
2125
2126 return ret;
2127 }
2128
2129 struct domain_context_mapping_data {
2130 struct dmar_domain *domain;
2131 struct intel_iommu *iommu;
2132 };
2133
2134 static int domain_context_mapping_cb(struct pci_dev *pdev,
2135 u16 alias, void *opaque)
2136 {
2137 struct domain_context_mapping_data *data = opaque;
2138
2139 return domain_context_mapping_one(data->domain, data->iommu,
2140 PCI_BUS_NUM(alias), alias & 0xff);
2141 }
2142
2143 static int
2144 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2145 {
2146 struct intel_iommu *iommu;
2147 u8 bus, devfn;
2148 struct domain_context_mapping_data data;
2149
2150 iommu = device_to_iommu(dev, &bus, &devfn);
2151 if (!iommu)
2152 return -ENODEV;
2153
2154 if (!dev_is_pci(dev))
2155 return domain_context_mapping_one(domain, iommu, bus, devfn);
2156
2157 data.domain = domain;
2158 data.iommu = iommu;
2159
2160 return pci_for_each_dma_alias(to_pci_dev(dev),
2161 &domain_context_mapping_cb, &data);
2162 }
2163
2164 static int domain_context_mapped_cb(struct pci_dev *pdev,
2165 u16 alias, void *opaque)
2166 {
2167 struct intel_iommu *iommu = opaque;
2168
2169 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2170 }
2171
2172 static int domain_context_mapped(struct device *dev)
2173 {
2174 struct intel_iommu *iommu;
2175 u8 bus, devfn;
2176
2177 iommu = device_to_iommu(dev, &bus, &devfn);
2178 if (!iommu)
2179 return -ENODEV;
2180
2181 if (!dev_is_pci(dev))
2182 return device_context_mapped(iommu, bus, devfn);
2183
2184 return !pci_for_each_dma_alias(to_pci_dev(dev),
2185 domain_context_mapped_cb, iommu);
2186 }
2187
2188 /* Returns a number of VTD pages, but aligned to MM page size */
2189 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2190 size_t size)
2191 {
2192 host_addr &= ~PAGE_MASK;
2193 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2194 }
2195
2196 /* Return largest possible superpage level for a given mapping */
2197 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2198 unsigned long iov_pfn,
2199 unsigned long phy_pfn,
2200 unsigned long pages)
2201 {
2202 int support, level = 1;
2203 unsigned long pfnmerge;
2204
2205 support = domain->iommu_superpage;
2206
2207 /* To use a large page, the virtual *and* physical addresses
2208 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2209 of them will mean we have to use smaller pages. So just
2210 merge them and check both at once. */
2211 pfnmerge = iov_pfn | phy_pfn;
2212
2213 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2214 pages >>= VTD_STRIDE_SHIFT;
2215 if (!pages)
2216 break;
2217 pfnmerge >>= VTD_STRIDE_SHIFT;
2218 level++;
2219 support--;
2220 }
2221 return level;
2222 }
2223
2224 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2225 struct scatterlist *sg, unsigned long phys_pfn,
2226 unsigned long nr_pages, int prot)
2227 {
2228 struct dma_pte *first_pte = NULL, *pte = NULL;
2229 phys_addr_t uninitialized_var(pteval);
2230 unsigned long sg_res = 0;
2231 unsigned int largepage_lvl = 0;
2232 unsigned long lvl_pages = 0;
2233
2234 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2235
2236 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2237 return -EINVAL;
2238
2239 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2240
2241 if (!sg) {
2242 sg_res = nr_pages;
2243 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2244 }
2245
2246 while (nr_pages > 0) {
2247 uint64_t tmp;
2248
2249 if (!sg_res) {
2250 unsigned int pgoff = sg->offset & ~PAGE_MASK;
2251
2252 sg_res = aligned_nrpages(sg->offset, sg->length);
2253 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + pgoff;
2254 sg->dma_length = sg->length;
2255 pteval = (sg_phys(sg) - pgoff) | prot;
2256 phys_pfn = pteval >> VTD_PAGE_SHIFT;
2257 }
2258
2259 if (!pte) {
2260 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2261
2262 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2263 if (!pte)
2264 return -ENOMEM;
2265 /* It is large page*/
2266 if (largepage_lvl > 1) {
2267 unsigned long nr_superpages, end_pfn;
2268
2269 pteval |= DMA_PTE_LARGE_PAGE;
2270 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2271
2272 nr_superpages = sg_res / lvl_pages;
2273 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2274
2275 /*
2276 * Ensure that old small page tables are
2277 * removed to make room for superpage(s).
2278 * We're adding new large pages, so make sure
2279 * we don't remove their parent tables.
2280 */
2281 dma_pte_free_pagetable(domain, iov_pfn, end_pfn,
2282 largepage_lvl + 1);
2283 } else {
2284 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2285 }
2286
2287 }
2288 /* We don't need lock here, nobody else
2289 * touches the iova range
2290 */
2291 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2292 if (tmp) {
2293 static int dumps = 5;
2294 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2295 iov_pfn, tmp, (unsigned long long)pteval);
2296 if (dumps) {
2297 dumps--;
2298 debug_dma_dump_mappings(NULL);
2299 }
2300 WARN_ON(1);
2301 }
2302
2303 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2304
2305 BUG_ON(nr_pages < lvl_pages);
2306 BUG_ON(sg_res < lvl_pages);
2307
2308 nr_pages -= lvl_pages;
2309 iov_pfn += lvl_pages;
2310 phys_pfn += lvl_pages;
2311 pteval += lvl_pages * VTD_PAGE_SIZE;
2312 sg_res -= lvl_pages;
2313
2314 /* If the next PTE would be the first in a new page, then we
2315 need to flush the cache on the entries we've just written.
2316 And then we'll need to recalculate 'pte', so clear it and
2317 let it get set again in the if (!pte) block above.
2318
2319 If we're done (!nr_pages) we need to flush the cache too.
2320
2321 Also if we've been setting superpages, we may need to
2322 recalculate 'pte' and switch back to smaller pages for the
2323 end of the mapping, if the trailing size is not enough to
2324 use another superpage (i.e. sg_res < lvl_pages). */
2325 pte++;
2326 if (!nr_pages || first_pte_in_page(pte) ||
2327 (largepage_lvl > 1 && sg_res < lvl_pages)) {
2328 domain_flush_cache(domain, first_pte,
2329 (void *)pte - (void *)first_pte);
2330 pte = NULL;
2331 }
2332
2333 if (!sg_res && nr_pages)
2334 sg = sg_next(sg);
2335 }
2336 return 0;
2337 }
2338
2339 static int domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2340 struct scatterlist *sg, unsigned long phys_pfn,
2341 unsigned long nr_pages, int prot)
2342 {
2343 int ret;
2344 struct intel_iommu *iommu;
2345
2346 /* Do the real mapping first */
2347 ret = __domain_mapping(domain, iov_pfn, sg, phys_pfn, nr_pages, prot);
2348 if (ret)
2349 return ret;
2350
2351 /* Notify about the new mapping */
2352 if (domain_type_is_vm(domain)) {
2353 /* VM typed domains can have more than one IOMMUs */
2354 int iommu_id;
2355 for_each_domain_iommu(iommu_id, domain) {
2356 iommu = g_iommus[iommu_id];
2357 __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
2358 }
2359 } else {
2360 /* General domains only have one IOMMU */
2361 iommu = domain_get_iommu(domain);
2362 __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
2363 }
2364
2365 return 0;
2366 }
2367
2368 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2369 struct scatterlist *sg, unsigned long nr_pages,
2370 int prot)
2371 {
2372 return domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2373 }
2374
2375 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2376 unsigned long phys_pfn, unsigned long nr_pages,
2377 int prot)
2378 {
2379 return domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2380 }
2381
2382 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2383 {
2384 unsigned long flags;
2385 struct context_entry *context;
2386 u16 did_old;
2387
2388 if (!iommu)
2389 return;
2390
2391 spin_lock_irqsave(&iommu->lock, flags);
2392 context = iommu_context_addr(iommu, bus, devfn, 0);
2393 if (!context) {
2394 spin_unlock_irqrestore(&iommu->lock, flags);
2395 return;
2396 }
2397 did_old = context_domain_id(context);
2398 context_clear_entry(context);
2399 __iommu_flush_cache(iommu, context, sizeof(*context));
2400 spin_unlock_irqrestore(&iommu->lock, flags);
2401 iommu->flush.flush_context(iommu,
2402 did_old,
2403 (((u16)bus) << 8) | devfn,
2404 DMA_CCMD_MASK_NOBIT,
2405 DMA_CCMD_DEVICE_INVL);
2406 iommu->flush.flush_iotlb(iommu,
2407 did_old,
2408 0,
2409 0,
2410 DMA_TLB_DSI_FLUSH);
2411 }
2412
2413 static inline void unlink_domain_info(struct device_domain_info *info)
2414 {
2415 assert_spin_locked(&device_domain_lock);
2416 list_del(&info->link);
2417 list_del(&info->global);
2418 if (info->dev)
2419 info->dev->archdata.iommu = NULL;
2420 }
2421
2422 static void domain_remove_dev_info(struct dmar_domain *domain)
2423 {
2424 struct device_domain_info *info, *tmp;
2425 unsigned long flags;
2426
2427 spin_lock_irqsave(&device_domain_lock, flags);
2428 list_for_each_entry_safe(info, tmp, &domain->devices, link)
2429 __dmar_remove_one_dev_info(info);
2430 spin_unlock_irqrestore(&device_domain_lock, flags);
2431 }
2432
2433 /*
2434 * find_domain
2435 * Note: we use struct device->archdata.iommu stores the info
2436 */
2437 static struct dmar_domain *find_domain(struct device *dev)
2438 {
2439 struct device_domain_info *info;
2440
2441 /* No lock here, assumes no domain exit in normal case */
2442 info = dev->archdata.iommu;
2443 if (likely(info))
2444 return info->domain;
2445 return NULL;
2446 }
2447
2448 static inline struct device_domain_info *
2449 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2450 {
2451 struct device_domain_info *info;
2452
2453 list_for_each_entry(info, &device_domain_list, global)
2454 if (info->iommu->segment == segment && info->bus == bus &&
2455 info->devfn == devfn)
2456 return info;
2457
2458 return NULL;
2459 }
2460
2461 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2462 int bus, int devfn,
2463 struct device *dev,
2464 struct dmar_domain *domain)
2465 {
2466 struct dmar_domain *found = NULL;
2467 struct device_domain_info *info;
2468 unsigned long flags;
2469 int ret;
2470
2471 info = alloc_devinfo_mem();
2472 if (!info)
2473 return NULL;
2474
2475 info->bus = bus;
2476 info->devfn = devfn;
2477 info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2478 info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2479 info->ats_qdep = 0;
2480 info->dev = dev;
2481 info->domain = domain;
2482 info->iommu = iommu;
2483 info->pasid_table = NULL;
2484
2485 if (dev && dev_is_pci(dev)) {
2486 struct pci_dev *pdev = to_pci_dev(info->dev);
2487
2488 if (!pci_ats_disabled() &&
2489 ecap_dev_iotlb_support(iommu->ecap) &&
2490 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2491 dmar_find_matched_atsr_unit(pdev))
2492 info->ats_supported = 1;
2493
2494 if (ecs_enabled(iommu)) {
2495 if (pasid_enabled(iommu)) {
2496 int features = pci_pasid_features(pdev);
2497 if (features >= 0)
2498 info->pasid_supported = features | 1;
2499 }
2500
2501 if (info->ats_supported && ecap_prs(iommu->ecap) &&
2502 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2503 info->pri_supported = 1;
2504 }
2505 }
2506
2507 spin_lock_irqsave(&device_domain_lock, flags);
2508 if (dev)
2509 found = find_domain(dev);
2510
2511 if (!found) {
2512 struct device_domain_info *info2;
2513 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2514 if (info2) {
2515 found = info2->domain;
2516 info2->dev = dev;
2517 }
2518 }
2519
2520 if (found) {
2521 spin_unlock_irqrestore(&device_domain_lock, flags);
2522 free_devinfo_mem(info);
2523 /* Caller must free the original domain */
2524 return found;
2525 }
2526
2527 spin_lock(&iommu->lock);
2528 ret = domain_attach_iommu(domain, iommu);
2529 spin_unlock(&iommu->lock);
2530
2531 if (ret) {
2532 spin_unlock_irqrestore(&device_domain_lock, flags);
2533 free_devinfo_mem(info);
2534 return NULL;
2535 }
2536
2537 list_add(&info->link, &domain->devices);
2538 list_add(&info->global, &device_domain_list);
2539 if (dev)
2540 dev->archdata.iommu = info;
2541
2542 if (dev && dev_is_pci(dev) && info->pasid_supported) {
2543 ret = intel_pasid_alloc_table(dev);
2544 if (ret) {
2545 pr_warn("No pasid table for %s, pasid disabled\n",
2546 dev_name(dev));
2547 info->pasid_supported = 0;
2548 }
2549 }
2550 spin_unlock_irqrestore(&device_domain_lock, flags);
2551
2552 if (dev && domain_context_mapping(domain, dev)) {
2553 pr_err("Domain context map for %s failed\n", dev_name(dev));
2554 dmar_remove_one_dev_info(domain, dev);
2555 return NULL;
2556 }
2557
2558 return domain;
2559 }
2560
2561 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2562 {
2563 *(u16 *)opaque = alias;
2564 return 0;
2565 }
2566
2567 static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
2568 {
2569 struct device_domain_info *info = NULL;
2570 struct dmar_domain *domain = NULL;
2571 struct intel_iommu *iommu;
2572 u16 dma_alias;
2573 unsigned long flags;
2574 u8 bus, devfn;
2575
2576 iommu = device_to_iommu(dev, &bus, &devfn);
2577 if (!iommu)
2578 return NULL;
2579
2580 if (dev_is_pci(dev)) {
2581 struct pci_dev *pdev = to_pci_dev(dev);
2582
2583 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2584
2585 spin_lock_irqsave(&device_domain_lock, flags);
2586 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2587 PCI_BUS_NUM(dma_alias),
2588 dma_alias & 0xff);
2589 if (info) {
2590 iommu = info->iommu;
2591 domain = info->domain;
2592 }
2593 spin_unlock_irqrestore(&device_domain_lock, flags);
2594
2595 /* DMA alias already has a domain, use it */
2596 if (info)
2597 goto out;
2598 }
2599
2600 /* Allocate and initialize new domain for the device */
2601 domain = alloc_domain(0);
2602 if (!domain)
2603 return NULL;
2604 if (domain_init(domain, iommu, gaw)) {
2605 domain_exit(domain);
2606 return NULL;
2607 }
2608
2609 out:
2610
2611 return domain;
2612 }
2613
2614 static struct dmar_domain *set_domain_for_dev(struct device *dev,
2615 struct dmar_domain *domain)
2616 {
2617 struct intel_iommu *iommu;
2618 struct dmar_domain *tmp;
2619 u16 req_id, dma_alias;
2620 u8 bus, devfn;
2621
2622 iommu = device_to_iommu(dev, &bus, &devfn);
2623 if (!iommu)
2624 return NULL;
2625
2626 req_id = ((u16)bus << 8) | devfn;
2627
2628 if (dev_is_pci(dev)) {
2629 struct pci_dev *pdev = to_pci_dev(dev);
2630
2631 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2632
2633 /* register PCI DMA alias device */
2634 if (req_id != dma_alias) {
2635 tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2636 dma_alias & 0xff, NULL, domain);
2637
2638 if (!tmp || tmp != domain)
2639 return tmp;
2640 }
2641 }
2642
2643 tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2644 if (!tmp || tmp != domain)
2645 return tmp;
2646
2647 return domain;
2648 }
2649
2650 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2651 {
2652 struct dmar_domain *domain, *tmp;
2653
2654 domain = find_domain(dev);
2655 if (domain)
2656 goto out;
2657
2658 domain = find_or_alloc_domain(dev, gaw);
2659 if (!domain)
2660 goto out;
2661
2662 tmp = set_domain_for_dev(dev, domain);
2663 if (!tmp || domain != tmp) {
2664 domain_exit(domain);
2665 domain = tmp;
2666 }
2667
2668 out:
2669
2670 return domain;
2671 }
2672
2673 static int iommu_domain_identity_map(struct dmar_domain *domain,
2674 unsigned long long start,
2675 unsigned long long end)
2676 {
2677 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2678 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2679
2680 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2681 dma_to_mm_pfn(last_vpfn))) {
2682 pr_err("Reserving iova failed\n");
2683 return -ENOMEM;
2684 }
2685
2686 pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2687 /*
2688 * RMRR range might have overlap with physical memory range,
2689 * clear it first
2690 */
2691 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2692
2693 return __domain_mapping(domain, first_vpfn, NULL,
2694 first_vpfn, last_vpfn - first_vpfn + 1,
2695 DMA_PTE_READ|DMA_PTE_WRITE);
2696 }
2697
2698 static int domain_prepare_identity_map(struct device *dev,
2699 struct dmar_domain *domain,
2700 unsigned long long start,
2701 unsigned long long end)
2702 {
2703 /* For _hardware_ passthrough, don't bother. But for software
2704 passthrough, we do it anyway -- it may indicate a memory
2705 range which is reserved in E820, so which didn't get set
2706 up to start with in si_domain */
2707 if (domain == si_domain && hw_pass_through) {
2708 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2709 dev_name(dev), start, end);
2710 return 0;
2711 }
2712
2713 pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2714 dev_name(dev), start, end);
2715
2716 if (end < start) {
2717 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2718 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2719 dmi_get_system_info(DMI_BIOS_VENDOR),
2720 dmi_get_system_info(DMI_BIOS_VERSION),
2721 dmi_get_system_info(DMI_PRODUCT_VERSION));
2722 return -EIO;
2723 }
2724
2725 if (end >> agaw_to_width(domain->agaw)) {
2726 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2727 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2728 agaw_to_width(domain->agaw),
2729 dmi_get_system_info(DMI_BIOS_VENDOR),
2730 dmi_get_system_info(DMI_BIOS_VERSION),
2731 dmi_get_system_info(DMI_PRODUCT_VERSION));
2732 return -EIO;
2733 }
2734
2735 return iommu_domain_identity_map(domain, start, end);
2736 }
2737
2738 static int iommu_prepare_identity_map(struct device *dev,
2739 unsigned long long start,
2740 unsigned long long end)
2741 {
2742 struct dmar_domain *domain;
2743 int ret;
2744
2745 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2746 if (!domain)
2747 return -ENOMEM;
2748
2749 ret = domain_prepare_identity_map(dev, domain, start, end);
2750 if (ret)
2751 domain_exit(domain);
2752
2753 return ret;
2754 }
2755
2756 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2757 struct device *dev)
2758 {
2759 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2760 return 0;
2761 return iommu_prepare_identity_map(dev, rmrr->base_address,
2762 rmrr->end_address);
2763 }
2764
2765 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2766 static inline void iommu_prepare_isa(void)
2767 {
2768 struct pci_dev *pdev;
2769 int ret;
2770
2771 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2772 if (!pdev)
2773 return;
2774
2775 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2776 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2777
2778 if (ret)
2779 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2780
2781 pci_dev_put(pdev);
2782 }
2783 #else
2784 static inline void iommu_prepare_isa(void)
2785 {
2786 return;
2787 }
2788 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2789
2790 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2791
2792 static int __init si_domain_init(int hw)
2793 {
2794 int nid, ret = 0;
2795
2796 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2797 if (!si_domain)
2798 return -EFAULT;
2799
2800 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2801 domain_exit(si_domain);
2802 return -EFAULT;
2803 }
2804
2805 pr_debug("Identity mapping domain allocated\n");
2806
2807 if (hw)
2808 return 0;
2809
2810 for_each_online_node(nid) {
2811 unsigned long start_pfn, end_pfn;
2812 int i;
2813
2814 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2815 ret = iommu_domain_identity_map(si_domain,
2816 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2817 if (ret)
2818 return ret;
2819 }
2820 }
2821
2822 return 0;
2823 }
2824
2825 static int identity_mapping(struct device *dev)
2826 {
2827 struct device_domain_info *info;
2828
2829 if (likely(!iommu_identity_mapping))
2830 return 0;
2831
2832 info = dev->archdata.iommu;
2833 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2834 return (info->domain == si_domain);
2835
2836 return 0;
2837 }
2838
2839 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2840 {
2841 struct dmar_domain *ndomain;
2842 struct intel_iommu *iommu;
2843 u8 bus, devfn;
2844
2845 iommu = device_to_iommu(dev, &bus, &devfn);
2846 if (!iommu)
2847 return -ENODEV;
2848
2849 ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2850 if (ndomain != domain)
2851 return -EBUSY;
2852
2853 return 0;
2854 }
2855
2856 static bool device_has_rmrr(struct device *dev)
2857 {
2858 struct dmar_rmrr_unit *rmrr;
2859 struct device *tmp;
2860 int i;
2861
2862 rcu_read_lock();
2863 for_each_rmrr_units(rmrr) {
2864 /*
2865 * Return TRUE if this RMRR contains the device that
2866 * is passed in.
2867 */
2868 for_each_active_dev_scope(rmrr->devices,
2869 rmrr->devices_cnt, i, tmp)
2870 if (tmp == dev) {
2871 rcu_read_unlock();
2872 return true;
2873 }
2874 }
2875 rcu_read_unlock();
2876 return false;
2877 }
2878
2879 /*
2880 * There are a couple cases where we need to restrict the functionality of
2881 * devices associated with RMRRs. The first is when evaluating a device for
2882 * identity mapping because problems exist when devices are moved in and out
2883 * of domains and their respective RMRR information is lost. This means that
2884 * a device with associated RMRRs will never be in a "passthrough" domain.
2885 * The second is use of the device through the IOMMU API. This interface
2886 * expects to have full control of the IOVA space for the device. We cannot
2887 * satisfy both the requirement that RMRR access is maintained and have an
2888 * unencumbered IOVA space. We also have no ability to quiesce the device's
2889 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2890 * We therefore prevent devices associated with an RMRR from participating in
2891 * the IOMMU API, which eliminates them from device assignment.
2892 *
2893 * In both cases we assume that PCI USB devices with RMRRs have them largely
2894 * for historical reasons and that the RMRR space is not actively used post
2895 * boot. This exclusion may change if vendors begin to abuse it.
2896 *
2897 * The same exception is made for graphics devices, with the requirement that
2898 * any use of the RMRR regions will be torn down before assigning the device
2899 * to a guest.
2900 */
2901 static bool device_is_rmrr_locked(struct device *dev)
2902 {
2903 if (!device_has_rmrr(dev))
2904 return false;
2905
2906 if (dev_is_pci(dev)) {
2907 struct pci_dev *pdev = to_pci_dev(dev);
2908
2909 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2910 return false;
2911 }
2912
2913 return true;
2914 }
2915
2916 static int iommu_should_identity_map(struct device *dev, int startup)
2917 {
2918
2919 if (dev_is_pci(dev)) {
2920 struct pci_dev *pdev = to_pci_dev(dev);
2921
2922 if (device_is_rmrr_locked(dev))
2923 return 0;
2924
2925 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2926 return 1;
2927
2928 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2929 return 1;
2930
2931 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2932 return 0;
2933
2934 /*
2935 * We want to start off with all devices in the 1:1 domain, and
2936 * take them out later if we find they can't access all of memory.
2937 *
2938 * However, we can't do this for PCI devices behind bridges,
2939 * because all PCI devices behind the same bridge will end up
2940 * with the same source-id on their transactions.
2941 *
2942 * Practically speaking, we can't change things around for these
2943 * devices at run-time, because we can't be sure there'll be no
2944 * DMA transactions in flight for any of their siblings.
2945 *
2946 * So PCI devices (unless they're on the root bus) as well as
2947 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2948 * the 1:1 domain, just in _case_ one of their siblings turns out
2949 * not to be able to map all of memory.
2950 */
2951 if (!pci_is_pcie(pdev)) {
2952 if (!pci_is_root_bus(pdev->bus))
2953 return 0;
2954 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2955 return 0;
2956 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2957 return 0;
2958 } else {
2959 if (device_has_rmrr(dev))
2960 return 0;
2961 }
2962
2963 /*
2964 * At boot time, we don't yet know if devices will be 64-bit capable.
2965 * Assume that they will — if they turn out not to be, then we can
2966 * take them out of the 1:1 domain later.
2967 */
2968 if (!startup) {
2969 /*
2970 * If the device's dma_mask is less than the system's memory
2971 * size then this is not a candidate for identity mapping.
2972 */
2973 u64 dma_mask = *dev->dma_mask;
2974
2975 if (dev->coherent_dma_mask &&
2976 dev->coherent_dma_mask < dma_mask)
2977 dma_mask = dev->coherent_dma_mask;
2978
2979 return dma_mask >= dma_get_required_mask(dev);
2980 }
2981
2982 return 1;
2983 }
2984
2985 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2986 {
2987 int ret;
2988
2989 if (!iommu_should_identity_map(dev, 1))
2990 return 0;
2991
2992 ret = domain_add_dev_info(si_domain, dev);
2993 if (!ret)
2994 pr_info("%s identity mapping for device %s\n",
2995 hw ? "Hardware" : "Software", dev_name(dev));
2996 else if (ret == -ENODEV)
2997 /* device not associated with an iommu */
2998 ret = 0;
2999
3000 return ret;
3001 }
3002
3003
3004 static int __init iommu_prepare_static_identity_mapping(int hw)
3005 {
3006 struct pci_dev *pdev = NULL;
3007 struct dmar_drhd_unit *drhd;
3008 struct intel_iommu *iommu;
3009 struct device *dev;
3010 int i;
3011 int ret = 0;
3012
3013 for_each_pci_dev(pdev) {
3014 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
3015 if (ret)
3016 return ret;
3017 }
3018
3019 for_each_active_iommu(iommu, drhd)
3020 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
3021 struct acpi_device_physical_node *pn;
3022 struct acpi_device *adev;
3023
3024 if (dev->bus != &acpi_bus_type)
3025 continue;
3026
3027 adev= to_acpi_device(dev);
3028 mutex_lock(&adev->physical_node_lock);
3029 list_for_each_entry(pn, &adev->physical_node_list, node) {
3030 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
3031 if (ret)
3032 break;
3033 }
3034 mutex_unlock(&adev->physical_node_lock);
3035 if (ret)
3036 return ret;
3037 }
3038
3039 return 0;
3040 }
3041
3042 static void intel_iommu_init_qi(struct intel_iommu *iommu)
3043 {
3044 /*
3045 * Start from the sane iommu hardware state.
3046 * If the queued invalidation is already initialized by us
3047 * (for example, while enabling interrupt-remapping) then
3048 * we got the things already rolling from a sane state.
3049 */
3050 if (!iommu->qi) {
3051 /*
3052 * Clear any previous faults.
3053 */
3054 dmar_fault(-1, iommu);
3055 /*
3056 * Disable queued invalidation if supported and already enabled
3057 * before OS handover.
3058 */
3059 dmar_disable_qi(iommu);
3060 }
3061
3062 if (dmar_enable_qi(iommu)) {
3063 /*
3064 * Queued Invalidate not enabled, use Register Based Invalidate
3065 */
3066 iommu->flush.flush_context = __iommu_flush_context;
3067 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
3068 pr_info("%s: Using Register based invalidation\n",
3069 iommu->name);
3070 } else {
3071 iommu->flush.flush_context = qi_flush_context;
3072 iommu->flush.flush_iotlb = qi_flush_iotlb;
3073 pr_info("%s: Using Queued invalidation\n", iommu->name);
3074 }
3075 }
3076
3077 static int copy_context_table(struct intel_iommu *iommu,
3078 struct root_entry *old_re,
3079 struct context_entry **tbl,
3080 int bus, bool ext)
3081 {
3082 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
3083 struct context_entry *new_ce = NULL, ce;
3084 struct context_entry *old_ce = NULL;
3085 struct root_entry re;
3086 phys_addr_t old_ce_phys;
3087
3088 tbl_idx = ext ? bus * 2 : bus;
3089 memcpy(&re, old_re, sizeof(re));
3090
3091 for (devfn = 0; devfn < 256; devfn++) {
3092 /* First calculate the correct index */
3093 idx = (ext ? devfn * 2 : devfn) % 256;
3094
3095 if (idx == 0) {
3096 /* First save what we may have and clean up */
3097 if (new_ce) {
3098 tbl[tbl_idx] = new_ce;
3099 __iommu_flush_cache(iommu, new_ce,
3100 VTD_PAGE_SIZE);
3101 pos = 1;
3102 }
3103
3104 if (old_ce)
3105 memunmap(old_ce);
3106
3107 ret = 0;
3108 if (devfn < 0x80)
3109 old_ce_phys = root_entry_lctp(&re);
3110 else
3111 old_ce_phys = root_entry_uctp(&re);
3112
3113 if (!old_ce_phys) {
3114 if (ext && devfn == 0) {
3115 /* No LCTP, try UCTP */
3116 devfn = 0x7f;
3117 continue;
3118 } else {
3119 goto out;
3120 }
3121 }
3122
3123 ret = -ENOMEM;
3124 old_ce = memremap(old_ce_phys, PAGE_SIZE,
3125 MEMREMAP_WB);
3126 if (!old_ce)
3127 goto out;
3128
3129 new_ce = alloc_pgtable_page(iommu->node);
3130 if (!new_ce)
3131 goto out_unmap;
3132
3133 ret = 0;
3134 }
3135
3136 /* Now copy the context entry */
3137 memcpy(&ce, old_ce + idx, sizeof(ce));
3138
3139 if (!__context_present(&ce))
3140 continue;
3141
3142 did = context_domain_id(&ce);
3143 if (did >= 0 && did < cap_ndoms(iommu->cap))
3144 set_bit(did, iommu->domain_ids);
3145
3146 /*
3147 * We need a marker for copied context entries. This
3148 * marker needs to work for the old format as well as
3149 * for extended context entries.
3150 *
3151 * Bit 67 of the context entry is used. In the old
3152 * format this bit is available to software, in the
3153 * extended format it is the PGE bit, but PGE is ignored
3154 * by HW if PASIDs are disabled (and thus still
3155 * available).
3156 *
3157 * So disable PASIDs first and then mark the entry
3158 * copied. This means that we don't copy PASID
3159 * translations from the old kernel, but this is fine as
3160 * faults there are not fatal.
3161 */
3162 context_clear_pasid_enable(&ce);
3163 context_set_copied(&ce);
3164
3165 new_ce[idx] = ce;
3166 }
3167
3168 tbl[tbl_idx + pos] = new_ce;
3169
3170 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3171
3172 out_unmap:
3173 memunmap(old_ce);
3174
3175 out:
3176 return ret;
3177 }
3178
3179 static int copy_translation_tables(struct intel_iommu *iommu)
3180 {
3181 struct context_entry **ctxt_tbls;
3182 struct root_entry *old_rt;
3183 phys_addr_t old_rt_phys;
3184 int ctxt_table_entries;
3185 unsigned long flags;
3186 u64 rtaddr_reg;
3187 int bus, ret;
3188 bool new_ext, ext;
3189
3190 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3191 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
3192 new_ext = !!ecap_ecs(iommu->ecap);
3193
3194 /*
3195 * The RTT bit can only be changed when translation is disabled,
3196 * but disabling translation means to open a window for data
3197 * corruption. So bail out and don't copy anything if we would
3198 * have to change the bit.
3199 */
3200 if (new_ext != ext)
3201 return -EINVAL;
3202
3203 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3204 if (!old_rt_phys)
3205 return -EINVAL;
3206
3207 old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
3208 if (!old_rt)
3209 return -ENOMEM;
3210
3211 /* This is too big for the stack - allocate it from slab */
3212 ctxt_table_entries = ext ? 512 : 256;
3213 ret = -ENOMEM;
3214 ctxt_tbls = kcalloc(ctxt_table_entries, sizeof(void *), GFP_KERNEL);
3215 if (!ctxt_tbls)
3216 goto out_unmap;
3217
3218 for (bus = 0; bus < 256; bus++) {
3219 ret = copy_context_table(iommu, &old_rt[bus],
3220 ctxt_tbls, bus, ext);
3221 if (ret) {
3222 pr_err("%s: Failed to copy context table for bus %d\n",
3223 iommu->name, bus);
3224 continue;
3225 }
3226 }
3227
3228 spin_lock_irqsave(&iommu->lock, flags);
3229
3230 /* Context tables are copied, now write them to the root_entry table */
3231 for (bus = 0; bus < 256; bus++) {
3232 int idx = ext ? bus * 2 : bus;
3233 u64 val;
3234
3235 if (ctxt_tbls[idx]) {
3236 val = virt_to_phys(ctxt_tbls[idx]) | 1;
3237 iommu->root_entry[bus].lo = val;
3238 }
3239
3240 if (!ext || !ctxt_tbls[idx + 1])
3241 continue;
3242
3243 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3244 iommu->root_entry[bus].hi = val;
3245 }
3246
3247 spin_unlock_irqrestore(&iommu->lock, flags);
3248
3249 kfree(ctxt_tbls);
3250
3251 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3252
3253 ret = 0;
3254
3255 out_unmap:
3256 memunmap(old_rt);
3257
3258 return ret;
3259 }
3260
3261 static int __init init_dmars(void)
3262 {
3263 struct dmar_drhd_unit *drhd;
3264 struct dmar_rmrr_unit *rmrr;
3265 bool copied_tables = false;
3266 struct device *dev;
3267 struct intel_iommu *iommu;
3268 int i, ret;
3269
3270 /*
3271 * for each drhd
3272 * allocate root
3273 * initialize and program root entry to not present
3274 * endfor
3275 */
3276 for_each_drhd_unit(drhd) {
3277 /*
3278 * lock not needed as this is only incremented in the single
3279 * threaded kernel __init code path all other access are read
3280 * only
3281 */
3282 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3283 g_num_of_iommus++;
3284 continue;
3285 }
3286 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3287 }
3288
3289 /* Preallocate enough resources for IOMMU hot-addition */
3290 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3291 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3292
3293 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3294 GFP_KERNEL);
3295 if (!g_iommus) {
3296 pr_err("Allocating global iommu array failed\n");
3297 ret = -ENOMEM;
3298 goto error;
3299 }
3300
3301 for_each_active_iommu(iommu, drhd) {
3302 /*
3303 * Find the max pasid size of all IOMMU's in the system.
3304 * We need to ensure the system pasid table is no bigger
3305 * than the smallest supported.
3306 */
3307 if (pasid_enabled(iommu)) {
3308 u32 temp = 2 << ecap_pss(iommu->ecap);
3309
3310 intel_pasid_max_id = min_t(u32, temp,
3311 intel_pasid_max_id);
3312 }
3313
3314 g_iommus[iommu->seq_id] = iommu;
3315
3316 intel_iommu_init_qi(iommu);
3317
3318 ret = iommu_init_domains(iommu);
3319 if (ret)
3320 goto free_iommu;
3321
3322 init_translation_status(iommu);
3323
3324 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3325 iommu_disable_translation(iommu);
3326 clear_translation_pre_enabled(iommu);
3327 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3328 iommu->name);
3329 }
3330
3331 /*
3332 * TBD:
3333 * we could share the same root & context tables
3334 * among all IOMMU's. Need to Split it later.
3335 */
3336 ret = iommu_alloc_root_entry(iommu);
3337 if (ret)
3338 goto free_iommu;
3339
3340 if (translation_pre_enabled(iommu)) {
3341 pr_info("Translation already enabled - trying to copy translation structures\n");
3342
3343 ret = copy_translation_tables(iommu);
3344 if (ret) {
3345 /*
3346 * We found the IOMMU with translation
3347 * enabled - but failed to copy over the
3348 * old root-entry table. Try to proceed
3349 * by disabling translation now and
3350 * allocating a clean root-entry table.
3351 * This might cause DMAR faults, but
3352 * probably the dump will still succeed.
3353 */
3354 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3355 iommu->name);
3356 iommu_disable_translation(iommu);
3357 clear_translation_pre_enabled(iommu);
3358 } else {
3359 pr_info("Copied translation tables from previous kernel for %s\n",
3360 iommu->name);
3361 copied_tables = true;
3362 }
3363 }
3364
3365 if (!ecap_pass_through(iommu->ecap))
3366 hw_pass_through = 0;
3367 #ifdef CONFIG_INTEL_IOMMU_SVM
3368 if (pasid_enabled(iommu))
3369 intel_svm_init(iommu);
3370 #endif
3371 }
3372
3373 /*
3374 * Now that qi is enabled on all iommus, set the root entry and flush
3375 * caches. This is required on some Intel X58 chipsets, otherwise the
3376 * flush_context function will loop forever and the boot hangs.
3377 */
3378 for_each_active_iommu(iommu, drhd) {
3379 iommu_flush_write_buffer(iommu);
3380 iommu_set_root_entry(iommu);
3381 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3382 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3383 }
3384
3385 if (iommu_pass_through)
3386 iommu_identity_mapping |= IDENTMAP_ALL;
3387
3388 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3389 dmar_map_gfx = 0;
3390 #endif
3391
3392 if (!dmar_map_gfx)
3393 iommu_identity_mapping |= IDENTMAP_GFX;
3394
3395 check_tylersburg_isoch();
3396
3397 if (iommu_identity_mapping) {
3398 ret = si_domain_init(hw_pass_through);
3399 if (ret)
3400 goto free_iommu;
3401 }
3402
3403
3404 /*
3405 * If we copied translations from a previous kernel in the kdump
3406 * case, we can not assign the devices to domains now, as that
3407 * would eliminate the old mappings. So skip this part and defer
3408 * the assignment to device driver initialization time.
3409 */
3410 if (copied_tables)
3411 goto domains_done;
3412
3413 /*
3414 * If pass through is not set or not enabled, setup context entries for
3415 * identity mappings for rmrr, gfx, and isa and may fall back to static
3416 * identity mapping if iommu_identity_mapping is set.
3417 */
3418 if (iommu_identity_mapping) {
3419 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3420 if (ret) {
3421 pr_crit("Failed to setup IOMMU pass-through\n");
3422 goto free_iommu;
3423 }
3424 }
3425 /*
3426 * For each rmrr
3427 * for each dev attached to rmrr
3428 * do
3429 * locate drhd for dev, alloc domain for dev
3430 * allocate free domain
3431 * allocate page table entries for rmrr
3432 * if context not allocated for bus
3433 * allocate and init context
3434 * set present in root table for this bus
3435 * init context with domain, translation etc
3436 * endfor
3437 * endfor
3438 */
3439 pr_info("Setting RMRR:\n");
3440 for_each_rmrr_units(rmrr) {
3441 /* some BIOS lists non-exist devices in DMAR table. */
3442 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3443 i, dev) {
3444 ret = iommu_prepare_rmrr_dev(rmrr, dev);
3445 if (ret)
3446 pr_err("Mapping reserved region failed\n");
3447 }
3448 }
3449
3450 iommu_prepare_isa();
3451
3452 domains_done:
3453
3454 /*
3455 * for each drhd
3456 * enable fault log
3457 * global invalidate context cache
3458 * global invalidate iotlb
3459 * enable translation
3460 */
3461 for_each_iommu(iommu, drhd) {
3462 if (drhd->ignored) {
3463 /*
3464 * we always have to disable PMRs or DMA may fail on
3465 * this device
3466 */
3467 if (force_on)
3468 iommu_disable_protect_mem_regions(iommu);
3469 continue;
3470 }
3471
3472 iommu_flush_write_buffer(iommu);
3473
3474 #ifdef CONFIG_INTEL_IOMMU_SVM
3475 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
3476 ret = intel_svm_enable_prq(iommu);
3477 if (ret)
3478 goto free_iommu;
3479 }
3480 #endif
3481 ret = dmar_set_interrupt(iommu);
3482 if (ret)
3483 goto free_iommu;
3484
3485 if (!translation_pre_enabled(iommu))
3486 iommu_enable_translation(iommu);
3487
3488 iommu_disable_protect_mem_regions(iommu);
3489 }
3490
3491 return 0;
3492
3493 free_iommu:
3494 for_each_active_iommu(iommu, drhd) {
3495 disable_dmar_iommu(iommu);
3496 free_dmar_iommu(iommu);
3497 }
3498
3499 kfree(g_iommus);
3500
3501 error:
3502 return ret;
3503 }
3504
3505 /* This takes a number of _MM_ pages, not VTD pages */
3506 static unsigned long intel_alloc_iova(struct device *dev,
3507 struct dmar_domain *domain,
3508 unsigned long nrpages, uint64_t dma_mask)
3509 {
3510 unsigned long iova_pfn = 0;
3511
3512 /* Restrict dma_mask to the width that the iommu can handle */
3513 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3514 /* Ensure we reserve the whole size-aligned region */
3515 nrpages = __roundup_pow_of_two(nrpages);
3516
3517 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3518 /*
3519 * First try to allocate an io virtual address in
3520 * DMA_BIT_MASK(32) and if that fails then try allocating
3521 * from higher range
3522 */
3523 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3524 IOVA_PFN(DMA_BIT_MASK(32)), false);
3525 if (iova_pfn)
3526 return iova_pfn;
3527 }
3528 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3529 IOVA_PFN(dma_mask), true);
3530 if (unlikely(!iova_pfn)) {
3531 pr_err("Allocating %ld-page iova for %s failed",
3532 nrpages, dev_name(dev));
3533 return 0;
3534 }
3535
3536 return iova_pfn;
3537 }
3538
3539 struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3540 {
3541 struct dmar_domain *domain, *tmp;
3542 struct dmar_rmrr_unit *rmrr;
3543 struct device *i_dev;
3544 int i, ret;
3545
3546 domain = find_domain(dev);
3547 if (domain)
3548 goto out;
3549
3550 domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3551 if (!domain)
3552 goto out;
3553
3554 /* We have a new domain - setup possible RMRRs for the device */
3555 rcu_read_lock();
3556 for_each_rmrr_units(rmrr) {
3557 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3558 i, i_dev) {
3559 if (i_dev != dev)
3560 continue;
3561
3562 ret = domain_prepare_identity_map(dev, domain,
3563 rmrr->base_address,
3564 rmrr->end_address);
3565 if (ret)
3566 dev_err(dev, "Mapping reserved region failed\n");
3567 }
3568 }
3569 rcu_read_unlock();
3570
3571 tmp = set_domain_for_dev(dev, domain);
3572 if (!tmp || domain != tmp) {
3573 domain_exit(domain);
3574 domain = tmp;
3575 }
3576
3577 out:
3578
3579 if (!domain)
3580 pr_err("Allocating domain for %s failed\n", dev_name(dev));
3581
3582
3583 return domain;
3584 }
3585
3586 /* Check if the dev needs to go through non-identity map and unmap process.*/
3587 static int iommu_no_mapping(struct device *dev)
3588 {
3589 int found;
3590
3591 if (iommu_dummy(dev))
3592 return 1;
3593
3594 if (!iommu_identity_mapping)
3595 return 0;
3596
3597 found = identity_mapping(dev);
3598 if (found) {
3599 if (iommu_should_identity_map(dev, 0))
3600 return 1;
3601 else {
3602 /*
3603 * 32 bit DMA is removed from si_domain and fall back
3604 * to non-identity mapping.
3605 */
3606 dmar_remove_one_dev_info(si_domain, dev);
3607 pr_info("32bit %s uses non-identity mapping\n",
3608 dev_name(dev));
3609 return 0;
3610 }
3611 } else {
3612 /*
3613 * In case of a detached 64 bit DMA device from vm, the device
3614 * is put into si_domain for identity mapping.
3615 */
3616 if (iommu_should_identity_map(dev, 0)) {
3617 int ret;
3618 ret = domain_add_dev_info(si_domain, dev);
3619 if (!ret) {
3620 pr_info("64bit %s uses identity mapping\n",
3621 dev_name(dev));
3622 return 1;
3623 }
3624 }
3625 }
3626
3627 return 0;
3628 }
3629
3630 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3631 size_t size, int dir, u64 dma_mask)
3632 {
3633 struct dmar_domain *domain;
3634 phys_addr_t start_paddr;
3635 unsigned long iova_pfn;
3636 int prot = 0;
3637 int ret;
3638 struct intel_iommu *iommu;
3639 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3640
3641 BUG_ON(dir == DMA_NONE);
3642
3643 if (iommu_no_mapping(dev))
3644 return paddr;
3645
3646 domain = get_valid_domain_for_dev(dev);
3647 if (!domain)
3648 return 0;
3649
3650 iommu = domain_get_iommu(domain);
3651 size = aligned_nrpages(paddr, size);
3652
3653 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3654 if (!iova_pfn)
3655 goto error;
3656
3657 /*
3658 * Check if DMAR supports zero-length reads on write only
3659 * mappings..
3660 */
3661 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3662 !cap_zlr(iommu->cap))
3663 prot |= DMA_PTE_READ;
3664 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3665 prot |= DMA_PTE_WRITE;
3666 /*
3667 * paddr - (paddr + size) might be partial page, we should map the whole
3668 * page. Note: if two part of one page are separately mapped, we
3669 * might have two guest_addr mapping to the same host paddr, but this
3670 * is not a big problem
3671 */
3672 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
3673 mm_to_dma_pfn(paddr_pfn), size, prot);
3674 if (ret)
3675 goto error;
3676
3677 start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
3678 start_paddr += paddr & ~PAGE_MASK;
3679 return start_paddr;
3680
3681 error:
3682 if (iova_pfn)
3683 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3684 pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
3685 dev_name(dev), size, (unsigned long long)paddr, dir);
3686 return 0;
3687 }
3688
3689 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3690 unsigned long offset, size_t size,
3691 enum dma_data_direction dir,
3692 unsigned long attrs)
3693 {
3694 return __intel_map_single(dev, page_to_phys(page) + offset, size,
3695 dir, *dev->dma_mask);
3696 }
3697
3698 static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
3699 {
3700 struct dmar_domain *domain;
3701 unsigned long start_pfn, last_pfn;
3702 unsigned long nrpages;
3703 unsigned long iova_pfn;
3704 struct intel_iommu *iommu;
3705 struct page *freelist;
3706
3707 if (iommu_no_mapping(dev))
3708 return;
3709
3710 domain = find_domain(dev);
3711 BUG_ON(!domain);
3712
3713 iommu = domain_get_iommu(domain);
3714
3715 iova_pfn = IOVA_PFN(dev_addr);
3716
3717 nrpages = aligned_nrpages(dev_addr, size);
3718 start_pfn = mm_to_dma_pfn(iova_pfn);
3719 last_pfn = start_pfn + nrpages - 1;
3720
3721 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3722 dev_name(dev), start_pfn, last_pfn);
3723
3724 freelist = domain_unmap(domain, start_pfn, last_pfn);
3725
3726 if (intel_iommu_strict || !has_iova_flush_queue(&domain->iovad)) {
3727 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3728 nrpages, !freelist, 0);
3729 /* free iova */
3730 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
3731 dma_free_pagelist(freelist);
3732 } else {
3733 queue_iova(&domain->iovad, iova_pfn, nrpages,
3734 (unsigned long)freelist);
3735 /*
3736 * queue up the release of the unmap to save the 1/6th of the
3737 * cpu used up by the iotlb flush operation...
3738 */
3739 }
3740 }
3741
3742 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3743 size_t size, enum dma_data_direction dir,
3744 unsigned long attrs)
3745 {
3746 intel_unmap(dev, dev_addr, size);
3747 }
3748
3749 static void *intel_alloc_coherent(struct device *dev, size_t size,
3750 dma_addr_t *dma_handle, gfp_t flags,
3751 unsigned long attrs)
3752 {
3753 struct page *page = NULL;
3754 int order;
3755
3756 size = PAGE_ALIGN(size);
3757 order = get_order(size);
3758
3759 if (!iommu_no_mapping(dev))
3760 flags &= ~(GFP_DMA | GFP_DMA32);
3761 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3762 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3763 flags |= GFP_DMA;
3764 else
3765 flags |= GFP_DMA32;
3766 }
3767
3768 if (gfpflags_allow_blocking(flags)) {
3769 unsigned int count = size >> PAGE_SHIFT;
3770
3771 page = dma_alloc_from_contiguous(dev, count, order,
3772 flags & __GFP_NOWARN);
3773 if (page && iommu_no_mapping(dev) &&
3774 page_to_phys(page) + size > dev->coherent_dma_mask) {
3775 dma_release_from_contiguous(dev, page, count);
3776 page = NULL;
3777 }
3778 }
3779
3780 if (!page)
3781 page = alloc_pages(flags, order);
3782 if (!page)
3783 return NULL;
3784 memset(page_address(page), 0, size);
3785
3786 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3787 DMA_BIDIRECTIONAL,
3788 dev->coherent_dma_mask);
3789 if (*dma_handle)
3790 return page_address(page);
3791 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3792 __free_pages(page, order);
3793
3794 return NULL;
3795 }
3796
3797 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3798 dma_addr_t dma_handle, unsigned long attrs)
3799 {
3800 int order;
3801 struct page *page = virt_to_page(vaddr);
3802
3803 size = PAGE_ALIGN(size);
3804 order = get_order(size);
3805
3806 intel_unmap(dev, dma_handle, size);
3807 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3808 __free_pages(page, order);
3809 }
3810
3811 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3812 int nelems, enum dma_data_direction dir,
3813 unsigned long attrs)
3814 {
3815 dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
3816 unsigned long nrpages = 0;
3817 struct scatterlist *sg;
3818 int i;
3819
3820 for_each_sg(sglist, sg, nelems, i) {
3821 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3822 }
3823
3824 intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
3825 }
3826
3827 static int intel_nontranslate_map_sg(struct device *hddev,
3828 struct scatterlist *sglist, int nelems, int dir)
3829 {
3830 int i;
3831 struct scatterlist *sg;
3832
3833 for_each_sg(sglist, sg, nelems, i) {
3834 BUG_ON(!sg_page(sg));
3835 sg->dma_address = sg_phys(sg);
3836 sg->dma_length = sg->length;
3837 }
3838 return nelems;
3839 }
3840
3841 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3842 enum dma_data_direction dir, unsigned long attrs)
3843 {
3844 int i;
3845 struct dmar_domain *domain;
3846 size_t size = 0;
3847 int prot = 0;
3848 unsigned long iova_pfn;
3849 int ret;
3850 struct scatterlist *sg;
3851 unsigned long start_vpfn;
3852 struct intel_iommu *iommu;
3853
3854 BUG_ON(dir == DMA_NONE);
3855 if (iommu_no_mapping(dev))
3856 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3857
3858 domain = get_valid_domain_for_dev(dev);
3859 if (!domain)
3860 return 0;
3861
3862 iommu = domain_get_iommu(domain);
3863
3864 for_each_sg(sglist, sg, nelems, i)
3865 size += aligned_nrpages(sg->offset, sg->length);
3866
3867 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3868 *dev->dma_mask);
3869 if (!iova_pfn) {
3870 sglist->dma_length = 0;
3871 return 0;
3872 }
3873
3874 /*
3875 * Check if DMAR supports zero-length reads on write only
3876 * mappings..
3877 */
3878 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3879 !cap_zlr(iommu->cap))
3880 prot |= DMA_PTE_READ;
3881 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3882 prot |= DMA_PTE_WRITE;
3883
3884 start_vpfn = mm_to_dma_pfn(iova_pfn);
3885
3886 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3887 if (unlikely(ret)) {
3888 dma_pte_free_pagetable(domain, start_vpfn,
3889 start_vpfn + size - 1,
3890 agaw_to_level(domain->agaw) + 1);
3891 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3892 return 0;
3893 }
3894
3895 return nelems;
3896 }
3897
3898 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3899 {
3900 return !dma_addr;
3901 }
3902
3903 const struct dma_map_ops intel_dma_ops = {
3904 .alloc = intel_alloc_coherent,
3905 .free = intel_free_coherent,
3906 .map_sg = intel_map_sg,
3907 .unmap_sg = intel_unmap_sg,
3908 .map_page = intel_map_page,
3909 .unmap_page = intel_unmap_page,
3910 .mapping_error = intel_mapping_error,
3911 #ifdef CONFIG_X86
3912 .dma_supported = dma_direct_supported,
3913 #endif
3914 };
3915
3916 static inline int iommu_domain_cache_init(void)
3917 {
3918 int ret = 0;
3919
3920 iommu_domain_cache = kmem_cache_create("iommu_domain",
3921 sizeof(struct dmar_domain),
3922 0,
3923 SLAB_HWCACHE_ALIGN,
3924
3925 NULL);
3926 if (!iommu_domain_cache) {
3927 pr_err("Couldn't create iommu_domain cache\n");
3928 ret = -ENOMEM;
3929 }
3930
3931 return ret;
3932 }
3933
3934 static inline int iommu_devinfo_cache_init(void)
3935 {
3936 int ret = 0;
3937
3938 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3939 sizeof(struct device_domain_info),
3940 0,
3941 SLAB_HWCACHE_ALIGN,
3942 NULL);
3943 if (!iommu_devinfo_cache) {
3944 pr_err("Couldn't create devinfo cache\n");
3945 ret = -ENOMEM;
3946 }
3947
3948 return ret;
3949 }
3950
3951 static int __init iommu_init_mempool(void)
3952 {
3953 int ret;
3954 ret = iova_cache_get();
3955 if (ret)
3956 return ret;
3957
3958 ret = iommu_domain_cache_init();
3959 if (ret)
3960 goto domain_error;
3961
3962 ret = iommu_devinfo_cache_init();
3963 if (!ret)
3964 return ret;
3965
3966 kmem_cache_destroy(iommu_domain_cache);
3967 domain_error:
3968 iova_cache_put();
3969
3970 return -ENOMEM;
3971 }
3972
3973 static void __init iommu_exit_mempool(void)
3974 {
3975 kmem_cache_destroy(iommu_devinfo_cache);
3976 kmem_cache_destroy(iommu_domain_cache);
3977 iova_cache_put();
3978 }
3979
3980 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3981 {
3982 struct dmar_drhd_unit *drhd;
3983 u32 vtbar;
3984 int rc;
3985
3986 /* We know that this device on this chipset has its own IOMMU.
3987 * If we find it under a different IOMMU, then the BIOS is lying
3988 * to us. Hope that the IOMMU for this device is actually
3989 * disabled, and it needs no translation...
3990 */
3991 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3992 if (rc) {
3993 /* "can't" happen */
3994 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3995 return;
3996 }
3997 vtbar &= 0xffff0000;
3998
3999 /* we know that the this iommu should be at offset 0xa000 from vtbar */
4000 drhd = dmar_find_matched_drhd_unit(pdev);
4001 if (!drhd || drhd->reg_base_addr - vtbar != 0xa000) {
4002 pr_warn_once(FW_BUG "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n");
4003 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
4004 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4005 }
4006 }
4007 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
4008
4009 static void __init init_no_remapping_devices(void)
4010 {
4011 struct dmar_drhd_unit *drhd;
4012 struct device *dev;
4013 int i;
4014
4015 for_each_drhd_unit(drhd) {
4016 if (!drhd->include_all) {
4017 for_each_active_dev_scope(drhd->devices,
4018 drhd->devices_cnt, i, dev)
4019 break;
4020 /* ignore DMAR unit if no devices exist */
4021 if (i == drhd->devices_cnt)
4022 drhd->ignored = 1;
4023 }
4024 }
4025
4026 for_each_active_drhd_unit(drhd) {
4027 if (drhd->include_all)
4028 continue;
4029
4030 for_each_active_dev_scope(drhd->devices,
4031 drhd->devices_cnt, i, dev)
4032 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
4033 break;
4034 if (i < drhd->devices_cnt)
4035 continue;
4036
4037 /* This IOMMU has *only* gfx devices. Either bypass it or
4038 set the gfx_mapped flag, as appropriate */
4039 if (!dmar_map_gfx) {
4040 drhd->ignored = 1;
4041 for_each_active_dev_scope(drhd->devices,
4042 drhd->devices_cnt, i, dev)
4043 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4044 }
4045 }
4046 }
4047
4048 #ifdef CONFIG_SUSPEND
4049 static int init_iommu_hw(void)
4050 {
4051 struct dmar_drhd_unit *drhd;
4052 struct intel_iommu *iommu = NULL;
4053
4054 for_each_active_iommu(iommu, drhd)
4055 if (iommu->qi)
4056 dmar_reenable_qi(iommu);
4057
4058 for_each_iommu(iommu, drhd) {
4059 if (drhd->ignored) {
4060 /*
4061 * we always have to disable PMRs or DMA may fail on
4062 * this device
4063 */
4064 if (force_on)
4065 iommu_disable_protect_mem_regions(iommu);
4066 continue;
4067 }
4068
4069 iommu_flush_write_buffer(iommu);
4070
4071 iommu_set_root_entry(iommu);
4072
4073 iommu->flush.flush_context(iommu, 0, 0, 0,
4074 DMA_CCMD_GLOBAL_INVL);
4075 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4076 iommu_enable_translation(iommu);
4077 iommu_disable_protect_mem_regions(iommu);
4078 }
4079
4080 return 0;
4081 }
4082
4083 static void iommu_flush_all(void)
4084 {
4085 struct dmar_drhd_unit *drhd;
4086 struct intel_iommu *iommu;
4087
4088 for_each_active_iommu(iommu, drhd) {
4089 iommu->flush.flush_context(iommu, 0, 0, 0,
4090 DMA_CCMD_GLOBAL_INVL);
4091 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
4092 DMA_TLB_GLOBAL_FLUSH);
4093 }
4094 }
4095
4096 static int iommu_suspend(void)
4097 {
4098 struct dmar_drhd_unit *drhd;
4099 struct intel_iommu *iommu = NULL;
4100 unsigned long flag;
4101
4102 for_each_active_iommu(iommu, drhd) {
4103 iommu->iommu_state = kcalloc(MAX_SR_DMAR_REGS, sizeof(u32),
4104 GFP_ATOMIC);
4105 if (!iommu->iommu_state)
4106 goto nomem;
4107 }
4108
4109 iommu_flush_all();
4110
4111 for_each_active_iommu(iommu, drhd) {
4112 iommu_disable_translation(iommu);
4113
4114 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4115
4116 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4117 readl(iommu->reg + DMAR_FECTL_REG);
4118 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4119 readl(iommu->reg + DMAR_FEDATA_REG);
4120 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4121 readl(iommu->reg + DMAR_FEADDR_REG);
4122 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4123 readl(iommu->reg + DMAR_FEUADDR_REG);
4124
4125 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4126 }
4127 return 0;
4128
4129 nomem:
4130 for_each_active_iommu(iommu, drhd)
4131 kfree(iommu->iommu_state);
4132
4133 return -ENOMEM;
4134 }
4135
4136 static void iommu_resume(void)
4137 {
4138 struct dmar_drhd_unit *drhd;
4139 struct intel_iommu *iommu = NULL;
4140 unsigned long flag;
4141
4142 if (init_iommu_hw()) {
4143 if (force_on)
4144 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4145 else
4146 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
4147 return;
4148 }
4149
4150 for_each_active_iommu(iommu, drhd) {
4151
4152 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4153
4154 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4155 iommu->reg + DMAR_FECTL_REG);
4156 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4157 iommu->reg + DMAR_FEDATA_REG);
4158 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4159 iommu->reg + DMAR_FEADDR_REG);
4160 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4161 iommu->reg + DMAR_FEUADDR_REG);
4162
4163 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4164 }
4165
4166 for_each_active_iommu(iommu, drhd)
4167 kfree(iommu->iommu_state);
4168 }
4169
4170 static struct syscore_ops iommu_syscore_ops = {
4171 .resume = iommu_resume,
4172 .suspend = iommu_suspend,
4173 };
4174
4175 static void __init init_iommu_pm_ops(void)
4176 {
4177 register_syscore_ops(&iommu_syscore_ops);
4178 }
4179
4180 #else
4181 static inline void init_iommu_pm_ops(void) {}
4182 #endif /* CONFIG_PM */
4183
4184
4185 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
4186 {
4187 struct acpi_dmar_reserved_memory *rmrr;
4188 struct dmar_rmrr_unit *rmrru;
4189 size_t length;
4190
4191 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4192 if (!rmrru)
4193 goto out;
4194
4195 rmrru->hdr = header;
4196 rmrr = (struct acpi_dmar_reserved_memory *)header;
4197 rmrru->base_address = rmrr->base_address;
4198 rmrru->end_address = rmrr->end_address;
4199
4200 length = rmrr->end_address - rmrr->base_address + 1;
4201
4202 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4203 ((void *)rmrr) + rmrr->header.length,
4204 &rmrru->devices_cnt);
4205 if (rmrru->devices_cnt && rmrru->devices == NULL)
4206 goto free_rmrru;
4207
4208 list_add(&rmrru->list, &dmar_rmrr_units);
4209
4210 return 0;
4211 free_rmrru:
4212 kfree(rmrru);
4213 out:
4214 return -ENOMEM;
4215 }
4216
4217 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4218 {
4219 struct dmar_atsr_unit *atsru;
4220 struct acpi_dmar_atsr *tmp;
4221
4222 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4223 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4224 if (atsr->segment != tmp->segment)
4225 continue;
4226 if (atsr->header.length != tmp->header.length)
4227 continue;
4228 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4229 return atsru;
4230 }
4231
4232 return NULL;
4233 }
4234
4235 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4236 {
4237 struct acpi_dmar_atsr *atsr;
4238 struct dmar_atsr_unit *atsru;
4239
4240 if (system_state >= SYSTEM_RUNNING && !intel_iommu_enabled)
4241 return 0;
4242
4243 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4244 atsru = dmar_find_atsr(atsr);
4245 if (atsru)
4246 return 0;
4247
4248 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4249 if (!atsru)
4250 return -ENOMEM;
4251
4252 /*
4253 * If memory is allocated from slab by ACPI _DSM method, we need to
4254 * copy the memory content because the memory buffer will be freed
4255 * on return.
4256 */
4257 atsru->hdr = (void *)(atsru + 1);
4258 memcpy(atsru->hdr, hdr, hdr->length);
4259 atsru->include_all = atsr->flags & 0x1;
4260 if (!atsru->include_all) {
4261 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4262 (void *)atsr + atsr->header.length,
4263 &atsru->devices_cnt);
4264 if (atsru->devices_cnt && atsru->devices == NULL) {
4265 kfree(atsru);
4266 return -ENOMEM;
4267 }
4268 }
4269
4270 list_add_rcu(&atsru->list, &dmar_atsr_units);
4271
4272 return 0;
4273 }
4274
4275 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4276 {
4277 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4278 kfree(atsru);
4279 }
4280
4281 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4282 {
4283 struct acpi_dmar_atsr *atsr;
4284 struct dmar_atsr_unit *atsru;
4285
4286 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4287 atsru = dmar_find_atsr(atsr);
4288 if (atsru) {
4289 list_del_rcu(&atsru->list);
4290 synchronize_rcu();
4291 intel_iommu_free_atsr(atsru);
4292 }
4293
4294 return 0;
4295 }
4296
4297 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4298 {
4299 int i;
4300 struct device *dev;
4301 struct acpi_dmar_atsr *atsr;
4302 struct dmar_atsr_unit *atsru;
4303
4304 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4305 atsru = dmar_find_atsr(atsr);
4306 if (!atsru)
4307 return 0;
4308
4309 if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
4310 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4311 i, dev)
4312 return -EBUSY;
4313 }
4314
4315 return 0;
4316 }
4317
4318 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4319 {
4320 int sp, ret = 0;
4321 struct intel_iommu *iommu = dmaru->iommu;
4322
4323 if (g_iommus[iommu->seq_id])
4324 return 0;
4325
4326 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4327 pr_warn("%s: Doesn't support hardware pass through.\n",
4328 iommu->name);
4329 return -ENXIO;
4330 }
4331 if (!ecap_sc_support(iommu->ecap) &&
4332 domain_update_iommu_snooping(iommu)) {
4333 pr_warn("%s: Doesn't support snooping.\n",
4334 iommu->name);
4335 return -ENXIO;
4336 }
4337 sp = domain_update_iommu_superpage(iommu) - 1;
4338 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4339 pr_warn("%s: Doesn't support large page.\n",
4340 iommu->name);
4341 return -ENXIO;
4342 }
4343
4344 /*
4345 * Disable translation if already enabled prior to OS handover.
4346 */
4347 if (iommu->gcmd & DMA_GCMD_TE)
4348 iommu_disable_translation(iommu);
4349
4350 g_iommus[iommu->seq_id] = iommu;
4351 ret = iommu_init_domains(iommu);
4352 if (ret == 0)
4353 ret = iommu_alloc_root_entry(iommu);
4354 if (ret)
4355 goto out;
4356
4357 #ifdef CONFIG_INTEL_IOMMU_SVM
4358 if (pasid_enabled(iommu))
4359 intel_svm_init(iommu);
4360 #endif
4361
4362 if (dmaru->ignored) {
4363 /*
4364 * we always have to disable PMRs or DMA may fail on this device
4365 */
4366 if (force_on)
4367 iommu_disable_protect_mem_regions(iommu);
4368 return 0;
4369 }
4370
4371 intel_iommu_init_qi(iommu);
4372 iommu_flush_write_buffer(iommu);
4373
4374 #ifdef CONFIG_INTEL_IOMMU_SVM
4375 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
4376 ret = intel_svm_enable_prq(iommu);
4377 if (ret)
4378 goto disable_iommu;
4379 }
4380 #endif
4381 ret = dmar_set_interrupt(iommu);
4382 if (ret)
4383 goto disable_iommu;
4384
4385 iommu_set_root_entry(iommu);
4386 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4387 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4388 iommu_enable_translation(iommu);
4389
4390 iommu_disable_protect_mem_regions(iommu);
4391 return 0;
4392
4393 disable_iommu:
4394 disable_dmar_iommu(iommu);
4395 out:
4396 free_dmar_iommu(iommu);
4397 return ret;
4398 }
4399
4400 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4401 {
4402 int ret = 0;
4403 struct intel_iommu *iommu = dmaru->iommu;
4404
4405 if (!intel_iommu_enabled)
4406 return 0;
4407 if (iommu == NULL)
4408 return -EINVAL;
4409
4410 if (insert) {
4411 ret = intel_iommu_add(dmaru);
4412 } else {
4413 disable_dmar_iommu(iommu);
4414 free_dmar_iommu(iommu);
4415 }
4416
4417 return ret;
4418 }
4419
4420 static void intel_iommu_free_dmars(void)
4421 {
4422 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4423 struct dmar_atsr_unit *atsru, *atsr_n;
4424
4425 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4426 list_del(&rmrru->list);
4427 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4428 kfree(rmrru);
4429 }
4430
4431 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4432 list_del(&atsru->list);
4433 intel_iommu_free_atsr(atsru);
4434 }
4435 }
4436
4437 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4438 {
4439 int i, ret = 1;
4440 struct pci_bus *bus;
4441 struct pci_dev *bridge = NULL;
4442 struct device *tmp;
4443 struct acpi_dmar_atsr *atsr;
4444 struct dmar_atsr_unit *atsru;
4445
4446 dev = pci_physfn(dev);
4447 for (bus = dev->bus; bus; bus = bus->parent) {
4448 bridge = bus->self;
4449 /* If it's an integrated device, allow ATS */
4450 if (!bridge)
4451 return 1;
4452 /* Connected via non-PCIe: no ATS */
4453 if (!pci_is_pcie(bridge) ||
4454 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4455 return 0;
4456 /* If we found the root port, look it up in the ATSR */
4457 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4458 break;
4459 }
4460
4461 rcu_read_lock();
4462 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4463 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4464 if (atsr->segment != pci_domain_nr(dev->bus))
4465 continue;
4466
4467 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4468 if (tmp == &bridge->dev)
4469 goto out;
4470
4471 if (atsru->include_all)
4472 goto out;
4473 }
4474 ret = 0;
4475 out:
4476 rcu_read_unlock();
4477
4478 return ret;
4479 }
4480
4481 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4482 {
4483 int ret = 0;
4484 struct dmar_rmrr_unit *rmrru;
4485 struct dmar_atsr_unit *atsru;
4486 struct acpi_dmar_atsr *atsr;
4487 struct acpi_dmar_reserved_memory *rmrr;
4488
4489 if (!intel_iommu_enabled && system_state >= SYSTEM_RUNNING)
4490 return 0;
4491
4492 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4493 rmrr = container_of(rmrru->hdr,
4494 struct acpi_dmar_reserved_memory, header);
4495 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4496 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4497 ((void *)rmrr) + rmrr->header.length,
4498 rmrr->segment, rmrru->devices,
4499 rmrru->devices_cnt);
4500 if(ret < 0)
4501 return ret;
4502 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4503 dmar_remove_dev_scope(info, rmrr->segment,
4504 rmrru->devices, rmrru->devices_cnt);
4505 }
4506 }
4507
4508 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4509 if (atsru->include_all)
4510 continue;
4511
4512 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4513 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4514 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4515 (void *)atsr + atsr->header.length,
4516 atsr->segment, atsru->devices,
4517 atsru->devices_cnt);
4518 if (ret > 0)
4519 break;
4520 else if(ret < 0)
4521 return ret;
4522 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4523 if (dmar_remove_dev_scope(info, atsr->segment,
4524 atsru->devices, atsru->devices_cnt))
4525 break;
4526 }
4527 }
4528
4529 return 0;
4530 }
4531
4532 /*
4533 * Here we only respond to action of unbound device from driver.
4534 *
4535 * Added device is not attached to its DMAR domain here yet. That will happen
4536 * when mapping the device to iova.
4537 */
4538 static int device_notifier(struct notifier_block *nb,
4539 unsigned long action, void *data)
4540 {
4541 struct device *dev = data;
4542 struct dmar_domain *domain;
4543
4544 if (iommu_dummy(dev))
4545 return 0;
4546
4547 if (action != BUS_NOTIFY_REMOVED_DEVICE)
4548 return 0;
4549
4550 domain = find_domain(dev);
4551 if (!domain)
4552 return 0;
4553
4554 dmar_remove_one_dev_info(domain, dev);
4555 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4556 domain_exit(domain);
4557
4558 return 0;
4559 }
4560
4561 static struct notifier_block device_nb = {
4562 .notifier_call = device_notifier,
4563 };
4564
4565 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4566 unsigned long val, void *v)
4567 {
4568 struct memory_notify *mhp = v;
4569 unsigned long long start, end;
4570 unsigned long start_vpfn, last_vpfn;
4571
4572 switch (val) {
4573 case MEM_GOING_ONLINE:
4574 start = mhp->start_pfn << PAGE_SHIFT;
4575 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4576 if (iommu_domain_identity_map(si_domain, start, end)) {
4577 pr_warn("Failed to build identity map for [%llx-%llx]\n",
4578 start, end);
4579 return NOTIFY_BAD;
4580 }
4581 break;
4582
4583 case MEM_OFFLINE:
4584 case MEM_CANCEL_ONLINE:
4585 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4586 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4587 while (start_vpfn <= last_vpfn) {
4588 struct iova *iova;
4589 struct dmar_drhd_unit *drhd;
4590 struct intel_iommu *iommu;
4591 struct page *freelist;
4592
4593 iova = find_iova(&si_domain->iovad, start_vpfn);
4594 if (iova == NULL) {
4595 pr_debug("Failed get IOVA for PFN %lx\n",
4596 start_vpfn);
4597 break;
4598 }
4599
4600 iova = split_and_remove_iova(&si_domain->iovad, iova,
4601 start_vpfn, last_vpfn);
4602 if (iova == NULL) {
4603 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4604 start_vpfn, last_vpfn);
4605 return NOTIFY_BAD;
4606 }
4607
4608 freelist = domain_unmap(si_domain, iova->pfn_lo,
4609 iova->pfn_hi);
4610
4611 rcu_read_lock();
4612 for_each_active_iommu(iommu, drhd)
4613 iommu_flush_iotlb_psi(iommu, si_domain,
4614 iova->pfn_lo, iova_size(iova),
4615 !freelist, 0);
4616 rcu_read_unlock();
4617 dma_free_pagelist(freelist);
4618
4619 start_vpfn = iova->pfn_hi + 1;
4620 free_iova_mem(iova);
4621 }
4622 break;
4623 }
4624
4625 return NOTIFY_OK;
4626 }
4627
4628 static struct notifier_block intel_iommu_memory_nb = {
4629 .notifier_call = intel_iommu_memory_notifier,
4630 .priority = 0
4631 };
4632
4633 static void free_all_cpu_cached_iovas(unsigned int cpu)
4634 {
4635 int i;
4636
4637 for (i = 0; i < g_num_of_iommus; i++) {
4638 struct intel_iommu *iommu = g_iommus[i];
4639 struct dmar_domain *domain;
4640 int did;
4641
4642 if (!iommu)
4643 continue;
4644
4645 for (did = 0; did < cap_ndoms(iommu->cap); did++) {
4646 domain = get_iommu_domain(iommu, (u16)did);
4647
4648 if (!domain)
4649 continue;
4650 free_cpu_cached_iovas(cpu, &domain->iovad);
4651 }
4652 }
4653 }
4654
4655 static int intel_iommu_cpu_dead(unsigned int cpu)
4656 {
4657 free_all_cpu_cached_iovas(cpu);
4658 return 0;
4659 }
4660
4661 static void intel_disable_iommus(void)
4662 {
4663 struct intel_iommu *iommu = NULL;
4664 struct dmar_drhd_unit *drhd;
4665
4666 for_each_iommu(iommu, drhd)
4667 iommu_disable_translation(iommu);
4668 }
4669
4670 static inline struct intel_iommu *dev_to_intel_iommu(struct device *dev)
4671 {
4672 struct iommu_device *iommu_dev = dev_to_iommu_device(dev);
4673
4674 return container_of(iommu_dev, struct intel_iommu, iommu);
4675 }
4676
4677 static ssize_t intel_iommu_show_version(struct device *dev,
4678 struct device_attribute *attr,
4679 char *buf)
4680 {
4681 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4682 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4683 return sprintf(buf, "%d:%d\n",
4684 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4685 }
4686 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4687
4688 static ssize_t intel_iommu_show_address(struct device *dev,
4689 struct device_attribute *attr,
4690 char *buf)
4691 {
4692 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4693 return sprintf(buf, "%llx\n", iommu->reg_phys);
4694 }
4695 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4696
4697 static ssize_t intel_iommu_show_cap(struct device *dev,
4698 struct device_attribute *attr,
4699 char *buf)
4700 {
4701 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4702 return sprintf(buf, "%llx\n", iommu->cap);
4703 }
4704 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4705
4706 static ssize_t intel_iommu_show_ecap(struct device *dev,
4707 struct device_attribute *attr,
4708 char *buf)
4709 {
4710 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4711 return sprintf(buf, "%llx\n", iommu->ecap);
4712 }
4713 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4714
4715 static ssize_t intel_iommu_show_ndoms(struct device *dev,
4716 struct device_attribute *attr,
4717 char *buf)
4718 {
4719 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4720 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4721 }
4722 static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4723
4724 static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4725 struct device_attribute *attr,
4726 char *buf)
4727 {
4728 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4729 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4730 cap_ndoms(iommu->cap)));
4731 }
4732 static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4733
4734 static struct attribute *intel_iommu_attrs[] = {
4735 &dev_attr_version.attr,
4736 &dev_attr_address.attr,
4737 &dev_attr_cap.attr,
4738 &dev_attr_ecap.attr,
4739 &dev_attr_domains_supported.attr,
4740 &dev_attr_domains_used.attr,
4741 NULL,
4742 };
4743
4744 static struct attribute_group intel_iommu_group = {
4745 .name = "intel-iommu",
4746 .attrs = intel_iommu_attrs,
4747 };
4748
4749 const struct attribute_group *intel_iommu_groups[] = {
4750 &intel_iommu_group,
4751 NULL,
4752 };
4753
4754 int __init intel_iommu_init(void)
4755 {
4756 int ret = -ENODEV;
4757 struct dmar_drhd_unit *drhd;
4758 struct intel_iommu *iommu;
4759
4760 /* VT-d is required for a TXT/tboot launch, so enforce that */
4761 force_on = tboot_force_iommu();
4762
4763 if (iommu_init_mempool()) {
4764 if (force_on)
4765 panic("tboot: Failed to initialize iommu memory\n");
4766 return -ENOMEM;
4767 }
4768
4769 down_write(&dmar_global_lock);
4770 if (dmar_table_init()) {
4771 if (force_on)
4772 panic("tboot: Failed to initialize DMAR table\n");
4773 goto out_free_dmar;
4774 }
4775
4776 if (dmar_dev_scope_init() < 0) {
4777 if (force_on)
4778 panic("tboot: Failed to initialize DMAR device scope\n");
4779 goto out_free_dmar;
4780 }
4781
4782 up_write(&dmar_global_lock);
4783
4784 /*
4785 * The bus notifier takes the dmar_global_lock, so lockdep will
4786 * complain later when we register it under the lock.
4787 */
4788 dmar_register_bus_notifier();
4789
4790 down_write(&dmar_global_lock);
4791
4792 if (no_iommu || dmar_disabled) {
4793 /*
4794 * We exit the function here to ensure IOMMU's remapping and
4795 * mempool aren't setup, which means that the IOMMU's PMRs
4796 * won't be disabled via the call to init_dmars(). So disable
4797 * it explicitly here. The PMRs were setup by tboot prior to
4798 * calling SENTER, but the kernel is expected to reset/tear
4799 * down the PMRs.
4800 */
4801 if (intel_iommu_tboot_noforce) {
4802 for_each_iommu(iommu, drhd)
4803 iommu_disable_protect_mem_regions(iommu);
4804 }
4805
4806 /*
4807 * Make sure the IOMMUs are switched off, even when we
4808 * boot into a kexec kernel and the previous kernel left
4809 * them enabled
4810 */
4811 intel_disable_iommus();
4812 goto out_free_dmar;
4813 }
4814
4815 if (list_empty(&dmar_rmrr_units))
4816 pr_info("No RMRR found\n");
4817
4818 if (list_empty(&dmar_atsr_units))
4819 pr_info("No ATSR found\n");
4820
4821 if (dmar_init_reserved_ranges()) {
4822 if (force_on)
4823 panic("tboot: Failed to reserve iommu ranges\n");
4824 goto out_free_reserved_range;
4825 }
4826
4827 if (dmar_map_gfx)
4828 intel_iommu_gfx_mapped = 1;
4829
4830 init_no_remapping_devices();
4831
4832 ret = init_dmars();
4833 if (ret) {
4834 if (force_on)
4835 panic("tboot: Failed to initialize DMARs\n");
4836 pr_err("Initialization failed\n");
4837 goto out_free_reserved_range;
4838 }
4839 up_write(&dmar_global_lock);
4840 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4841
4842 #if defined(CONFIG_X86) && defined(CONFIG_SWIOTLB)
4843 swiotlb = 0;
4844 #endif
4845 dma_ops = &intel_dma_ops;
4846
4847 init_iommu_pm_ops();
4848
4849 for_each_active_iommu(iommu, drhd) {
4850 iommu_device_sysfs_add(&iommu->iommu, NULL,
4851 intel_iommu_groups,
4852 "%s", iommu->name);
4853 iommu_device_set_ops(&iommu->iommu, &intel_iommu_ops);
4854 iommu_device_register(&iommu->iommu);
4855 }
4856
4857 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4858 bus_register_notifier(&pci_bus_type, &device_nb);
4859 if (si_domain && !hw_pass_through)
4860 register_memory_notifier(&intel_iommu_memory_nb);
4861 cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL,
4862 intel_iommu_cpu_dead);
4863 intel_iommu_enabled = 1;
4864
4865 return 0;
4866
4867 out_free_reserved_range:
4868 put_iova_domain(&reserved_iova_list);
4869 out_free_dmar:
4870 intel_iommu_free_dmars();
4871 up_write(&dmar_global_lock);
4872 iommu_exit_mempool();
4873 return ret;
4874 }
4875
4876 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4877 {
4878 struct intel_iommu *iommu = opaque;
4879
4880 domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4881 return 0;
4882 }
4883
4884 /*
4885 * NB - intel-iommu lacks any sort of reference counting for the users of
4886 * dependent devices. If multiple endpoints have intersecting dependent
4887 * devices, unbinding the driver from any one of them will possibly leave
4888 * the others unable to operate.
4889 */
4890 static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
4891 {
4892 if (!iommu || !dev || !dev_is_pci(dev))
4893 return;
4894
4895 pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
4896 }
4897
4898 static void __dmar_remove_one_dev_info(struct device_domain_info *info)
4899 {
4900 struct intel_iommu *iommu;
4901 unsigned long flags;
4902
4903 assert_spin_locked(&device_domain_lock);
4904
4905 if (WARN_ON(!info))
4906 return;
4907
4908 iommu = info->iommu;
4909
4910 if (info->dev) {
4911 iommu_disable_dev_iotlb(info);
4912 domain_context_clear(iommu, info->dev);
4913 intel_pasid_free_table(info->dev);
4914 }
4915
4916 unlink_domain_info(info);
4917
4918 spin_lock_irqsave(&iommu->lock, flags);
4919 domain_detach_iommu(info->domain, iommu);
4920 spin_unlock_irqrestore(&iommu->lock, flags);
4921
4922 free_devinfo_mem(info);
4923 }
4924
4925 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4926 struct device *dev)
4927 {
4928 struct device_domain_info *info;
4929 unsigned long flags;
4930
4931 spin_lock_irqsave(&device_domain_lock, flags);
4932 info = dev->archdata.iommu;
4933 __dmar_remove_one_dev_info(info);
4934 spin_unlock_irqrestore(&device_domain_lock, flags);
4935 }
4936
4937 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4938 {
4939 int adjust_width;
4940
4941 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
4942 domain_reserve_special_ranges(domain);
4943
4944 /* calculate AGAW */
4945 domain->gaw = guest_width;
4946 adjust_width = guestwidth_to_adjustwidth(guest_width);
4947 domain->agaw = width_to_agaw(adjust_width);
4948
4949 domain->iommu_coherency = 0;
4950 domain->iommu_snooping = 0;
4951 domain->iommu_superpage = 0;
4952 domain->max_addr = 0;
4953
4954 /* always allocate the top pgd */
4955 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
4956 if (!domain->pgd)
4957 return -ENOMEM;
4958 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4959 return 0;
4960 }
4961
4962 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
4963 {
4964 struct dmar_domain *dmar_domain;
4965 struct iommu_domain *domain;
4966
4967 if (type != IOMMU_DOMAIN_UNMANAGED)
4968 return NULL;
4969
4970 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
4971 if (!dmar_domain) {
4972 pr_err("Can't allocate dmar_domain\n");
4973 return NULL;
4974 }
4975 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4976 pr_err("Domain initialization failed\n");
4977 domain_exit(dmar_domain);
4978 return NULL;
4979 }
4980 domain_update_iommu_cap(dmar_domain);
4981
4982 domain = &dmar_domain->domain;
4983 domain->geometry.aperture_start = 0;
4984 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4985 domain->geometry.force_aperture = true;
4986
4987 return domain;
4988 }
4989
4990 static void intel_iommu_domain_free(struct iommu_domain *domain)
4991 {
4992 domain_exit(to_dmar_domain(domain));
4993 }
4994
4995 static int intel_iommu_attach_device(struct iommu_domain *domain,
4996 struct device *dev)
4997 {
4998 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4999 struct intel_iommu *iommu;
5000 int addr_width;
5001 u8 bus, devfn;
5002
5003 if (device_is_rmrr_locked(dev)) {
5004 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
5005 return -EPERM;
5006 }
5007
5008 /* normally dev is not mapped */
5009 if (unlikely(domain_context_mapped(dev))) {
5010 struct dmar_domain *old_domain;
5011
5012 old_domain = find_domain(dev);
5013 if (old_domain) {
5014 rcu_read_lock();
5015 dmar_remove_one_dev_info(old_domain, dev);
5016 rcu_read_unlock();
5017
5018 if (!domain_type_is_vm_or_si(old_domain) &&
5019 list_empty(&old_domain->devices))
5020 domain_exit(old_domain);
5021 }
5022 }
5023
5024 iommu = device_to_iommu(dev, &bus, &devfn);
5025 if (!iommu)
5026 return -ENODEV;
5027
5028 /* check if this iommu agaw is sufficient for max mapped address */
5029 addr_width = agaw_to_width(iommu->agaw);
5030 if (addr_width > cap_mgaw(iommu->cap))
5031 addr_width = cap_mgaw(iommu->cap);
5032
5033 if (dmar_domain->max_addr > (1LL << addr_width)) {
5034 pr_err("%s: iommu width (%d) is not "
5035 "sufficient for the mapped address (%llx)\n",
5036 __func__, addr_width, dmar_domain->max_addr);
5037 return -EFAULT;
5038 }
5039 dmar_domain->gaw = addr_width;
5040
5041 /*
5042 * Knock out extra levels of page tables if necessary
5043 */
5044 while (iommu->agaw < dmar_domain->agaw) {
5045 struct dma_pte *pte;
5046
5047 pte = dmar_domain->pgd;
5048 if (dma_pte_present(pte)) {
5049 dmar_domain->pgd = (struct dma_pte *)
5050 phys_to_virt(dma_pte_addr(pte));
5051 free_pgtable_page(pte);
5052 }
5053 dmar_domain->agaw--;
5054 }
5055
5056 return domain_add_dev_info(dmar_domain, dev);
5057 }
5058
5059 static void intel_iommu_detach_device(struct iommu_domain *domain,
5060 struct device *dev)
5061 {
5062 dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
5063 }
5064
5065 static int intel_iommu_map(struct iommu_domain *domain,
5066 unsigned long iova, phys_addr_t hpa,
5067 size_t size, int iommu_prot)
5068 {
5069 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5070 u64 max_addr;
5071 int prot = 0;
5072 int ret;
5073
5074 if (iommu_prot & IOMMU_READ)
5075 prot |= DMA_PTE_READ;
5076 if (iommu_prot & IOMMU_WRITE)
5077 prot |= DMA_PTE_WRITE;
5078 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
5079 prot |= DMA_PTE_SNP;
5080
5081 max_addr = iova + size;
5082 if (dmar_domain->max_addr < max_addr) {
5083 u64 end;
5084
5085 /* check if minimum agaw is sufficient for mapped address */
5086 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
5087 if (end < max_addr) {
5088 pr_err("%s: iommu width (%d) is not "
5089 "sufficient for the mapped address (%llx)\n",
5090 __func__, dmar_domain->gaw, max_addr);
5091 return -EFAULT;
5092 }
5093 dmar_domain->max_addr = max_addr;
5094 }
5095 /* Round up size to next multiple of PAGE_SIZE, if it and
5096 the low bits of hpa would take us onto the next page */
5097 size = aligned_nrpages(hpa, size);
5098 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
5099 hpa >> VTD_PAGE_SHIFT, size, prot);
5100 return ret;
5101 }
5102
5103 static size_t intel_iommu_unmap(struct iommu_domain *domain,
5104 unsigned long iova, size_t size)
5105 {
5106 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5107 struct page *freelist = NULL;
5108 unsigned long start_pfn, last_pfn;
5109 unsigned int npages;
5110 int iommu_id, level = 0;
5111
5112 /* Cope with horrid API which requires us to unmap more than the
5113 size argument if it happens to be a large-page mapping. */
5114 BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
5115
5116 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
5117 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
5118
5119 start_pfn = iova >> VTD_PAGE_SHIFT;
5120 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5121
5122 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
5123
5124 npages = last_pfn - start_pfn + 1;
5125
5126 for_each_domain_iommu(iommu_id, dmar_domain)
5127 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5128 start_pfn, npages, !freelist, 0);
5129
5130 dma_free_pagelist(freelist);
5131
5132 if (dmar_domain->max_addr == iova + size)
5133 dmar_domain->max_addr = iova;
5134
5135 return size;
5136 }
5137
5138 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
5139 dma_addr_t iova)
5140 {
5141 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5142 struct dma_pte *pte;
5143 int level = 0;
5144 u64 phys = 0;
5145
5146 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
5147 if (pte && dma_pte_present(pte))
5148 phys = dma_pte_addr(pte) +
5149 (iova & (BIT_MASK(level_to_offset_bits(level) +
5150 VTD_PAGE_SHIFT) - 1));
5151
5152 return phys;
5153 }
5154
5155 static bool intel_iommu_capable(enum iommu_cap cap)
5156 {
5157 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5158 return domain_update_iommu_snooping(NULL) == 1;
5159 if (cap == IOMMU_CAP_INTR_REMAP)
5160 return irq_remapping_enabled == 1;
5161
5162 return false;
5163 }
5164
5165 static int intel_iommu_add_device(struct device *dev)
5166 {
5167 struct intel_iommu *iommu;
5168 struct iommu_group *group;
5169 u8 bus, devfn;
5170
5171 iommu = device_to_iommu(dev, &bus, &devfn);
5172 if (!iommu)
5173 return -ENODEV;
5174
5175 iommu_device_link(&iommu->iommu, dev);
5176
5177 group = iommu_group_get_for_dev(dev);
5178
5179 if (IS_ERR(group))
5180 return PTR_ERR(group);
5181
5182 iommu_group_put(group);
5183 return 0;
5184 }
5185
5186 static void intel_iommu_remove_device(struct device *dev)
5187 {
5188 struct intel_iommu *iommu;
5189 u8 bus, devfn;
5190
5191 iommu = device_to_iommu(dev, &bus, &devfn);
5192 if (!iommu)
5193 return;
5194
5195 iommu_group_remove_device(dev);
5196
5197 iommu_device_unlink(&iommu->iommu, dev);
5198 }
5199
5200 static void intel_iommu_get_resv_regions(struct device *device,
5201 struct list_head *head)
5202 {
5203 int prot = DMA_PTE_READ | DMA_PTE_WRITE;
5204 struct iommu_resv_region *reg;
5205 struct dmar_rmrr_unit *rmrr;
5206 struct device *i_dev;
5207 int i;
5208
5209 down_read(&dmar_global_lock);
5210 for_each_rmrr_units(rmrr) {
5211 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
5212 i, i_dev) {
5213 struct iommu_resv_region *resv;
5214 size_t length;
5215
5216 if (i_dev != device)
5217 continue;
5218
5219 length = rmrr->end_address - rmrr->base_address + 1;
5220 resv = iommu_alloc_resv_region(rmrr->base_address,
5221 length, prot,
5222 IOMMU_RESV_DIRECT);
5223 if (!resv)
5224 break;
5225
5226 list_add_tail(&resv->list, head);
5227 }
5228 }
5229 up_read(&dmar_global_lock);
5230
5231 reg = iommu_alloc_resv_region(IOAPIC_RANGE_START,
5232 IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1,
5233 0, IOMMU_RESV_MSI);
5234 if (!reg)
5235 return;
5236 list_add_tail(&reg->list, head);
5237 }
5238
5239 static void intel_iommu_put_resv_regions(struct device *dev,
5240 struct list_head *head)
5241 {
5242 struct iommu_resv_region *entry, *next;
5243
5244 list_for_each_entry_safe(entry, next, head, list)
5245 kfree(entry);
5246 }
5247
5248 #ifdef CONFIG_INTEL_IOMMU_SVM
5249 #define MAX_NR_PASID_BITS (20)
5250 static inline unsigned long intel_iommu_get_pts(struct device *dev)
5251 {
5252 int pts, max_pasid;
5253
5254 max_pasid = intel_pasid_get_dev_max_id(dev);
5255 pts = find_first_bit((unsigned long *)&max_pasid, MAX_NR_PASID_BITS);
5256 if (pts < 5)
5257 return 0;
5258
5259 return pts - 5;
5260 }
5261
5262 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5263 {
5264 struct device_domain_info *info;
5265 struct context_entry *context;
5266 struct dmar_domain *domain;
5267 unsigned long flags;
5268 u64 ctx_lo;
5269 int ret;
5270
5271 domain = get_valid_domain_for_dev(sdev->dev);
5272 if (!domain)
5273 return -EINVAL;
5274
5275 spin_lock_irqsave(&device_domain_lock, flags);
5276 spin_lock(&iommu->lock);
5277
5278 ret = -EINVAL;
5279 info = sdev->dev->archdata.iommu;
5280 if (!info || !info->pasid_supported)
5281 goto out;
5282
5283 context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5284 if (WARN_ON(!context))
5285 goto out;
5286
5287 ctx_lo = context[0].lo;
5288
5289 sdev->did = domain->iommu_did[iommu->seq_id];
5290 sdev->sid = PCI_DEVID(info->bus, info->devfn);
5291
5292 if (!(ctx_lo & CONTEXT_PASIDE)) {
5293 if (iommu->pasid_state_table)
5294 context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
5295 context[1].lo = (u64)virt_to_phys(info->pasid_table->table) |
5296 intel_iommu_get_pts(sdev->dev);
5297
5298 wmb();
5299 /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both
5300 * extended to permit requests-with-PASID if the PASIDE bit
5301 * is set. which makes sense. For CONTEXT_TT_PASS_THROUGH,
5302 * however, the PASIDE bit is ignored and requests-with-PASID
5303 * are unconditionally blocked. Which makes less sense.
5304 * So convert from CONTEXT_TT_PASS_THROUGH to one of the new
5305 * "guest mode" translation types depending on whether ATS
5306 * is available or not. Annoyingly, we can't use the new
5307 * modes *unless* PASIDE is set. */
5308 if ((ctx_lo & CONTEXT_TT_MASK) == (CONTEXT_TT_PASS_THROUGH << 2)) {
5309 ctx_lo &= ~CONTEXT_TT_MASK;
5310 if (info->ats_supported)
5311 ctx_lo |= CONTEXT_TT_PT_PASID_DEV_IOTLB << 2;
5312 else
5313 ctx_lo |= CONTEXT_TT_PT_PASID << 2;
5314 }
5315 ctx_lo |= CONTEXT_PASIDE;
5316 if (iommu->pasid_state_table)
5317 ctx_lo |= CONTEXT_DINVE;
5318 if (info->pri_supported)
5319 ctx_lo |= CONTEXT_PRS;
5320 context[0].lo = ctx_lo;
5321 wmb();
5322 iommu->flush.flush_context(iommu, sdev->did, sdev->sid,
5323 DMA_CCMD_MASK_NOBIT,
5324 DMA_CCMD_DEVICE_INVL);
5325 }
5326
5327 /* Enable PASID support in the device, if it wasn't already */
5328 if (!info->pasid_enabled)
5329 iommu_enable_dev_iotlb(info);
5330
5331 if (info->ats_enabled) {
5332 sdev->dev_iotlb = 1;
5333 sdev->qdep = info->ats_qdep;
5334 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5335 sdev->qdep = 0;
5336 }
5337 ret = 0;
5338
5339 out:
5340 spin_unlock(&iommu->lock);
5341 spin_unlock_irqrestore(&device_domain_lock, flags);
5342
5343 return ret;
5344 }
5345
5346 struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5347 {
5348 struct intel_iommu *iommu;
5349 u8 bus, devfn;
5350
5351 if (iommu_dummy(dev)) {
5352 dev_warn(dev,
5353 "No IOMMU translation for device; cannot enable SVM\n");
5354 return NULL;
5355 }
5356
5357 iommu = device_to_iommu(dev, &bus, &devfn);
5358 if ((!iommu)) {
5359 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
5360 return NULL;
5361 }
5362
5363 return iommu;
5364 }
5365 #endif /* CONFIG_INTEL_IOMMU_SVM */
5366
5367 const struct iommu_ops intel_iommu_ops = {
5368 .capable = intel_iommu_capable,
5369 .domain_alloc = intel_iommu_domain_alloc,
5370 .domain_free = intel_iommu_domain_free,
5371 .attach_dev = intel_iommu_attach_device,
5372 .detach_dev = intel_iommu_detach_device,
5373 .map = intel_iommu_map,
5374 .unmap = intel_iommu_unmap,
5375 .iova_to_phys = intel_iommu_iova_to_phys,
5376 .add_device = intel_iommu_add_device,
5377 .remove_device = intel_iommu_remove_device,
5378 .get_resv_regions = intel_iommu_get_resv_regions,
5379 .put_resv_regions = intel_iommu_put_resv_regions,
5380 .device_group = pci_device_group,
5381 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
5382 };
5383
5384 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5385 {
5386 /* G4x/GM45 integrated gfx dmar support is totally busted. */
5387 pr_info("Disabling IOMMU for graphics on this chipset\n");
5388 dmar_map_gfx = 0;
5389 }
5390
5391 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5392 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5393 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5394 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5395 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5396 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5397 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5398
5399 static void quirk_iommu_rwbf(struct pci_dev *dev)
5400 {
5401 /*
5402 * Mobile 4 Series Chipset neglects to set RWBF capability,
5403 * but needs it. Same seems to hold for the desktop versions.
5404 */
5405 pr_info("Forcing write-buffer flush capability\n");
5406 rwbf_quirk = 1;
5407 }
5408
5409 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
5410 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5411 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5412 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5413 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5414 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5415 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
5416
5417 #define GGC 0x52
5418 #define GGC_MEMORY_SIZE_MASK (0xf << 8)
5419 #define GGC_MEMORY_SIZE_NONE (0x0 << 8)
5420 #define GGC_MEMORY_SIZE_1M (0x1 << 8)
5421 #define GGC_MEMORY_SIZE_2M (0x3 << 8)
5422 #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
5423 #define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
5424 #define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
5425 #define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
5426
5427 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
5428 {
5429 unsigned short ggc;
5430
5431 if (pci_read_config_word(dev, GGC, &ggc))
5432 return;
5433
5434 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
5435 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
5436 dmar_map_gfx = 0;
5437 } else if (dmar_map_gfx) {
5438 /* we have to ensure the gfx device is idle before we flush */
5439 pr_info("Disabling batched IOTLB flush on Ironlake\n");
5440 intel_iommu_strict = 1;
5441 }
5442 }
5443 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5444 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5445 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5446 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5447
5448 /* On Tylersburg chipsets, some BIOSes have been known to enable the
5449 ISOCH DMAR unit for the Azalia sound device, but not give it any
5450 TLB entries, which causes it to deadlock. Check for that. We do
5451 this in a function called from init_dmars(), instead of in a PCI
5452 quirk, because we don't want to print the obnoxious "BIOS broken"
5453 message if VT-d is actually disabled.
5454 */
5455 static void __init check_tylersburg_isoch(void)
5456 {
5457 struct pci_dev *pdev;
5458 uint32_t vtisochctrl;
5459
5460 /* If there's no Azalia in the system anyway, forget it. */
5461 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5462 if (!pdev)
5463 return;
5464 pci_dev_put(pdev);
5465
5466 /* System Management Registers. Might be hidden, in which case
5467 we can't do the sanity check. But that's OK, because the
5468 known-broken BIOSes _don't_ actually hide it, so far. */
5469 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5470 if (!pdev)
5471 return;
5472
5473 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5474 pci_dev_put(pdev);
5475 return;
5476 }
5477
5478 pci_dev_put(pdev);
5479
5480 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5481 if (vtisochctrl & 1)
5482 return;
5483
5484 /* Drop all bits other than the number of TLB entries */
5485 vtisochctrl &= 0x1c;
5486
5487 /* If we have the recommended number of TLB entries (16), fine. */
5488 if (vtisochctrl == 0x10)
5489 return;
5490
5491 /* Zero TLB entries? You get to ride the short bus to school. */
5492 if (!vtisochctrl) {
5493 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5494 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5495 dmi_get_system_info(DMI_BIOS_VENDOR),
5496 dmi_get_system_info(DMI_BIOS_VERSION),
5497 dmi_get_system_info(DMI_PRODUCT_VERSION));
5498 iommu_identity_mapping |= IDENTMAP_AZALIA;
5499 return;
5500 }
5501
5502 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
5503 vtisochctrl);
5504 }
Linux with added FPC-III support