?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
treewide: remove redundant IS_ERR() before error code check
[linux/fpc-iii.git] / drivers / iommu / intel-iommu.c
blob35a4a3abedc6a611f304d6b79961c0fd16481f5b
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright © 2006-2014 Intel Corporation.
4 *
5 * Authors: David Woodhouse <dwmw2@infradead.org>,
6 * Ashok Raj <ashok.raj@intel.com>,
7 * Shaohua Li <shaohua.li@intel.com>,
8 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
9 * Fenghua Yu <fenghua.yu@intel.com>
10 * Joerg Roedel <jroedel@suse.de>
11 */
12
13 #define pr_fmt(fmt) "DMAR: " fmt
14 #define dev_fmt(fmt) pr_fmt(fmt)
15
16 #include <linux/init.h>
17 #include <linux/bitmap.h>
18 #include <linux/debugfs.h>
19 #include <linux/export.h>
20 #include <linux/slab.h>
21 #include <linux/irq.h>
22 #include <linux/interrupt.h>
23 #include <linux/spinlock.h>
24 #include <linux/pci.h>
25 #include <linux/dmar.h>
26 #include <linux/dma-mapping.h>
27 #include <linux/mempool.h>
28 #include <linux/memory.h>
29 #include <linux/cpu.h>
30 #include <linux/timer.h>
31 #include <linux/io.h>
32 #include <linux/iova.h>
33 #include <linux/iommu.h>
34 #include <linux/intel-iommu.h>
35 #include <linux/syscore_ops.h>
36 #include <linux/tboot.h>
37 #include <linux/dmi.h>
38 #include <linux/pci-ats.h>
39 #include <linux/memblock.h>
40 #include <linux/dma-contiguous.h>
41 #include <linux/dma-direct.h>
42 #include <linux/crash_dump.h>
43 #include <linux/numa.h>
44 #include <linux/swiotlb.h>
45 #include <asm/irq_remapping.h>
46 #include <asm/cacheflush.h>
47 #include <asm/iommu.h>
48 #include <trace/events/intel_iommu.h>
49
50 #include "irq_remapping.h"
51 #include "intel-pasid.h"
52
53 #define ROOT_SIZE VTD_PAGE_SIZE
54 #define CONTEXT_SIZE VTD_PAGE_SIZE
55
56 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
57 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
58 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
59 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
60
61 #define IOAPIC_RANGE_START (0xfee00000)
62 #define IOAPIC_RANGE_END (0xfeefffff)
63 #define IOVA_START_ADDR (0x1000)
64
65 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 57
66
67 #define MAX_AGAW_WIDTH 64
68 #define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
69
70 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
71 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
72
73 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
74 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
75 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
76 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
77 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
78
79 /* IO virtual address start page frame number */
80 #define IOVA_START_PFN (1)
81
82 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
83
84 /* page table handling */
85 #define LEVEL_STRIDE (9)
86 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
87
88 /*
89 * This bitmap is used to advertise the page sizes our hardware support
90 * to the IOMMU core, which will then use this information to split
91 * physically contiguous memory regions it is mapping into page sizes
92 * that we support.
93 *
94 * Traditionally the IOMMU core just handed us the mappings directly,
95 * after making sure the size is an order of a 4KiB page and that the
96 * mapping has natural alignment.
97 *
98 * To retain this behavior, we currently advertise that we support
99 * all page sizes that are an order of 4KiB.
100 *
101 * If at some point we'd like to utilize the IOMMU core's new behavior,
102 * we could change this to advertise the real page sizes we support.
103 */
104 #define INTEL_IOMMU_PGSIZES (~0xFFFUL)
105
106 static inline int agaw_to_level(int agaw)
107 {
108 return agaw + 2;
109 }
110
111 static inline int agaw_to_width(int agaw)
112 {
113 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
114 }
115
116 static inline int width_to_agaw(int width)
117 {
118 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
119 }
120
121 static inline unsigned int level_to_offset_bits(int level)
122 {
123 return (level - 1) * LEVEL_STRIDE;
124 }
125
126 static inline int pfn_level_offset(unsigned long pfn, int level)
127 {
128 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
129 }
130
131 static inline unsigned long level_mask(int level)
132 {
133 return -1UL << level_to_offset_bits(level);
134 }
135
136 static inline unsigned long level_size(int level)
137 {
138 return 1UL << level_to_offset_bits(level);
139 }
140
141 static inline unsigned long align_to_level(unsigned long pfn, int level)
142 {
143 return (pfn + level_size(level) - 1) & level_mask(level);
144 }
145
146 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
147 {
148 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
149 }
150
151 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
152 are never going to work. */
153 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
154 {
155 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
156 }
157
158 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
159 {
160 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
161 }
162 static inline unsigned long page_to_dma_pfn(struct page *pg)
163 {
164 return mm_to_dma_pfn(page_to_pfn(pg));
165 }
166 static inline unsigned long virt_to_dma_pfn(void *p)
167 {
168 return page_to_dma_pfn(virt_to_page(p));
169 }
170
171 /* global iommu list, set NULL for ignored DMAR units */
172 static struct intel_iommu **g_iommus;
173
174 static void __init check_tylersburg_isoch(void);
175 static int rwbf_quirk;
176
177 /*
178 * set to 1 to panic kernel if can't successfully enable VT-d
179 * (used when kernel is launched w/ TXT)
180 */
181 static int force_on = 0;
182 int intel_iommu_tboot_noforce;
183 static int no_platform_optin;
184
185 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
186
187 /*
188 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
189 * if marked present.
190 */
191 static phys_addr_t root_entry_lctp(struct root_entry *re)
192 {
193 if (!(re->lo & 1))
194 return 0;
195
196 return re->lo & VTD_PAGE_MASK;
197 }
198
199 /*
200 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
201 * if marked present.
202 */
203 static phys_addr_t root_entry_uctp(struct root_entry *re)
204 {
205 if (!(re->hi & 1))
206 return 0;
207
208 return re->hi & VTD_PAGE_MASK;
209 }
210
211 static inline void context_clear_pasid_enable(struct context_entry *context)
212 {
213 context->lo &= ~(1ULL << 11);
214 }
215
216 static inline bool context_pasid_enabled(struct context_entry *context)
217 {
218 return !!(context->lo & (1ULL << 11));
219 }
220
221 static inline void context_set_copied(struct context_entry *context)
222 {
223 context->hi |= (1ull << 3);
224 }
225
226 static inline bool context_copied(struct context_entry *context)
227 {
228 return !!(context->hi & (1ULL << 3));
229 }
230
231 static inline bool __context_present(struct context_entry *context)
232 {
233 return (context->lo & 1);
234 }
235
236 bool context_present(struct context_entry *context)
237 {
238 return context_pasid_enabled(context) ?
239 __context_present(context) :
240 __context_present(context) && !context_copied(context);
241 }
242
243 static inline void context_set_present(struct context_entry *context)
244 {
245 context->lo |= 1;
246 }
247
248 static inline void context_set_fault_enable(struct context_entry *context)
249 {
250 context->lo &= (((u64)-1) << 2) | 1;
251 }
252
253 static inline void context_set_translation_type(struct context_entry *context,
254 unsigned long value)
255 {
256 context->lo &= (((u64)-1) << 4) | 3;
257 context->lo |= (value & 3) << 2;
258 }
259
260 static inline void context_set_address_root(struct context_entry *context,
261 unsigned long value)
262 {
263 context->lo &= ~VTD_PAGE_MASK;
264 context->lo |= value & VTD_PAGE_MASK;
265 }
266
267 static inline void context_set_address_width(struct context_entry *context,
268 unsigned long value)
269 {
270 context->hi |= value & 7;
271 }
272
273 static inline void context_set_domain_id(struct context_entry *context,
274 unsigned long value)
275 {
276 context->hi |= (value & ((1 << 16) - 1)) << 8;
277 }
278
279 static inline int context_domain_id(struct context_entry *c)
280 {
281 return((c->hi >> 8) & 0xffff);
282 }
283
284 static inline void context_clear_entry(struct context_entry *context)
285 {
286 context->lo = 0;
287 context->hi = 0;
288 }
289
290 /*
291 * This domain is a statically identity mapping domain.
292 * 1. This domain creats a static 1:1 mapping to all usable memory.
293 * 2. It maps to each iommu if successful.
294 * 3. Each iommu mapps to this domain if successful.
295 */
296 static struct dmar_domain *si_domain;
297 static int hw_pass_through = 1;
298
299 /* si_domain contains mulitple devices */
300 #define DOMAIN_FLAG_STATIC_IDENTITY BIT(0)
301
302 /*
303 * This is a DMA domain allocated through the iommu domain allocation
304 * interface. But one or more devices belonging to this domain have
305 * been chosen to use a private domain. We should avoid to use the
306 * map/unmap/iova_to_phys APIs on it.
307 */
308 #define DOMAIN_FLAG_LOSE_CHILDREN BIT(1)
309
310 #define for_each_domain_iommu(idx, domain) \
311 for (idx = 0; idx < g_num_of_iommus; idx++) \
312 if (domain->iommu_refcnt[idx])
313
314 struct dmar_rmrr_unit {
315 struct list_head list; /* list of rmrr units */
316 struct acpi_dmar_header *hdr; /* ACPI header */
317 u64 base_address; /* reserved base address*/
318 u64 end_address; /* reserved end address */
319 struct dmar_dev_scope *devices; /* target devices */
320 int devices_cnt; /* target device count */
321 };
322
323 struct dmar_atsr_unit {
324 struct list_head list; /* list of ATSR units */
325 struct acpi_dmar_header *hdr; /* ACPI header */
326 struct dmar_dev_scope *devices; /* target devices */
327 int devices_cnt; /* target device count */
328 u8 include_all:1; /* include all ports */
329 };
330
331 static LIST_HEAD(dmar_atsr_units);
332 static LIST_HEAD(dmar_rmrr_units);
333
334 #define for_each_rmrr_units(rmrr) \
335 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
336
337 /* bitmap for indexing intel_iommus */
338 static int g_num_of_iommus;
339
340 static void domain_exit(struct dmar_domain *domain);
341 static void domain_remove_dev_info(struct dmar_domain *domain);
342 static void dmar_remove_one_dev_info(struct device *dev);
343 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
344 static void domain_context_clear(struct intel_iommu *iommu,
345 struct device *dev);
346 static int domain_detach_iommu(struct dmar_domain *domain,
347 struct intel_iommu *iommu);
348 static bool device_is_rmrr_locked(struct device *dev);
349 static int intel_iommu_attach_device(struct iommu_domain *domain,
350 struct device *dev);
351 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
352 dma_addr_t iova);
353
354 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
355 int dmar_disabled = 0;
356 #else
357 int dmar_disabled = 1;
358 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
359
360 int intel_iommu_sm;
361 int intel_iommu_enabled = 0;
362 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
363
364 static int dmar_map_gfx = 1;
365 static int dmar_forcedac;
366 static int intel_iommu_strict;
367 static int intel_iommu_superpage = 1;
368 static int iommu_identity_mapping;
369 static int intel_no_bounce;
370
371 #define IDENTMAP_ALL 1
372 #define IDENTMAP_GFX 2
373 #define IDENTMAP_AZALIA 4
374
375 int intel_iommu_gfx_mapped;
376 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
377
378 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
379 #define DEFER_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-2))
380 static DEFINE_SPINLOCK(device_domain_lock);
381 static LIST_HEAD(device_domain_list);
382
383 #define device_needs_bounce(d) (!intel_no_bounce && dev_is_pci(d) && \
384 to_pci_dev(d)->untrusted)
385
386 /*
387 * Iterate over elements in device_domain_list and call the specified
388 * callback @fn against each element.
389 */
390 int for_each_device_domain(int (*fn)(struct device_domain_info *info,
391 void *data), void *data)
392 {
393 int ret = 0;
394 unsigned long flags;
395 struct device_domain_info *info;
396
397 spin_lock_irqsave(&device_domain_lock, flags);
398 list_for_each_entry(info, &device_domain_list, global) {
399 ret = fn(info, data);
400 if (ret) {
401 spin_unlock_irqrestore(&device_domain_lock, flags);
402 return ret;
403 }
404 }
405 spin_unlock_irqrestore(&device_domain_lock, flags);
406
407 return 0;
408 }
409
410 const struct iommu_ops intel_iommu_ops;
411
412 static bool translation_pre_enabled(struct intel_iommu *iommu)
413 {
414 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
415 }
416
417 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
418 {
419 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
420 }
421
422 static void init_translation_status(struct intel_iommu *iommu)
423 {
424 u32 gsts;
425
426 gsts = readl(iommu->reg + DMAR_GSTS_REG);
427 if (gsts & DMA_GSTS_TES)
428 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
429 }
430
431 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
432 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
433 {
434 return container_of(dom, struct dmar_domain, domain);
435 }
436
437 static int __init intel_iommu_setup(char *str)
438 {
439 if (!str)
440 return -EINVAL;
441 while (*str) {
442 if (!strncmp(str, "on", 2)) {
443 dmar_disabled = 0;
444 pr_info("IOMMU enabled\n");
445 } else if (!strncmp(str, "off", 3)) {
446 dmar_disabled = 1;
447 no_platform_optin = 1;
448 pr_info("IOMMU disabled\n");
449 } else if (!strncmp(str, "igfx_off", 8)) {
450 dmar_map_gfx = 0;
451 pr_info("Disable GFX device mapping\n");
452 } else if (!strncmp(str, "forcedac", 8)) {
453 pr_info("Forcing DAC for PCI devices\n");
454 dmar_forcedac = 1;
455 } else if (!strncmp(str, "strict", 6)) {
456 pr_info("Disable batched IOTLB flush\n");
457 intel_iommu_strict = 1;
458 } else if (!strncmp(str, "sp_off", 6)) {
459 pr_info("Disable supported super page\n");
460 intel_iommu_superpage = 0;
461 } else if (!strncmp(str, "sm_on", 5)) {
462 pr_info("Intel-IOMMU: scalable mode supported\n");
463 intel_iommu_sm = 1;
464 } else if (!strncmp(str, "tboot_noforce", 13)) {
465 printk(KERN_INFO
466 "Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
467 intel_iommu_tboot_noforce = 1;
468 } else if (!strncmp(str, "nobounce", 8)) {
469 pr_info("Intel-IOMMU: No bounce buffer. This could expose security risks of DMA attacks\n");
470 intel_no_bounce = 1;
471 }
472
473 str += strcspn(str, ",");
474 while (*str == ',')
475 str++;
476 }
477 return 0;
478 }
479 __setup("intel_iommu=", intel_iommu_setup);
480
481 static struct kmem_cache *iommu_domain_cache;
482 static struct kmem_cache *iommu_devinfo_cache;
483
484 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
485 {
486 struct dmar_domain **domains;
487 int idx = did >> 8;
488
489 domains = iommu->domains[idx];
490 if (!domains)
491 return NULL;
492
493 return domains[did & 0xff];
494 }
495
496 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
497 struct dmar_domain *domain)
498 {
499 struct dmar_domain **domains;
500 int idx = did >> 8;
501
502 if (!iommu->domains[idx]) {
503 size_t size = 256 * sizeof(struct dmar_domain *);
504 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
505 }
506
507 domains = iommu->domains[idx];
508 if (WARN_ON(!domains))
509 return;
510 else
511 domains[did & 0xff] = domain;
512 }
513
514 void *alloc_pgtable_page(int node)
515 {
516 struct page *page;
517 void *vaddr = NULL;
518
519 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
520 if (page)
521 vaddr = page_address(page);
522 return vaddr;
523 }
524
525 void free_pgtable_page(void *vaddr)
526 {
527 free_page((unsigned long)vaddr);
528 }
529
530 static inline void *alloc_domain_mem(void)
531 {
532 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
533 }
534
535 static void free_domain_mem(void *vaddr)
536 {
537 kmem_cache_free(iommu_domain_cache, vaddr);
538 }
539
540 static inline void * alloc_devinfo_mem(void)
541 {
542 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
543 }
544
545 static inline void free_devinfo_mem(void *vaddr)
546 {
547 kmem_cache_free(iommu_devinfo_cache, vaddr);
548 }
549
550 static inline int domain_type_is_si(struct dmar_domain *domain)
551 {
552 return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
553 }
554
555 static inline int domain_pfn_supported(struct dmar_domain *domain,
556 unsigned long pfn)
557 {
558 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
559
560 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
561 }
562
563 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
564 {
565 unsigned long sagaw;
566 int agaw = -1;
567
568 sagaw = cap_sagaw(iommu->cap);
569 for (agaw = width_to_agaw(max_gaw);
570 agaw >= 0; agaw--) {
571 if (test_bit(agaw, &sagaw))
572 break;
573 }
574
575 return agaw;
576 }
577
578 /*
579 * Calculate max SAGAW for each iommu.
580 */
581 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
582 {
583 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
584 }
585
586 /*
587 * calculate agaw for each iommu.
588 * "SAGAW" may be different across iommus, use a default agaw, and
589 * get a supported less agaw for iommus that don't support the default agaw.
590 */
591 int iommu_calculate_agaw(struct intel_iommu *iommu)
592 {
593 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
594 }
595
596 /* This functionin only returns single iommu in a domain */
597 struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
598 {
599 int iommu_id;
600
601 /* si_domain and vm domain should not get here. */
602 if (WARN_ON(domain->domain.type != IOMMU_DOMAIN_DMA))
603 return NULL;
604
605 for_each_domain_iommu(iommu_id, domain)
606 break;
607
608 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
609 return NULL;
610
611 return g_iommus[iommu_id];
612 }
613
614 static void domain_update_iommu_coherency(struct dmar_domain *domain)
615 {
616 struct dmar_drhd_unit *drhd;
617 struct intel_iommu *iommu;
618 bool found = false;
619 int i;
620
621 domain->iommu_coherency = 1;
622
623 for_each_domain_iommu(i, domain) {
624 found = true;
625 if (!ecap_coherent(g_iommus[i]->ecap)) {
626 domain->iommu_coherency = 0;
627 break;
628 }
629 }
630 if (found)
631 return;
632
633 /* No hardware attached; use lowest common denominator */
634 rcu_read_lock();
635 for_each_active_iommu(iommu, drhd) {
636 if (!ecap_coherent(iommu->ecap)) {
637 domain->iommu_coherency = 0;
638 break;
639 }
640 }
641 rcu_read_unlock();
642 }
643
644 static int domain_update_iommu_snooping(struct intel_iommu *skip)
645 {
646 struct dmar_drhd_unit *drhd;
647 struct intel_iommu *iommu;
648 int ret = 1;
649
650 rcu_read_lock();
651 for_each_active_iommu(iommu, drhd) {
652 if (iommu != skip) {
653 if (!ecap_sc_support(iommu->ecap)) {
654 ret = 0;
655 break;
656 }
657 }
658 }
659 rcu_read_unlock();
660
661 return ret;
662 }
663
664 static int domain_update_iommu_superpage(struct intel_iommu *skip)
665 {
666 struct dmar_drhd_unit *drhd;
667 struct intel_iommu *iommu;
668 int mask = 0xf;
669
670 if (!intel_iommu_superpage) {
671 return 0;
672 }
673
674 /* set iommu_superpage to the smallest common denominator */
675 rcu_read_lock();
676 for_each_active_iommu(iommu, drhd) {
677 if (iommu != skip) {
678 mask &= cap_super_page_val(iommu->cap);
679 if (!mask)
680 break;
681 }
682 }
683 rcu_read_unlock();
684
685 return fls(mask);
686 }
687
688 /* Some capabilities may be different across iommus */
689 static void domain_update_iommu_cap(struct dmar_domain *domain)
690 {
691 domain_update_iommu_coherency(domain);
692 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
693 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
694 }
695
696 struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,
697 u8 devfn, int alloc)
698 {
699 struct root_entry *root = &iommu->root_entry[bus];
700 struct context_entry *context;
701 u64 *entry;
702
703 entry = &root->lo;
704 if (sm_supported(iommu)) {
705 if (devfn >= 0x80) {
706 devfn -= 0x80;
707 entry = &root->hi;
708 }
709 devfn *= 2;
710 }
711 if (*entry & 1)
712 context = phys_to_virt(*entry & VTD_PAGE_MASK);
713 else {
714 unsigned long phy_addr;
715 if (!alloc)
716 return NULL;
717
718 context = alloc_pgtable_page(iommu->node);
719 if (!context)
720 return NULL;
721
722 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
723 phy_addr = virt_to_phys((void *)context);
724 *entry = phy_addr | 1;
725 __iommu_flush_cache(iommu, entry, sizeof(*entry));
726 }
727 return &context[devfn];
728 }
729
730 static int iommu_dummy(struct device *dev)
731 {
732 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
733 }
734
735 /**
736 * is_downstream_to_pci_bridge - test if a device belongs to the PCI
737 * sub-hierarchy of a candidate PCI-PCI bridge
738 * @dev: candidate PCI device belonging to @bridge PCI sub-hierarchy
739 * @bridge: the candidate PCI-PCI bridge
740 *
741 * Return: true if @dev belongs to @bridge PCI sub-hierarchy, else false.
742 */
743 static bool
744 is_downstream_to_pci_bridge(struct device *dev, struct device *bridge)
745 {
746 struct pci_dev *pdev, *pbridge;
747
748 if (!dev_is_pci(dev) || !dev_is_pci(bridge))
749 return false;
750
751 pdev = to_pci_dev(dev);
752 pbridge = to_pci_dev(bridge);
753
754 if (pbridge->subordinate &&
755 pbridge->subordinate->number <= pdev->bus->number &&
756 pbridge->subordinate->busn_res.end >= pdev->bus->number)
757 return true;
758
759 return false;
760 }
761
762 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
763 {
764 struct dmar_drhd_unit *drhd = NULL;
765 struct intel_iommu *iommu;
766 struct device *tmp;
767 struct pci_dev *pdev = NULL;
768 u16 segment = 0;
769 int i;
770
771 if (iommu_dummy(dev))
772 return NULL;
773
774 if (dev_is_pci(dev)) {
775 struct pci_dev *pf_pdev;
776
777 pdev = pci_real_dma_dev(to_pci_dev(dev));
778
779 /* VFs aren't listed in scope tables; we need to look up
780 * the PF instead to find the IOMMU. */
781 pf_pdev = pci_physfn(pdev);
782 dev = &pf_pdev->dev;
783 segment = pci_domain_nr(pdev->bus);
784 } else if (has_acpi_companion(dev))
785 dev = &ACPI_COMPANION(dev)->dev;
786
787 rcu_read_lock();
788 for_each_active_iommu(iommu, drhd) {
789 if (pdev && segment != drhd->segment)
790 continue;
791
792 for_each_active_dev_scope(drhd->devices,
793 drhd->devices_cnt, i, tmp) {
794 if (tmp == dev) {
795 /* For a VF use its original BDF# not that of the PF
796 * which we used for the IOMMU lookup. Strictly speaking
797 * we could do this for all PCI devices; we only need to
798 * get the BDF# from the scope table for ACPI matches. */
799 if (pdev && pdev->is_virtfn)
800 goto got_pdev;
801
802 *bus = drhd->devices[i].bus;
803 *devfn = drhd->devices[i].devfn;
804 goto out;
805 }
806
807 if (is_downstream_to_pci_bridge(dev, tmp))
808 goto got_pdev;
809 }
810
811 if (pdev && drhd->include_all) {
812 got_pdev:
813 *bus = pdev->bus->number;
814 *devfn = pdev->devfn;
815 goto out;
816 }
817 }
818 iommu = NULL;
819 out:
820 rcu_read_unlock();
821
822 return iommu;
823 }
824
825 static void domain_flush_cache(struct dmar_domain *domain,
826 void *addr, int size)
827 {
828 if (!domain->iommu_coherency)
829 clflush_cache_range(addr, size);
830 }
831
832 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
833 {
834 struct context_entry *context;
835 int ret = 0;
836 unsigned long flags;
837
838 spin_lock_irqsave(&iommu->lock, flags);
839 context = iommu_context_addr(iommu, bus, devfn, 0);
840 if (context)
841 ret = context_present(context);
842 spin_unlock_irqrestore(&iommu->lock, flags);
843 return ret;
844 }
845
846 static void free_context_table(struct intel_iommu *iommu)
847 {
848 int i;
849 unsigned long flags;
850 struct context_entry *context;
851
852 spin_lock_irqsave(&iommu->lock, flags);
853 if (!iommu->root_entry) {
854 goto out;
855 }
856 for (i = 0; i < ROOT_ENTRY_NR; i++) {
857 context = iommu_context_addr(iommu, i, 0, 0);
858 if (context)
859 free_pgtable_page(context);
860
861 if (!sm_supported(iommu))
862 continue;
863
864 context = iommu_context_addr(iommu, i, 0x80, 0);
865 if (context)
866 free_pgtable_page(context);
867
868 }
869 free_pgtable_page(iommu->root_entry);
870 iommu->root_entry = NULL;
871 out:
872 spin_unlock_irqrestore(&iommu->lock, flags);
873 }
874
875 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
876 unsigned long pfn, int *target_level)
877 {
878 struct dma_pte *parent, *pte;
879 int level = agaw_to_level(domain->agaw);
880 int offset;
881
882 BUG_ON(!domain->pgd);
883
884 if (!domain_pfn_supported(domain, pfn))
885 /* Address beyond IOMMU's addressing capabilities. */
886 return NULL;
887
888 parent = domain->pgd;
889
890 while (1) {
891 void *tmp_page;
892
893 offset = pfn_level_offset(pfn, level);
894 pte = &parent[offset];
895 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
896 break;
897 if (level == *target_level)
898 break;
899
900 if (!dma_pte_present(pte)) {
901 uint64_t pteval;
902
903 tmp_page = alloc_pgtable_page(domain->nid);
904
905 if (!tmp_page)
906 return NULL;
907
908 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
909 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
910 if (cmpxchg64(&pte->val, 0ULL, pteval))
911 /* Someone else set it while we were thinking; use theirs. */
912 free_pgtable_page(tmp_page);
913 else
914 domain_flush_cache(domain, pte, sizeof(*pte));
915 }
916 if (level == 1)
917 break;
918
919 parent = phys_to_virt(dma_pte_addr(pte));
920 level--;
921 }
922
923 if (!*target_level)
924 *target_level = level;
925
926 return pte;
927 }
928
929 /* return address's pte at specific level */
930 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
931 unsigned long pfn,
932 int level, int *large_page)
933 {
934 struct dma_pte *parent, *pte;
935 int total = agaw_to_level(domain->agaw);
936 int offset;
937
938 parent = domain->pgd;
939 while (level <= total) {
940 offset = pfn_level_offset(pfn, total);
941 pte = &parent[offset];
942 if (level == total)
943 return pte;
944
945 if (!dma_pte_present(pte)) {
946 *large_page = total;
947 break;
948 }
949
950 if (dma_pte_superpage(pte)) {
951 *large_page = total;
952 return pte;
953 }
954
955 parent = phys_to_virt(dma_pte_addr(pte));
956 total--;
957 }
958 return NULL;
959 }
960
961 /* clear last level pte, a tlb flush should be followed */
962 static void dma_pte_clear_range(struct dmar_domain *domain,
963 unsigned long start_pfn,
964 unsigned long last_pfn)
965 {
966 unsigned int large_page;
967 struct dma_pte *first_pte, *pte;
968
969 BUG_ON(!domain_pfn_supported(domain, start_pfn));
970 BUG_ON(!domain_pfn_supported(domain, last_pfn));
971 BUG_ON(start_pfn > last_pfn);
972
973 /* we don't need lock here; nobody else touches the iova range */
974 do {
975 large_page = 1;
976 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
977 if (!pte) {
978 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
979 continue;
980 }
981 do {
982 dma_clear_pte(pte);
983 start_pfn += lvl_to_nr_pages(large_page);
984 pte++;
985 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
986
987 domain_flush_cache(domain, first_pte,
988 (void *)pte - (void *)first_pte);
989
990 } while (start_pfn && start_pfn <= last_pfn);
991 }
992
993 static void dma_pte_free_level(struct dmar_domain *domain, int level,
994 int retain_level, struct dma_pte *pte,
995 unsigned long pfn, unsigned long start_pfn,
996 unsigned long last_pfn)
997 {
998 pfn = max(start_pfn, pfn);
999 pte = &pte[pfn_level_offset(pfn, level)];
1000
1001 do {
1002 unsigned long level_pfn;
1003 struct dma_pte *level_pte;
1004
1005 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1006 goto next;
1007
1008 level_pfn = pfn & level_mask(level);
1009 level_pte = phys_to_virt(dma_pte_addr(pte));
1010
1011 if (level > 2) {
1012 dma_pte_free_level(domain, level - 1, retain_level,
1013 level_pte, level_pfn, start_pfn,
1014 last_pfn);
1015 }
1016
1017 /*
1018 * Free the page table if we're below the level we want to
1019 * retain and the range covers the entire table.
1020 */
1021 if (level < retain_level && !(start_pfn > level_pfn ||
1022 last_pfn < level_pfn + level_size(level) - 1)) {
1023 dma_clear_pte(pte);
1024 domain_flush_cache(domain, pte, sizeof(*pte));
1025 free_pgtable_page(level_pte);
1026 }
1027 next:
1028 pfn += level_size(level);
1029 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1030 }
1031
1032 /*
1033 * clear last level (leaf) ptes and free page table pages below the
1034 * level we wish to keep intact.
1035 */
1036 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1037 unsigned long start_pfn,
1038 unsigned long last_pfn,
1039 int retain_level)
1040 {
1041 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1042 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1043 BUG_ON(start_pfn > last_pfn);
1044
1045 dma_pte_clear_range(domain, start_pfn, last_pfn);
1046
1047 /* We don't need lock here; nobody else touches the iova range */
1048 dma_pte_free_level(domain, agaw_to_level(domain->agaw), retain_level,
1049 domain->pgd, 0, start_pfn, last_pfn);
1050
1051 /* free pgd */
1052 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1053 free_pgtable_page(domain->pgd);
1054 domain->pgd = NULL;
1055 }
1056 }
1057
1058 /* When a page at a given level is being unlinked from its parent, we don't
1059 need to *modify* it at all. All we need to do is make a list of all the
1060 pages which can be freed just as soon as we've flushed the IOTLB and we
1061 know the hardware page-walk will no longer touch them.
1062 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1063 be freed. */
1064 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1065 int level, struct dma_pte *pte,
1066 struct page *freelist)
1067 {
1068 struct page *pg;
1069
1070 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1071 pg->freelist = freelist;
1072 freelist = pg;
1073
1074 if (level == 1)
1075 return freelist;
1076
1077 pte = page_address(pg);
1078 do {
1079 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1080 freelist = dma_pte_list_pagetables(domain, level - 1,
1081 pte, freelist);
1082 pte++;
1083 } while (!first_pte_in_page(pte));
1084
1085 return freelist;
1086 }
1087
1088 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1089 struct dma_pte *pte, unsigned long pfn,
1090 unsigned long start_pfn,
1091 unsigned long last_pfn,
1092 struct page *freelist)
1093 {
1094 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1095
1096 pfn = max(start_pfn, pfn);
1097 pte = &pte[pfn_level_offset(pfn, level)];
1098
1099 do {
1100 unsigned long level_pfn;
1101
1102 if (!dma_pte_present(pte))
1103 goto next;
1104
1105 level_pfn = pfn & level_mask(level);
1106
1107 /* If range covers entire pagetable, free it */
1108 if (start_pfn <= level_pfn &&
1109 last_pfn >= level_pfn + level_size(level) - 1) {
1110 /* These suborbinate page tables are going away entirely. Don't
1111 bother to clear them; we're just going to *free* them. */
1112 if (level > 1 && !dma_pte_superpage(pte))
1113 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1114
1115 dma_clear_pte(pte);
1116 if (!first_pte)
1117 first_pte = pte;
1118 last_pte = pte;
1119 } else if (level > 1) {
1120 /* Recurse down into a level that isn't *entirely* obsolete */
1121 freelist = dma_pte_clear_level(domain, level - 1,
1122 phys_to_virt(dma_pte_addr(pte)),
1123 level_pfn, start_pfn, last_pfn,
1124 freelist);
1125 }
1126 next:
1127 pfn += level_size(level);
1128 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1129
1130 if (first_pte)
1131 domain_flush_cache(domain, first_pte,
1132 (void *)++last_pte - (void *)first_pte);
1133
1134 return freelist;
1135 }
1136
1137 /* We can't just free the pages because the IOMMU may still be walking
1138 the page tables, and may have cached the intermediate levels. The
1139 pages can only be freed after the IOTLB flush has been done. */
1140 static struct page *domain_unmap(struct dmar_domain *domain,
1141 unsigned long start_pfn,
1142 unsigned long last_pfn)
1143 {
1144 struct page *freelist;
1145
1146 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1147 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1148 BUG_ON(start_pfn > last_pfn);
1149
1150 /* we don't need lock here; nobody else touches the iova range */
1151 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1152 domain->pgd, 0, start_pfn, last_pfn, NULL);
1153
1154 /* free pgd */
1155 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1156 struct page *pgd_page = virt_to_page(domain->pgd);
1157 pgd_page->freelist = freelist;
1158 freelist = pgd_page;
1159
1160 domain->pgd = NULL;
1161 }
1162
1163 return freelist;
1164 }
1165
1166 static void dma_free_pagelist(struct page *freelist)
1167 {
1168 struct page *pg;
1169
1170 while ((pg = freelist)) {
1171 freelist = pg->freelist;
1172 free_pgtable_page(page_address(pg));
1173 }
1174 }
1175
1176 static void iova_entry_free(unsigned long data)
1177 {
1178 struct page *freelist = (struct page *)data;
1179
1180 dma_free_pagelist(freelist);
1181 }
1182
1183 /* iommu handling */
1184 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1185 {
1186 struct root_entry *root;
1187 unsigned long flags;
1188
1189 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1190 if (!root) {
1191 pr_err("Allocating root entry for %s failed\n",
1192 iommu->name);
1193 return -ENOMEM;
1194 }
1195
1196 __iommu_flush_cache(iommu, root, ROOT_SIZE);
1197
1198 spin_lock_irqsave(&iommu->lock, flags);
1199 iommu->root_entry = root;
1200 spin_unlock_irqrestore(&iommu->lock, flags);
1201
1202 return 0;
1203 }
1204
1205 static void iommu_set_root_entry(struct intel_iommu *iommu)
1206 {
1207 u64 addr;
1208 u32 sts;
1209 unsigned long flag;
1210
1211 addr = virt_to_phys(iommu->root_entry);
1212 if (sm_supported(iommu))
1213 addr |= DMA_RTADDR_SMT;
1214
1215 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1216 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1217
1218 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1219
1220 /* Make sure hardware complete it */
1221 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1222 readl, (sts & DMA_GSTS_RTPS), sts);
1223
1224 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1225 }
1226
1227 void iommu_flush_write_buffer(struct intel_iommu *iommu)
1228 {
1229 u32 val;
1230 unsigned long flag;
1231
1232 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1233 return;
1234
1235 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1236 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1237
1238 /* Make sure hardware complete it */
1239 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1240 readl, (!(val & DMA_GSTS_WBFS)), val);
1241
1242 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1243 }
1244
1245 /* return value determine if we need a write buffer flush */
1246 static void __iommu_flush_context(struct intel_iommu *iommu,
1247 u16 did, u16 source_id, u8 function_mask,
1248 u64 type)
1249 {
1250 u64 val = 0;
1251 unsigned long flag;
1252
1253 switch (type) {
1254 case DMA_CCMD_GLOBAL_INVL:
1255 val = DMA_CCMD_GLOBAL_INVL;
1256 break;
1257 case DMA_CCMD_DOMAIN_INVL:
1258 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1259 break;
1260 case DMA_CCMD_DEVICE_INVL:
1261 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1262 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1263 break;
1264 default:
1265 BUG();
1266 }
1267 val |= DMA_CCMD_ICC;
1268
1269 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1270 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1271
1272 /* Make sure hardware complete it */
1273 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1274 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1275
1276 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1277 }
1278
1279 /* return value determine if we need a write buffer flush */
1280 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1281 u64 addr, unsigned int size_order, u64 type)
1282 {
1283 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1284 u64 val = 0, val_iva = 0;
1285 unsigned long flag;
1286
1287 switch (type) {
1288 case DMA_TLB_GLOBAL_FLUSH:
1289 /* global flush doesn't need set IVA_REG */
1290 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1291 break;
1292 case DMA_TLB_DSI_FLUSH:
1293 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1294 break;
1295 case DMA_TLB_PSI_FLUSH:
1296 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1297 /* IH bit is passed in as part of address */
1298 val_iva = size_order | addr;
1299 break;
1300 default:
1301 BUG();
1302 }
1303 /* Note: set drain read/write */
1304 #if 0
1305 /*
1306 * This is probably to be super secure.. Looks like we can
1307 * ignore it without any impact.
1308 */
1309 if (cap_read_drain(iommu->cap))
1310 val |= DMA_TLB_READ_DRAIN;
1311 #endif
1312 if (cap_write_drain(iommu->cap))
1313 val |= DMA_TLB_WRITE_DRAIN;
1314
1315 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1316 /* Note: Only uses first TLB reg currently */
1317 if (val_iva)
1318 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1319 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1320
1321 /* Make sure hardware complete it */
1322 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1323 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1324
1325 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1326
1327 /* check IOTLB invalidation granularity */
1328 if (DMA_TLB_IAIG(val) == 0)
1329 pr_err("Flush IOTLB failed\n");
1330 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1331 pr_debug("TLB flush request %Lx, actual %Lx\n",
1332 (unsigned long long)DMA_TLB_IIRG(type),
1333 (unsigned long long)DMA_TLB_IAIG(val));
1334 }
1335
1336 static struct device_domain_info *
1337 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1338 u8 bus, u8 devfn)
1339 {
1340 struct device_domain_info *info;
1341
1342 assert_spin_locked(&device_domain_lock);
1343
1344 if (!iommu->qi)
1345 return NULL;
1346
1347 list_for_each_entry(info, &domain->devices, link)
1348 if (info->iommu == iommu && info->bus == bus &&
1349 info->devfn == devfn) {
1350 if (info->ats_supported && info->dev)
1351 return info;
1352 break;
1353 }
1354
1355 return NULL;
1356 }
1357
1358 static void domain_update_iotlb(struct dmar_domain *domain)
1359 {
1360 struct device_domain_info *info;
1361 bool has_iotlb_device = false;
1362
1363 assert_spin_locked(&device_domain_lock);
1364
1365 list_for_each_entry(info, &domain->devices, link) {
1366 struct pci_dev *pdev;
1367
1368 if (!info->dev || !dev_is_pci(info->dev))
1369 continue;
1370
1371 pdev = to_pci_dev(info->dev);
1372 if (pdev->ats_enabled) {
1373 has_iotlb_device = true;
1374 break;
1375 }
1376 }
1377
1378 domain->has_iotlb_device = has_iotlb_device;
1379 }
1380
1381 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1382 {
1383 struct pci_dev *pdev;
1384
1385 assert_spin_locked(&device_domain_lock);
1386
1387 if (!info || !dev_is_pci(info->dev))
1388 return;
1389
1390 pdev = to_pci_dev(info->dev);
1391 /* For IOMMU that supports device IOTLB throttling (DIT), we assign
1392 * PFSID to the invalidation desc of a VF such that IOMMU HW can gauge
1393 * queue depth at PF level. If DIT is not set, PFSID will be treated as
1394 * reserved, which should be set to 0.
1395 */
1396 if (!ecap_dit(info->iommu->ecap))
1397 info->pfsid = 0;
1398 else {
1399 struct pci_dev *pf_pdev;
1400
1401 /* pdev will be returned if device is not a vf */
1402 pf_pdev = pci_physfn(pdev);
1403 info->pfsid = pci_dev_id(pf_pdev);
1404 }
1405
1406 #ifdef CONFIG_INTEL_IOMMU_SVM
1407 /* The PCIe spec, in its wisdom, declares that the behaviour of
1408 the device if you enable PASID support after ATS support is
1409 undefined. So always enable PASID support on devices which
1410 have it, even if we can't yet know if we're ever going to
1411 use it. */
1412 if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1413 info->pasid_enabled = 1;
1414
1415 if (info->pri_supported &&
1416 (info->pasid_enabled ? pci_prg_resp_pasid_required(pdev) : 1) &&
1417 !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1418 info->pri_enabled = 1;
1419 #endif
1420 if (!pdev->untrusted && info->ats_supported &&
1421 pci_ats_page_aligned(pdev) &&
1422 !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1423 info->ats_enabled = 1;
1424 domain_update_iotlb(info->domain);
1425 info->ats_qdep = pci_ats_queue_depth(pdev);
1426 }
1427 }
1428
1429 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1430 {
1431 struct pci_dev *pdev;
1432
1433 assert_spin_locked(&device_domain_lock);
1434
1435 if (!dev_is_pci(info->dev))
1436 return;
1437
1438 pdev = to_pci_dev(info->dev);
1439
1440 if (info->ats_enabled) {
1441 pci_disable_ats(pdev);
1442 info->ats_enabled = 0;
1443 domain_update_iotlb(info->domain);
1444 }
1445 #ifdef CONFIG_INTEL_IOMMU_SVM
1446 if (info->pri_enabled) {
1447 pci_disable_pri(pdev);
1448 info->pri_enabled = 0;
1449 }
1450 if (info->pasid_enabled) {
1451 pci_disable_pasid(pdev);
1452 info->pasid_enabled = 0;
1453 }
1454 #endif
1455 }
1456
1457 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1458 u64 addr, unsigned mask)
1459 {
1460 u16 sid, qdep;
1461 unsigned long flags;
1462 struct device_domain_info *info;
1463
1464 if (!domain->has_iotlb_device)
1465 return;
1466
1467 spin_lock_irqsave(&device_domain_lock, flags);
1468 list_for_each_entry(info, &domain->devices, link) {
1469 if (!info->ats_enabled)
1470 continue;
1471
1472 sid = info->bus << 8 | info->devfn;
1473 qdep = info->ats_qdep;
1474 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
1475 qdep, addr, mask);
1476 }
1477 spin_unlock_irqrestore(&device_domain_lock, flags);
1478 }
1479
1480 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1481 struct dmar_domain *domain,
1482 unsigned long pfn, unsigned int pages,
1483 int ih, int map)
1484 {
1485 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1486 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1487 u16 did = domain->iommu_did[iommu->seq_id];
1488
1489 BUG_ON(pages == 0);
1490
1491 if (ih)
1492 ih = 1 << 6;
1493 /*
1494 * Fallback to domain selective flush if no PSI support or the size is
1495 * too big.
1496 * PSI requires page size to be 2 ^ x, and the base address is naturally
1497 * aligned to the size
1498 */
1499 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1500 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1501 DMA_TLB_DSI_FLUSH);
1502 else
1503 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1504 DMA_TLB_PSI_FLUSH);
1505
1506 /*
1507 * In caching mode, changes of pages from non-present to present require
1508 * flush. However, device IOTLB doesn't need to be flushed in this case.
1509 */
1510 if (!cap_caching_mode(iommu->cap) || !map)
1511 iommu_flush_dev_iotlb(domain, addr, mask);
1512 }
1513
1514 /* Notification for newly created mappings */
1515 static inline void __mapping_notify_one(struct intel_iommu *iommu,
1516 struct dmar_domain *domain,
1517 unsigned long pfn, unsigned int pages)
1518 {
1519 /* It's a non-present to present mapping. Only flush if caching mode */
1520 if (cap_caching_mode(iommu->cap))
1521 iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1);
1522 else
1523 iommu_flush_write_buffer(iommu);
1524 }
1525
1526 static void iommu_flush_iova(struct iova_domain *iovad)
1527 {
1528 struct dmar_domain *domain;
1529 int idx;
1530
1531 domain = container_of(iovad, struct dmar_domain, iovad);
1532
1533 for_each_domain_iommu(idx, domain) {
1534 struct intel_iommu *iommu = g_iommus[idx];
1535 u16 did = domain->iommu_did[iommu->seq_id];
1536
1537 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
1538
1539 if (!cap_caching_mode(iommu->cap))
1540 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1541 0, MAX_AGAW_PFN_WIDTH);
1542 }
1543 }
1544
1545 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1546 {
1547 u32 pmen;
1548 unsigned long flags;
1549
1550 if (!cap_plmr(iommu->cap) && !cap_phmr(iommu->cap))
1551 return;
1552
1553 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1554 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1555 pmen &= ~DMA_PMEN_EPM;
1556 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1557
1558 /* wait for the protected region status bit to clear */
1559 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1560 readl, !(pmen & DMA_PMEN_PRS), pmen);
1561
1562 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1563 }
1564
1565 static void iommu_enable_translation(struct intel_iommu *iommu)
1566 {
1567 u32 sts;
1568 unsigned long flags;
1569
1570 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1571 iommu->gcmd |= DMA_GCMD_TE;
1572 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1573
1574 /* Make sure hardware complete it */
1575 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1576 readl, (sts & DMA_GSTS_TES), sts);
1577
1578 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1579 }
1580
1581 static void iommu_disable_translation(struct intel_iommu *iommu)
1582 {
1583 u32 sts;
1584 unsigned long flag;
1585
1586 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1587 iommu->gcmd &= ~DMA_GCMD_TE;
1588 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1589
1590 /* Make sure hardware complete it */
1591 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1592 readl, (!(sts & DMA_GSTS_TES)), sts);
1593
1594 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1595 }
1596
1597 static int iommu_init_domains(struct intel_iommu *iommu)
1598 {
1599 u32 ndomains, nlongs;
1600 size_t size;
1601
1602 ndomains = cap_ndoms(iommu->cap);
1603 pr_debug("%s: Number of Domains supported <%d>\n",
1604 iommu->name, ndomains);
1605 nlongs = BITS_TO_LONGS(ndomains);
1606
1607 spin_lock_init(&iommu->lock);
1608
1609 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1610 if (!iommu->domain_ids) {
1611 pr_err("%s: Allocating domain id array failed\n",
1612 iommu->name);
1613 return -ENOMEM;
1614 }
1615
1616 size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
1617 iommu->domains = kzalloc(size, GFP_KERNEL);
1618
1619 if (iommu->domains) {
1620 size = 256 * sizeof(struct dmar_domain *);
1621 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1622 }
1623
1624 if (!iommu->domains || !iommu->domains[0]) {
1625 pr_err("%s: Allocating domain array failed\n",
1626 iommu->name);
1627 kfree(iommu->domain_ids);
1628 kfree(iommu->domains);
1629 iommu->domain_ids = NULL;
1630 iommu->domains = NULL;
1631 return -ENOMEM;
1632 }
1633
1634 /*
1635 * If Caching mode is set, then invalid translations are tagged
1636 * with domain-id 0, hence we need to pre-allocate it. We also
1637 * use domain-id 0 as a marker for non-allocated domain-id, so
1638 * make sure it is not used for a real domain.
1639 */
1640 set_bit(0, iommu->domain_ids);
1641
1642 /*
1643 * Vt-d spec rev3.0 (section 6.2.3.1) requires that each pasid
1644 * entry for first-level or pass-through translation modes should
1645 * be programmed with a domain id different from those used for
1646 * second-level or nested translation. We reserve a domain id for
1647 * this purpose.
1648 */
1649 if (sm_supported(iommu))
1650 set_bit(FLPT_DEFAULT_DID, iommu->domain_ids);
1651
1652 return 0;
1653 }
1654
1655 static void disable_dmar_iommu(struct intel_iommu *iommu)
1656 {
1657 struct device_domain_info *info, *tmp;
1658 unsigned long flags;
1659
1660 if (!iommu->domains || !iommu->domain_ids)
1661 return;
1662
1663 spin_lock_irqsave(&device_domain_lock, flags);
1664 list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1665 if (info->iommu != iommu)
1666 continue;
1667
1668 if (!info->dev || !info->domain)
1669 continue;
1670
1671 __dmar_remove_one_dev_info(info);
1672 }
1673 spin_unlock_irqrestore(&device_domain_lock, flags);
1674
1675 if (iommu->gcmd & DMA_GCMD_TE)
1676 iommu_disable_translation(iommu);
1677 }
1678
1679 static void free_dmar_iommu(struct intel_iommu *iommu)
1680 {
1681 if ((iommu->domains) && (iommu->domain_ids)) {
1682 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
1683 int i;
1684
1685 for (i = 0; i < elems; i++)
1686 kfree(iommu->domains[i]);
1687 kfree(iommu->domains);
1688 kfree(iommu->domain_ids);
1689 iommu->domains = NULL;
1690 iommu->domain_ids = NULL;
1691 }
1692
1693 g_iommus[iommu->seq_id] = NULL;
1694
1695 /* free context mapping */
1696 free_context_table(iommu);
1697
1698 #ifdef CONFIG_INTEL_IOMMU_SVM
1699 if (pasid_supported(iommu)) {
1700 if (ecap_prs(iommu->ecap))
1701 intel_svm_finish_prq(iommu);
1702 }
1703 #endif
1704 }
1705
1706 static struct dmar_domain *alloc_domain(int flags)
1707 {
1708 struct dmar_domain *domain;
1709
1710 domain = alloc_domain_mem();
1711 if (!domain)
1712 return NULL;
1713
1714 memset(domain, 0, sizeof(*domain));
1715 domain->nid = NUMA_NO_NODE;
1716 domain->flags = flags;
1717 domain->has_iotlb_device = false;
1718 INIT_LIST_HEAD(&domain->devices);
1719
1720 return domain;
1721 }
1722
1723 /* Must be called with iommu->lock */
1724 static int domain_attach_iommu(struct dmar_domain *domain,
1725 struct intel_iommu *iommu)
1726 {
1727 unsigned long ndomains;
1728 int num;
1729
1730 assert_spin_locked(&device_domain_lock);
1731 assert_spin_locked(&iommu->lock);
1732
1733 domain->iommu_refcnt[iommu->seq_id] += 1;
1734 domain->iommu_count += 1;
1735 if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1736 ndomains = cap_ndoms(iommu->cap);
1737 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1738
1739 if (num >= ndomains) {
1740 pr_err("%s: No free domain ids\n", iommu->name);
1741 domain->iommu_refcnt[iommu->seq_id] -= 1;
1742 domain->iommu_count -= 1;
1743 return -ENOSPC;
1744 }
1745
1746 set_bit(num, iommu->domain_ids);
1747 set_iommu_domain(iommu, num, domain);
1748
1749 domain->iommu_did[iommu->seq_id] = num;
1750 domain->nid = iommu->node;
1751
1752 domain_update_iommu_cap(domain);
1753 }
1754
1755 return 0;
1756 }
1757
1758 static int domain_detach_iommu(struct dmar_domain *domain,
1759 struct intel_iommu *iommu)
1760 {
1761 int num, count;
1762
1763 assert_spin_locked(&device_domain_lock);
1764 assert_spin_locked(&iommu->lock);
1765
1766 domain->iommu_refcnt[iommu->seq_id] -= 1;
1767 count = --domain->iommu_count;
1768 if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1769 num = domain->iommu_did[iommu->seq_id];
1770 clear_bit(num, iommu->domain_ids);
1771 set_iommu_domain(iommu, num, NULL);
1772
1773 domain_update_iommu_cap(domain);
1774 domain->iommu_did[iommu->seq_id] = 0;
1775 }
1776
1777 return count;
1778 }
1779
1780 static struct iova_domain reserved_iova_list;
1781 static struct lock_class_key reserved_rbtree_key;
1782
1783 static int dmar_init_reserved_ranges(void)
1784 {
1785 struct pci_dev *pdev = NULL;
1786 struct iova *iova;
1787 int i;
1788
1789 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN);
1790
1791 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1792 &reserved_rbtree_key);
1793
1794 /* IOAPIC ranges shouldn't be accessed by DMA */
1795 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1796 IOVA_PFN(IOAPIC_RANGE_END));
1797 if (!iova) {
1798 pr_err("Reserve IOAPIC range failed\n");
1799 return -ENODEV;
1800 }
1801
1802 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1803 for_each_pci_dev(pdev) {
1804 struct resource *r;
1805
1806 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1807 r = &pdev->resource[i];
1808 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1809 continue;
1810 iova = reserve_iova(&reserved_iova_list,
1811 IOVA_PFN(r->start),
1812 IOVA_PFN(r->end));
1813 if (!iova) {
1814 pci_err(pdev, "Reserve iova for %pR failed\n", r);
1815 return -ENODEV;
1816 }
1817 }
1818 }
1819 return 0;
1820 }
1821
1822 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1823 {
1824 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1825 }
1826
1827 static inline int guestwidth_to_adjustwidth(int gaw)
1828 {
1829 int agaw;
1830 int r = (gaw - 12) % 9;
1831
1832 if (r == 0)
1833 agaw = gaw;
1834 else
1835 agaw = gaw + 9 - r;
1836 if (agaw > 64)
1837 agaw = 64;
1838 return agaw;
1839 }
1840
1841 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1842 int guest_width)
1843 {
1844 int adjust_width, agaw;
1845 unsigned long sagaw;
1846 int err;
1847
1848 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
1849
1850 err = init_iova_flush_queue(&domain->iovad,
1851 iommu_flush_iova, iova_entry_free);
1852 if (err)
1853 return err;
1854
1855 domain_reserve_special_ranges(domain);
1856
1857 /* calculate AGAW */
1858 if (guest_width > cap_mgaw(iommu->cap))
1859 guest_width = cap_mgaw(iommu->cap);
1860 domain->gaw = guest_width;
1861 adjust_width = guestwidth_to_adjustwidth(guest_width);
1862 agaw = width_to_agaw(adjust_width);
1863 sagaw = cap_sagaw(iommu->cap);
1864 if (!test_bit(agaw, &sagaw)) {
1865 /* hardware doesn't support it, choose a bigger one */
1866 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1867 agaw = find_next_bit(&sagaw, 5, agaw);
1868 if (agaw >= 5)
1869 return -ENODEV;
1870 }
1871 domain->agaw = agaw;
1872
1873 if (ecap_coherent(iommu->ecap))
1874 domain->iommu_coherency = 1;
1875 else
1876 domain->iommu_coherency = 0;
1877
1878 if (ecap_sc_support(iommu->ecap))
1879 domain->iommu_snooping = 1;
1880 else
1881 domain->iommu_snooping = 0;
1882
1883 if (intel_iommu_superpage)
1884 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1885 else
1886 domain->iommu_superpage = 0;
1887
1888 domain->nid = iommu->node;
1889
1890 /* always allocate the top pgd */
1891 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1892 if (!domain->pgd)
1893 return -ENOMEM;
1894 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1895 return 0;
1896 }
1897
1898 static void domain_exit(struct dmar_domain *domain)
1899 {
1900
1901 /* Remove associated devices and clear attached or cached domains */
1902 domain_remove_dev_info(domain);
1903
1904 /* destroy iovas */
1905 put_iova_domain(&domain->iovad);
1906
1907 if (domain->pgd) {
1908 struct page *freelist;
1909
1910 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1911 dma_free_pagelist(freelist);
1912 }
1913
1914 free_domain_mem(domain);
1915 }
1916
1917 /*
1918 * Get the PASID directory size for scalable mode context entry.
1919 * Value of X in the PDTS field of a scalable mode context entry
1920 * indicates PASID directory with 2^(X + 7) entries.
1921 */
1922 static inline unsigned long context_get_sm_pds(struct pasid_table *table)
1923 {
1924 int pds, max_pde;
1925
1926 max_pde = table->max_pasid >> PASID_PDE_SHIFT;
1927 pds = find_first_bit((unsigned long *)&max_pde, MAX_NR_PASID_BITS);
1928 if (pds < 7)
1929 return 0;
1930
1931 return pds - 7;
1932 }
1933
1934 /*
1935 * Set the RID_PASID field of a scalable mode context entry. The
1936 * IOMMU hardware will use the PASID value set in this field for
1937 * DMA translations of DMA requests without PASID.
1938 */
1939 static inline void
1940 context_set_sm_rid2pasid(struct context_entry *context, unsigned long pasid)
1941 {
1942 context->hi |= pasid & ((1 << 20) - 1);
1943 context->hi |= (1 << 20);
1944 }
1945
1946 /*
1947 * Set the DTE(Device-TLB Enable) field of a scalable mode context
1948 * entry.
1949 */
1950 static inline void context_set_sm_dte(struct context_entry *context)
1951 {
1952 context->lo |= (1 << 2);
1953 }
1954
1955 /*
1956 * Set the PRE(Page Request Enable) field of a scalable mode context
1957 * entry.
1958 */
1959 static inline void context_set_sm_pre(struct context_entry *context)
1960 {
1961 context->lo |= (1 << 4);
1962 }
1963
1964 /* Convert value to context PASID directory size field coding. */
1965 #define context_pdts(pds) (((pds) & 0x7) << 9)
1966
1967 static int domain_context_mapping_one(struct dmar_domain *domain,
1968 struct intel_iommu *iommu,
1969 struct pasid_table *table,
1970 u8 bus, u8 devfn)
1971 {
1972 u16 did = domain->iommu_did[iommu->seq_id];
1973 int translation = CONTEXT_TT_MULTI_LEVEL;
1974 struct device_domain_info *info = NULL;
1975 struct context_entry *context;
1976 unsigned long flags;
1977 int ret;
1978
1979 WARN_ON(did == 0);
1980
1981 if (hw_pass_through && domain_type_is_si(domain))
1982 translation = CONTEXT_TT_PASS_THROUGH;
1983
1984 pr_debug("Set context mapping for %02x:%02x.%d\n",
1985 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1986
1987 BUG_ON(!domain->pgd);
1988
1989 spin_lock_irqsave(&device_domain_lock, flags);
1990 spin_lock(&iommu->lock);
1991
1992 ret = -ENOMEM;
1993 context = iommu_context_addr(iommu, bus, devfn, 1);
1994 if (!context)
1995 goto out_unlock;
1996
1997 ret = 0;
1998 if (context_present(context))
1999 goto out_unlock;
2000
2001 /*
2002 * For kdump cases, old valid entries may be cached due to the
2003 * in-flight DMA and copied pgtable, but there is no unmapping
2004 * behaviour for them, thus we need an explicit cache flush for
2005 * the newly-mapped device. For kdump, at this point, the device
2006 * is supposed to finish reset at its driver probe stage, so no
2007 * in-flight DMA will exist, and we don't need to worry anymore
2008 * hereafter.
2009 */
2010 if (context_copied(context)) {
2011 u16 did_old = context_domain_id(context);
2012
2013 if (did_old < cap_ndoms(iommu->cap)) {
2014 iommu->flush.flush_context(iommu, did_old,
2015 (((u16)bus) << 8) | devfn,
2016 DMA_CCMD_MASK_NOBIT,
2017 DMA_CCMD_DEVICE_INVL);
2018 iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
2019 DMA_TLB_DSI_FLUSH);
2020 }
2021 }
2022
2023 context_clear_entry(context);
2024
2025 if (sm_supported(iommu)) {
2026 unsigned long pds;
2027
2028 WARN_ON(!table);
2029
2030 /* Setup the PASID DIR pointer: */
2031 pds = context_get_sm_pds(table);
2032 context->lo = (u64)virt_to_phys(table->table) |
2033 context_pdts(pds);
2034
2035 /* Setup the RID_PASID field: */
2036 context_set_sm_rid2pasid(context, PASID_RID2PASID);
2037
2038 /*
2039 * Setup the Device-TLB enable bit and Page request
2040 * Enable bit:
2041 */
2042 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2043 if (info && info->ats_supported)
2044 context_set_sm_dte(context);
2045 if (info && info->pri_supported)
2046 context_set_sm_pre(context);
2047 } else {
2048 struct dma_pte *pgd = domain->pgd;
2049 int agaw;
2050
2051 context_set_domain_id(context, did);
2052
2053 if (translation != CONTEXT_TT_PASS_THROUGH) {
2054 /*
2055 * Skip top levels of page tables for iommu which has
2056 * less agaw than default. Unnecessary for PT mode.
2057 */
2058 for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
2059 ret = -ENOMEM;
2060 pgd = phys_to_virt(dma_pte_addr(pgd));
2061 if (!dma_pte_present(pgd))
2062 goto out_unlock;
2063 }
2064
2065 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2066 if (info && info->ats_supported)
2067 translation = CONTEXT_TT_DEV_IOTLB;
2068 else
2069 translation = CONTEXT_TT_MULTI_LEVEL;
2070
2071 context_set_address_root(context, virt_to_phys(pgd));
2072 context_set_address_width(context, agaw);
2073 } else {
2074 /*
2075 * In pass through mode, AW must be programmed to
2076 * indicate the largest AGAW value supported by
2077 * hardware. And ASR is ignored by hardware.
2078 */
2079 context_set_address_width(context, iommu->msagaw);
2080 }
2081
2082 context_set_translation_type(context, translation);
2083 }
2084
2085 context_set_fault_enable(context);
2086 context_set_present(context);
2087 domain_flush_cache(domain, context, sizeof(*context));
2088
2089 /*
2090 * It's a non-present to present mapping. If hardware doesn't cache
2091 * non-present entry we only need to flush the write-buffer. If the
2092 * _does_ cache non-present entries, then it does so in the special
2093 * domain #0, which we have to flush:
2094 */
2095 if (cap_caching_mode(iommu->cap)) {
2096 iommu->flush.flush_context(iommu, 0,
2097 (((u16)bus) << 8) | devfn,
2098 DMA_CCMD_MASK_NOBIT,
2099 DMA_CCMD_DEVICE_INVL);
2100 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2101 } else {
2102 iommu_flush_write_buffer(iommu);
2103 }
2104 iommu_enable_dev_iotlb(info);
2105
2106 ret = 0;
2107
2108 out_unlock:
2109 spin_unlock(&iommu->lock);
2110 spin_unlock_irqrestore(&device_domain_lock, flags);
2111
2112 return ret;
2113 }
2114
2115 struct domain_context_mapping_data {
2116 struct dmar_domain *domain;
2117 struct intel_iommu *iommu;
2118 struct pasid_table *table;
2119 };
2120
2121 static int domain_context_mapping_cb(struct pci_dev *pdev,
2122 u16 alias, void *opaque)
2123 {
2124 struct domain_context_mapping_data *data = opaque;
2125
2126 return domain_context_mapping_one(data->domain, data->iommu,
2127 data->table, PCI_BUS_NUM(alias),
2128 alias & 0xff);
2129 }
2130
2131 static int
2132 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2133 {
2134 struct domain_context_mapping_data data;
2135 struct pasid_table *table;
2136 struct intel_iommu *iommu;
2137 u8 bus, devfn;
2138
2139 iommu = device_to_iommu(dev, &bus, &devfn);
2140 if (!iommu)
2141 return -ENODEV;
2142
2143 table = intel_pasid_get_table(dev);
2144
2145 if (!dev_is_pci(dev))
2146 return domain_context_mapping_one(domain, iommu, table,
2147 bus, devfn);
2148
2149 data.domain = domain;
2150 data.iommu = iommu;
2151 data.table = table;
2152
2153 return pci_for_each_dma_alias(to_pci_dev(dev),
2154 &domain_context_mapping_cb, &data);
2155 }
2156
2157 static int domain_context_mapped_cb(struct pci_dev *pdev,
2158 u16 alias, void *opaque)
2159 {
2160 struct intel_iommu *iommu = opaque;
2161
2162 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2163 }
2164
2165 static int domain_context_mapped(struct device *dev)
2166 {
2167 struct intel_iommu *iommu;
2168 u8 bus, devfn;
2169
2170 iommu = device_to_iommu(dev, &bus, &devfn);
2171 if (!iommu)
2172 return -ENODEV;
2173
2174 if (!dev_is_pci(dev))
2175 return device_context_mapped(iommu, bus, devfn);
2176
2177 return !pci_for_each_dma_alias(to_pci_dev(dev),
2178 domain_context_mapped_cb, iommu);
2179 }
2180
2181 /* Returns a number of VTD pages, but aligned to MM page size */
2182 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2183 size_t size)
2184 {
2185 host_addr &= ~PAGE_MASK;
2186 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2187 }
2188
2189 /* Return largest possible superpage level for a given mapping */
2190 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2191 unsigned long iov_pfn,
2192 unsigned long phy_pfn,
2193 unsigned long pages)
2194 {
2195 int support, level = 1;
2196 unsigned long pfnmerge;
2197
2198 support = domain->iommu_superpage;
2199
2200 /* To use a large page, the virtual *and* physical addresses
2201 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2202 of them will mean we have to use smaller pages. So just
2203 merge them and check both at once. */
2204 pfnmerge = iov_pfn | phy_pfn;
2205
2206 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2207 pages >>= VTD_STRIDE_SHIFT;
2208 if (!pages)
2209 break;
2210 pfnmerge >>= VTD_STRIDE_SHIFT;
2211 level++;
2212 support--;
2213 }
2214 return level;
2215 }
2216
2217 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2218 struct scatterlist *sg, unsigned long phys_pfn,
2219 unsigned long nr_pages, int prot)
2220 {
2221 struct dma_pte *first_pte = NULL, *pte = NULL;
2222 phys_addr_t uninitialized_var(pteval);
2223 unsigned long sg_res = 0;
2224 unsigned int largepage_lvl = 0;
2225 unsigned long lvl_pages = 0;
2226
2227 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2228
2229 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2230 return -EINVAL;
2231
2232 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2233
2234 if (!sg) {
2235 sg_res = nr_pages;
2236 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2237 }
2238
2239 while (nr_pages > 0) {
2240 uint64_t tmp;
2241
2242 if (!sg_res) {
2243 unsigned int pgoff = sg->offset & ~PAGE_MASK;
2244
2245 sg_res = aligned_nrpages(sg->offset, sg->length);
2246 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + pgoff;
2247 sg->dma_length = sg->length;
2248 pteval = (sg_phys(sg) - pgoff) | prot;
2249 phys_pfn = pteval >> VTD_PAGE_SHIFT;
2250 }
2251
2252 if (!pte) {
2253 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2254
2255 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2256 if (!pte)
2257 return -ENOMEM;
2258 /* It is large page*/
2259 if (largepage_lvl > 1) {
2260 unsigned long nr_superpages, end_pfn;
2261
2262 pteval |= DMA_PTE_LARGE_PAGE;
2263 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2264
2265 nr_superpages = sg_res / lvl_pages;
2266 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2267
2268 /*
2269 * Ensure that old small page tables are
2270 * removed to make room for superpage(s).
2271 * We're adding new large pages, so make sure
2272 * we don't remove their parent tables.
2273 */
2274 dma_pte_free_pagetable(domain, iov_pfn, end_pfn,
2275 largepage_lvl + 1);
2276 } else {
2277 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2278 }
2279
2280 }
2281 /* We don't need lock here, nobody else
2282 * touches the iova range
2283 */
2284 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2285 if (tmp) {
2286 static int dumps = 5;
2287 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2288 iov_pfn, tmp, (unsigned long long)pteval);
2289 if (dumps) {
2290 dumps--;
2291 debug_dma_dump_mappings(NULL);
2292 }
2293 WARN_ON(1);
2294 }
2295
2296 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2297
2298 BUG_ON(nr_pages < lvl_pages);
2299 BUG_ON(sg_res < lvl_pages);
2300
2301 nr_pages -= lvl_pages;
2302 iov_pfn += lvl_pages;
2303 phys_pfn += lvl_pages;
2304 pteval += lvl_pages * VTD_PAGE_SIZE;
2305 sg_res -= lvl_pages;
2306
2307 /* If the next PTE would be the first in a new page, then we
2308 need to flush the cache on the entries we've just written.
2309 And then we'll need to recalculate 'pte', so clear it and
2310 let it get set again in the if (!pte) block above.
2311
2312 If we're done (!nr_pages) we need to flush the cache too.
2313
2314 Also if we've been setting superpages, we may need to
2315 recalculate 'pte' and switch back to smaller pages for the
2316 end of the mapping, if the trailing size is not enough to
2317 use another superpage (i.e. sg_res < lvl_pages). */
2318 pte++;
2319 if (!nr_pages || first_pte_in_page(pte) ||
2320 (largepage_lvl > 1 && sg_res < lvl_pages)) {
2321 domain_flush_cache(domain, first_pte,
2322 (void *)pte - (void *)first_pte);
2323 pte = NULL;
2324 }
2325
2326 if (!sg_res && nr_pages)
2327 sg = sg_next(sg);
2328 }
2329 return 0;
2330 }
2331
2332 static int domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2333 struct scatterlist *sg, unsigned long phys_pfn,
2334 unsigned long nr_pages, int prot)
2335 {
2336 int iommu_id, ret;
2337 struct intel_iommu *iommu;
2338
2339 /* Do the real mapping first */
2340 ret = __domain_mapping(domain, iov_pfn, sg, phys_pfn, nr_pages, prot);
2341 if (ret)
2342 return ret;
2343
2344 for_each_domain_iommu(iommu_id, domain) {
2345 iommu = g_iommus[iommu_id];
2346 __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
2347 }
2348
2349 return 0;
2350 }
2351
2352 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2353 struct scatterlist *sg, unsigned long nr_pages,
2354 int prot)
2355 {
2356 return domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2357 }
2358
2359 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2360 unsigned long phys_pfn, unsigned long nr_pages,
2361 int prot)
2362 {
2363 return domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2364 }
2365
2366 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2367 {
2368 unsigned long flags;
2369 struct context_entry *context;
2370 u16 did_old;
2371
2372 if (!iommu)
2373 return;
2374
2375 spin_lock_irqsave(&iommu->lock, flags);
2376 context = iommu_context_addr(iommu, bus, devfn, 0);
2377 if (!context) {
2378 spin_unlock_irqrestore(&iommu->lock, flags);
2379 return;
2380 }
2381 did_old = context_domain_id(context);
2382 context_clear_entry(context);
2383 __iommu_flush_cache(iommu, context, sizeof(*context));
2384 spin_unlock_irqrestore(&iommu->lock, flags);
2385 iommu->flush.flush_context(iommu,
2386 did_old,
2387 (((u16)bus) << 8) | devfn,
2388 DMA_CCMD_MASK_NOBIT,
2389 DMA_CCMD_DEVICE_INVL);
2390 iommu->flush.flush_iotlb(iommu,
2391 did_old,
2392 0,
2393 0,
2394 DMA_TLB_DSI_FLUSH);
2395 }
2396
2397 static inline void unlink_domain_info(struct device_domain_info *info)
2398 {
2399 assert_spin_locked(&device_domain_lock);
2400 list_del(&info->link);
2401 list_del(&info->global);
2402 if (info->dev)
2403 info->dev->archdata.iommu = NULL;
2404 }
2405
2406 static void domain_remove_dev_info(struct dmar_domain *domain)
2407 {
2408 struct device_domain_info *info, *tmp;
2409 unsigned long flags;
2410
2411 spin_lock_irqsave(&device_domain_lock, flags);
2412 list_for_each_entry_safe(info, tmp, &domain->devices, link)
2413 __dmar_remove_one_dev_info(info);
2414 spin_unlock_irqrestore(&device_domain_lock, flags);
2415 }
2416
2417 static struct dmar_domain *find_domain(struct device *dev)
2418 {
2419 struct device_domain_info *info;
2420
2421 if (unlikely(dev->archdata.iommu == DEFER_DEVICE_DOMAIN_INFO ||
2422 dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO))
2423 return NULL;
2424
2425 if (dev_is_pci(dev))
2426 dev = &pci_real_dma_dev(to_pci_dev(dev))->dev;
2427
2428 /* No lock here, assumes no domain exit in normal case */
2429 info = dev->archdata.iommu;
2430 if (likely(info))
2431 return info->domain;
2432
2433 return NULL;
2434 }
2435
2436 static struct dmar_domain *deferred_attach_domain(struct device *dev)
2437 {
2438 if (unlikely(dev->archdata.iommu == DEFER_DEVICE_DOMAIN_INFO)) {
2439 struct iommu_domain *domain;
2440
2441 dev->archdata.iommu = NULL;
2442 domain = iommu_get_domain_for_dev(dev);
2443 if (domain)
2444 intel_iommu_attach_device(domain, dev);
2445 }
2446
2447 return find_domain(dev);
2448 }
2449
2450 static inline struct device_domain_info *
2451 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2452 {
2453 struct device_domain_info *info;
2454
2455 list_for_each_entry(info, &device_domain_list, global)
2456 if (info->iommu->segment == segment && info->bus == bus &&
2457 info->devfn == devfn)
2458 return info;
2459
2460 return NULL;
2461 }
2462
2463 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2464 int bus, int devfn,
2465 struct device *dev,
2466 struct dmar_domain *domain)
2467 {
2468 struct dmar_domain *found = NULL;
2469 struct device_domain_info *info;
2470 unsigned long flags;
2471 int ret;
2472
2473 info = alloc_devinfo_mem();
2474 if (!info)
2475 return NULL;
2476
2477 info->bus = bus;
2478 info->devfn = devfn;
2479 info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2480 info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2481 info->ats_qdep = 0;
2482 info->dev = dev;
2483 info->domain = domain;
2484 info->iommu = iommu;
2485 info->pasid_table = NULL;
2486 info->auxd_enabled = 0;
2487 INIT_LIST_HEAD(&info->auxiliary_domains);
2488
2489 if (dev && dev_is_pci(dev)) {
2490 struct pci_dev *pdev = to_pci_dev(info->dev);
2491
2492 if (!pdev->untrusted &&
2493 !pci_ats_disabled() &&
2494 ecap_dev_iotlb_support(iommu->ecap) &&
2495 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2496 dmar_find_matched_atsr_unit(pdev))
2497 info->ats_supported = 1;
2498
2499 if (sm_supported(iommu)) {
2500 if (pasid_supported(iommu)) {
2501 int features = pci_pasid_features(pdev);
2502 if (features >= 0)
2503 info->pasid_supported = features | 1;
2504 }
2505
2506 if (info->ats_supported && ecap_prs(iommu->ecap) &&
2507 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2508 info->pri_supported = 1;
2509 }
2510 }
2511
2512 spin_lock_irqsave(&device_domain_lock, flags);
2513 if (dev)
2514 found = find_domain(dev);
2515
2516 if (!found) {
2517 struct device_domain_info *info2;
2518 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2519 if (info2) {
2520 found = info2->domain;
2521 info2->dev = dev;
2522 }
2523 }
2524
2525 if (found) {
2526 spin_unlock_irqrestore(&device_domain_lock, flags);
2527 free_devinfo_mem(info);
2528 /* Caller must free the original domain */
2529 return found;
2530 }
2531
2532 spin_lock(&iommu->lock);
2533 ret = domain_attach_iommu(domain, iommu);
2534 spin_unlock(&iommu->lock);
2535
2536 if (ret) {
2537 spin_unlock_irqrestore(&device_domain_lock, flags);
2538 free_devinfo_mem(info);
2539 return NULL;
2540 }
2541
2542 list_add(&info->link, &domain->devices);
2543 list_add(&info->global, &device_domain_list);
2544 if (dev)
2545 dev->archdata.iommu = info;
2546 spin_unlock_irqrestore(&device_domain_lock, flags);
2547
2548 /* PASID table is mandatory for a PCI device in scalable mode. */
2549 if (dev && dev_is_pci(dev) && sm_supported(iommu)) {
2550 ret = intel_pasid_alloc_table(dev);
2551 if (ret) {
2552 dev_err(dev, "PASID table allocation failed\n");
2553 dmar_remove_one_dev_info(dev);
2554 return NULL;
2555 }
2556
2557 /* Setup the PASID entry for requests without PASID: */
2558 spin_lock(&iommu->lock);
2559 if (hw_pass_through && domain_type_is_si(domain))
2560 ret = intel_pasid_setup_pass_through(iommu, domain,
2561 dev, PASID_RID2PASID);
2562 else
2563 ret = intel_pasid_setup_second_level(iommu, domain,
2564 dev, PASID_RID2PASID);
2565 spin_unlock(&iommu->lock);
2566 if (ret) {
2567 dev_err(dev, "Setup RID2PASID failed\n");
2568 dmar_remove_one_dev_info(dev);
2569 return NULL;
2570 }
2571 }
2572
2573 if (dev && domain_context_mapping(domain, dev)) {
2574 dev_err(dev, "Domain context map failed\n");
2575 dmar_remove_one_dev_info(dev);
2576 return NULL;
2577 }
2578
2579 return domain;
2580 }
2581
2582 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2583 {
2584 *(u16 *)opaque = alias;
2585 return 0;
2586 }
2587
2588 static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
2589 {
2590 struct device_domain_info *info;
2591 struct dmar_domain *domain = NULL;
2592 struct intel_iommu *iommu;
2593 u16 dma_alias;
2594 unsigned long flags;
2595 u8 bus, devfn;
2596
2597 iommu = device_to_iommu(dev, &bus, &devfn);
2598 if (!iommu)
2599 return NULL;
2600
2601 if (dev_is_pci(dev)) {
2602 struct pci_dev *pdev = to_pci_dev(dev);
2603
2604 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2605
2606 spin_lock_irqsave(&device_domain_lock, flags);
2607 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2608 PCI_BUS_NUM(dma_alias),
2609 dma_alias & 0xff);
2610 if (info) {
2611 iommu = info->iommu;
2612 domain = info->domain;
2613 }
2614 spin_unlock_irqrestore(&device_domain_lock, flags);
2615
2616 /* DMA alias already has a domain, use it */
2617 if (info)
2618 goto out;
2619 }
2620
2621 /* Allocate and initialize new domain for the device */
2622 domain = alloc_domain(0);
2623 if (!domain)
2624 return NULL;
2625 if (domain_init(domain, iommu, gaw)) {
2626 domain_exit(domain);
2627 return NULL;
2628 }
2629
2630 out:
2631 return domain;
2632 }
2633
2634 static struct dmar_domain *set_domain_for_dev(struct device *dev,
2635 struct dmar_domain *domain)
2636 {
2637 struct intel_iommu *iommu;
2638 struct dmar_domain *tmp;
2639 u16 req_id, dma_alias;
2640 u8 bus, devfn;
2641
2642 iommu = device_to_iommu(dev, &bus, &devfn);
2643 if (!iommu)
2644 return NULL;
2645
2646 req_id = ((u16)bus << 8) | devfn;
2647
2648 if (dev_is_pci(dev)) {
2649 struct pci_dev *pdev = to_pci_dev(dev);
2650
2651 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2652
2653 /* register PCI DMA alias device */
2654 if (req_id != dma_alias) {
2655 tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2656 dma_alias & 0xff, NULL, domain);
2657
2658 if (!tmp || tmp != domain)
2659 return tmp;
2660 }
2661 }
2662
2663 tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2664 if (!tmp || tmp != domain)
2665 return tmp;
2666
2667 return domain;
2668 }
2669
2670 static int iommu_domain_identity_map(struct dmar_domain *domain,
2671 unsigned long long start,
2672 unsigned long long end)
2673 {
2674 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2675 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2676
2677 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2678 dma_to_mm_pfn(last_vpfn))) {
2679 pr_err("Reserving iova failed\n");
2680 return -ENOMEM;
2681 }
2682
2683 pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2684 /*
2685 * RMRR range might have overlap with physical memory range,
2686 * clear it first
2687 */
2688 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2689
2690 return __domain_mapping(domain, first_vpfn, NULL,
2691 first_vpfn, last_vpfn - first_vpfn + 1,
2692 DMA_PTE_READ|DMA_PTE_WRITE);
2693 }
2694
2695 static int domain_prepare_identity_map(struct device *dev,
2696 struct dmar_domain *domain,
2697 unsigned long long start,
2698 unsigned long long end)
2699 {
2700 /* For _hardware_ passthrough, don't bother. But for software
2701 passthrough, we do it anyway -- it may indicate a memory
2702 range which is reserved in E820, so which didn't get set
2703 up to start with in si_domain */
2704 if (domain == si_domain && hw_pass_through) {
2705 dev_warn(dev, "Ignoring identity map for HW passthrough [0x%Lx - 0x%Lx]\n",
2706 start, end);
2707 return 0;
2708 }
2709
2710 dev_info(dev, "Setting identity map [0x%Lx - 0x%Lx]\n", start, end);
2711
2712 if (end < start) {
2713 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2714 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2715 dmi_get_system_info(DMI_BIOS_VENDOR),
2716 dmi_get_system_info(DMI_BIOS_VERSION),
2717 dmi_get_system_info(DMI_PRODUCT_VERSION));
2718 return -EIO;
2719 }
2720
2721 if (end >> agaw_to_width(domain->agaw)) {
2722 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2723 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2724 agaw_to_width(domain->agaw),
2725 dmi_get_system_info(DMI_BIOS_VENDOR),
2726 dmi_get_system_info(DMI_BIOS_VERSION),
2727 dmi_get_system_info(DMI_PRODUCT_VERSION));
2728 return -EIO;
2729 }
2730
2731 return iommu_domain_identity_map(domain, start, end);
2732 }
2733
2734 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2735
2736 static int __init si_domain_init(int hw)
2737 {
2738 struct dmar_rmrr_unit *rmrr;
2739 struct device *dev;
2740 int i, nid, ret;
2741
2742 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2743 if (!si_domain)
2744 return -EFAULT;
2745
2746 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2747 domain_exit(si_domain);
2748 return -EFAULT;
2749 }
2750
2751 if (hw)
2752 return 0;
2753
2754 for_each_online_node(nid) {
2755 unsigned long start_pfn, end_pfn;
2756 int i;
2757
2758 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2759 ret = iommu_domain_identity_map(si_domain,
2760 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2761 if (ret)
2762 return ret;
2763 }
2764 }
2765
2766 /*
2767 * Normally we use DMA domains for devices which have RMRRs. But we
2768 * loose this requirement for graphic and usb devices. Identity map
2769 * the RMRRs for graphic and USB devices so that they could use the
2770 * si_domain.
2771 */
2772 for_each_rmrr_units(rmrr) {
2773 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
2774 i, dev) {
2775 unsigned long long start = rmrr->base_address;
2776 unsigned long long end = rmrr->end_address;
2777
2778 if (device_is_rmrr_locked(dev))
2779 continue;
2780
2781 if (WARN_ON(end < start ||
2782 end >> agaw_to_width(si_domain->agaw)))
2783 continue;
2784
2785 ret = iommu_domain_identity_map(si_domain, start, end);
2786 if (ret)
2787 return ret;
2788 }
2789 }
2790
2791 return 0;
2792 }
2793
2794 static int identity_mapping(struct device *dev)
2795 {
2796 struct device_domain_info *info;
2797
2798 info = dev->archdata.iommu;
2799 if (info && info != DUMMY_DEVICE_DOMAIN_INFO && info != DEFER_DEVICE_DOMAIN_INFO)
2800 return (info->domain == si_domain);
2801
2802 return 0;
2803 }
2804
2805 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2806 {
2807 struct dmar_domain *ndomain;
2808 struct intel_iommu *iommu;
2809 u8 bus, devfn;
2810
2811 iommu = device_to_iommu(dev, &bus, &devfn);
2812 if (!iommu)
2813 return -ENODEV;
2814
2815 ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2816 if (ndomain != domain)
2817 return -EBUSY;
2818
2819 return 0;
2820 }
2821
2822 static bool device_has_rmrr(struct device *dev)
2823 {
2824 struct dmar_rmrr_unit *rmrr;
2825 struct device *tmp;
2826 int i;
2827
2828 rcu_read_lock();
2829 for_each_rmrr_units(rmrr) {
2830 /*
2831 * Return TRUE if this RMRR contains the device that
2832 * is passed in.
2833 */
2834 for_each_active_dev_scope(rmrr->devices,
2835 rmrr->devices_cnt, i, tmp)
2836 if (tmp == dev ||
2837 is_downstream_to_pci_bridge(dev, tmp)) {
2838 rcu_read_unlock();
2839 return true;
2840 }
2841 }
2842 rcu_read_unlock();
2843 return false;
2844 }
2845
2846 /**
2847 * device_rmrr_is_relaxable - Test whether the RMRR of this device
2848 * is relaxable (ie. is allowed to be not enforced under some conditions)
2849 * @dev: device handle
2850 *
2851 * We assume that PCI USB devices with RMRRs have them largely
2852 * for historical reasons and that the RMRR space is not actively used post
2853 * boot. This exclusion may change if vendors begin to abuse it.
2854 *
2855 * The same exception is made for graphics devices, with the requirement that
2856 * any use of the RMRR regions will be torn down before assigning the device
2857 * to a guest.
2858 *
2859 * Return: true if the RMRR is relaxable, false otherwise
2860 */
2861 static bool device_rmrr_is_relaxable(struct device *dev)
2862 {
2863 struct pci_dev *pdev;
2864
2865 if (!dev_is_pci(dev))
2866 return false;
2867
2868 pdev = to_pci_dev(dev);
2869 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2870 return true;
2871 else
2872 return false;
2873 }
2874
2875 /*
2876 * There are a couple cases where we need to restrict the functionality of
2877 * devices associated with RMRRs. The first is when evaluating a device for
2878 * identity mapping because problems exist when devices are moved in and out
2879 * of domains and their respective RMRR information is lost. This means that
2880 * a device with associated RMRRs will never be in a "passthrough" domain.
2881 * The second is use of the device through the IOMMU API. This interface
2882 * expects to have full control of the IOVA space for the device. We cannot
2883 * satisfy both the requirement that RMRR access is maintained and have an
2884 * unencumbered IOVA space. We also have no ability to quiesce the device's
2885 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2886 * We therefore prevent devices associated with an RMRR from participating in
2887 * the IOMMU API, which eliminates them from device assignment.
2888 *
2889 * In both cases, devices which have relaxable RMRRs are not concerned by this
2890 * restriction. See device_rmrr_is_relaxable comment.
2891 */
2892 static bool device_is_rmrr_locked(struct device *dev)
2893 {
2894 if (!device_has_rmrr(dev))
2895 return false;
2896
2897 if (device_rmrr_is_relaxable(dev))
2898 return false;
2899
2900 return true;
2901 }
2902
2903 /*
2904 * Return the required default domain type for a specific device.
2905 *
2906 * @dev: the device in query
2907 * @startup: true if this is during early boot
2908 *
2909 * Returns:
2910 * - IOMMU_DOMAIN_DMA: device requires a dynamic mapping domain
2911 * - IOMMU_DOMAIN_IDENTITY: device requires an identical mapping domain
2912 * - 0: both identity and dynamic domains work for this device
2913 */
2914 static int device_def_domain_type(struct device *dev)
2915 {
2916 if (dev_is_pci(dev)) {
2917 struct pci_dev *pdev = to_pci_dev(dev);
2918
2919 if (device_is_rmrr_locked(dev))
2920 return IOMMU_DOMAIN_DMA;
2921
2922 /*
2923 * Prevent any device marked as untrusted from getting
2924 * placed into the statically identity mapping domain.
2925 */
2926 if (pdev->untrusted)
2927 return IOMMU_DOMAIN_DMA;
2928
2929 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2930 return IOMMU_DOMAIN_IDENTITY;
2931
2932 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2933 return IOMMU_DOMAIN_IDENTITY;
2934
2935 /*
2936 * We want to start off with all devices in the 1:1 domain, and
2937 * take them out later if we find they can't access all of memory.
2938 *
2939 * However, we can't do this for PCI devices behind bridges,
2940 * because all PCI devices behind the same bridge will end up
2941 * with the same source-id on their transactions.
2942 *
2943 * Practically speaking, we can't change things around for these
2944 * devices at run-time, because we can't be sure there'll be no
2945 * DMA transactions in flight for any of their siblings.
2946 *
2947 * So PCI devices (unless they're on the root bus) as well as
2948 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2949 * the 1:1 domain, just in _case_ one of their siblings turns out
2950 * not to be able to map all of memory.
2951 */
2952 if (!pci_is_pcie(pdev)) {
2953 if (!pci_is_root_bus(pdev->bus))
2954 return IOMMU_DOMAIN_DMA;
2955 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2956 return IOMMU_DOMAIN_DMA;
2957 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2958 return IOMMU_DOMAIN_DMA;
2959 } else {
2960 if (device_has_rmrr(dev))
2961 return IOMMU_DOMAIN_DMA;
2962 }
2963
2964 return (iommu_identity_mapping & IDENTMAP_ALL) ?
2965 IOMMU_DOMAIN_IDENTITY : 0;
2966 }
2967
2968 static void intel_iommu_init_qi(struct intel_iommu *iommu)
2969 {
2970 /*
2971 * Start from the sane iommu hardware state.
2972 * If the queued invalidation is already initialized by us
2973 * (for example, while enabling interrupt-remapping) then
2974 * we got the things already rolling from a sane state.
2975 */
2976 if (!iommu->qi) {
2977 /*
2978 * Clear any previous faults.
2979 */
2980 dmar_fault(-1, iommu);
2981 /*
2982 * Disable queued invalidation if supported and already enabled
2983 * before OS handover.
2984 */
2985 dmar_disable_qi(iommu);
2986 }
2987
2988 if (dmar_enable_qi(iommu)) {
2989 /*
2990 * Queued Invalidate not enabled, use Register Based Invalidate
2991 */
2992 iommu->flush.flush_context = __iommu_flush_context;
2993 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2994 pr_info("%s: Using Register based invalidation\n",
2995 iommu->name);
2996 } else {
2997 iommu->flush.flush_context = qi_flush_context;
2998 iommu->flush.flush_iotlb = qi_flush_iotlb;
2999 pr_info("%s: Using Queued invalidation\n", iommu->name);
3000 }
3001 }
3002
3003 static int copy_context_table(struct intel_iommu *iommu,
3004 struct root_entry *old_re,
3005 struct context_entry **tbl,
3006 int bus, bool ext)
3007 {
3008 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
3009 struct context_entry *new_ce = NULL, ce;
3010 struct context_entry *old_ce = NULL;
3011 struct root_entry re;
3012 phys_addr_t old_ce_phys;
3013
3014 tbl_idx = ext ? bus * 2 : bus;
3015 memcpy(&re, old_re, sizeof(re));
3016
3017 for (devfn = 0; devfn < 256; devfn++) {
3018 /* First calculate the correct index */
3019 idx = (ext ? devfn * 2 : devfn) % 256;
3020
3021 if (idx == 0) {
3022 /* First save what we may have and clean up */
3023 if (new_ce) {
3024 tbl[tbl_idx] = new_ce;
3025 __iommu_flush_cache(iommu, new_ce,
3026 VTD_PAGE_SIZE);
3027 pos = 1;
3028 }
3029
3030 if (old_ce)
3031 memunmap(old_ce);
3032
3033 ret = 0;
3034 if (devfn < 0x80)
3035 old_ce_phys = root_entry_lctp(&re);
3036 else
3037 old_ce_phys = root_entry_uctp(&re);
3038
3039 if (!old_ce_phys) {
3040 if (ext && devfn == 0) {
3041 /* No LCTP, try UCTP */
3042 devfn = 0x7f;
3043 continue;
3044 } else {
3045 goto out;
3046 }
3047 }
3048
3049 ret = -ENOMEM;
3050 old_ce = memremap(old_ce_phys, PAGE_SIZE,
3051 MEMREMAP_WB);
3052 if (!old_ce)
3053 goto out;
3054
3055 new_ce = alloc_pgtable_page(iommu->node);
3056 if (!new_ce)
3057 goto out_unmap;
3058
3059 ret = 0;
3060 }
3061
3062 /* Now copy the context entry */
3063 memcpy(&ce, old_ce + idx, sizeof(ce));
3064
3065 if (!__context_present(&ce))
3066 continue;
3067
3068 did = context_domain_id(&ce);
3069 if (did >= 0 && did < cap_ndoms(iommu->cap))
3070 set_bit(did, iommu->domain_ids);
3071
3072 /*
3073 * We need a marker for copied context entries. This
3074 * marker needs to work for the old format as well as
3075 * for extended context entries.
3076 *
3077 * Bit 67 of the context entry is used. In the old
3078 * format this bit is available to software, in the
3079 * extended format it is the PGE bit, but PGE is ignored
3080 * by HW if PASIDs are disabled (and thus still
3081 * available).
3082 *
3083 * So disable PASIDs first and then mark the entry
3084 * copied. This means that we don't copy PASID
3085 * translations from the old kernel, but this is fine as
3086 * faults there are not fatal.
3087 */
3088 context_clear_pasid_enable(&ce);
3089 context_set_copied(&ce);
3090
3091 new_ce[idx] = ce;
3092 }
3093
3094 tbl[tbl_idx + pos] = new_ce;
3095
3096 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3097
3098 out_unmap:
3099 memunmap(old_ce);
3100
3101 out:
3102 return ret;
3103 }
3104
3105 static int copy_translation_tables(struct intel_iommu *iommu)
3106 {
3107 struct context_entry **ctxt_tbls;
3108 struct root_entry *old_rt;
3109 phys_addr_t old_rt_phys;
3110 int ctxt_table_entries;
3111 unsigned long flags;
3112 u64 rtaddr_reg;
3113 int bus, ret;
3114 bool new_ext, ext;
3115
3116 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3117 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
3118 new_ext = !!ecap_ecs(iommu->ecap);
3119
3120 /*
3121 * The RTT bit can only be changed when translation is disabled,
3122 * but disabling translation means to open a window for data
3123 * corruption. So bail out and don't copy anything if we would
3124 * have to change the bit.
3125 */
3126 if (new_ext != ext)
3127 return -EINVAL;
3128
3129 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3130 if (!old_rt_phys)
3131 return -EINVAL;
3132
3133 old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
3134 if (!old_rt)
3135 return -ENOMEM;
3136
3137 /* This is too big for the stack - allocate it from slab */
3138 ctxt_table_entries = ext ? 512 : 256;
3139 ret = -ENOMEM;
3140 ctxt_tbls = kcalloc(ctxt_table_entries, sizeof(void *), GFP_KERNEL);
3141 if (!ctxt_tbls)
3142 goto out_unmap;
3143
3144 for (bus = 0; bus < 256; bus++) {
3145 ret = copy_context_table(iommu, &old_rt[bus],
3146 ctxt_tbls, bus, ext);
3147 if (ret) {
3148 pr_err("%s: Failed to copy context table for bus %d\n",
3149 iommu->name, bus);
3150 continue;
3151 }
3152 }
3153
3154 spin_lock_irqsave(&iommu->lock, flags);
3155
3156 /* Context tables are copied, now write them to the root_entry table */
3157 for (bus = 0; bus < 256; bus++) {
3158 int idx = ext ? bus * 2 : bus;
3159 u64 val;
3160
3161 if (ctxt_tbls[idx]) {
3162 val = virt_to_phys(ctxt_tbls[idx]) | 1;
3163 iommu->root_entry[bus].lo = val;
3164 }
3165
3166 if (!ext || !ctxt_tbls[idx + 1])
3167 continue;
3168
3169 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3170 iommu->root_entry[bus].hi = val;
3171 }
3172
3173 spin_unlock_irqrestore(&iommu->lock, flags);
3174
3175 kfree(ctxt_tbls);
3176
3177 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3178
3179 ret = 0;
3180
3181 out_unmap:
3182 memunmap(old_rt);
3183
3184 return ret;
3185 }
3186
3187 static int __init init_dmars(void)
3188 {
3189 struct dmar_drhd_unit *drhd;
3190 struct intel_iommu *iommu;
3191 int ret;
3192
3193 /*
3194 * for each drhd
3195 * allocate root
3196 * initialize and program root entry to not present
3197 * endfor
3198 */
3199 for_each_drhd_unit(drhd) {
3200 /*
3201 * lock not needed as this is only incremented in the single
3202 * threaded kernel __init code path all other access are read
3203 * only
3204 */
3205 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3206 g_num_of_iommus++;
3207 continue;
3208 }
3209 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3210 }
3211
3212 /* Preallocate enough resources for IOMMU hot-addition */
3213 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3214 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3215
3216 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3217 GFP_KERNEL);
3218 if (!g_iommus) {
3219 pr_err("Allocating global iommu array failed\n");
3220 ret = -ENOMEM;
3221 goto error;
3222 }
3223
3224 for_each_iommu(iommu, drhd) {
3225 if (drhd->ignored) {
3226 iommu_disable_translation(iommu);
3227 continue;
3228 }
3229
3230 /*
3231 * Find the max pasid size of all IOMMU's in the system.
3232 * We need to ensure the system pasid table is no bigger
3233 * than the smallest supported.
3234 */
3235 if (pasid_supported(iommu)) {
3236 u32 temp = 2 << ecap_pss(iommu->ecap);
3237
3238 intel_pasid_max_id = min_t(u32, temp,
3239 intel_pasid_max_id);
3240 }
3241
3242 g_iommus[iommu->seq_id] = iommu;
3243
3244 intel_iommu_init_qi(iommu);
3245
3246 ret = iommu_init_domains(iommu);
3247 if (ret)
3248 goto free_iommu;
3249
3250 init_translation_status(iommu);
3251
3252 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3253 iommu_disable_translation(iommu);
3254 clear_translation_pre_enabled(iommu);
3255 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3256 iommu->name);
3257 }
3258
3259 /*
3260 * TBD:
3261 * we could share the same root & context tables
3262 * among all IOMMU's. Need to Split it later.
3263 */
3264 ret = iommu_alloc_root_entry(iommu);
3265 if (ret)
3266 goto free_iommu;
3267
3268 if (translation_pre_enabled(iommu)) {
3269 pr_info("Translation already enabled - trying to copy translation structures\n");
3270
3271 ret = copy_translation_tables(iommu);
3272 if (ret) {
3273 /*
3274 * We found the IOMMU with translation
3275 * enabled - but failed to copy over the
3276 * old root-entry table. Try to proceed
3277 * by disabling translation now and
3278 * allocating a clean root-entry table.
3279 * This might cause DMAR faults, but
3280 * probably the dump will still succeed.
3281 */
3282 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3283 iommu->name);
3284 iommu_disable_translation(iommu);
3285 clear_translation_pre_enabled(iommu);
3286 } else {
3287 pr_info("Copied translation tables from previous kernel for %s\n",
3288 iommu->name);
3289 }
3290 }
3291
3292 if (!ecap_pass_through(iommu->ecap))
3293 hw_pass_through = 0;
3294 #ifdef CONFIG_INTEL_IOMMU_SVM
3295 if (pasid_supported(iommu))
3296 intel_svm_init(iommu);
3297 #endif
3298 }
3299
3300 /*
3301 * Now that qi is enabled on all iommus, set the root entry and flush
3302 * caches. This is required on some Intel X58 chipsets, otherwise the
3303 * flush_context function will loop forever and the boot hangs.
3304 */
3305 for_each_active_iommu(iommu, drhd) {
3306 iommu_flush_write_buffer(iommu);
3307 iommu_set_root_entry(iommu);
3308 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3309 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3310 }
3311
3312 if (iommu_default_passthrough())
3313 iommu_identity_mapping |= IDENTMAP_ALL;
3314
3315 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3316 dmar_map_gfx = 0;
3317 #endif
3318
3319 if (!dmar_map_gfx)
3320 iommu_identity_mapping |= IDENTMAP_GFX;
3321
3322 check_tylersburg_isoch();
3323
3324 ret = si_domain_init(hw_pass_through);
3325 if (ret)
3326 goto free_iommu;
3327
3328 /*
3329 * for each drhd
3330 * enable fault log
3331 * global invalidate context cache
3332 * global invalidate iotlb
3333 * enable translation
3334 */
3335 for_each_iommu(iommu, drhd) {
3336 if (drhd->ignored) {
3337 /*
3338 * we always have to disable PMRs or DMA may fail on
3339 * this device
3340 */
3341 if (force_on)
3342 iommu_disable_protect_mem_regions(iommu);
3343 continue;
3344 }
3345
3346 iommu_flush_write_buffer(iommu);
3347
3348 #ifdef CONFIG_INTEL_IOMMU_SVM
3349 if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) {
3350 /*
3351 * Call dmar_alloc_hwirq() with dmar_global_lock held,
3352 * could cause possible lock race condition.
3353 */
3354 up_write(&dmar_global_lock);
3355 ret = intel_svm_enable_prq(iommu);
3356 down_write(&dmar_global_lock);
3357 if (ret)
3358 goto free_iommu;
3359 }
3360 #endif
3361 ret = dmar_set_interrupt(iommu);
3362 if (ret)
3363 goto free_iommu;
3364 }
3365
3366 return 0;
3367
3368 free_iommu:
3369 for_each_active_iommu(iommu, drhd) {
3370 disable_dmar_iommu(iommu);
3371 free_dmar_iommu(iommu);
3372 }
3373
3374 kfree(g_iommus);
3375
3376 error:
3377 return ret;
3378 }
3379
3380 /* This takes a number of _MM_ pages, not VTD pages */
3381 static unsigned long intel_alloc_iova(struct device *dev,
3382 struct dmar_domain *domain,
3383 unsigned long nrpages, uint64_t dma_mask)
3384 {
3385 unsigned long iova_pfn;
3386
3387 /* Restrict dma_mask to the width that the iommu can handle */
3388 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3389 /* Ensure we reserve the whole size-aligned region */
3390 nrpages = __roundup_pow_of_two(nrpages);
3391
3392 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3393 /*
3394 * First try to allocate an io virtual address in
3395 * DMA_BIT_MASK(32) and if that fails then try allocating
3396 * from higher range
3397 */
3398 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3399 IOVA_PFN(DMA_BIT_MASK(32)), false);
3400 if (iova_pfn)
3401 return iova_pfn;
3402 }
3403 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3404 IOVA_PFN(dma_mask), true);
3405 if (unlikely(!iova_pfn)) {
3406 dev_err(dev, "Allocating %ld-page iova failed", nrpages);
3407 return 0;
3408 }
3409
3410 return iova_pfn;
3411 }
3412
3413 static struct dmar_domain *get_private_domain_for_dev(struct device *dev)
3414 {
3415 struct dmar_domain *domain, *tmp;
3416 struct dmar_rmrr_unit *rmrr;
3417 struct device *i_dev;
3418 int i, ret;
3419
3420 /* Device shouldn't be attached by any domains. */
3421 domain = find_domain(dev);
3422 if (domain)
3423 return NULL;
3424
3425 domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3426 if (!domain)
3427 goto out;
3428
3429 /* We have a new domain - setup possible RMRRs for the device */
3430 rcu_read_lock();
3431 for_each_rmrr_units(rmrr) {
3432 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3433 i, i_dev) {
3434 if (i_dev != dev)
3435 continue;
3436
3437 ret = domain_prepare_identity_map(dev, domain,
3438 rmrr->base_address,
3439 rmrr->end_address);
3440 if (ret)
3441 dev_err(dev, "Mapping reserved region failed\n");
3442 }
3443 }
3444 rcu_read_unlock();
3445
3446 tmp = set_domain_for_dev(dev, domain);
3447 if (!tmp || domain != tmp) {
3448 domain_exit(domain);
3449 domain = tmp;
3450 }
3451
3452 out:
3453 if (!domain)
3454 dev_err(dev, "Allocating domain failed\n");
3455 else
3456 domain->domain.type = IOMMU_DOMAIN_DMA;
3457
3458 return domain;
3459 }
3460
3461 /* Check if the dev needs to go through non-identity map and unmap process.*/
3462 static bool iommu_need_mapping(struct device *dev)
3463 {
3464 int ret;
3465
3466 if (iommu_dummy(dev))
3467 return false;
3468
3469 ret = identity_mapping(dev);
3470 if (ret) {
3471 u64 dma_mask = *dev->dma_mask;
3472
3473 if (dev->coherent_dma_mask && dev->coherent_dma_mask < dma_mask)
3474 dma_mask = dev->coherent_dma_mask;
3475
3476 if (dma_mask >= dma_direct_get_required_mask(dev))
3477 return false;
3478
3479 /*
3480 * 32 bit DMA is removed from si_domain and fall back to
3481 * non-identity mapping.
3482 */
3483 dmar_remove_one_dev_info(dev);
3484 ret = iommu_request_dma_domain_for_dev(dev);
3485 if (ret) {
3486 struct iommu_domain *domain;
3487 struct dmar_domain *dmar_domain;
3488
3489 domain = iommu_get_domain_for_dev(dev);
3490 if (domain) {
3491 dmar_domain = to_dmar_domain(domain);
3492 dmar_domain->flags |= DOMAIN_FLAG_LOSE_CHILDREN;
3493 }
3494 dmar_remove_one_dev_info(dev);
3495 get_private_domain_for_dev(dev);
3496 }
3497
3498 dev_info(dev, "32bit DMA uses non-identity mapping\n");
3499 }
3500
3501 return true;
3502 }
3503
3504 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3505 size_t size, int dir, u64 dma_mask)
3506 {
3507 struct dmar_domain *domain;
3508 phys_addr_t start_paddr;
3509 unsigned long iova_pfn;
3510 int prot = 0;
3511 int ret;
3512 struct intel_iommu *iommu;
3513 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3514
3515 BUG_ON(dir == DMA_NONE);
3516
3517 domain = deferred_attach_domain(dev);
3518 if (!domain)
3519 return DMA_MAPPING_ERROR;
3520
3521 iommu = domain_get_iommu(domain);
3522 size = aligned_nrpages(paddr, size);
3523
3524 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3525 if (!iova_pfn)
3526 goto error;
3527
3528 /*
3529 * Check if DMAR supports zero-length reads on write only
3530 * mappings..
3531 */
3532 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3533 !cap_zlr(iommu->cap))
3534 prot |= DMA_PTE_READ;
3535 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3536 prot |= DMA_PTE_WRITE;
3537 /*
3538 * paddr - (paddr + size) might be partial page, we should map the whole
3539 * page. Note: if two part of one page are separately mapped, we
3540 * might have two guest_addr mapping to the same host paddr, but this
3541 * is not a big problem
3542 */
3543 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
3544 mm_to_dma_pfn(paddr_pfn), size, prot);
3545 if (ret)
3546 goto error;
3547
3548 start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
3549 start_paddr += paddr & ~PAGE_MASK;
3550
3551 trace_map_single(dev, start_paddr, paddr, size << VTD_PAGE_SHIFT);
3552
3553 return start_paddr;
3554
3555 error:
3556 if (iova_pfn)
3557 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3558 dev_err(dev, "Device request: %zx@%llx dir %d --- failed\n",
3559 size, (unsigned long long)paddr, dir);
3560 return DMA_MAPPING_ERROR;
3561 }
3562
3563 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3564 unsigned long offset, size_t size,
3565 enum dma_data_direction dir,
3566 unsigned long attrs)
3567 {
3568 if (iommu_need_mapping(dev))
3569 return __intel_map_single(dev, page_to_phys(page) + offset,
3570 size, dir, *dev->dma_mask);
3571 return dma_direct_map_page(dev, page, offset, size, dir, attrs);
3572 }
3573
3574 static dma_addr_t intel_map_resource(struct device *dev, phys_addr_t phys_addr,
3575 size_t size, enum dma_data_direction dir,
3576 unsigned long attrs)
3577 {
3578 if (iommu_need_mapping(dev))
3579 return __intel_map_single(dev, phys_addr, size, dir,
3580 *dev->dma_mask);
3581 return dma_direct_map_resource(dev, phys_addr, size, dir, attrs);
3582 }
3583
3584 static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
3585 {
3586 struct dmar_domain *domain;
3587 unsigned long start_pfn, last_pfn;
3588 unsigned long nrpages;
3589 unsigned long iova_pfn;
3590 struct intel_iommu *iommu;
3591 struct page *freelist;
3592 struct pci_dev *pdev = NULL;
3593
3594 domain = find_domain(dev);
3595 BUG_ON(!domain);
3596
3597 iommu = domain_get_iommu(domain);
3598
3599 iova_pfn = IOVA_PFN(dev_addr);
3600
3601 nrpages = aligned_nrpages(dev_addr, size);
3602 start_pfn = mm_to_dma_pfn(iova_pfn);
3603 last_pfn = start_pfn + nrpages - 1;
3604
3605 if (dev_is_pci(dev))
3606 pdev = to_pci_dev(dev);
3607
3608 freelist = domain_unmap(domain, start_pfn, last_pfn);
3609 if (intel_iommu_strict || (pdev && pdev->untrusted) ||
3610 !has_iova_flush_queue(&domain->iovad)) {
3611 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3612 nrpages, !freelist, 0);
3613 /* free iova */
3614 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
3615 dma_free_pagelist(freelist);
3616 } else {
3617 queue_iova(&domain->iovad, iova_pfn, nrpages,
3618 (unsigned long)freelist);
3619 /*
3620 * queue up the release of the unmap to save the 1/6th of the
3621 * cpu used up by the iotlb flush operation...
3622 */
3623 }
3624
3625 trace_unmap_single(dev, dev_addr, size);
3626 }
3627
3628 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3629 size_t size, enum dma_data_direction dir,
3630 unsigned long attrs)
3631 {
3632 if (iommu_need_mapping(dev))
3633 intel_unmap(dev, dev_addr, size);
3634 else
3635 dma_direct_unmap_page(dev, dev_addr, size, dir, attrs);
3636 }
3637
3638 static void intel_unmap_resource(struct device *dev, dma_addr_t dev_addr,
3639 size_t size, enum dma_data_direction dir, unsigned long attrs)
3640 {
3641 if (iommu_need_mapping(dev))
3642 intel_unmap(dev, dev_addr, size);
3643 }
3644
3645 static void *intel_alloc_coherent(struct device *dev, size_t size,
3646 dma_addr_t *dma_handle, gfp_t flags,
3647 unsigned long attrs)
3648 {
3649 struct page *page = NULL;
3650 int order;
3651
3652 if (!iommu_need_mapping(dev))
3653 return dma_direct_alloc(dev, size, dma_handle, flags, attrs);
3654
3655 size = PAGE_ALIGN(size);
3656 order = get_order(size);
3657
3658 if (gfpflags_allow_blocking(flags)) {
3659 unsigned int count = size >> PAGE_SHIFT;
3660
3661 page = dma_alloc_from_contiguous(dev, count, order,
3662 flags & __GFP_NOWARN);
3663 }
3664
3665 if (!page)
3666 page = alloc_pages(flags, order);
3667 if (!page)
3668 return NULL;
3669 memset(page_address(page), 0, size);
3670
3671 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3672 DMA_BIDIRECTIONAL,
3673 dev->coherent_dma_mask);
3674 if (*dma_handle != DMA_MAPPING_ERROR)
3675 return page_address(page);
3676 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3677 __free_pages(page, order);
3678
3679 return NULL;
3680 }
3681
3682 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3683 dma_addr_t dma_handle, unsigned long attrs)
3684 {
3685 int order;
3686 struct page *page = virt_to_page(vaddr);
3687
3688 if (!iommu_need_mapping(dev))
3689 return dma_direct_free(dev, size, vaddr, dma_handle, attrs);
3690
3691 size = PAGE_ALIGN(size);
3692 order = get_order(size);
3693
3694 intel_unmap(dev, dma_handle, size);
3695 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3696 __free_pages(page, order);
3697 }
3698
3699 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3700 int nelems, enum dma_data_direction dir,
3701 unsigned long attrs)
3702 {
3703 dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
3704 unsigned long nrpages = 0;
3705 struct scatterlist *sg;
3706 int i;
3707
3708 if (!iommu_need_mapping(dev))
3709 return dma_direct_unmap_sg(dev, sglist, nelems, dir, attrs);
3710
3711 for_each_sg(sglist, sg, nelems, i) {
3712 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3713 }
3714
3715 intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
3716
3717 trace_unmap_sg(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
3718 }
3719
3720 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3721 enum dma_data_direction dir, unsigned long attrs)
3722 {
3723 int i;
3724 struct dmar_domain *domain;
3725 size_t size = 0;
3726 int prot = 0;
3727 unsigned long iova_pfn;
3728 int ret;
3729 struct scatterlist *sg;
3730 unsigned long start_vpfn;
3731 struct intel_iommu *iommu;
3732
3733 BUG_ON(dir == DMA_NONE);
3734 if (!iommu_need_mapping(dev))
3735 return dma_direct_map_sg(dev, sglist, nelems, dir, attrs);
3736
3737 domain = deferred_attach_domain(dev);
3738 if (!domain)
3739 return 0;
3740
3741 iommu = domain_get_iommu(domain);
3742
3743 for_each_sg(sglist, sg, nelems, i)
3744 size += aligned_nrpages(sg->offset, sg->length);
3745
3746 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3747 *dev->dma_mask);
3748 if (!iova_pfn) {
3749 sglist->dma_length = 0;
3750 return 0;
3751 }
3752
3753 /*
3754 * Check if DMAR supports zero-length reads on write only
3755 * mappings..
3756 */
3757 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3758 !cap_zlr(iommu->cap))
3759 prot |= DMA_PTE_READ;
3760 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3761 prot |= DMA_PTE_WRITE;
3762
3763 start_vpfn = mm_to_dma_pfn(iova_pfn);
3764
3765 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3766 if (unlikely(ret)) {
3767 dma_pte_free_pagetable(domain, start_vpfn,
3768 start_vpfn + size - 1,
3769 agaw_to_level(domain->agaw) + 1);
3770 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3771 return 0;
3772 }
3773
3774 trace_map_sg(dev, iova_pfn << PAGE_SHIFT,
3775 sg_phys(sglist), size << VTD_PAGE_SHIFT);
3776
3777 return nelems;
3778 }
3779
3780 static u64 intel_get_required_mask(struct device *dev)
3781 {
3782 if (!iommu_need_mapping(dev))
3783 return dma_direct_get_required_mask(dev);
3784 return DMA_BIT_MASK(32);
3785 }
3786
3787 static const struct dma_map_ops intel_dma_ops = {
3788 .alloc = intel_alloc_coherent,
3789 .free = intel_free_coherent,
3790 .map_sg = intel_map_sg,
3791 .unmap_sg = intel_unmap_sg,
3792 .map_page = intel_map_page,
3793 .unmap_page = intel_unmap_page,
3794 .map_resource = intel_map_resource,
3795 .unmap_resource = intel_unmap_resource,
3796 .dma_supported = dma_direct_supported,
3797 .mmap = dma_common_mmap,
3798 .get_sgtable = dma_common_get_sgtable,
3799 .get_required_mask = intel_get_required_mask,
3800 };
3801
3802 static void
3803 bounce_sync_single(struct device *dev, dma_addr_t addr, size_t size,
3804 enum dma_data_direction dir, enum dma_sync_target target)
3805 {
3806 struct dmar_domain *domain;
3807 phys_addr_t tlb_addr;
3808
3809 domain = find_domain(dev);
3810 if (WARN_ON(!domain))
3811 return;
3812
3813 tlb_addr = intel_iommu_iova_to_phys(&domain->domain, addr);
3814 if (is_swiotlb_buffer(tlb_addr))
3815 swiotlb_tbl_sync_single(dev, tlb_addr, size, dir, target);
3816 }
3817
3818 static dma_addr_t
3819 bounce_map_single(struct device *dev, phys_addr_t paddr, size_t size,
3820 enum dma_data_direction dir, unsigned long attrs,
3821 u64 dma_mask)
3822 {
3823 size_t aligned_size = ALIGN(size, VTD_PAGE_SIZE);
3824 struct dmar_domain *domain;
3825 struct intel_iommu *iommu;
3826 unsigned long iova_pfn;
3827 unsigned long nrpages;
3828 phys_addr_t tlb_addr;
3829 int prot = 0;
3830 int ret;
3831
3832 domain = deferred_attach_domain(dev);
3833 if (WARN_ON(dir == DMA_NONE || !domain))
3834 return DMA_MAPPING_ERROR;
3835
3836 iommu = domain_get_iommu(domain);
3837 if (WARN_ON(!iommu))
3838 return DMA_MAPPING_ERROR;
3839
3840 nrpages = aligned_nrpages(0, size);
3841 iova_pfn = intel_alloc_iova(dev, domain,
3842 dma_to_mm_pfn(nrpages), dma_mask);
3843 if (!iova_pfn)
3844 return DMA_MAPPING_ERROR;
3845
3846 /*
3847 * Check if DMAR supports zero-length reads on write only
3848 * mappings..
3849 */
3850 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL ||
3851 !cap_zlr(iommu->cap))
3852 prot |= DMA_PTE_READ;
3853 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3854 prot |= DMA_PTE_WRITE;
3855
3856 /*
3857 * If both the physical buffer start address and size are
3858 * page aligned, we don't need to use a bounce page.
3859 */
3860 if (!IS_ALIGNED(paddr | size, VTD_PAGE_SIZE)) {
3861 tlb_addr = swiotlb_tbl_map_single(dev,
3862 __phys_to_dma(dev, io_tlb_start),
3863 paddr, size, aligned_size, dir, attrs);
3864 if (tlb_addr == DMA_MAPPING_ERROR) {
3865 goto swiotlb_error;
3866 } else {
3867 /* Cleanup the padding area. */
3868 void *padding_start = phys_to_virt(tlb_addr);
3869 size_t padding_size = aligned_size;
3870
3871 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
3872 (dir == DMA_TO_DEVICE ||
3873 dir == DMA_BIDIRECTIONAL)) {
3874 padding_start += size;
3875 padding_size -= size;
3876 }
3877
3878 memset(padding_start, 0, padding_size);
3879 }
3880 } else {
3881 tlb_addr = paddr;
3882 }
3883
3884 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
3885 tlb_addr >> VTD_PAGE_SHIFT, nrpages, prot);
3886 if (ret)
3887 goto mapping_error;
3888
3889 trace_bounce_map_single(dev, iova_pfn << PAGE_SHIFT, paddr, size);
3890
3891 return (phys_addr_t)iova_pfn << PAGE_SHIFT;
3892
3893 mapping_error:
3894 if (is_swiotlb_buffer(tlb_addr))
3895 swiotlb_tbl_unmap_single(dev, tlb_addr, size,
3896 aligned_size, dir, attrs);
3897 swiotlb_error:
3898 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
3899 dev_err(dev, "Device bounce map: %zx@%llx dir %d --- failed\n",
3900 size, (unsigned long long)paddr, dir);
3901
3902 return DMA_MAPPING_ERROR;
3903 }
3904
3905 static void
3906 bounce_unmap_single(struct device *dev, dma_addr_t dev_addr, size_t size,
3907 enum dma_data_direction dir, unsigned long attrs)
3908 {
3909 size_t aligned_size = ALIGN(size, VTD_PAGE_SIZE);
3910 struct dmar_domain *domain;
3911 phys_addr_t tlb_addr;
3912
3913 domain = find_domain(dev);
3914 if (WARN_ON(!domain))
3915 return;
3916
3917 tlb_addr = intel_iommu_iova_to_phys(&domain->domain, dev_addr);
3918 if (WARN_ON(!tlb_addr))
3919 return;
3920
3921 intel_unmap(dev, dev_addr, size);
3922 if (is_swiotlb_buffer(tlb_addr))
3923 swiotlb_tbl_unmap_single(dev, tlb_addr, size,
3924 aligned_size, dir, attrs);
3925
3926 trace_bounce_unmap_single(dev, dev_addr, size);
3927 }
3928
3929 static dma_addr_t
3930 bounce_map_page(struct device *dev, struct page *page, unsigned long offset,
3931 size_t size, enum dma_data_direction dir, unsigned long attrs)
3932 {
3933 return bounce_map_single(dev, page_to_phys(page) + offset,
3934 size, dir, attrs, *dev->dma_mask);
3935 }
3936
3937 static dma_addr_t
3938 bounce_map_resource(struct device *dev, phys_addr_t phys_addr, size_t size,
3939 enum dma_data_direction dir, unsigned long attrs)
3940 {
3941 return bounce_map_single(dev, phys_addr, size,
3942 dir, attrs, *dev->dma_mask);
3943 }
3944
3945 static void
3946 bounce_unmap_page(struct device *dev, dma_addr_t dev_addr, size_t size,
3947 enum dma_data_direction dir, unsigned long attrs)
3948 {
3949 bounce_unmap_single(dev, dev_addr, size, dir, attrs);
3950 }
3951
3952 static void
3953 bounce_unmap_resource(struct device *dev, dma_addr_t dev_addr, size_t size,
3954 enum dma_data_direction dir, unsigned long attrs)
3955 {
3956 bounce_unmap_single(dev, dev_addr, size, dir, attrs);
3957 }
3958
3959 static void
3960 bounce_unmap_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3961 enum dma_data_direction dir, unsigned long attrs)
3962 {
3963 struct scatterlist *sg;
3964 int i;
3965
3966 for_each_sg(sglist, sg, nelems, i)
3967 bounce_unmap_page(dev, sg->dma_address,
3968 sg_dma_len(sg), dir, attrs);
3969 }
3970
3971 static int
3972 bounce_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3973 enum dma_data_direction dir, unsigned long attrs)
3974 {
3975 int i;
3976 struct scatterlist *sg;
3977
3978 for_each_sg(sglist, sg, nelems, i) {
3979 sg->dma_address = bounce_map_page(dev, sg_page(sg),
3980 sg->offset, sg->length,
3981 dir, attrs);
3982 if (sg->dma_address == DMA_MAPPING_ERROR)
3983 goto out_unmap;
3984 sg_dma_len(sg) = sg->length;
3985 }
3986
3987 return nelems;
3988
3989 out_unmap:
3990 bounce_unmap_sg(dev, sglist, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);
3991 return 0;
3992 }
3993
3994 static void
3995 bounce_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
3996 size_t size, enum dma_data_direction dir)
3997 {
3998 bounce_sync_single(dev, addr, size, dir, SYNC_FOR_CPU);
3999 }
4000
4001 static void
4002 bounce_sync_single_for_device(struct device *dev, dma_addr_t addr,
4003 size_t size, enum dma_data_direction dir)
4004 {
4005 bounce_sync_single(dev, addr, size, dir, SYNC_FOR_DEVICE);
4006 }
4007
4008 static void
4009 bounce_sync_sg_for_cpu(struct device *dev, struct scatterlist *sglist,
4010 int nelems, enum dma_data_direction dir)
4011 {
4012 struct scatterlist *sg;
4013 int i;
4014
4015 for_each_sg(sglist, sg, nelems, i)
4016 bounce_sync_single(dev, sg_dma_address(sg),
4017 sg_dma_len(sg), dir, SYNC_FOR_CPU);
4018 }
4019
4020 static void
4021 bounce_sync_sg_for_device(struct device *dev, struct scatterlist *sglist,
4022 int nelems, enum dma_data_direction dir)
4023 {
4024 struct scatterlist *sg;
4025 int i;
4026
4027 for_each_sg(sglist, sg, nelems, i)
4028 bounce_sync_single(dev, sg_dma_address(sg),
4029 sg_dma_len(sg), dir, SYNC_FOR_DEVICE);
4030 }
4031
4032 static const struct dma_map_ops bounce_dma_ops = {
4033 .alloc = intel_alloc_coherent,
4034 .free = intel_free_coherent,
4035 .map_sg = bounce_map_sg,
4036 .unmap_sg = bounce_unmap_sg,
4037 .map_page = bounce_map_page,
4038 .unmap_page = bounce_unmap_page,
4039 .sync_single_for_cpu = bounce_sync_single_for_cpu,
4040 .sync_single_for_device = bounce_sync_single_for_device,
4041 .sync_sg_for_cpu = bounce_sync_sg_for_cpu,
4042 .sync_sg_for_device = bounce_sync_sg_for_device,
4043 .map_resource = bounce_map_resource,
4044 .unmap_resource = bounce_unmap_resource,
4045 .dma_supported = dma_direct_supported,
4046 };
4047
4048 static inline int iommu_domain_cache_init(void)
4049 {
4050 int ret = 0;
4051
4052 iommu_domain_cache = kmem_cache_create("iommu_domain",
4053 sizeof(struct dmar_domain),
4054 0,
4055 SLAB_HWCACHE_ALIGN,
4056
4057 NULL);
4058 if (!iommu_domain_cache) {
4059 pr_err("Couldn't create iommu_domain cache\n");
4060 ret = -ENOMEM;
4061 }
4062
4063 return ret;
4064 }
4065
4066 static inline int iommu_devinfo_cache_init(void)
4067 {
4068 int ret = 0;
4069
4070 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
4071 sizeof(struct device_domain_info),
4072 0,
4073 SLAB_HWCACHE_ALIGN,
4074 NULL);
4075 if (!iommu_devinfo_cache) {
4076 pr_err("Couldn't create devinfo cache\n");
4077 ret = -ENOMEM;
4078 }
4079
4080 return ret;
4081 }
4082
4083 static int __init iommu_init_mempool(void)
4084 {
4085 int ret;
4086 ret = iova_cache_get();
4087 if (ret)
4088 return ret;
4089
4090 ret = iommu_domain_cache_init();
4091 if (ret)
4092 goto domain_error;
4093
4094 ret = iommu_devinfo_cache_init();
4095 if (!ret)
4096 return ret;
4097
4098 kmem_cache_destroy(iommu_domain_cache);
4099 domain_error:
4100 iova_cache_put();
4101
4102 return -ENOMEM;
4103 }
4104
4105 static void __init iommu_exit_mempool(void)
4106 {
4107 kmem_cache_destroy(iommu_devinfo_cache);
4108 kmem_cache_destroy(iommu_domain_cache);
4109 iova_cache_put();
4110 }
4111
4112 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
4113 {
4114 struct dmar_drhd_unit *drhd;
4115 u32 vtbar;
4116 int rc;
4117
4118 /* We know that this device on this chipset has its own IOMMU.
4119 * If we find it under a different IOMMU, then the BIOS is lying
4120 * to us. Hope that the IOMMU for this device is actually
4121 * disabled, and it needs no translation...
4122 */
4123 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
4124 if (rc) {
4125 /* "can't" happen */
4126 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
4127 return;
4128 }
4129 vtbar &= 0xffff0000;
4130
4131 /* we know that the this iommu should be at offset 0xa000 from vtbar */
4132 drhd = dmar_find_matched_drhd_unit(pdev);
4133 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
4134 TAINT_FIRMWARE_WORKAROUND,
4135 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
4136 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4137 }
4138 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
4139
4140 static void __init init_no_remapping_devices(void)
4141 {
4142 struct dmar_drhd_unit *drhd;
4143 struct device *dev;
4144 int i;
4145
4146 for_each_drhd_unit(drhd) {
4147 if (!drhd->include_all) {
4148 for_each_active_dev_scope(drhd->devices,
4149 drhd->devices_cnt, i, dev)
4150 break;
4151 /* ignore DMAR unit if no devices exist */
4152 if (i == drhd->devices_cnt)
4153 drhd->ignored = 1;
4154 }
4155 }
4156
4157 for_each_active_drhd_unit(drhd) {
4158 if (drhd->include_all)
4159 continue;
4160
4161 for_each_active_dev_scope(drhd->devices,
4162 drhd->devices_cnt, i, dev)
4163 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
4164 break;
4165 if (i < drhd->devices_cnt)
4166 continue;
4167
4168 /* This IOMMU has *only* gfx devices. Either bypass it or
4169 set the gfx_mapped flag, as appropriate */
4170 if (!dmar_map_gfx) {
4171 drhd->ignored = 1;
4172 for_each_active_dev_scope(drhd->devices,
4173 drhd->devices_cnt, i, dev)
4174 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4175 }
4176 }
4177 }
4178
4179 #ifdef CONFIG_SUSPEND
4180 static int init_iommu_hw(void)
4181 {
4182 struct dmar_drhd_unit *drhd;
4183 struct intel_iommu *iommu = NULL;
4184
4185 for_each_active_iommu(iommu, drhd)
4186 if (iommu->qi)
4187 dmar_reenable_qi(iommu);
4188
4189 for_each_iommu(iommu, drhd) {
4190 if (drhd->ignored) {
4191 /*
4192 * we always have to disable PMRs or DMA may fail on
4193 * this device
4194 */
4195 if (force_on)
4196 iommu_disable_protect_mem_regions(iommu);
4197 continue;
4198 }
4199
4200 iommu_flush_write_buffer(iommu);
4201
4202 iommu_set_root_entry(iommu);
4203
4204 iommu->flush.flush_context(iommu, 0, 0, 0,
4205 DMA_CCMD_GLOBAL_INVL);
4206 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4207 iommu_enable_translation(iommu);
4208 iommu_disable_protect_mem_regions(iommu);
4209 }
4210
4211 return 0;
4212 }
4213
4214 static void iommu_flush_all(void)
4215 {
4216 struct dmar_drhd_unit *drhd;
4217 struct intel_iommu *iommu;
4218
4219 for_each_active_iommu(iommu, drhd) {
4220 iommu->flush.flush_context(iommu, 0, 0, 0,
4221 DMA_CCMD_GLOBAL_INVL);
4222 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
4223 DMA_TLB_GLOBAL_FLUSH);
4224 }
4225 }
4226
4227 static int iommu_suspend(void)
4228 {
4229 struct dmar_drhd_unit *drhd;
4230 struct intel_iommu *iommu = NULL;
4231 unsigned long flag;
4232
4233 for_each_active_iommu(iommu, drhd) {
4234 iommu->iommu_state = kcalloc(MAX_SR_DMAR_REGS, sizeof(u32),
4235 GFP_ATOMIC);
4236 if (!iommu->iommu_state)
4237 goto nomem;
4238 }
4239
4240 iommu_flush_all();
4241
4242 for_each_active_iommu(iommu, drhd) {
4243 iommu_disable_translation(iommu);
4244
4245 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4246
4247 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4248 readl(iommu->reg + DMAR_FECTL_REG);
4249 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4250 readl(iommu->reg + DMAR_FEDATA_REG);
4251 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4252 readl(iommu->reg + DMAR_FEADDR_REG);
4253 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4254 readl(iommu->reg + DMAR_FEUADDR_REG);
4255
4256 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4257 }
4258 return 0;
4259
4260 nomem:
4261 for_each_active_iommu(iommu, drhd)
4262 kfree(iommu->iommu_state);
4263
4264 return -ENOMEM;
4265 }
4266
4267 static void iommu_resume(void)
4268 {
4269 struct dmar_drhd_unit *drhd;
4270 struct intel_iommu *iommu = NULL;
4271 unsigned long flag;
4272
4273 if (init_iommu_hw()) {
4274 if (force_on)
4275 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4276 else
4277 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
4278 return;
4279 }
4280
4281 for_each_active_iommu(iommu, drhd) {
4282
4283 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4284
4285 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4286 iommu->reg + DMAR_FECTL_REG);
4287 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4288 iommu->reg + DMAR_FEDATA_REG);
4289 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4290 iommu->reg + DMAR_FEADDR_REG);
4291 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4292 iommu->reg + DMAR_FEUADDR_REG);
4293
4294 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4295 }
4296
4297 for_each_active_iommu(iommu, drhd)
4298 kfree(iommu->iommu_state);
4299 }
4300
4301 static struct syscore_ops iommu_syscore_ops = {
4302 .resume = iommu_resume,
4303 .suspend = iommu_suspend,
4304 };
4305
4306 static void __init init_iommu_pm_ops(void)
4307 {
4308 register_syscore_ops(&iommu_syscore_ops);
4309 }
4310
4311 #else
4312 static inline void init_iommu_pm_ops(void) {}
4313 #endif /* CONFIG_PM */
4314
4315 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
4316 {
4317 struct acpi_dmar_reserved_memory *rmrr;
4318 struct dmar_rmrr_unit *rmrru;
4319 int ret;
4320
4321 rmrr = (struct acpi_dmar_reserved_memory *)header;
4322 ret = arch_rmrr_sanity_check(rmrr);
4323 if (ret)
4324 return ret;
4325
4326 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4327 if (!rmrru)
4328 goto out;
4329
4330 rmrru->hdr = header;
4331
4332 rmrru->base_address = rmrr->base_address;
4333 rmrru->end_address = rmrr->end_address;
4334
4335 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4336 ((void *)rmrr) + rmrr->header.length,
4337 &rmrru->devices_cnt);
4338 if (rmrru->devices_cnt && rmrru->devices == NULL)
4339 goto free_rmrru;
4340
4341 list_add(&rmrru->list, &dmar_rmrr_units);
4342
4343 return 0;
4344 free_rmrru:
4345 kfree(rmrru);
4346 out:
4347 return -ENOMEM;
4348 }
4349
4350 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4351 {
4352 struct dmar_atsr_unit *atsru;
4353 struct acpi_dmar_atsr *tmp;
4354
4355 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4356 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4357 if (atsr->segment != tmp->segment)
4358 continue;
4359 if (atsr->header.length != tmp->header.length)
4360 continue;
4361 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4362 return atsru;
4363 }
4364
4365 return NULL;
4366 }
4367
4368 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4369 {
4370 struct acpi_dmar_atsr *atsr;
4371 struct dmar_atsr_unit *atsru;
4372
4373 if (system_state >= SYSTEM_RUNNING && !intel_iommu_enabled)
4374 return 0;
4375
4376 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4377 atsru = dmar_find_atsr(atsr);
4378 if (atsru)
4379 return 0;
4380
4381 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4382 if (!atsru)
4383 return -ENOMEM;
4384
4385 /*
4386 * If memory is allocated from slab by ACPI _DSM method, we need to
4387 * copy the memory content because the memory buffer will be freed
4388 * on return.
4389 */
4390 atsru->hdr = (void *)(atsru + 1);
4391 memcpy(atsru->hdr, hdr, hdr->length);
4392 atsru->include_all = atsr->flags & 0x1;
4393 if (!atsru->include_all) {
4394 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4395 (void *)atsr + atsr->header.length,
4396 &atsru->devices_cnt);
4397 if (atsru->devices_cnt && atsru->devices == NULL) {
4398 kfree(atsru);
4399 return -ENOMEM;
4400 }
4401 }
4402
4403 list_add_rcu(&atsru->list, &dmar_atsr_units);
4404
4405 return 0;
4406 }
4407
4408 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4409 {
4410 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4411 kfree(atsru);
4412 }
4413
4414 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4415 {
4416 struct acpi_dmar_atsr *atsr;
4417 struct dmar_atsr_unit *atsru;
4418
4419 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4420 atsru = dmar_find_atsr(atsr);
4421 if (atsru) {
4422 list_del_rcu(&atsru->list);
4423 synchronize_rcu();
4424 intel_iommu_free_atsr(atsru);
4425 }
4426
4427 return 0;
4428 }
4429
4430 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4431 {
4432 int i;
4433 struct device *dev;
4434 struct acpi_dmar_atsr *atsr;
4435 struct dmar_atsr_unit *atsru;
4436
4437 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4438 atsru = dmar_find_atsr(atsr);
4439 if (!atsru)
4440 return 0;
4441
4442 if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
4443 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4444 i, dev)
4445 return -EBUSY;
4446 }
4447
4448 return 0;
4449 }
4450
4451 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4452 {
4453 int sp, ret;
4454 struct intel_iommu *iommu = dmaru->iommu;
4455
4456 if (g_iommus[iommu->seq_id])
4457 return 0;
4458
4459 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4460 pr_warn("%s: Doesn't support hardware pass through.\n",
4461 iommu->name);
4462 return -ENXIO;
4463 }
4464 if (!ecap_sc_support(iommu->ecap) &&
4465 domain_update_iommu_snooping(iommu)) {
4466 pr_warn("%s: Doesn't support snooping.\n",
4467 iommu->name);
4468 return -ENXIO;
4469 }
4470 sp = domain_update_iommu_superpage(iommu) - 1;
4471 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4472 pr_warn("%s: Doesn't support large page.\n",
4473 iommu->name);
4474 return -ENXIO;
4475 }
4476
4477 /*
4478 * Disable translation if already enabled prior to OS handover.
4479 */
4480 if (iommu->gcmd & DMA_GCMD_TE)
4481 iommu_disable_translation(iommu);
4482
4483 g_iommus[iommu->seq_id] = iommu;
4484 ret = iommu_init_domains(iommu);
4485 if (ret == 0)
4486 ret = iommu_alloc_root_entry(iommu);
4487 if (ret)
4488 goto out;
4489
4490 #ifdef CONFIG_INTEL_IOMMU_SVM
4491 if (pasid_supported(iommu))
4492 intel_svm_init(iommu);
4493 #endif
4494
4495 if (dmaru->ignored) {
4496 /*
4497 * we always have to disable PMRs or DMA may fail on this device
4498 */
4499 if (force_on)
4500 iommu_disable_protect_mem_regions(iommu);
4501 return 0;
4502 }
4503
4504 intel_iommu_init_qi(iommu);
4505 iommu_flush_write_buffer(iommu);
4506
4507 #ifdef CONFIG_INTEL_IOMMU_SVM
4508 if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) {
4509 ret = intel_svm_enable_prq(iommu);
4510 if (ret)
4511 goto disable_iommu;
4512 }
4513 #endif
4514 ret = dmar_set_interrupt(iommu);
4515 if (ret)
4516 goto disable_iommu;
4517
4518 iommu_set_root_entry(iommu);
4519 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4520 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4521 iommu_enable_translation(iommu);
4522
4523 iommu_disable_protect_mem_regions(iommu);
4524 return 0;
4525
4526 disable_iommu:
4527 disable_dmar_iommu(iommu);
4528 out:
4529 free_dmar_iommu(iommu);
4530 return ret;
4531 }
4532
4533 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4534 {
4535 int ret = 0;
4536 struct intel_iommu *iommu = dmaru->iommu;
4537
4538 if (!intel_iommu_enabled)
4539 return 0;
4540 if (iommu == NULL)
4541 return -EINVAL;
4542
4543 if (insert) {
4544 ret = intel_iommu_add(dmaru);
4545 } else {
4546 disable_dmar_iommu(iommu);
4547 free_dmar_iommu(iommu);
4548 }
4549
4550 return ret;
4551 }
4552
4553 static void intel_iommu_free_dmars(void)
4554 {
4555 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4556 struct dmar_atsr_unit *atsru, *atsr_n;
4557
4558 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4559 list_del(&rmrru->list);
4560 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4561 kfree(rmrru);
4562 }
4563
4564 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4565 list_del(&atsru->list);
4566 intel_iommu_free_atsr(atsru);
4567 }
4568 }
4569
4570 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4571 {
4572 int i, ret = 1;
4573 struct pci_bus *bus;
4574 struct pci_dev *bridge = NULL;
4575 struct device *tmp;
4576 struct acpi_dmar_atsr *atsr;
4577 struct dmar_atsr_unit *atsru;
4578
4579 dev = pci_physfn(dev);
4580 for (bus = dev->bus; bus; bus = bus->parent) {
4581 bridge = bus->self;
4582 /* If it's an integrated device, allow ATS */
4583 if (!bridge)
4584 return 1;
4585 /* Connected via non-PCIe: no ATS */
4586 if (!pci_is_pcie(bridge) ||
4587 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4588 return 0;
4589 /* If we found the root port, look it up in the ATSR */
4590 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4591 break;
4592 }
4593
4594 rcu_read_lock();
4595 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4596 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4597 if (atsr->segment != pci_domain_nr(dev->bus))
4598 continue;
4599
4600 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4601 if (tmp == &bridge->dev)
4602 goto out;
4603
4604 if (atsru->include_all)
4605 goto out;
4606 }
4607 ret = 0;
4608 out:
4609 rcu_read_unlock();
4610
4611 return ret;
4612 }
4613
4614 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4615 {
4616 int ret;
4617 struct dmar_rmrr_unit *rmrru;
4618 struct dmar_atsr_unit *atsru;
4619 struct acpi_dmar_atsr *atsr;
4620 struct acpi_dmar_reserved_memory *rmrr;
4621
4622 if (!intel_iommu_enabled && system_state >= SYSTEM_RUNNING)
4623 return 0;
4624
4625 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4626 rmrr = container_of(rmrru->hdr,
4627 struct acpi_dmar_reserved_memory, header);
4628 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4629 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4630 ((void *)rmrr) + rmrr->header.length,
4631 rmrr->segment, rmrru->devices,
4632 rmrru->devices_cnt);
4633 if (ret < 0)
4634 return ret;
4635 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4636 dmar_remove_dev_scope(info, rmrr->segment,
4637 rmrru->devices, rmrru->devices_cnt);
4638 }
4639 }
4640
4641 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4642 if (atsru->include_all)
4643 continue;
4644
4645 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4646 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4647 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4648 (void *)atsr + atsr->header.length,
4649 atsr->segment, atsru->devices,
4650 atsru->devices_cnt);
4651 if (ret > 0)
4652 break;
4653 else if (ret < 0)
4654 return ret;
4655 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4656 if (dmar_remove_dev_scope(info, atsr->segment,
4657 atsru->devices, atsru->devices_cnt))
4658 break;
4659 }
4660 }
4661
4662 return 0;
4663 }
4664
4665 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4666 unsigned long val, void *v)
4667 {
4668 struct memory_notify *mhp = v;
4669 unsigned long long start, end;
4670 unsigned long start_vpfn, last_vpfn;
4671
4672 switch (val) {
4673 case MEM_GOING_ONLINE:
4674 start = mhp->start_pfn << PAGE_SHIFT;
4675 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4676 if (iommu_domain_identity_map(si_domain, start, end)) {
4677 pr_warn("Failed to build identity map for [%llx-%llx]\n",
4678 start, end);
4679 return NOTIFY_BAD;
4680 }
4681 break;
4682
4683 case MEM_OFFLINE:
4684 case MEM_CANCEL_ONLINE:
4685 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4686 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4687 while (start_vpfn <= last_vpfn) {
4688 struct iova *iova;
4689 struct dmar_drhd_unit *drhd;
4690 struct intel_iommu *iommu;
4691 struct page *freelist;
4692
4693 iova = find_iova(&si_domain->iovad, start_vpfn);
4694 if (iova == NULL) {
4695 pr_debug("Failed get IOVA for PFN %lx\n",
4696 start_vpfn);
4697 break;
4698 }
4699
4700 iova = split_and_remove_iova(&si_domain->iovad, iova,
4701 start_vpfn, last_vpfn);
4702 if (iova == NULL) {
4703 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4704 start_vpfn, last_vpfn);
4705 return NOTIFY_BAD;
4706 }
4707
4708 freelist = domain_unmap(si_domain, iova->pfn_lo,
4709 iova->pfn_hi);
4710
4711 rcu_read_lock();
4712 for_each_active_iommu(iommu, drhd)
4713 iommu_flush_iotlb_psi(iommu, si_domain,
4714 iova->pfn_lo, iova_size(iova),
4715 !freelist, 0);
4716 rcu_read_unlock();
4717 dma_free_pagelist(freelist);
4718
4719 start_vpfn = iova->pfn_hi + 1;
4720 free_iova_mem(iova);
4721 }
4722 break;
4723 }
4724
4725 return NOTIFY_OK;
4726 }
4727
4728 static struct notifier_block intel_iommu_memory_nb = {
4729 .notifier_call = intel_iommu_memory_notifier,
4730 .priority = 0
4731 };
4732
4733 static void free_all_cpu_cached_iovas(unsigned int cpu)
4734 {
4735 int i;
4736
4737 for (i = 0; i < g_num_of_iommus; i++) {
4738 struct intel_iommu *iommu = g_iommus[i];
4739 struct dmar_domain *domain;
4740 int did;
4741
4742 if (!iommu)
4743 continue;
4744
4745 for (did = 0; did < cap_ndoms(iommu->cap); did++) {
4746 domain = get_iommu_domain(iommu, (u16)did);
4747
4748 if (!domain)
4749 continue;
4750 free_cpu_cached_iovas(cpu, &domain->iovad);
4751 }
4752 }
4753 }
4754
4755 static int intel_iommu_cpu_dead(unsigned int cpu)
4756 {
4757 free_all_cpu_cached_iovas(cpu);
4758 return 0;
4759 }
4760
4761 static void intel_disable_iommus(void)
4762 {
4763 struct intel_iommu *iommu = NULL;
4764 struct dmar_drhd_unit *drhd;
4765
4766 for_each_iommu(iommu, drhd)
4767 iommu_disable_translation(iommu);
4768 }
4769
4770 void intel_iommu_shutdown(void)
4771 {
4772 struct dmar_drhd_unit *drhd;
4773 struct intel_iommu *iommu = NULL;
4774
4775 if (no_iommu || dmar_disabled)
4776 return;
4777
4778 down_write(&dmar_global_lock);
4779
4780 /* Disable PMRs explicitly here. */
4781 for_each_iommu(iommu, drhd)
4782 iommu_disable_protect_mem_regions(iommu);
4783
4784 /* Make sure the IOMMUs are switched off */
4785 intel_disable_iommus();
4786
4787 up_write(&dmar_global_lock);
4788 }
4789
4790 static inline struct intel_iommu *dev_to_intel_iommu(struct device *dev)
4791 {
4792 struct iommu_device *iommu_dev = dev_to_iommu_device(dev);
4793
4794 return container_of(iommu_dev, struct intel_iommu, iommu);
4795 }
4796
4797 static ssize_t intel_iommu_show_version(struct device *dev,
4798 struct device_attribute *attr,
4799 char *buf)
4800 {
4801 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4802 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4803 return sprintf(buf, "%d:%d\n",
4804 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4805 }
4806 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4807
4808 static ssize_t intel_iommu_show_address(struct device *dev,
4809 struct device_attribute *attr,
4810 char *buf)
4811 {
4812 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4813 return sprintf(buf, "%llx\n", iommu->reg_phys);
4814 }
4815 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4816
4817 static ssize_t intel_iommu_show_cap(struct device *dev,
4818 struct device_attribute *attr,
4819 char *buf)
4820 {
4821 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4822 return sprintf(buf, "%llx\n", iommu->cap);
4823 }
4824 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4825
4826 static ssize_t intel_iommu_show_ecap(struct device *dev,
4827 struct device_attribute *attr,
4828 char *buf)
4829 {
4830 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4831 return sprintf(buf, "%llx\n", iommu->ecap);
4832 }
4833 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4834
4835 static ssize_t intel_iommu_show_ndoms(struct device *dev,
4836 struct device_attribute *attr,
4837 char *buf)
4838 {
4839 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4840 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4841 }
4842 static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4843
4844 static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4845 struct device_attribute *attr,
4846 char *buf)
4847 {
4848 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4849 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4850 cap_ndoms(iommu->cap)));
4851 }
4852 static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4853
4854 static struct attribute *intel_iommu_attrs[] = {
4855 &dev_attr_version.attr,
4856 &dev_attr_address.attr,
4857 &dev_attr_cap.attr,
4858 &dev_attr_ecap.attr,
4859 &dev_attr_domains_supported.attr,
4860 &dev_attr_domains_used.attr,
4861 NULL,
4862 };
4863
4864 static struct attribute_group intel_iommu_group = {
4865 .name = "intel-iommu",
4866 .attrs = intel_iommu_attrs,
4867 };
4868
4869 const struct attribute_group *intel_iommu_groups[] = {
4870 &intel_iommu_group,
4871 NULL,
4872 };
4873
4874 static inline bool has_untrusted_dev(void)
4875 {
4876 struct pci_dev *pdev = NULL;
4877
4878 for_each_pci_dev(pdev)
4879 if (pdev->untrusted)
4880 return true;
4881
4882 return false;
4883 }
4884
4885 static int __init platform_optin_force_iommu(void)
4886 {
4887 if (!dmar_platform_optin() || no_platform_optin || !has_untrusted_dev())
4888 return 0;
4889
4890 if (no_iommu || dmar_disabled)
4891 pr_info("Intel-IOMMU force enabled due to platform opt in\n");
4892
4893 /*
4894 * If Intel-IOMMU is disabled by default, we will apply identity
4895 * map for all devices except those marked as being untrusted.
4896 */
4897 if (dmar_disabled)
4898 iommu_identity_mapping |= IDENTMAP_ALL;
4899
4900 dmar_disabled = 0;
4901 no_iommu = 0;
4902
4903 return 1;
4904 }
4905
4906 static int __init probe_acpi_namespace_devices(void)
4907 {
4908 struct dmar_drhd_unit *drhd;
4909 /* To avoid a -Wunused-but-set-variable warning. */
4910 struct intel_iommu *iommu __maybe_unused;
4911 struct device *dev;
4912 int i, ret = 0;
4913
4914 for_each_active_iommu(iommu, drhd) {
4915 for_each_active_dev_scope(drhd->devices,
4916 drhd->devices_cnt, i, dev) {
4917 struct acpi_device_physical_node *pn;
4918 struct iommu_group *group;
4919 struct acpi_device *adev;
4920
4921 if (dev->bus != &acpi_bus_type)
4922 continue;
4923
4924 adev = to_acpi_device(dev);
4925 mutex_lock(&adev->physical_node_lock);
4926 list_for_each_entry(pn,
4927 &adev->physical_node_list, node) {
4928 group = iommu_group_get(pn->dev);
4929 if (group) {
4930 iommu_group_put(group);
4931 continue;
4932 }
4933
4934 pn->dev->bus->iommu_ops = &intel_iommu_ops;
4935 ret = iommu_probe_device(pn->dev);
4936 if (ret)
4937 break;
4938 }
4939 mutex_unlock(&adev->physical_node_lock);
4940
4941 if (ret)
4942 return ret;
4943 }
4944 }
4945
4946 return 0;
4947 }
4948
4949 int __init intel_iommu_init(void)
4950 {
4951 int ret = -ENODEV;
4952 struct dmar_drhd_unit *drhd;
4953 struct intel_iommu *iommu;
4954
4955 /*
4956 * Intel IOMMU is required for a TXT/tboot launch or platform
4957 * opt in, so enforce that.
4958 */
4959 force_on = tboot_force_iommu() || platform_optin_force_iommu();
4960
4961 if (iommu_init_mempool()) {
4962 if (force_on)
4963 panic("tboot: Failed to initialize iommu memory\n");
4964 return -ENOMEM;
4965 }
4966
4967 down_write(&dmar_global_lock);
4968 if (dmar_table_init()) {
4969 if (force_on)
4970 panic("tboot: Failed to initialize DMAR table\n");
4971 goto out_free_dmar;
4972 }
4973
4974 if (dmar_dev_scope_init() < 0) {
4975 if (force_on)
4976 panic("tboot: Failed to initialize DMAR device scope\n");
4977 goto out_free_dmar;
4978 }
4979
4980 up_write(&dmar_global_lock);
4981
4982 /*
4983 * The bus notifier takes the dmar_global_lock, so lockdep will
4984 * complain later when we register it under the lock.
4985 */
4986 dmar_register_bus_notifier();
4987
4988 down_write(&dmar_global_lock);
4989
4990 if (no_iommu || dmar_disabled) {
4991 /*
4992 * We exit the function here to ensure IOMMU's remapping and
4993 * mempool aren't setup, which means that the IOMMU's PMRs
4994 * won't be disabled via the call to init_dmars(). So disable
4995 * it explicitly here. The PMRs were setup by tboot prior to
4996 * calling SENTER, but the kernel is expected to reset/tear
4997 * down the PMRs.
4998 */
4999 if (intel_iommu_tboot_noforce) {
5000 for_each_iommu(iommu, drhd)
5001 iommu_disable_protect_mem_regions(iommu);
5002 }
5003
5004 /*
5005 * Make sure the IOMMUs are switched off, even when we
5006 * boot into a kexec kernel and the previous kernel left
5007 * them enabled
5008 */
5009 intel_disable_iommus();
5010 goto out_free_dmar;
5011 }
5012
5013 if (list_empty(&dmar_rmrr_units))
5014 pr_info("No RMRR found\n");
5015
5016 if (list_empty(&dmar_atsr_units))
5017 pr_info("No ATSR found\n");
5018
5019 if (dmar_init_reserved_ranges()) {
5020 if (force_on)
5021 panic("tboot: Failed to reserve iommu ranges\n");
5022 goto out_free_reserved_range;
5023 }
5024
5025 if (dmar_map_gfx)
5026 intel_iommu_gfx_mapped = 1;
5027
5028 init_no_remapping_devices();
5029
5030 ret = init_dmars();
5031 if (ret) {
5032 if (force_on)
5033 panic("tboot: Failed to initialize DMARs\n");
5034 pr_err("Initialization failed\n");
5035 goto out_free_reserved_range;
5036 }
5037 up_write(&dmar_global_lock);
5038
5039 #if defined(CONFIG_X86) && defined(CONFIG_SWIOTLB)
5040 /*
5041 * If the system has no untrusted device or the user has decided
5042 * to disable the bounce page mechanisms, we don't need swiotlb.
5043 * Mark this and the pre-allocated bounce pages will be released
5044 * later.
5045 */
5046 if (!has_untrusted_dev() || intel_no_bounce)
5047 swiotlb = 0;
5048 #endif
5049 dma_ops = &intel_dma_ops;
5050
5051 init_iommu_pm_ops();
5052
5053 for_each_active_iommu(iommu, drhd) {
5054 iommu_device_sysfs_add(&iommu->iommu, NULL,
5055 intel_iommu_groups,
5056 "%s", iommu->name);
5057 iommu_device_set_ops(&iommu->iommu, &intel_iommu_ops);
5058 iommu_device_register(&iommu->iommu);
5059 }
5060
5061 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
5062 if (si_domain && !hw_pass_through)
5063 register_memory_notifier(&intel_iommu_memory_nb);
5064 cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL,
5065 intel_iommu_cpu_dead);
5066
5067 down_read(&dmar_global_lock);
5068 if (probe_acpi_namespace_devices())
5069 pr_warn("ACPI name space devices didn't probe correctly\n");
5070 up_read(&dmar_global_lock);
5071
5072 /* Finally, we enable the DMA remapping hardware. */
5073 for_each_iommu(iommu, drhd) {
5074 if (!drhd->ignored && !translation_pre_enabled(iommu))
5075 iommu_enable_translation(iommu);
5076
5077 iommu_disable_protect_mem_regions(iommu);
5078 }
5079 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
5080
5081 intel_iommu_enabled = 1;
5082 intel_iommu_debugfs_init();
5083
5084 return 0;
5085
5086 out_free_reserved_range:
5087 put_iova_domain(&reserved_iova_list);
5088 out_free_dmar:
5089 intel_iommu_free_dmars();
5090 up_write(&dmar_global_lock);
5091 iommu_exit_mempool();
5092 return ret;
5093 }
5094
5095 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
5096 {
5097 struct intel_iommu *iommu = opaque;
5098
5099 domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
5100 return 0;
5101 }
5102
5103 /*
5104 * NB - intel-iommu lacks any sort of reference counting for the users of
5105 * dependent devices. If multiple endpoints have intersecting dependent
5106 * devices, unbinding the driver from any one of them will possibly leave
5107 * the others unable to operate.
5108 */
5109 static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
5110 {
5111 if (!iommu || !dev || !dev_is_pci(dev))
5112 return;
5113
5114 pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
5115 }
5116
5117 static void __dmar_remove_one_dev_info(struct device_domain_info *info)
5118 {
5119 struct dmar_domain *domain;
5120 struct intel_iommu *iommu;
5121 unsigned long flags;
5122
5123 assert_spin_locked(&device_domain_lock);
5124
5125 if (WARN_ON(!info))
5126 return;
5127
5128 iommu = info->iommu;
5129 domain = info->domain;
5130
5131 if (info->dev) {
5132 if (dev_is_pci(info->dev) && sm_supported(iommu))
5133 intel_pasid_tear_down_entry(iommu, info->dev,
5134 PASID_RID2PASID);
5135
5136 iommu_disable_dev_iotlb(info);
5137 domain_context_clear(iommu, info->dev);
5138 intel_pasid_free_table(info->dev);
5139 }
5140
5141 unlink_domain_info(info);
5142
5143 spin_lock_irqsave(&iommu->lock, flags);
5144 domain_detach_iommu(domain, iommu);
5145 spin_unlock_irqrestore(&iommu->lock, flags);
5146
5147 /* free the private domain */
5148 if (domain->flags & DOMAIN_FLAG_LOSE_CHILDREN &&
5149 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
5150 list_empty(&domain->devices))
5151 domain_exit(info->domain);
5152
5153 free_devinfo_mem(info);
5154 }
5155
5156 static void dmar_remove_one_dev_info(struct device *dev)
5157 {
5158 struct device_domain_info *info;
5159 unsigned long flags;
5160
5161 spin_lock_irqsave(&device_domain_lock, flags);
5162 info = dev->archdata.iommu;
5163 if (info && info != DEFER_DEVICE_DOMAIN_INFO
5164 && info != DUMMY_DEVICE_DOMAIN_INFO)
5165 __dmar_remove_one_dev_info(info);
5166 spin_unlock_irqrestore(&device_domain_lock, flags);
5167 }
5168
5169 static int md_domain_init(struct dmar_domain *domain, int guest_width)
5170 {
5171 int adjust_width;
5172
5173 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
5174 domain_reserve_special_ranges(domain);
5175
5176 /* calculate AGAW */
5177 domain->gaw = guest_width;
5178 adjust_width = guestwidth_to_adjustwidth(guest_width);
5179 domain->agaw = width_to_agaw(adjust_width);
5180
5181 domain->iommu_coherency = 0;
5182 domain->iommu_snooping = 0;
5183 domain->iommu_superpage = 0;
5184 domain->max_addr = 0;
5185
5186 /* always allocate the top pgd */
5187 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5188 if (!domain->pgd)
5189 return -ENOMEM;
5190 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
5191 return 0;
5192 }
5193
5194 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
5195 {
5196 struct dmar_domain *dmar_domain;
5197 struct iommu_domain *domain;
5198
5199 switch (type) {
5200 case IOMMU_DOMAIN_DMA:
5201 /* fallthrough */
5202 case IOMMU_DOMAIN_UNMANAGED:
5203 dmar_domain = alloc_domain(0);
5204 if (!dmar_domain) {
5205 pr_err("Can't allocate dmar_domain\n");
5206 return NULL;
5207 }
5208 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
5209 pr_err("Domain initialization failed\n");
5210 domain_exit(dmar_domain);
5211 return NULL;
5212 }
5213
5214 if (type == IOMMU_DOMAIN_DMA &&
5215 init_iova_flush_queue(&dmar_domain->iovad,
5216 iommu_flush_iova, iova_entry_free)) {
5217 pr_warn("iova flush queue initialization failed\n");
5218 intel_iommu_strict = 1;
5219 }
5220
5221 domain_update_iommu_cap(dmar_domain);
5222
5223 domain = &dmar_domain->domain;
5224 domain->geometry.aperture_start = 0;
5225 domain->geometry.aperture_end =
5226 __DOMAIN_MAX_ADDR(dmar_domain->gaw);
5227 domain->geometry.force_aperture = true;
5228
5229 return domain;
5230 case IOMMU_DOMAIN_IDENTITY:
5231 return &si_domain->domain;
5232 default:
5233 return NULL;
5234 }
5235
5236 return NULL;
5237 }
5238
5239 static void intel_iommu_domain_free(struct iommu_domain *domain)
5240 {
5241 if (domain != &si_domain->domain)
5242 domain_exit(to_dmar_domain(domain));
5243 }
5244
5245 /*
5246 * Check whether a @domain could be attached to the @dev through the
5247 * aux-domain attach/detach APIs.
5248 */
5249 static inline bool
5250 is_aux_domain(struct device *dev, struct iommu_domain *domain)
5251 {
5252 struct device_domain_info *info = dev->archdata.iommu;
5253
5254 return info && info->auxd_enabled &&
5255 domain->type == IOMMU_DOMAIN_UNMANAGED;
5256 }
5257
5258 static void auxiliary_link_device(struct dmar_domain *domain,
5259 struct device *dev)
5260 {
5261 struct device_domain_info *info = dev->archdata.iommu;
5262
5263 assert_spin_locked(&device_domain_lock);
5264 if (WARN_ON(!info))
5265 return;
5266
5267 domain->auxd_refcnt++;
5268 list_add(&domain->auxd, &info->auxiliary_domains);
5269 }
5270
5271 static void auxiliary_unlink_device(struct dmar_domain *domain,
5272 struct device *dev)
5273 {
5274 struct device_domain_info *info = dev->archdata.iommu;
5275
5276 assert_spin_locked(&device_domain_lock);
5277 if (WARN_ON(!info))
5278 return;
5279
5280 list_del(&domain->auxd);
5281 domain->auxd_refcnt--;
5282
5283 if (!domain->auxd_refcnt && domain->default_pasid > 0)
5284 intel_pasid_free_id(domain->default_pasid);
5285 }
5286
5287 static int aux_domain_add_dev(struct dmar_domain *domain,
5288 struct device *dev)
5289 {
5290 int ret;
5291 u8 bus, devfn;
5292 unsigned long flags;
5293 struct intel_iommu *iommu;
5294
5295 iommu = device_to_iommu(dev, &bus, &devfn);
5296 if (!iommu)
5297 return -ENODEV;
5298
5299 if (domain->default_pasid <= 0) {
5300 int pasid;
5301
5302 pasid = intel_pasid_alloc_id(domain, PASID_MIN,
5303 pci_max_pasids(to_pci_dev(dev)),
5304 GFP_KERNEL);
5305 if (pasid <= 0) {
5306 pr_err("Can't allocate default pasid\n");
5307 return -ENODEV;
5308 }
5309 domain->default_pasid = pasid;
5310 }
5311
5312 spin_lock_irqsave(&device_domain_lock, flags);
5313 /*
5314 * iommu->lock must be held to attach domain to iommu and setup the
5315 * pasid entry for second level translation.
5316 */
5317 spin_lock(&iommu->lock);
5318 ret = domain_attach_iommu(domain, iommu);
5319 if (ret)
5320 goto attach_failed;
5321
5322 /* Setup the PASID entry for mediated devices: */
5323 ret = intel_pasid_setup_second_level(iommu, domain, dev,
5324 domain->default_pasid);
5325 if (ret)
5326 goto table_failed;
5327 spin_unlock(&iommu->lock);
5328
5329 auxiliary_link_device(domain, dev);
5330
5331 spin_unlock_irqrestore(&device_domain_lock, flags);
5332
5333 return 0;
5334
5335 table_failed:
5336 domain_detach_iommu(domain, iommu);
5337 attach_failed:
5338 spin_unlock(&iommu->lock);
5339 spin_unlock_irqrestore(&device_domain_lock, flags);
5340 if (!domain->auxd_refcnt && domain->default_pasid > 0)
5341 intel_pasid_free_id(domain->default_pasid);
5342
5343 return ret;
5344 }
5345
5346 static void aux_domain_remove_dev(struct dmar_domain *domain,
5347 struct device *dev)
5348 {
5349 struct device_domain_info *info;
5350 struct intel_iommu *iommu;
5351 unsigned long flags;
5352
5353 if (!is_aux_domain(dev, &domain->domain))
5354 return;
5355
5356 spin_lock_irqsave(&device_domain_lock, flags);
5357 info = dev->archdata.iommu;
5358 iommu = info->iommu;
5359
5360 auxiliary_unlink_device(domain, dev);
5361
5362 spin_lock(&iommu->lock);
5363 intel_pasid_tear_down_entry(iommu, dev, domain->default_pasid);
5364 domain_detach_iommu(domain, iommu);
5365 spin_unlock(&iommu->lock);
5366
5367 spin_unlock_irqrestore(&device_domain_lock, flags);
5368 }
5369
5370 static int prepare_domain_attach_device(struct iommu_domain *domain,
5371 struct device *dev)
5372 {
5373 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5374 struct intel_iommu *iommu;
5375 int addr_width;
5376 u8 bus, devfn;
5377
5378 iommu = device_to_iommu(dev, &bus, &devfn);
5379 if (!iommu)
5380 return -ENODEV;
5381
5382 /* check if this iommu agaw is sufficient for max mapped address */
5383 addr_width = agaw_to_width(iommu->agaw);
5384 if (addr_width > cap_mgaw(iommu->cap))
5385 addr_width = cap_mgaw(iommu->cap);
5386
5387 if (dmar_domain->max_addr > (1LL << addr_width)) {
5388 dev_err(dev, "%s: iommu width (%d) is not "
5389 "sufficient for the mapped address (%llx)\n",
5390 __func__, addr_width, dmar_domain->max_addr);
5391 return -EFAULT;
5392 }
5393 dmar_domain->gaw = addr_width;
5394
5395 /*
5396 * Knock out extra levels of page tables if necessary
5397 */
5398 while (iommu->agaw < dmar_domain->agaw) {
5399 struct dma_pte *pte;
5400
5401 pte = dmar_domain->pgd;
5402 if (dma_pte_present(pte)) {
5403 dmar_domain->pgd = (struct dma_pte *)
5404 phys_to_virt(dma_pte_addr(pte));
5405 free_pgtable_page(pte);
5406 }
5407 dmar_domain->agaw--;
5408 }
5409
5410 return 0;
5411 }
5412
5413 static int intel_iommu_attach_device(struct iommu_domain *domain,
5414 struct device *dev)
5415 {
5416 int ret;
5417
5418 if (domain->type == IOMMU_DOMAIN_UNMANAGED &&
5419 device_is_rmrr_locked(dev)) {
5420 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
5421 return -EPERM;
5422 }
5423
5424 if (is_aux_domain(dev, domain))
5425 return -EPERM;
5426
5427 /* normally dev is not mapped */
5428 if (unlikely(domain_context_mapped(dev))) {
5429 struct dmar_domain *old_domain;
5430
5431 old_domain = find_domain(dev);
5432 if (old_domain)
5433 dmar_remove_one_dev_info(dev);
5434 }
5435
5436 ret = prepare_domain_attach_device(domain, dev);
5437 if (ret)
5438 return ret;
5439
5440 return domain_add_dev_info(to_dmar_domain(domain), dev);
5441 }
5442
5443 static int intel_iommu_aux_attach_device(struct iommu_domain *domain,
5444 struct device *dev)
5445 {
5446 int ret;
5447
5448 if (!is_aux_domain(dev, domain))
5449 return -EPERM;
5450
5451 ret = prepare_domain_attach_device(domain, dev);
5452 if (ret)
5453 return ret;
5454
5455 return aux_domain_add_dev(to_dmar_domain(domain), dev);
5456 }
5457
5458 static void intel_iommu_detach_device(struct iommu_domain *domain,
5459 struct device *dev)
5460 {
5461 dmar_remove_one_dev_info(dev);
5462 }
5463
5464 static void intel_iommu_aux_detach_device(struct iommu_domain *domain,
5465 struct device *dev)
5466 {
5467 aux_domain_remove_dev(to_dmar_domain(domain), dev);
5468 }
5469
5470 static int intel_iommu_map(struct iommu_domain *domain,
5471 unsigned long iova, phys_addr_t hpa,
5472 size_t size, int iommu_prot, gfp_t gfp)
5473 {
5474 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5475 u64 max_addr;
5476 int prot = 0;
5477 int ret;
5478
5479 if (iommu_prot & IOMMU_READ)
5480 prot |= DMA_PTE_READ;
5481 if (iommu_prot & IOMMU_WRITE)
5482 prot |= DMA_PTE_WRITE;
5483 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
5484 prot |= DMA_PTE_SNP;
5485
5486 max_addr = iova + size;
5487 if (dmar_domain->max_addr < max_addr) {
5488 u64 end;
5489
5490 /* check if minimum agaw is sufficient for mapped address */
5491 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
5492 if (end < max_addr) {
5493 pr_err("%s: iommu width (%d) is not "
5494 "sufficient for the mapped address (%llx)\n",
5495 __func__, dmar_domain->gaw, max_addr);
5496 return -EFAULT;
5497 }
5498 dmar_domain->max_addr = max_addr;
5499 }
5500 /* Round up size to next multiple of PAGE_SIZE, if it and
5501 the low bits of hpa would take us onto the next page */
5502 size = aligned_nrpages(hpa, size);
5503 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
5504 hpa >> VTD_PAGE_SHIFT, size, prot);
5505 return ret;
5506 }
5507
5508 static size_t intel_iommu_unmap(struct iommu_domain *domain,
5509 unsigned long iova, size_t size,
5510 struct iommu_iotlb_gather *gather)
5511 {
5512 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5513 struct page *freelist = NULL;
5514 unsigned long start_pfn, last_pfn;
5515 unsigned int npages;
5516 int iommu_id, level = 0;
5517
5518 /* Cope with horrid API which requires us to unmap more than the
5519 size argument if it happens to be a large-page mapping. */
5520 BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
5521
5522 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
5523 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
5524
5525 start_pfn = iova >> VTD_PAGE_SHIFT;
5526 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5527
5528 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
5529
5530 npages = last_pfn - start_pfn + 1;
5531
5532 for_each_domain_iommu(iommu_id, dmar_domain)
5533 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5534 start_pfn, npages, !freelist, 0);
5535
5536 dma_free_pagelist(freelist);
5537
5538 if (dmar_domain->max_addr == iova + size)
5539 dmar_domain->max_addr = iova;
5540
5541 return size;
5542 }
5543
5544 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
5545 dma_addr_t iova)
5546 {
5547 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5548 struct dma_pte *pte;
5549 int level = 0;
5550 u64 phys = 0;
5551
5552 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
5553 if (pte)
5554 phys = dma_pte_addr(pte);
5555
5556 return phys;
5557 }
5558
5559 static inline bool scalable_mode_support(void)
5560 {
5561 struct dmar_drhd_unit *drhd;
5562 struct intel_iommu *iommu;
5563 bool ret = true;
5564
5565 rcu_read_lock();
5566 for_each_active_iommu(iommu, drhd) {
5567 if (!sm_supported(iommu)) {
5568 ret = false;
5569 break;
5570 }
5571 }
5572 rcu_read_unlock();
5573
5574 return ret;
5575 }
5576
5577 static inline bool iommu_pasid_support(void)
5578 {
5579 struct dmar_drhd_unit *drhd;
5580 struct intel_iommu *iommu;
5581 bool ret = true;
5582
5583 rcu_read_lock();
5584 for_each_active_iommu(iommu, drhd) {
5585 if (!pasid_supported(iommu)) {
5586 ret = false;
5587 break;
5588 }
5589 }
5590 rcu_read_unlock();
5591
5592 return ret;
5593 }
5594
5595 static bool intel_iommu_capable(enum iommu_cap cap)
5596 {
5597 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5598 return domain_update_iommu_snooping(NULL) == 1;
5599 if (cap == IOMMU_CAP_INTR_REMAP)
5600 return irq_remapping_enabled == 1;
5601
5602 return false;
5603 }
5604
5605 static int intel_iommu_add_device(struct device *dev)
5606 {
5607 struct dmar_domain *dmar_domain;
5608 struct iommu_domain *domain;
5609 struct intel_iommu *iommu;
5610 struct iommu_group *group;
5611 u8 bus, devfn;
5612 int ret;
5613
5614 iommu = device_to_iommu(dev, &bus, &devfn);
5615 if (!iommu)
5616 return -ENODEV;
5617
5618 iommu_device_link(&iommu->iommu, dev);
5619
5620 if (translation_pre_enabled(iommu))
5621 dev->archdata.iommu = DEFER_DEVICE_DOMAIN_INFO;
5622
5623 group = iommu_group_get_for_dev(dev);
5624
5625 if (IS_ERR(group)) {
5626 ret = PTR_ERR(group);
5627 goto unlink;
5628 }
5629
5630 iommu_group_put(group);
5631
5632 domain = iommu_get_domain_for_dev(dev);
5633 dmar_domain = to_dmar_domain(domain);
5634 if (domain->type == IOMMU_DOMAIN_DMA) {
5635 if (device_def_domain_type(dev) == IOMMU_DOMAIN_IDENTITY) {
5636 ret = iommu_request_dm_for_dev(dev);
5637 if (ret) {
5638 dmar_remove_one_dev_info(dev);
5639 dmar_domain->flags |= DOMAIN_FLAG_LOSE_CHILDREN;
5640 domain_add_dev_info(si_domain, dev);
5641 dev_info(dev,
5642 "Device uses a private identity domain.\n");
5643 }
5644 }
5645 } else {
5646 if (device_def_domain_type(dev) == IOMMU_DOMAIN_DMA) {
5647 ret = iommu_request_dma_domain_for_dev(dev);
5648 if (ret) {
5649 dmar_remove_one_dev_info(dev);
5650 dmar_domain->flags |= DOMAIN_FLAG_LOSE_CHILDREN;
5651 if (!get_private_domain_for_dev(dev)) {
5652 dev_warn(dev,
5653 "Failed to get a private domain.\n");
5654 ret = -ENOMEM;
5655 goto unlink;
5656 }
5657
5658 dev_info(dev,
5659 "Device uses a private dma domain.\n");
5660 }
5661 }
5662 }
5663
5664 if (device_needs_bounce(dev)) {
5665 dev_info(dev, "Use Intel IOMMU bounce page dma_ops\n");
5666 set_dma_ops(dev, &bounce_dma_ops);
5667 }
5668
5669 return 0;
5670
5671 unlink:
5672 iommu_device_unlink(&iommu->iommu, dev);
5673 return ret;
5674 }
5675
5676 static void intel_iommu_remove_device(struct device *dev)
5677 {
5678 struct intel_iommu *iommu;
5679 u8 bus, devfn;
5680
5681 iommu = device_to_iommu(dev, &bus, &devfn);
5682 if (!iommu)
5683 return;
5684
5685 dmar_remove_one_dev_info(dev);
5686
5687 iommu_group_remove_device(dev);
5688
5689 iommu_device_unlink(&iommu->iommu, dev);
5690
5691 if (device_needs_bounce(dev))
5692 set_dma_ops(dev, NULL);
5693 }
5694
5695 static void intel_iommu_get_resv_regions(struct device *device,
5696 struct list_head *head)
5697 {
5698 int prot = DMA_PTE_READ | DMA_PTE_WRITE;
5699 struct iommu_resv_region *reg;
5700 struct dmar_rmrr_unit *rmrr;
5701 struct device *i_dev;
5702 int i;
5703
5704 down_read(&dmar_global_lock);
5705 for_each_rmrr_units(rmrr) {
5706 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
5707 i, i_dev) {
5708 struct iommu_resv_region *resv;
5709 enum iommu_resv_type type;
5710 size_t length;
5711
5712 if (i_dev != device &&
5713 !is_downstream_to_pci_bridge(device, i_dev))
5714 continue;
5715
5716 length = rmrr->end_address - rmrr->base_address + 1;
5717
5718 type = device_rmrr_is_relaxable(device) ?
5719 IOMMU_RESV_DIRECT_RELAXABLE : IOMMU_RESV_DIRECT;
5720
5721 resv = iommu_alloc_resv_region(rmrr->base_address,
5722 length, prot, type);
5723 if (!resv)
5724 break;
5725
5726 list_add_tail(&resv->list, head);
5727 }
5728 }
5729 up_read(&dmar_global_lock);
5730
5731 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
5732 if (dev_is_pci(device)) {
5733 struct pci_dev *pdev = to_pci_dev(device);
5734
5735 if ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA) {
5736 reg = iommu_alloc_resv_region(0, 1UL << 24, prot,
5737 IOMMU_RESV_DIRECT_RELAXABLE);
5738 if (reg)
5739 list_add_tail(&reg->list, head);
5740 }
5741 }
5742 #endif /* CONFIG_INTEL_IOMMU_FLOPPY_WA */
5743
5744 reg = iommu_alloc_resv_region(IOAPIC_RANGE_START,
5745 IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1,
5746 0, IOMMU_RESV_MSI);
5747 if (!reg)
5748 return;
5749 list_add_tail(&reg->list, head);
5750 }
5751
5752 static void intel_iommu_put_resv_regions(struct device *dev,
5753 struct list_head *head)
5754 {
5755 struct iommu_resv_region *entry, *next;
5756
5757 list_for_each_entry_safe(entry, next, head, list)
5758 kfree(entry);
5759 }
5760
5761 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev)
5762 {
5763 struct device_domain_info *info;
5764 struct context_entry *context;
5765 struct dmar_domain *domain;
5766 unsigned long flags;
5767 u64 ctx_lo;
5768 int ret;
5769
5770 domain = find_domain(dev);
5771 if (!domain)
5772 return -EINVAL;
5773
5774 spin_lock_irqsave(&device_domain_lock, flags);
5775 spin_lock(&iommu->lock);
5776
5777 ret = -EINVAL;
5778 info = dev->archdata.iommu;
5779 if (!info || !info->pasid_supported)
5780 goto out;
5781
5782 context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5783 if (WARN_ON(!context))
5784 goto out;
5785
5786 ctx_lo = context[0].lo;
5787
5788 if (!(ctx_lo & CONTEXT_PASIDE)) {
5789 ctx_lo |= CONTEXT_PASIDE;
5790 context[0].lo = ctx_lo;
5791 wmb();
5792 iommu->flush.flush_context(iommu,
5793 domain->iommu_did[iommu->seq_id],
5794 PCI_DEVID(info->bus, info->devfn),
5795 DMA_CCMD_MASK_NOBIT,
5796 DMA_CCMD_DEVICE_INVL);
5797 }
5798
5799 /* Enable PASID support in the device, if it wasn't already */
5800 if (!info->pasid_enabled)
5801 iommu_enable_dev_iotlb(info);
5802
5803 ret = 0;
5804
5805 out:
5806 spin_unlock(&iommu->lock);
5807 spin_unlock_irqrestore(&device_domain_lock, flags);
5808
5809 return ret;
5810 }
5811
5812 static void intel_iommu_apply_resv_region(struct device *dev,
5813 struct iommu_domain *domain,
5814 struct iommu_resv_region *region)
5815 {
5816 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5817 unsigned long start, end;
5818
5819 start = IOVA_PFN(region->start);
5820 end = IOVA_PFN(region->start + region->length - 1);
5821
5822 WARN_ON_ONCE(!reserve_iova(&dmar_domain->iovad, start, end));
5823 }
5824
5825 static struct iommu_group *intel_iommu_device_group(struct device *dev)
5826 {
5827 if (dev_is_pci(dev))
5828 return pci_device_group(dev);
5829 return generic_device_group(dev);
5830 }
5831
5832 #ifdef CONFIG_INTEL_IOMMU_SVM
5833 struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5834 {
5835 struct intel_iommu *iommu;
5836 u8 bus, devfn;
5837
5838 if (iommu_dummy(dev)) {
5839 dev_warn(dev,
5840 "No IOMMU translation for device; cannot enable SVM\n");
5841 return NULL;
5842 }
5843
5844 iommu = device_to_iommu(dev, &bus, &devfn);
5845 if ((!iommu)) {
5846 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
5847 return NULL;
5848 }
5849
5850 return iommu;
5851 }
5852 #endif /* CONFIG_INTEL_IOMMU_SVM */
5853
5854 static int intel_iommu_enable_auxd(struct device *dev)
5855 {
5856 struct device_domain_info *info;
5857 struct intel_iommu *iommu;
5858 unsigned long flags;
5859 u8 bus, devfn;
5860 int ret;
5861
5862 iommu = device_to_iommu(dev, &bus, &devfn);
5863 if (!iommu || dmar_disabled)
5864 return -EINVAL;
5865
5866 if (!sm_supported(iommu) || !pasid_supported(iommu))
5867 return -EINVAL;
5868
5869 ret = intel_iommu_enable_pasid(iommu, dev);
5870 if (ret)
5871 return -ENODEV;
5872
5873 spin_lock_irqsave(&device_domain_lock, flags);
5874 info = dev->archdata.iommu;
5875 info->auxd_enabled = 1;
5876 spin_unlock_irqrestore(&device_domain_lock, flags);
5877
5878 return 0;
5879 }
5880
5881 static int intel_iommu_disable_auxd(struct device *dev)
5882 {
5883 struct device_domain_info *info;
5884 unsigned long flags;
5885
5886 spin_lock_irqsave(&device_domain_lock, flags);
5887 info = dev->archdata.iommu;
5888 if (!WARN_ON(!info))
5889 info->auxd_enabled = 0;
5890 spin_unlock_irqrestore(&device_domain_lock, flags);
5891
5892 return 0;
5893 }
5894
5895 /*
5896 * A PCI express designated vendor specific extended capability is defined
5897 * in the section 3.7 of Intel scalable I/O virtualization technical spec
5898 * for system software and tools to detect endpoint devices supporting the
5899 * Intel scalable IO virtualization without host driver dependency.
5900 *
5901 * Returns the address of the matching extended capability structure within
5902 * the device's PCI configuration space or 0 if the device does not support
5903 * it.
5904 */
5905 static int siov_find_pci_dvsec(struct pci_dev *pdev)
5906 {
5907 int pos;
5908 u16 vendor, id;
5909
5910 pos = pci_find_next_ext_capability(pdev, 0, 0x23);
5911 while (pos) {
5912 pci_read_config_word(pdev, pos + 4, &vendor);
5913 pci_read_config_word(pdev, pos + 8, &id);
5914 if (vendor == PCI_VENDOR_ID_INTEL && id == 5)
5915 return pos;
5916
5917 pos = pci_find_next_ext_capability(pdev, pos, 0x23);
5918 }
5919
5920 return 0;
5921 }
5922
5923 static bool
5924 intel_iommu_dev_has_feat(struct device *dev, enum iommu_dev_features feat)
5925 {
5926 if (feat == IOMMU_DEV_FEAT_AUX) {
5927 int ret;
5928
5929 if (!dev_is_pci(dev) || dmar_disabled ||
5930 !scalable_mode_support() || !iommu_pasid_support())
5931 return false;
5932
5933 ret = pci_pasid_features(to_pci_dev(dev));
5934 if (ret < 0)
5935 return false;
5936
5937 return !!siov_find_pci_dvsec(to_pci_dev(dev));
5938 }
5939
5940 return false;
5941 }
5942
5943 static int
5944 intel_iommu_dev_enable_feat(struct device *dev, enum iommu_dev_features feat)
5945 {
5946 if (feat == IOMMU_DEV_FEAT_AUX)
5947 return intel_iommu_enable_auxd(dev);
5948
5949 return -ENODEV;
5950 }
5951
5952 static int
5953 intel_iommu_dev_disable_feat(struct device *dev, enum iommu_dev_features feat)
5954 {
5955 if (feat == IOMMU_DEV_FEAT_AUX)
5956 return intel_iommu_disable_auxd(dev);
5957
5958 return -ENODEV;
5959 }
5960
5961 static bool
5962 intel_iommu_dev_feat_enabled(struct device *dev, enum iommu_dev_features feat)
5963 {
5964 struct device_domain_info *info = dev->archdata.iommu;
5965
5966 if (feat == IOMMU_DEV_FEAT_AUX)
5967 return scalable_mode_support() && info && info->auxd_enabled;
5968
5969 return false;
5970 }
5971
5972 static int
5973 intel_iommu_aux_get_pasid(struct iommu_domain *domain, struct device *dev)
5974 {
5975 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5976
5977 return dmar_domain->default_pasid > 0 ?
5978 dmar_domain->default_pasid : -EINVAL;
5979 }
5980
5981 static bool intel_iommu_is_attach_deferred(struct iommu_domain *domain,
5982 struct device *dev)
5983 {
5984 return dev->archdata.iommu == DEFER_DEVICE_DOMAIN_INFO;
5985 }
5986
5987 const struct iommu_ops intel_iommu_ops = {
5988 .capable = intel_iommu_capable,
5989 .domain_alloc = intel_iommu_domain_alloc,
5990 .domain_free = intel_iommu_domain_free,
5991 .attach_dev = intel_iommu_attach_device,
5992 .detach_dev = intel_iommu_detach_device,
5993 .aux_attach_dev = intel_iommu_aux_attach_device,
5994 .aux_detach_dev = intel_iommu_aux_detach_device,
5995 .aux_get_pasid = intel_iommu_aux_get_pasid,
5996 .map = intel_iommu_map,
5997 .unmap = intel_iommu_unmap,
5998 .iova_to_phys = intel_iommu_iova_to_phys,
5999 .add_device = intel_iommu_add_device,
6000 .remove_device = intel_iommu_remove_device,
6001 .get_resv_regions = intel_iommu_get_resv_regions,
6002 .put_resv_regions = intel_iommu_put_resv_regions,
6003 .apply_resv_region = intel_iommu_apply_resv_region,
6004 .device_group = intel_iommu_device_group,
6005 .dev_has_feat = intel_iommu_dev_has_feat,
6006 .dev_feat_enabled = intel_iommu_dev_feat_enabled,
6007 .dev_enable_feat = intel_iommu_dev_enable_feat,
6008 .dev_disable_feat = intel_iommu_dev_disable_feat,
6009 .is_attach_deferred = intel_iommu_is_attach_deferred,
6010 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
6011 };
6012
6013 static void quirk_iommu_igfx(struct pci_dev *dev)
6014 {
6015 pci_info(dev, "Disabling IOMMU for graphics on this chipset\n");
6016 dmar_map_gfx = 0;
6017 }
6018
6019 /* G4x/GM45 integrated gfx dmar support is totally busted. */
6020 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_igfx);
6021 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_igfx);
6022 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_igfx);
6023 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_igfx);
6024 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_igfx);
6025 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_igfx);
6026 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_igfx);
6027
6028 /* Broadwell igfx malfunctions with dmar */
6029 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1606, quirk_iommu_igfx);
6030 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160B, quirk_iommu_igfx);
6031 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160E, quirk_iommu_igfx);
6032 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1602, quirk_iommu_igfx);
6033 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160A, quirk_iommu_igfx);
6034 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160D, quirk_iommu_igfx);
6035 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1616, quirk_iommu_igfx);
6036 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161B, quirk_iommu_igfx);
6037 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161E, quirk_iommu_igfx);
6038 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1612, quirk_iommu_igfx);
6039 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161A, quirk_iommu_igfx);
6040 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161D, quirk_iommu_igfx);
6041 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1626, quirk_iommu_igfx);
6042 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162B, quirk_iommu_igfx);
6043 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162E, quirk_iommu_igfx);
6044 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1622, quirk_iommu_igfx);
6045 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162A, quirk_iommu_igfx);
6046 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162D, quirk_iommu_igfx);
6047 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1636, quirk_iommu_igfx);
6048 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163B, quirk_iommu_igfx);
6049 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163E, quirk_iommu_igfx);
6050 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1632, quirk_iommu_igfx);
6051 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163A, quirk_iommu_igfx);
6052 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163D, quirk_iommu_igfx);
6053
6054 static void quirk_iommu_rwbf(struct pci_dev *dev)
6055 {
6056 /*
6057 * Mobile 4 Series Chipset neglects to set RWBF capability,
6058 * but needs it. Same seems to hold for the desktop versions.
6059 */
6060 pci_info(dev, "Forcing write-buffer flush capability\n");
6061 rwbf_quirk = 1;
6062 }
6063
6064 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
6065 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
6066 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
6067 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
6068 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
6069 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
6070 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
6071
6072 #define GGC 0x52
6073 #define GGC_MEMORY_SIZE_MASK (0xf << 8)
6074 #define GGC_MEMORY_SIZE_NONE (0x0 << 8)
6075 #define GGC_MEMORY_SIZE_1M (0x1 << 8)
6076 #define GGC_MEMORY_SIZE_2M (0x3 << 8)
6077 #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
6078 #define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
6079 #define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
6080 #define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
6081
6082 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
6083 {
6084 unsigned short ggc;
6085
6086 if (pci_read_config_word(dev, GGC, &ggc))
6087 return;
6088
6089 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
6090 pci_info(dev, "BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
6091 dmar_map_gfx = 0;
6092 } else if (dmar_map_gfx) {
6093 /* we have to ensure the gfx device is idle before we flush */
6094 pci_info(dev, "Disabling batched IOTLB flush on Ironlake\n");
6095 intel_iommu_strict = 1;
6096 }
6097 }
6098 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
6099 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
6100 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
6101 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
6102
6103 /* On Tylersburg chipsets, some BIOSes have been known to enable the
6104 ISOCH DMAR unit for the Azalia sound device, but not give it any
6105 TLB entries, which causes it to deadlock. Check for that. We do
6106 this in a function called from init_dmars(), instead of in a PCI
6107 quirk, because we don't want to print the obnoxious "BIOS broken"
6108 message if VT-d is actually disabled.
6109 */
6110 static void __init check_tylersburg_isoch(void)
6111 {
6112 struct pci_dev *pdev;
6113 uint32_t vtisochctrl;
6114
6115 /* If there's no Azalia in the system anyway, forget it. */
6116 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
6117 if (!pdev)
6118 return;
6119 pci_dev_put(pdev);
6120
6121 /* System Management Registers. Might be hidden, in which case
6122 we can't do the sanity check. But that's OK, because the
6123 known-broken BIOSes _don't_ actually hide it, so far. */
6124 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
6125 if (!pdev)
6126 return;
6127
6128 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
6129 pci_dev_put(pdev);
6130 return;
6131 }
6132
6133 pci_dev_put(pdev);
6134
6135 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
6136 if (vtisochctrl & 1)
6137 return;
6138
6139 /* Drop all bits other than the number of TLB entries */
6140 vtisochctrl &= 0x1c;
6141
6142 /* If we have the recommended number of TLB entries (16), fine. */
6143 if (vtisochctrl == 0x10)
6144 return;
6145
6146 /* Zero TLB entries? You get to ride the short bus to school. */
6147 if (!vtisochctrl) {
6148 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
6149 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
6150 dmi_get_system_info(DMI_BIOS_VENDOR),
6151 dmi_get_system_info(DMI_BIOS_VERSION),
6152 dmi_get_system_info(DMI_PRODUCT_VERSION));
6153 iommu_identity_mapping |= IDENTMAP_AZALIA;
6154 return;
6155 }
6156
6157 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
6158 vtisochctrl);
6159 }
Linux with added FPC-III support