?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
ARM: dts: add 'dr_mode' property to hsotg devices for exynos boards
[linux/fpc-iii.git] / drivers / iommu / intel-iommu.c
blob1232336b960edb163278d59510a56e51840df9b9
1 /*
2 * Copyright © 2006-2014 Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
18 */
19
20 #include <linux/init.h>
21 #include <linux/bitmap.h>
22 #include <linux/debugfs.h>
23 #include <linux/export.h>
24 #include <linux/slab.h>
25 #include <linux/irq.h>
26 #include <linux/interrupt.h>
27 #include <linux/spinlock.h>
28 #include <linux/pci.h>
29 #include <linux/dmar.h>
30 #include <linux/dma-mapping.h>
31 #include <linux/mempool.h>
32 #include <linux/memory.h>
33 #include <linux/timer.h>
34 #include <linux/iova.h>
35 #include <linux/iommu.h>
36 #include <linux/intel-iommu.h>
37 #include <linux/syscore_ops.h>
38 #include <linux/tboot.h>
39 #include <linux/dmi.h>
40 #include <linux/pci-ats.h>
41 #include <linux/memblock.h>
42 #include <linux/dma-contiguous.h>
43 #include <asm/irq_remapping.h>
44 #include <asm/cacheflush.h>
45 #include <asm/iommu.h>
46
47 #include "irq_remapping.h"
48
49 #define ROOT_SIZE VTD_PAGE_SIZE
50 #define CONTEXT_SIZE VTD_PAGE_SIZE
51
52 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
53 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
54 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
55
56 #define IOAPIC_RANGE_START (0xfee00000)
57 #define IOAPIC_RANGE_END (0xfeefffff)
58 #define IOVA_START_ADDR (0x1000)
59
60 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
61
62 #define MAX_AGAW_WIDTH 64
63 #define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
64
65 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
66 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
67
68 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
69 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
70 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
71 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
72 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
73
74 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
75 #define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
76 #define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
77
78 /* page table handling */
79 #define LEVEL_STRIDE (9)
80 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
81
82 /*
83 * This bitmap is used to advertise the page sizes our hardware support
84 * to the IOMMU core, which will then use this information to split
85 * physically contiguous memory regions it is mapping into page sizes
86 * that we support.
87 *
88 * Traditionally the IOMMU core just handed us the mappings directly,
89 * after making sure the size is an order of a 4KiB page and that the
90 * mapping has natural alignment.
91 *
92 * To retain this behavior, we currently advertise that we support
93 * all page sizes that are an order of 4KiB.
94 *
95 * If at some point we'd like to utilize the IOMMU core's new behavior,
96 * we could change this to advertise the real page sizes we support.
97 */
98 #define INTEL_IOMMU_PGSIZES (~0xFFFUL)
99
100 static inline int agaw_to_level(int agaw)
101 {
102 return agaw + 2;
103 }
104
105 static inline int agaw_to_width(int agaw)
106 {
107 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
108 }
109
110 static inline int width_to_agaw(int width)
111 {
112 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
113 }
114
115 static inline unsigned int level_to_offset_bits(int level)
116 {
117 return (level - 1) * LEVEL_STRIDE;
118 }
119
120 static inline int pfn_level_offset(unsigned long pfn, int level)
121 {
122 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
123 }
124
125 static inline unsigned long level_mask(int level)
126 {
127 return -1UL << level_to_offset_bits(level);
128 }
129
130 static inline unsigned long level_size(int level)
131 {
132 return 1UL << level_to_offset_bits(level);
133 }
134
135 static inline unsigned long align_to_level(unsigned long pfn, int level)
136 {
137 return (pfn + level_size(level) - 1) & level_mask(level);
138 }
139
140 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
141 {
142 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
143 }
144
145 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
146 are never going to work. */
147 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
148 {
149 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
150 }
151
152 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
153 {
154 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
155 }
156 static inline unsigned long page_to_dma_pfn(struct page *pg)
157 {
158 return mm_to_dma_pfn(page_to_pfn(pg));
159 }
160 static inline unsigned long virt_to_dma_pfn(void *p)
161 {
162 return page_to_dma_pfn(virt_to_page(p));
163 }
164
165 /* global iommu list, set NULL for ignored DMAR units */
166 static struct intel_iommu **g_iommus;
167
168 static void __init check_tylersburg_isoch(void);
169 static int rwbf_quirk;
170
171 /*
172 * set to 1 to panic kernel if can't successfully enable VT-d
173 * (used when kernel is launched w/ TXT)
174 */
175 static int force_on = 0;
176
177 /*
178 * 0: Present
179 * 1-11: Reserved
180 * 12-63: Context Ptr (12 - (haw-1))
181 * 64-127: Reserved
182 */
183 struct root_entry {
184 u64 val;
185 u64 rsvd1;
186 };
187 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
188 static inline bool root_present(struct root_entry *root)
189 {
190 return (root->val & 1);
191 }
192 static inline void set_root_present(struct root_entry *root)
193 {
194 root->val |= 1;
195 }
196 static inline void set_root_value(struct root_entry *root, unsigned long value)
197 {
198 root->val &= ~VTD_PAGE_MASK;
199 root->val |= value & VTD_PAGE_MASK;
200 }
201
202 static inline struct context_entry *
203 get_context_addr_from_root(struct root_entry *root)
204 {
205 return (struct context_entry *)
206 (root_present(root)?phys_to_virt(
207 root->val & VTD_PAGE_MASK) :
208 NULL);
209 }
210
211 /*
212 * low 64 bits:
213 * 0: present
214 * 1: fault processing disable
215 * 2-3: translation type
216 * 12-63: address space root
217 * high 64 bits:
218 * 0-2: address width
219 * 3-6: aval
220 * 8-23: domain id
221 */
222 struct context_entry {
223 u64 lo;
224 u64 hi;
225 };
226
227 static inline bool context_present(struct context_entry *context)
228 {
229 return (context->lo & 1);
230 }
231 static inline void context_set_present(struct context_entry *context)
232 {
233 context->lo |= 1;
234 }
235
236 static inline void context_set_fault_enable(struct context_entry *context)
237 {
238 context->lo &= (((u64)-1) << 2) | 1;
239 }
240
241 static inline void context_set_translation_type(struct context_entry *context,
242 unsigned long value)
243 {
244 context->lo &= (((u64)-1) << 4) | 3;
245 context->lo |= (value & 3) << 2;
246 }
247
248 static inline void context_set_address_root(struct context_entry *context,
249 unsigned long value)
250 {
251 context->lo &= ~VTD_PAGE_MASK;
252 context->lo |= value & VTD_PAGE_MASK;
253 }
254
255 static inline void context_set_address_width(struct context_entry *context,
256 unsigned long value)
257 {
258 context->hi |= value & 7;
259 }
260
261 static inline void context_set_domain_id(struct context_entry *context,
262 unsigned long value)
263 {
264 context->hi |= (value & ((1 << 16) - 1)) << 8;
265 }
266
267 static inline void context_clear_entry(struct context_entry *context)
268 {
269 context->lo = 0;
270 context->hi = 0;
271 }
272
273 /*
274 * 0: readable
275 * 1: writable
276 * 2-6: reserved
277 * 7: super page
278 * 8-10: available
279 * 11: snoop behavior
280 * 12-63: Host physcial address
281 */
282 struct dma_pte {
283 u64 val;
284 };
285
286 static inline void dma_clear_pte(struct dma_pte *pte)
287 {
288 pte->val = 0;
289 }
290
291 static inline u64 dma_pte_addr(struct dma_pte *pte)
292 {
293 #ifdef CONFIG_64BIT
294 return pte->val & VTD_PAGE_MASK;
295 #else
296 /* Must have a full atomic 64-bit read */
297 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
298 #endif
299 }
300
301 static inline bool dma_pte_present(struct dma_pte *pte)
302 {
303 return (pte->val & 3) != 0;
304 }
305
306 static inline bool dma_pte_superpage(struct dma_pte *pte)
307 {
308 return (pte->val & DMA_PTE_LARGE_PAGE);
309 }
310
311 static inline int first_pte_in_page(struct dma_pte *pte)
312 {
313 return !((unsigned long)pte & ~VTD_PAGE_MASK);
314 }
315
316 /*
317 * This domain is a statically identity mapping domain.
318 * 1. This domain creats a static 1:1 mapping to all usable memory.
319 * 2. It maps to each iommu if successful.
320 * 3. Each iommu mapps to this domain if successful.
321 */
322 static struct dmar_domain *si_domain;
323 static int hw_pass_through = 1;
324
325 /* domain represents a virtual machine, more than one devices
326 * across iommus may be owned in one domain, e.g. kvm guest.
327 */
328 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
329
330 /* si_domain contains mulitple devices */
331 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
332
333 struct dmar_domain {
334 int id; /* domain id */
335 int nid; /* node id */
336 DECLARE_BITMAP(iommu_bmp, DMAR_UNITS_SUPPORTED);
337 /* bitmap of iommus this domain uses*/
338
339 struct list_head devices; /* all devices' list */
340 struct iova_domain iovad; /* iova's that belong to this domain */
341
342 struct dma_pte *pgd; /* virtual address */
343 int gaw; /* max guest address width */
344
345 /* adjusted guest address width, 0 is level 2 30-bit */
346 int agaw;
347
348 int flags; /* flags to find out type of domain */
349
350 int iommu_coherency;/* indicate coherency of iommu access */
351 int iommu_snooping; /* indicate snooping control feature*/
352 int iommu_count; /* reference count of iommu */
353 int iommu_superpage;/* Level of superpages supported:
354 0 == 4KiB (no superpages), 1 == 2MiB,
355 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
356 spinlock_t iommu_lock; /* protect iommu set in domain */
357 u64 max_addr; /* maximum mapped address */
358 };
359
360 /* PCI domain-device relationship */
361 struct device_domain_info {
362 struct list_head link; /* link to domain siblings */
363 struct list_head global; /* link to global list */
364 u8 bus; /* PCI bus number */
365 u8 devfn; /* PCI devfn number */
366 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
367 struct intel_iommu *iommu; /* IOMMU used by this device */
368 struct dmar_domain *domain; /* pointer to domain */
369 };
370
371 struct dmar_rmrr_unit {
372 struct list_head list; /* list of rmrr units */
373 struct acpi_dmar_header *hdr; /* ACPI header */
374 u64 base_address; /* reserved base address*/
375 u64 end_address; /* reserved end address */
376 struct dmar_dev_scope *devices; /* target devices */
377 int devices_cnt; /* target device count */
378 };
379
380 struct dmar_atsr_unit {
381 struct list_head list; /* list of ATSR units */
382 struct acpi_dmar_header *hdr; /* ACPI header */
383 struct dmar_dev_scope *devices; /* target devices */
384 int devices_cnt; /* target device count */
385 u8 include_all:1; /* include all ports */
386 };
387
388 static LIST_HEAD(dmar_atsr_units);
389 static LIST_HEAD(dmar_rmrr_units);
390
391 #define for_each_rmrr_units(rmrr) \
392 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
393
394 static void flush_unmaps_timeout(unsigned long data);
395
396 static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
397
398 #define HIGH_WATER_MARK 250
399 struct deferred_flush_tables {
400 int next;
401 struct iova *iova[HIGH_WATER_MARK];
402 struct dmar_domain *domain[HIGH_WATER_MARK];
403 struct page *freelist[HIGH_WATER_MARK];
404 };
405
406 static struct deferred_flush_tables *deferred_flush;
407
408 /* bitmap for indexing intel_iommus */
409 static int g_num_of_iommus;
410
411 static DEFINE_SPINLOCK(async_umap_flush_lock);
412 static LIST_HEAD(unmaps_to_do);
413
414 static int timer_on;
415 static long list_size;
416
417 static void domain_exit(struct dmar_domain *domain);
418 static void domain_remove_dev_info(struct dmar_domain *domain);
419 static void domain_remove_one_dev_info(struct dmar_domain *domain,
420 struct device *dev);
421 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
422 struct device *dev);
423 static int domain_detach_iommu(struct dmar_domain *domain,
424 struct intel_iommu *iommu);
425
426 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
427 int dmar_disabled = 0;
428 #else
429 int dmar_disabled = 1;
430 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
431
432 int intel_iommu_enabled = 0;
433 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
434
435 static int dmar_map_gfx = 1;
436 static int dmar_forcedac;
437 static int intel_iommu_strict;
438 static int intel_iommu_superpage = 1;
439
440 int intel_iommu_gfx_mapped;
441 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
442
443 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
444 static DEFINE_SPINLOCK(device_domain_lock);
445 static LIST_HEAD(device_domain_list);
446
447 static const struct iommu_ops intel_iommu_ops;
448
449 static int __init intel_iommu_setup(char *str)
450 {
451 if (!str)
452 return -EINVAL;
453 while (*str) {
454 if (!strncmp(str, "on", 2)) {
455 dmar_disabled = 0;
456 printk(KERN_INFO "Intel-IOMMU: enabled\n");
457 } else if (!strncmp(str, "off", 3)) {
458 dmar_disabled = 1;
459 printk(KERN_INFO "Intel-IOMMU: disabled\n");
460 } else if (!strncmp(str, "igfx_off", 8)) {
461 dmar_map_gfx = 0;
462 printk(KERN_INFO
463 "Intel-IOMMU: disable GFX device mapping\n");
464 } else if (!strncmp(str, "forcedac", 8)) {
465 printk(KERN_INFO
466 "Intel-IOMMU: Forcing DAC for PCI devices\n");
467 dmar_forcedac = 1;
468 } else if (!strncmp(str, "strict", 6)) {
469 printk(KERN_INFO
470 "Intel-IOMMU: disable batched IOTLB flush\n");
471 intel_iommu_strict = 1;
472 } else if (!strncmp(str, "sp_off", 6)) {
473 printk(KERN_INFO
474 "Intel-IOMMU: disable supported super page\n");
475 intel_iommu_superpage = 0;
476 }
477
478 str += strcspn(str, ",");
479 while (*str == ',')
480 str++;
481 }
482 return 0;
483 }
484 __setup("intel_iommu=", intel_iommu_setup);
485
486 static struct kmem_cache *iommu_domain_cache;
487 static struct kmem_cache *iommu_devinfo_cache;
488 static struct kmem_cache *iommu_iova_cache;
489
490 static inline void *alloc_pgtable_page(int node)
491 {
492 struct page *page;
493 void *vaddr = NULL;
494
495 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
496 if (page)
497 vaddr = page_address(page);
498 return vaddr;
499 }
500
501 static inline void free_pgtable_page(void *vaddr)
502 {
503 free_page((unsigned long)vaddr);
504 }
505
506 static inline void *alloc_domain_mem(void)
507 {
508 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
509 }
510
511 static void free_domain_mem(void *vaddr)
512 {
513 kmem_cache_free(iommu_domain_cache, vaddr);
514 }
515
516 static inline void * alloc_devinfo_mem(void)
517 {
518 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
519 }
520
521 static inline void free_devinfo_mem(void *vaddr)
522 {
523 kmem_cache_free(iommu_devinfo_cache, vaddr);
524 }
525
526 struct iova *alloc_iova_mem(void)
527 {
528 return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
529 }
530
531 void free_iova_mem(struct iova *iova)
532 {
533 kmem_cache_free(iommu_iova_cache, iova);
534 }
535
536 static inline int domain_type_is_vm(struct dmar_domain *domain)
537 {
538 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
539 }
540
541 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
542 {
543 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
544 DOMAIN_FLAG_STATIC_IDENTITY);
545 }
546
547 static inline int domain_pfn_supported(struct dmar_domain *domain,
548 unsigned long pfn)
549 {
550 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
551
552 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
553 }
554
555 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
556 {
557 unsigned long sagaw;
558 int agaw = -1;
559
560 sagaw = cap_sagaw(iommu->cap);
561 for (agaw = width_to_agaw(max_gaw);
562 agaw >= 0; agaw--) {
563 if (test_bit(agaw, &sagaw))
564 break;
565 }
566
567 return agaw;
568 }
569
570 /*
571 * Calculate max SAGAW for each iommu.
572 */
573 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
574 {
575 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
576 }
577
578 /*
579 * calculate agaw for each iommu.
580 * "SAGAW" may be different across iommus, use a default agaw, and
581 * get a supported less agaw for iommus that don't support the default agaw.
582 */
583 int iommu_calculate_agaw(struct intel_iommu *iommu)
584 {
585 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
586 }
587
588 /* This functionin only returns single iommu in a domain */
589 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
590 {
591 int iommu_id;
592
593 /* si_domain and vm domain should not get here. */
594 BUG_ON(domain_type_is_vm_or_si(domain));
595 iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
596 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
597 return NULL;
598
599 return g_iommus[iommu_id];
600 }
601
602 static void domain_update_iommu_coherency(struct dmar_domain *domain)
603 {
604 struct dmar_drhd_unit *drhd;
605 struct intel_iommu *iommu;
606 int i, found = 0;
607
608 domain->iommu_coherency = 1;
609
610 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
611 found = 1;
612 if (!ecap_coherent(g_iommus[i]->ecap)) {
613 domain->iommu_coherency = 0;
614 break;
615 }
616 }
617 if (found)
618 return;
619
620 /* No hardware attached; use lowest common denominator */
621 rcu_read_lock();
622 for_each_active_iommu(iommu, drhd) {
623 if (!ecap_coherent(iommu->ecap)) {
624 domain->iommu_coherency = 0;
625 break;
626 }
627 }
628 rcu_read_unlock();
629 }
630
631 static int domain_update_iommu_snooping(struct intel_iommu *skip)
632 {
633 struct dmar_drhd_unit *drhd;
634 struct intel_iommu *iommu;
635 int ret = 1;
636
637 rcu_read_lock();
638 for_each_active_iommu(iommu, drhd) {
639 if (iommu != skip) {
640 if (!ecap_sc_support(iommu->ecap)) {
641 ret = 0;
642 break;
643 }
644 }
645 }
646 rcu_read_unlock();
647
648 return ret;
649 }
650
651 static int domain_update_iommu_superpage(struct intel_iommu *skip)
652 {
653 struct dmar_drhd_unit *drhd;
654 struct intel_iommu *iommu;
655 int mask = 0xf;
656
657 if (!intel_iommu_superpage) {
658 return 0;
659 }
660
661 /* set iommu_superpage to the smallest common denominator */
662 rcu_read_lock();
663 for_each_active_iommu(iommu, drhd) {
664 if (iommu != skip) {
665 mask &= cap_super_page_val(iommu->cap);
666 if (!mask)
667 break;
668 }
669 }
670 rcu_read_unlock();
671
672 return fls(mask);
673 }
674
675 /* Some capabilities may be different across iommus */
676 static void domain_update_iommu_cap(struct dmar_domain *domain)
677 {
678 domain_update_iommu_coherency(domain);
679 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
680 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
681 }
682
683 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
684 {
685 struct dmar_drhd_unit *drhd = NULL;
686 struct intel_iommu *iommu;
687 struct device *tmp;
688 struct pci_dev *ptmp, *pdev = NULL;
689 u16 segment = 0;
690 int i;
691
692 if (dev_is_pci(dev)) {
693 pdev = to_pci_dev(dev);
694 segment = pci_domain_nr(pdev->bus);
695 } else if (ACPI_COMPANION(dev))
696 dev = &ACPI_COMPANION(dev)->dev;
697
698 rcu_read_lock();
699 for_each_active_iommu(iommu, drhd) {
700 if (pdev && segment != drhd->segment)
701 continue;
702
703 for_each_active_dev_scope(drhd->devices,
704 drhd->devices_cnt, i, tmp) {
705 if (tmp == dev) {
706 *bus = drhd->devices[i].bus;
707 *devfn = drhd->devices[i].devfn;
708 goto out;
709 }
710
711 if (!pdev || !dev_is_pci(tmp))
712 continue;
713
714 ptmp = to_pci_dev(tmp);
715 if (ptmp->subordinate &&
716 ptmp->subordinate->number <= pdev->bus->number &&
717 ptmp->subordinate->busn_res.end >= pdev->bus->number)
718 goto got_pdev;
719 }
720
721 if (pdev && drhd->include_all) {
722 got_pdev:
723 *bus = pdev->bus->number;
724 *devfn = pdev->devfn;
725 goto out;
726 }
727 }
728 iommu = NULL;
729 out:
730 rcu_read_unlock();
731
732 return iommu;
733 }
734
735 static void domain_flush_cache(struct dmar_domain *domain,
736 void *addr, int size)
737 {
738 if (!domain->iommu_coherency)
739 clflush_cache_range(addr, size);
740 }
741
742 /* Gets context entry for a given bus and devfn */
743 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
744 u8 bus, u8 devfn)
745 {
746 struct root_entry *root;
747 struct context_entry *context;
748 unsigned long phy_addr;
749 unsigned long flags;
750
751 spin_lock_irqsave(&iommu->lock, flags);
752 root = &iommu->root_entry[bus];
753 context = get_context_addr_from_root(root);
754 if (!context) {
755 context = (struct context_entry *)
756 alloc_pgtable_page(iommu->node);
757 if (!context) {
758 spin_unlock_irqrestore(&iommu->lock, flags);
759 return NULL;
760 }
761 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
762 phy_addr = virt_to_phys((void *)context);
763 set_root_value(root, phy_addr);
764 set_root_present(root);
765 __iommu_flush_cache(iommu, root, sizeof(*root));
766 }
767 spin_unlock_irqrestore(&iommu->lock, flags);
768 return &context[devfn];
769 }
770
771 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
772 {
773 struct root_entry *root;
774 struct context_entry *context;
775 int ret;
776 unsigned long flags;
777
778 spin_lock_irqsave(&iommu->lock, flags);
779 root = &iommu->root_entry[bus];
780 context = get_context_addr_from_root(root);
781 if (!context) {
782 ret = 0;
783 goto out;
784 }
785 ret = context_present(&context[devfn]);
786 out:
787 spin_unlock_irqrestore(&iommu->lock, flags);
788 return ret;
789 }
790
791 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
792 {
793 struct root_entry *root;
794 struct context_entry *context;
795 unsigned long flags;
796
797 spin_lock_irqsave(&iommu->lock, flags);
798 root = &iommu->root_entry[bus];
799 context = get_context_addr_from_root(root);
800 if (context) {
801 context_clear_entry(&context[devfn]);
802 __iommu_flush_cache(iommu, &context[devfn], \
803 sizeof(*context));
804 }
805 spin_unlock_irqrestore(&iommu->lock, flags);
806 }
807
808 static void free_context_table(struct intel_iommu *iommu)
809 {
810 struct root_entry *root;
811 int i;
812 unsigned long flags;
813 struct context_entry *context;
814
815 spin_lock_irqsave(&iommu->lock, flags);
816 if (!iommu->root_entry) {
817 goto out;
818 }
819 for (i = 0; i < ROOT_ENTRY_NR; i++) {
820 root = &iommu->root_entry[i];
821 context = get_context_addr_from_root(root);
822 if (context)
823 free_pgtable_page(context);
824 }
825 free_pgtable_page(iommu->root_entry);
826 iommu->root_entry = NULL;
827 out:
828 spin_unlock_irqrestore(&iommu->lock, flags);
829 }
830
831 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
832 unsigned long pfn, int *target_level)
833 {
834 struct dma_pte *parent, *pte = NULL;
835 int level = agaw_to_level(domain->agaw);
836 int offset;
837
838 BUG_ON(!domain->pgd);
839
840 if (!domain_pfn_supported(domain, pfn))
841 /* Address beyond IOMMU's addressing capabilities. */
842 return NULL;
843
844 parent = domain->pgd;
845
846 while (1) {
847 void *tmp_page;
848
849 offset = pfn_level_offset(pfn, level);
850 pte = &parent[offset];
851 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
852 break;
853 if (level == *target_level)
854 break;
855
856 if (!dma_pte_present(pte)) {
857 uint64_t pteval;
858
859 tmp_page = alloc_pgtable_page(domain->nid);
860
861 if (!tmp_page)
862 return NULL;
863
864 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
865 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
866 if (cmpxchg64(&pte->val, 0ULL, pteval))
867 /* Someone else set it while we were thinking; use theirs. */
868 free_pgtable_page(tmp_page);
869 else
870 domain_flush_cache(domain, pte, sizeof(*pte));
871 }
872 if (level == 1)
873 break;
874
875 parent = phys_to_virt(dma_pte_addr(pte));
876 level--;
877 }
878
879 if (!*target_level)
880 *target_level = level;
881
882 return pte;
883 }
884
885
886 /* return address's pte at specific level */
887 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
888 unsigned long pfn,
889 int level, int *large_page)
890 {
891 struct dma_pte *parent, *pte = NULL;
892 int total = agaw_to_level(domain->agaw);
893 int offset;
894
895 parent = domain->pgd;
896 while (level <= total) {
897 offset = pfn_level_offset(pfn, total);
898 pte = &parent[offset];
899 if (level == total)
900 return pte;
901
902 if (!dma_pte_present(pte)) {
903 *large_page = total;
904 break;
905 }
906
907 if (dma_pte_superpage(pte)) {
908 *large_page = total;
909 return pte;
910 }
911
912 parent = phys_to_virt(dma_pte_addr(pte));
913 total--;
914 }
915 return NULL;
916 }
917
918 /* clear last level pte, a tlb flush should be followed */
919 static void dma_pte_clear_range(struct dmar_domain *domain,
920 unsigned long start_pfn,
921 unsigned long last_pfn)
922 {
923 unsigned int large_page = 1;
924 struct dma_pte *first_pte, *pte;
925
926 BUG_ON(!domain_pfn_supported(domain, start_pfn));
927 BUG_ON(!domain_pfn_supported(domain, last_pfn));
928 BUG_ON(start_pfn > last_pfn);
929
930 /* we don't need lock here; nobody else touches the iova range */
931 do {
932 large_page = 1;
933 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
934 if (!pte) {
935 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
936 continue;
937 }
938 do {
939 dma_clear_pte(pte);
940 start_pfn += lvl_to_nr_pages(large_page);
941 pte++;
942 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
943
944 domain_flush_cache(domain, first_pte,
945 (void *)pte - (void *)first_pte);
946
947 } while (start_pfn && start_pfn <= last_pfn);
948 }
949
950 static void dma_pte_free_level(struct dmar_domain *domain, int level,
951 struct dma_pte *pte, unsigned long pfn,
952 unsigned long start_pfn, unsigned long last_pfn)
953 {
954 pfn = max(start_pfn, pfn);
955 pte = &pte[pfn_level_offset(pfn, level)];
956
957 do {
958 unsigned long level_pfn;
959 struct dma_pte *level_pte;
960
961 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
962 goto next;
963
964 level_pfn = pfn & level_mask(level - 1);
965 level_pte = phys_to_virt(dma_pte_addr(pte));
966
967 if (level > 2)
968 dma_pte_free_level(domain, level - 1, level_pte,
969 level_pfn, start_pfn, last_pfn);
970
971 /* If range covers entire pagetable, free it */
972 if (!(start_pfn > level_pfn ||
973 last_pfn < level_pfn + level_size(level) - 1)) {
974 dma_clear_pte(pte);
975 domain_flush_cache(domain, pte, sizeof(*pte));
976 free_pgtable_page(level_pte);
977 }
978 next:
979 pfn += level_size(level);
980 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
981 }
982
983 /* free page table pages. last level pte should already be cleared */
984 static void dma_pte_free_pagetable(struct dmar_domain *domain,
985 unsigned long start_pfn,
986 unsigned long last_pfn)
987 {
988 BUG_ON(!domain_pfn_supported(domain, start_pfn));
989 BUG_ON(!domain_pfn_supported(domain, last_pfn));
990 BUG_ON(start_pfn > last_pfn);
991
992 dma_pte_clear_range(domain, start_pfn, last_pfn);
993
994 /* We don't need lock here; nobody else touches the iova range */
995 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
996 domain->pgd, 0, start_pfn, last_pfn);
997
998 /* free pgd */
999 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1000 free_pgtable_page(domain->pgd);
1001 domain->pgd = NULL;
1002 }
1003 }
1004
1005 /* When a page at a given level is being unlinked from its parent, we don't
1006 need to *modify* it at all. All we need to do is make a list of all the
1007 pages which can be freed just as soon as we've flushed the IOTLB and we
1008 know the hardware page-walk will no longer touch them.
1009 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1010 be freed. */
1011 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1012 int level, struct dma_pte *pte,
1013 struct page *freelist)
1014 {
1015 struct page *pg;
1016
1017 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1018 pg->freelist = freelist;
1019 freelist = pg;
1020
1021 if (level == 1)
1022 return freelist;
1023
1024 pte = page_address(pg);
1025 do {
1026 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1027 freelist = dma_pte_list_pagetables(domain, level - 1,
1028 pte, freelist);
1029 pte++;
1030 } while (!first_pte_in_page(pte));
1031
1032 return freelist;
1033 }
1034
1035 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1036 struct dma_pte *pte, unsigned long pfn,
1037 unsigned long start_pfn,
1038 unsigned long last_pfn,
1039 struct page *freelist)
1040 {
1041 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1042
1043 pfn = max(start_pfn, pfn);
1044 pte = &pte[pfn_level_offset(pfn, level)];
1045
1046 do {
1047 unsigned long level_pfn;
1048
1049 if (!dma_pte_present(pte))
1050 goto next;
1051
1052 level_pfn = pfn & level_mask(level);
1053
1054 /* If range covers entire pagetable, free it */
1055 if (start_pfn <= level_pfn &&
1056 last_pfn >= level_pfn + level_size(level) - 1) {
1057 /* These suborbinate page tables are going away entirely. Don't
1058 bother to clear them; we're just going to *free* them. */
1059 if (level > 1 && !dma_pte_superpage(pte))
1060 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1061
1062 dma_clear_pte(pte);
1063 if (!first_pte)
1064 first_pte = pte;
1065 last_pte = pte;
1066 } else if (level > 1) {
1067 /* Recurse down into a level that isn't *entirely* obsolete */
1068 freelist = dma_pte_clear_level(domain, level - 1,
1069 phys_to_virt(dma_pte_addr(pte)),
1070 level_pfn, start_pfn, last_pfn,
1071 freelist);
1072 }
1073 next:
1074 pfn += level_size(level);
1075 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1076
1077 if (first_pte)
1078 domain_flush_cache(domain, first_pte,
1079 (void *)++last_pte - (void *)first_pte);
1080
1081 return freelist;
1082 }
1083
1084 /* We can't just free the pages because the IOMMU may still be walking
1085 the page tables, and may have cached the intermediate levels. The
1086 pages can only be freed after the IOTLB flush has been done. */
1087 struct page *domain_unmap(struct dmar_domain *domain,
1088 unsigned long start_pfn,
1089 unsigned long last_pfn)
1090 {
1091 struct page *freelist = NULL;
1092
1093 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1094 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1095 BUG_ON(start_pfn > last_pfn);
1096
1097 /* we don't need lock here; nobody else touches the iova range */
1098 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1099 domain->pgd, 0, start_pfn, last_pfn, NULL);
1100
1101 /* free pgd */
1102 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1103 struct page *pgd_page = virt_to_page(domain->pgd);
1104 pgd_page->freelist = freelist;
1105 freelist = pgd_page;
1106
1107 domain->pgd = NULL;
1108 }
1109
1110 return freelist;
1111 }
1112
1113 void dma_free_pagelist(struct page *freelist)
1114 {
1115 struct page *pg;
1116
1117 while ((pg = freelist)) {
1118 freelist = pg->freelist;
1119 free_pgtable_page(page_address(pg));
1120 }
1121 }
1122
1123 /* iommu handling */
1124 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1125 {
1126 struct root_entry *root;
1127 unsigned long flags;
1128
1129 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1130 if (!root) {
1131 pr_err("IOMMU: allocating root entry for %s failed\n",
1132 iommu->name);
1133 return -ENOMEM;
1134 }
1135
1136 __iommu_flush_cache(iommu, root, ROOT_SIZE);
1137
1138 spin_lock_irqsave(&iommu->lock, flags);
1139 iommu->root_entry = root;
1140 spin_unlock_irqrestore(&iommu->lock, flags);
1141
1142 return 0;
1143 }
1144
1145 static void iommu_set_root_entry(struct intel_iommu *iommu)
1146 {
1147 void *addr;
1148 u32 sts;
1149 unsigned long flag;
1150
1151 addr = iommu->root_entry;
1152
1153 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1154 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
1155
1156 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1157
1158 /* Make sure hardware complete it */
1159 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1160 readl, (sts & DMA_GSTS_RTPS), sts);
1161
1162 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1163 }
1164
1165 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1166 {
1167 u32 val;
1168 unsigned long flag;
1169
1170 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1171 return;
1172
1173 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1174 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1175
1176 /* Make sure hardware complete it */
1177 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1178 readl, (!(val & DMA_GSTS_WBFS)), val);
1179
1180 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1181 }
1182
1183 /* return value determine if we need a write buffer flush */
1184 static void __iommu_flush_context(struct intel_iommu *iommu,
1185 u16 did, u16 source_id, u8 function_mask,
1186 u64 type)
1187 {
1188 u64 val = 0;
1189 unsigned long flag;
1190
1191 switch (type) {
1192 case DMA_CCMD_GLOBAL_INVL:
1193 val = DMA_CCMD_GLOBAL_INVL;
1194 break;
1195 case DMA_CCMD_DOMAIN_INVL:
1196 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1197 break;
1198 case DMA_CCMD_DEVICE_INVL:
1199 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1200 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1201 break;
1202 default:
1203 BUG();
1204 }
1205 val |= DMA_CCMD_ICC;
1206
1207 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1208 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1209
1210 /* Make sure hardware complete it */
1211 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1212 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1213
1214 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1215 }
1216
1217 /* return value determine if we need a write buffer flush */
1218 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1219 u64 addr, unsigned int size_order, u64 type)
1220 {
1221 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1222 u64 val = 0, val_iva = 0;
1223 unsigned long flag;
1224
1225 switch (type) {
1226 case DMA_TLB_GLOBAL_FLUSH:
1227 /* global flush doesn't need set IVA_REG */
1228 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1229 break;
1230 case DMA_TLB_DSI_FLUSH:
1231 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1232 break;
1233 case DMA_TLB_PSI_FLUSH:
1234 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1235 /* IH bit is passed in as part of address */
1236 val_iva = size_order | addr;
1237 break;
1238 default:
1239 BUG();
1240 }
1241 /* Note: set drain read/write */
1242 #if 0
1243 /*
1244 * This is probably to be super secure.. Looks like we can
1245 * ignore it without any impact.
1246 */
1247 if (cap_read_drain(iommu->cap))
1248 val |= DMA_TLB_READ_DRAIN;
1249 #endif
1250 if (cap_write_drain(iommu->cap))
1251 val |= DMA_TLB_WRITE_DRAIN;
1252
1253 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1254 /* Note: Only uses first TLB reg currently */
1255 if (val_iva)
1256 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1257 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1258
1259 /* Make sure hardware complete it */
1260 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1261 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1262
1263 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1264
1265 /* check IOTLB invalidation granularity */
1266 if (DMA_TLB_IAIG(val) == 0)
1267 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1268 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1269 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
1270 (unsigned long long)DMA_TLB_IIRG(type),
1271 (unsigned long long)DMA_TLB_IAIG(val));
1272 }
1273
1274 static struct device_domain_info *
1275 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1276 u8 bus, u8 devfn)
1277 {
1278 int found = 0;
1279 unsigned long flags;
1280 struct device_domain_info *info;
1281 struct pci_dev *pdev;
1282
1283 if (!ecap_dev_iotlb_support(iommu->ecap))
1284 return NULL;
1285
1286 if (!iommu->qi)
1287 return NULL;
1288
1289 spin_lock_irqsave(&device_domain_lock, flags);
1290 list_for_each_entry(info, &domain->devices, link)
1291 if (info->iommu == iommu && info->bus == bus &&
1292 info->devfn == devfn) {
1293 found = 1;
1294 break;
1295 }
1296 spin_unlock_irqrestore(&device_domain_lock, flags);
1297
1298 if (!found || !info->dev || !dev_is_pci(info->dev))
1299 return NULL;
1300
1301 pdev = to_pci_dev(info->dev);
1302
1303 if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
1304 return NULL;
1305
1306 if (!dmar_find_matched_atsr_unit(pdev))
1307 return NULL;
1308
1309 return info;
1310 }
1311
1312 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1313 {
1314 if (!info || !dev_is_pci(info->dev))
1315 return;
1316
1317 pci_enable_ats(to_pci_dev(info->dev), VTD_PAGE_SHIFT);
1318 }
1319
1320 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1321 {
1322 if (!info->dev || !dev_is_pci(info->dev) ||
1323 !pci_ats_enabled(to_pci_dev(info->dev)))
1324 return;
1325
1326 pci_disable_ats(to_pci_dev(info->dev));
1327 }
1328
1329 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1330 u64 addr, unsigned mask)
1331 {
1332 u16 sid, qdep;
1333 unsigned long flags;
1334 struct device_domain_info *info;
1335
1336 spin_lock_irqsave(&device_domain_lock, flags);
1337 list_for_each_entry(info, &domain->devices, link) {
1338 struct pci_dev *pdev;
1339 if (!info->dev || !dev_is_pci(info->dev))
1340 continue;
1341
1342 pdev = to_pci_dev(info->dev);
1343 if (!pci_ats_enabled(pdev))
1344 continue;
1345
1346 sid = info->bus << 8 | info->devfn;
1347 qdep = pci_ats_queue_depth(pdev);
1348 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1349 }
1350 spin_unlock_irqrestore(&device_domain_lock, flags);
1351 }
1352
1353 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1354 unsigned long pfn, unsigned int pages, int ih, int map)
1355 {
1356 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1357 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1358
1359 BUG_ON(pages == 0);
1360
1361 if (ih)
1362 ih = 1 << 6;
1363 /*
1364 * Fallback to domain selective flush if no PSI support or the size is
1365 * too big.
1366 * PSI requires page size to be 2 ^ x, and the base address is naturally
1367 * aligned to the size
1368 */
1369 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1370 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1371 DMA_TLB_DSI_FLUSH);
1372 else
1373 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1374 DMA_TLB_PSI_FLUSH);
1375
1376 /*
1377 * In caching mode, changes of pages from non-present to present require
1378 * flush. However, device IOTLB doesn't need to be flushed in this case.
1379 */
1380 if (!cap_caching_mode(iommu->cap) || !map)
1381 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
1382 }
1383
1384 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1385 {
1386 u32 pmen;
1387 unsigned long flags;
1388
1389 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1390 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1391 pmen &= ~DMA_PMEN_EPM;
1392 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1393
1394 /* wait for the protected region status bit to clear */
1395 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1396 readl, !(pmen & DMA_PMEN_PRS), pmen);
1397
1398 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1399 }
1400
1401 static void iommu_enable_translation(struct intel_iommu *iommu)
1402 {
1403 u32 sts;
1404 unsigned long flags;
1405
1406 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1407 iommu->gcmd |= DMA_GCMD_TE;
1408 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1409
1410 /* Make sure hardware complete it */
1411 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1412 readl, (sts & DMA_GSTS_TES), sts);
1413
1414 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1415 }
1416
1417 static void iommu_disable_translation(struct intel_iommu *iommu)
1418 {
1419 u32 sts;
1420 unsigned long flag;
1421
1422 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1423 iommu->gcmd &= ~DMA_GCMD_TE;
1424 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1425
1426 /* Make sure hardware complete it */
1427 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1428 readl, (!(sts & DMA_GSTS_TES)), sts);
1429
1430 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1431 }
1432
1433
1434 static int iommu_init_domains(struct intel_iommu *iommu)
1435 {
1436 unsigned long ndomains;
1437 unsigned long nlongs;
1438
1439 ndomains = cap_ndoms(iommu->cap);
1440 pr_debug("IOMMU%d: Number of Domains supported <%ld>\n",
1441 iommu->seq_id, ndomains);
1442 nlongs = BITS_TO_LONGS(ndomains);
1443
1444 spin_lock_init(&iommu->lock);
1445
1446 /* TBD: there might be 64K domains,
1447 * consider other allocation for future chip
1448 */
1449 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1450 if (!iommu->domain_ids) {
1451 pr_err("IOMMU%d: allocating domain id array failed\n",
1452 iommu->seq_id);
1453 return -ENOMEM;
1454 }
1455 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1456 GFP_KERNEL);
1457 if (!iommu->domains) {
1458 pr_err("IOMMU%d: allocating domain array failed\n",
1459 iommu->seq_id);
1460 kfree(iommu->domain_ids);
1461 iommu->domain_ids = NULL;
1462 return -ENOMEM;
1463 }
1464
1465 /*
1466 * if Caching mode is set, then invalid translations are tagged
1467 * with domainid 0. Hence we need to pre-allocate it.
1468 */
1469 if (cap_caching_mode(iommu->cap))
1470 set_bit(0, iommu->domain_ids);
1471 return 0;
1472 }
1473
1474 static void disable_dmar_iommu(struct intel_iommu *iommu)
1475 {
1476 struct dmar_domain *domain;
1477 int i;
1478
1479 if ((iommu->domains) && (iommu->domain_ids)) {
1480 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
1481 /*
1482 * Domain id 0 is reserved for invalid translation
1483 * if hardware supports caching mode.
1484 */
1485 if (cap_caching_mode(iommu->cap) && i == 0)
1486 continue;
1487
1488 domain = iommu->domains[i];
1489 clear_bit(i, iommu->domain_ids);
1490 if (domain_detach_iommu(domain, iommu) == 0 &&
1491 !domain_type_is_vm(domain))
1492 domain_exit(domain);
1493 }
1494 }
1495
1496 if (iommu->gcmd & DMA_GCMD_TE)
1497 iommu_disable_translation(iommu);
1498 }
1499
1500 static void free_dmar_iommu(struct intel_iommu *iommu)
1501 {
1502 if ((iommu->domains) && (iommu->domain_ids)) {
1503 kfree(iommu->domains);
1504 kfree(iommu->domain_ids);
1505 iommu->domains = NULL;
1506 iommu->domain_ids = NULL;
1507 }
1508
1509 g_iommus[iommu->seq_id] = NULL;
1510
1511 /* free context mapping */
1512 free_context_table(iommu);
1513 }
1514
1515 static struct dmar_domain *alloc_domain(int flags)
1516 {
1517 /* domain id for virtual machine, it won't be set in context */
1518 static atomic_t vm_domid = ATOMIC_INIT(0);
1519 struct dmar_domain *domain;
1520
1521 domain = alloc_domain_mem();
1522 if (!domain)
1523 return NULL;
1524
1525 memset(domain, 0, sizeof(*domain));
1526 domain->nid = -1;
1527 domain->flags = flags;
1528 spin_lock_init(&domain->iommu_lock);
1529 INIT_LIST_HEAD(&domain->devices);
1530 if (flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1531 domain->id = atomic_inc_return(&vm_domid);
1532
1533 return domain;
1534 }
1535
1536 static int __iommu_attach_domain(struct dmar_domain *domain,
1537 struct intel_iommu *iommu)
1538 {
1539 int num;
1540 unsigned long ndomains;
1541
1542 ndomains = cap_ndoms(iommu->cap);
1543 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1544 if (num < ndomains) {
1545 set_bit(num, iommu->domain_ids);
1546 iommu->domains[num] = domain;
1547 } else {
1548 num = -ENOSPC;
1549 }
1550
1551 return num;
1552 }
1553
1554 static int iommu_attach_domain(struct dmar_domain *domain,
1555 struct intel_iommu *iommu)
1556 {
1557 int num;
1558 unsigned long flags;
1559
1560 spin_lock_irqsave(&iommu->lock, flags);
1561 num = __iommu_attach_domain(domain, iommu);
1562 spin_unlock_irqrestore(&iommu->lock, flags);
1563 if (num < 0)
1564 pr_err("IOMMU: no free domain ids\n");
1565
1566 return num;
1567 }
1568
1569 static int iommu_attach_vm_domain(struct dmar_domain *domain,
1570 struct intel_iommu *iommu)
1571 {
1572 int num;
1573 unsigned long ndomains;
1574
1575 ndomains = cap_ndoms(iommu->cap);
1576 for_each_set_bit(num, iommu->domain_ids, ndomains)
1577 if (iommu->domains[num] == domain)
1578 return num;
1579
1580 return __iommu_attach_domain(domain, iommu);
1581 }
1582
1583 static void iommu_detach_domain(struct dmar_domain *domain,
1584 struct intel_iommu *iommu)
1585 {
1586 unsigned long flags;
1587 int num, ndomains;
1588
1589 spin_lock_irqsave(&iommu->lock, flags);
1590 if (domain_type_is_vm_or_si(domain)) {
1591 ndomains = cap_ndoms(iommu->cap);
1592 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1593 if (iommu->domains[num] == domain) {
1594 clear_bit(num, iommu->domain_ids);
1595 iommu->domains[num] = NULL;
1596 break;
1597 }
1598 }
1599 } else {
1600 clear_bit(domain->id, iommu->domain_ids);
1601 iommu->domains[domain->id] = NULL;
1602 }
1603 spin_unlock_irqrestore(&iommu->lock, flags);
1604 }
1605
1606 static void domain_attach_iommu(struct dmar_domain *domain,
1607 struct intel_iommu *iommu)
1608 {
1609 unsigned long flags;
1610
1611 spin_lock_irqsave(&domain->iommu_lock, flags);
1612 if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
1613 domain->iommu_count++;
1614 if (domain->iommu_count == 1)
1615 domain->nid = iommu->node;
1616 domain_update_iommu_cap(domain);
1617 }
1618 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1619 }
1620
1621 static int domain_detach_iommu(struct dmar_domain *domain,
1622 struct intel_iommu *iommu)
1623 {
1624 unsigned long flags;
1625 int count = INT_MAX;
1626
1627 spin_lock_irqsave(&domain->iommu_lock, flags);
1628 if (test_and_clear_bit(iommu->seq_id, domain->iommu_bmp)) {
1629 count = --domain->iommu_count;
1630 domain_update_iommu_cap(domain);
1631 }
1632 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1633
1634 return count;
1635 }
1636
1637 static struct iova_domain reserved_iova_list;
1638 static struct lock_class_key reserved_rbtree_key;
1639
1640 static int dmar_init_reserved_ranges(void)
1641 {
1642 struct pci_dev *pdev = NULL;
1643 struct iova *iova;
1644 int i;
1645
1646 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1647
1648 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1649 &reserved_rbtree_key);
1650
1651 /* IOAPIC ranges shouldn't be accessed by DMA */
1652 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1653 IOVA_PFN(IOAPIC_RANGE_END));
1654 if (!iova) {
1655 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1656 return -ENODEV;
1657 }
1658
1659 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1660 for_each_pci_dev(pdev) {
1661 struct resource *r;
1662
1663 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1664 r = &pdev->resource[i];
1665 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1666 continue;
1667 iova = reserve_iova(&reserved_iova_list,
1668 IOVA_PFN(r->start),
1669 IOVA_PFN(r->end));
1670 if (!iova) {
1671 printk(KERN_ERR "Reserve iova failed\n");
1672 return -ENODEV;
1673 }
1674 }
1675 }
1676 return 0;
1677 }
1678
1679 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1680 {
1681 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1682 }
1683
1684 static inline int guestwidth_to_adjustwidth(int gaw)
1685 {
1686 int agaw;
1687 int r = (gaw - 12) % 9;
1688
1689 if (r == 0)
1690 agaw = gaw;
1691 else
1692 agaw = gaw + 9 - r;
1693 if (agaw > 64)
1694 agaw = 64;
1695 return agaw;
1696 }
1697
1698 static int domain_init(struct dmar_domain *domain, int guest_width)
1699 {
1700 struct intel_iommu *iommu;
1701 int adjust_width, agaw;
1702 unsigned long sagaw;
1703
1704 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1705 domain_reserve_special_ranges(domain);
1706
1707 /* calculate AGAW */
1708 iommu = domain_get_iommu(domain);
1709 if (guest_width > cap_mgaw(iommu->cap))
1710 guest_width = cap_mgaw(iommu->cap);
1711 domain->gaw = guest_width;
1712 adjust_width = guestwidth_to_adjustwidth(guest_width);
1713 agaw = width_to_agaw(adjust_width);
1714 sagaw = cap_sagaw(iommu->cap);
1715 if (!test_bit(agaw, &sagaw)) {
1716 /* hardware doesn't support it, choose a bigger one */
1717 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1718 agaw = find_next_bit(&sagaw, 5, agaw);
1719 if (agaw >= 5)
1720 return -ENODEV;
1721 }
1722 domain->agaw = agaw;
1723
1724 if (ecap_coherent(iommu->ecap))
1725 domain->iommu_coherency = 1;
1726 else
1727 domain->iommu_coherency = 0;
1728
1729 if (ecap_sc_support(iommu->ecap))
1730 domain->iommu_snooping = 1;
1731 else
1732 domain->iommu_snooping = 0;
1733
1734 if (intel_iommu_superpage)
1735 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1736 else
1737 domain->iommu_superpage = 0;
1738
1739 domain->nid = iommu->node;
1740
1741 /* always allocate the top pgd */
1742 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1743 if (!domain->pgd)
1744 return -ENOMEM;
1745 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1746 return 0;
1747 }
1748
1749 static void domain_exit(struct dmar_domain *domain)
1750 {
1751 struct dmar_drhd_unit *drhd;
1752 struct intel_iommu *iommu;
1753 struct page *freelist = NULL;
1754
1755 /* Domain 0 is reserved, so dont process it */
1756 if (!domain)
1757 return;
1758
1759 /* Flush any lazy unmaps that may reference this domain */
1760 if (!intel_iommu_strict)
1761 flush_unmaps_timeout(0);
1762
1763 /* remove associated devices */
1764 domain_remove_dev_info(domain);
1765
1766 /* destroy iovas */
1767 put_iova_domain(&domain->iovad);
1768
1769 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1770
1771 /* clear attached or cached domains */
1772 rcu_read_lock();
1773 for_each_active_iommu(iommu, drhd)
1774 iommu_detach_domain(domain, iommu);
1775 rcu_read_unlock();
1776
1777 dma_free_pagelist(freelist);
1778
1779 free_domain_mem(domain);
1780 }
1781
1782 static int domain_context_mapping_one(struct dmar_domain *domain,
1783 struct intel_iommu *iommu,
1784 u8 bus, u8 devfn, int translation)
1785 {
1786 struct context_entry *context;
1787 unsigned long flags;
1788 struct dma_pte *pgd;
1789 int id;
1790 int agaw;
1791 struct device_domain_info *info = NULL;
1792
1793 pr_debug("Set context mapping for %02x:%02x.%d\n",
1794 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1795
1796 BUG_ON(!domain->pgd);
1797 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1798 translation != CONTEXT_TT_MULTI_LEVEL);
1799
1800 context = device_to_context_entry(iommu, bus, devfn);
1801 if (!context)
1802 return -ENOMEM;
1803 spin_lock_irqsave(&iommu->lock, flags);
1804 if (context_present(context)) {
1805 spin_unlock_irqrestore(&iommu->lock, flags);
1806 return 0;
1807 }
1808
1809 id = domain->id;
1810 pgd = domain->pgd;
1811
1812 if (domain_type_is_vm_or_si(domain)) {
1813 if (domain_type_is_vm(domain)) {
1814 id = iommu_attach_vm_domain(domain, iommu);
1815 if (id < 0) {
1816 spin_unlock_irqrestore(&iommu->lock, flags);
1817 pr_err("IOMMU: no free domain ids\n");
1818 return -EFAULT;
1819 }
1820 }
1821
1822 /* Skip top levels of page tables for
1823 * iommu which has less agaw than default.
1824 * Unnecessary for PT mode.
1825 */
1826 if (translation != CONTEXT_TT_PASS_THROUGH) {
1827 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1828 pgd = phys_to_virt(dma_pte_addr(pgd));
1829 if (!dma_pte_present(pgd)) {
1830 spin_unlock_irqrestore(&iommu->lock, flags);
1831 return -ENOMEM;
1832 }
1833 }
1834 }
1835 }
1836
1837 context_set_domain_id(context, id);
1838
1839 if (translation != CONTEXT_TT_PASS_THROUGH) {
1840 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
1841 translation = info ? CONTEXT_TT_DEV_IOTLB :
1842 CONTEXT_TT_MULTI_LEVEL;
1843 }
1844 /*
1845 * In pass through mode, AW must be programmed to indicate the largest
1846 * AGAW value supported by hardware. And ASR is ignored by hardware.
1847 */
1848 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
1849 context_set_address_width(context, iommu->msagaw);
1850 else {
1851 context_set_address_root(context, virt_to_phys(pgd));
1852 context_set_address_width(context, iommu->agaw);
1853 }
1854
1855 context_set_translation_type(context, translation);
1856 context_set_fault_enable(context);
1857 context_set_present(context);
1858 domain_flush_cache(domain, context, sizeof(*context));
1859
1860 /*
1861 * It's a non-present to present mapping. If hardware doesn't cache
1862 * non-present entry we only need to flush the write-buffer. If the
1863 * _does_ cache non-present entries, then it does so in the special
1864 * domain #0, which we have to flush:
1865 */
1866 if (cap_caching_mode(iommu->cap)) {
1867 iommu->flush.flush_context(iommu, 0,
1868 (((u16)bus) << 8) | devfn,
1869 DMA_CCMD_MASK_NOBIT,
1870 DMA_CCMD_DEVICE_INVL);
1871 iommu->flush.flush_iotlb(iommu, id, 0, 0, DMA_TLB_DSI_FLUSH);
1872 } else {
1873 iommu_flush_write_buffer(iommu);
1874 }
1875 iommu_enable_dev_iotlb(info);
1876 spin_unlock_irqrestore(&iommu->lock, flags);
1877
1878 domain_attach_iommu(domain, iommu);
1879
1880 return 0;
1881 }
1882
1883 struct domain_context_mapping_data {
1884 struct dmar_domain *domain;
1885 struct intel_iommu *iommu;
1886 int translation;
1887 };
1888
1889 static int domain_context_mapping_cb(struct pci_dev *pdev,
1890 u16 alias, void *opaque)
1891 {
1892 struct domain_context_mapping_data *data = opaque;
1893
1894 return domain_context_mapping_one(data->domain, data->iommu,
1895 PCI_BUS_NUM(alias), alias & 0xff,
1896 data->translation);
1897 }
1898
1899 static int
1900 domain_context_mapping(struct dmar_domain *domain, struct device *dev,
1901 int translation)
1902 {
1903 struct intel_iommu *iommu;
1904 u8 bus, devfn;
1905 struct domain_context_mapping_data data;
1906
1907 iommu = device_to_iommu(dev, &bus, &devfn);
1908 if (!iommu)
1909 return -ENODEV;
1910
1911 if (!dev_is_pci(dev))
1912 return domain_context_mapping_one(domain, iommu, bus, devfn,
1913 translation);
1914
1915 data.domain = domain;
1916 data.iommu = iommu;
1917 data.translation = translation;
1918
1919 return pci_for_each_dma_alias(to_pci_dev(dev),
1920 &domain_context_mapping_cb, &data);
1921 }
1922
1923 static int domain_context_mapped_cb(struct pci_dev *pdev,
1924 u16 alias, void *opaque)
1925 {
1926 struct intel_iommu *iommu = opaque;
1927
1928 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
1929 }
1930
1931 static int domain_context_mapped(struct device *dev)
1932 {
1933 struct intel_iommu *iommu;
1934 u8 bus, devfn;
1935
1936 iommu = device_to_iommu(dev, &bus, &devfn);
1937 if (!iommu)
1938 return -ENODEV;
1939
1940 if (!dev_is_pci(dev))
1941 return device_context_mapped(iommu, bus, devfn);
1942
1943 return !pci_for_each_dma_alias(to_pci_dev(dev),
1944 domain_context_mapped_cb, iommu);
1945 }
1946
1947 /* Returns a number of VTD pages, but aligned to MM page size */
1948 static inline unsigned long aligned_nrpages(unsigned long host_addr,
1949 size_t size)
1950 {
1951 host_addr &= ~PAGE_MASK;
1952 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1953 }
1954
1955 /* Return largest possible superpage level for a given mapping */
1956 static inline int hardware_largepage_caps(struct dmar_domain *domain,
1957 unsigned long iov_pfn,
1958 unsigned long phy_pfn,
1959 unsigned long pages)
1960 {
1961 int support, level = 1;
1962 unsigned long pfnmerge;
1963
1964 support = domain->iommu_superpage;
1965
1966 /* To use a large page, the virtual *and* physical addresses
1967 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1968 of them will mean we have to use smaller pages. So just
1969 merge them and check both at once. */
1970 pfnmerge = iov_pfn | phy_pfn;
1971
1972 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1973 pages >>= VTD_STRIDE_SHIFT;
1974 if (!pages)
1975 break;
1976 pfnmerge >>= VTD_STRIDE_SHIFT;
1977 level++;
1978 support--;
1979 }
1980 return level;
1981 }
1982
1983 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1984 struct scatterlist *sg, unsigned long phys_pfn,
1985 unsigned long nr_pages, int prot)
1986 {
1987 struct dma_pte *first_pte = NULL, *pte = NULL;
1988 phys_addr_t uninitialized_var(pteval);
1989 unsigned long sg_res = 0;
1990 unsigned int largepage_lvl = 0;
1991 unsigned long lvl_pages = 0;
1992
1993 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
1994
1995 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1996 return -EINVAL;
1997
1998 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1999
2000 if (!sg) {
2001 sg_res = nr_pages;
2002 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2003 }
2004
2005 while (nr_pages > 0) {
2006 uint64_t tmp;
2007
2008 if (!sg_res) {
2009 sg_res = aligned_nrpages(sg->offset, sg->length);
2010 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2011 sg->dma_length = sg->length;
2012 pteval = page_to_phys(sg_page(sg)) | prot;
2013 phys_pfn = pteval >> VTD_PAGE_SHIFT;
2014 }
2015
2016 if (!pte) {
2017 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2018
2019 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2020 if (!pte)
2021 return -ENOMEM;
2022 /* It is large page*/
2023 if (largepage_lvl > 1) {
2024 pteval |= DMA_PTE_LARGE_PAGE;
2025 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2026 /*
2027 * Ensure that old small page tables are
2028 * removed to make room for superpage,
2029 * if they exist.
2030 */
2031 dma_pte_free_pagetable(domain, iov_pfn,
2032 iov_pfn + lvl_pages - 1);
2033 } else {
2034 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2035 }
2036
2037 }
2038 /* We don't need lock here, nobody else
2039 * touches the iova range
2040 */
2041 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2042 if (tmp) {
2043 static int dumps = 5;
2044 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2045 iov_pfn, tmp, (unsigned long long)pteval);
2046 if (dumps) {
2047 dumps--;
2048 debug_dma_dump_mappings(NULL);
2049 }
2050 WARN_ON(1);
2051 }
2052
2053 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2054
2055 BUG_ON(nr_pages < lvl_pages);
2056 BUG_ON(sg_res < lvl_pages);
2057
2058 nr_pages -= lvl_pages;
2059 iov_pfn += lvl_pages;
2060 phys_pfn += lvl_pages;
2061 pteval += lvl_pages * VTD_PAGE_SIZE;
2062 sg_res -= lvl_pages;
2063
2064 /* If the next PTE would be the first in a new page, then we
2065 need to flush the cache on the entries we've just written.
2066 And then we'll need to recalculate 'pte', so clear it and
2067 let it get set again in the if (!pte) block above.
2068
2069 If we're done (!nr_pages) we need to flush the cache too.
2070
2071 Also if we've been setting superpages, we may need to
2072 recalculate 'pte' and switch back to smaller pages for the
2073 end of the mapping, if the trailing size is not enough to
2074 use another superpage (i.e. sg_res < lvl_pages). */
2075 pte++;
2076 if (!nr_pages || first_pte_in_page(pte) ||
2077 (largepage_lvl > 1 && sg_res < lvl_pages)) {
2078 domain_flush_cache(domain, first_pte,
2079 (void *)pte - (void *)first_pte);
2080 pte = NULL;
2081 }
2082
2083 if (!sg_res && nr_pages)
2084 sg = sg_next(sg);
2085 }
2086 return 0;
2087 }
2088
2089 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2090 struct scatterlist *sg, unsigned long nr_pages,
2091 int prot)
2092 {
2093 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2094 }
2095
2096 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2097 unsigned long phys_pfn, unsigned long nr_pages,
2098 int prot)
2099 {
2100 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2101 }
2102
2103 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
2104 {
2105 if (!iommu)
2106 return;
2107
2108 clear_context_table(iommu, bus, devfn);
2109 iommu->flush.flush_context(iommu, 0, 0, 0,
2110 DMA_CCMD_GLOBAL_INVL);
2111 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2112 }
2113
2114 static inline void unlink_domain_info(struct device_domain_info *info)
2115 {
2116 assert_spin_locked(&device_domain_lock);
2117 list_del(&info->link);
2118 list_del(&info->global);
2119 if (info->dev)
2120 info->dev->archdata.iommu = NULL;
2121 }
2122
2123 static void domain_remove_dev_info(struct dmar_domain *domain)
2124 {
2125 struct device_domain_info *info, *tmp;
2126 unsigned long flags;
2127
2128 spin_lock_irqsave(&device_domain_lock, flags);
2129 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
2130 unlink_domain_info(info);
2131 spin_unlock_irqrestore(&device_domain_lock, flags);
2132
2133 iommu_disable_dev_iotlb(info);
2134 iommu_detach_dev(info->iommu, info->bus, info->devfn);
2135
2136 if (domain_type_is_vm(domain)) {
2137 iommu_detach_dependent_devices(info->iommu, info->dev);
2138 domain_detach_iommu(domain, info->iommu);
2139 }
2140
2141 free_devinfo_mem(info);
2142 spin_lock_irqsave(&device_domain_lock, flags);
2143 }
2144 spin_unlock_irqrestore(&device_domain_lock, flags);
2145 }
2146
2147 /*
2148 * find_domain
2149 * Note: we use struct device->archdata.iommu stores the info
2150 */
2151 static struct dmar_domain *find_domain(struct device *dev)
2152 {
2153 struct device_domain_info *info;
2154
2155 /* No lock here, assumes no domain exit in normal case */
2156 info = dev->archdata.iommu;
2157 if (info)
2158 return info->domain;
2159 return NULL;
2160 }
2161
2162 static inline struct device_domain_info *
2163 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2164 {
2165 struct device_domain_info *info;
2166
2167 list_for_each_entry(info, &device_domain_list, global)
2168 if (info->iommu->segment == segment && info->bus == bus &&
2169 info->devfn == devfn)
2170 return info;
2171
2172 return NULL;
2173 }
2174
2175 static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu,
2176 int bus, int devfn,
2177 struct device *dev,
2178 struct dmar_domain *domain)
2179 {
2180 struct dmar_domain *found = NULL;
2181 struct device_domain_info *info;
2182 unsigned long flags;
2183
2184 info = alloc_devinfo_mem();
2185 if (!info)
2186 return NULL;
2187
2188 info->bus = bus;
2189 info->devfn = devfn;
2190 info->dev = dev;
2191 info->domain = domain;
2192 info->iommu = iommu;
2193
2194 spin_lock_irqsave(&device_domain_lock, flags);
2195 if (dev)
2196 found = find_domain(dev);
2197 else {
2198 struct device_domain_info *info2;
2199 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2200 if (info2)
2201 found = info2->domain;
2202 }
2203 if (found) {
2204 spin_unlock_irqrestore(&device_domain_lock, flags);
2205 free_devinfo_mem(info);
2206 /* Caller must free the original domain */
2207 return found;
2208 }
2209
2210 list_add(&info->link, &domain->devices);
2211 list_add(&info->global, &device_domain_list);
2212 if (dev)
2213 dev->archdata.iommu = info;
2214 spin_unlock_irqrestore(&device_domain_lock, flags);
2215
2216 return domain;
2217 }
2218
2219 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2220 {
2221 *(u16 *)opaque = alias;
2222 return 0;
2223 }
2224
2225 /* domain is initialized */
2226 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2227 {
2228 struct dmar_domain *domain, *tmp;
2229 struct intel_iommu *iommu;
2230 struct device_domain_info *info;
2231 u16 dma_alias;
2232 unsigned long flags;
2233 u8 bus, devfn;
2234
2235 domain = find_domain(dev);
2236 if (domain)
2237 return domain;
2238
2239 iommu = device_to_iommu(dev, &bus, &devfn);
2240 if (!iommu)
2241 return NULL;
2242
2243 if (dev_is_pci(dev)) {
2244 struct pci_dev *pdev = to_pci_dev(dev);
2245
2246 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2247
2248 spin_lock_irqsave(&device_domain_lock, flags);
2249 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2250 PCI_BUS_NUM(dma_alias),
2251 dma_alias & 0xff);
2252 if (info) {
2253 iommu = info->iommu;
2254 domain = info->domain;
2255 }
2256 spin_unlock_irqrestore(&device_domain_lock, flags);
2257
2258 /* DMA alias already has a domain, uses it */
2259 if (info)
2260 goto found_domain;
2261 }
2262
2263 /* Allocate and initialize new domain for the device */
2264 domain = alloc_domain(0);
2265 if (!domain)
2266 return NULL;
2267 domain->id = iommu_attach_domain(domain, iommu);
2268 if (domain->id < 0) {
2269 free_domain_mem(domain);
2270 return NULL;
2271 }
2272 domain_attach_iommu(domain, iommu);
2273 if (domain_init(domain, gaw)) {
2274 domain_exit(domain);
2275 return NULL;
2276 }
2277
2278 /* register PCI DMA alias device */
2279 if (dev_is_pci(dev)) {
2280 tmp = dmar_insert_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2281 dma_alias & 0xff, NULL, domain);
2282
2283 if (!tmp || tmp != domain) {
2284 domain_exit(domain);
2285 domain = tmp;
2286 }
2287
2288 if (!domain)
2289 return NULL;
2290 }
2291
2292 found_domain:
2293 tmp = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2294
2295 if (!tmp || tmp != domain) {
2296 domain_exit(domain);
2297 domain = tmp;
2298 }
2299
2300 return domain;
2301 }
2302
2303 static int iommu_identity_mapping;
2304 #define IDENTMAP_ALL 1
2305 #define IDENTMAP_GFX 2
2306 #define IDENTMAP_AZALIA 4
2307
2308 static int iommu_domain_identity_map(struct dmar_domain *domain,
2309 unsigned long long start,
2310 unsigned long long end)
2311 {
2312 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2313 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2314
2315 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2316 dma_to_mm_pfn(last_vpfn))) {
2317 printk(KERN_ERR "IOMMU: reserve iova failed\n");
2318 return -ENOMEM;
2319 }
2320
2321 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2322 start, end, domain->id);
2323 /*
2324 * RMRR range might have overlap with physical memory range,
2325 * clear it first
2326 */
2327 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2328
2329 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2330 last_vpfn - first_vpfn + 1,
2331 DMA_PTE_READ|DMA_PTE_WRITE);
2332 }
2333
2334 static int iommu_prepare_identity_map(struct device *dev,
2335 unsigned long long start,
2336 unsigned long long end)
2337 {
2338 struct dmar_domain *domain;
2339 int ret;
2340
2341 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2342 if (!domain)
2343 return -ENOMEM;
2344
2345 /* For _hardware_ passthrough, don't bother. But for software
2346 passthrough, we do it anyway -- it may indicate a memory
2347 range which is reserved in E820, so which didn't get set
2348 up to start with in si_domain */
2349 if (domain == si_domain && hw_pass_through) {
2350 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2351 dev_name(dev), start, end);
2352 return 0;
2353 }
2354
2355 printk(KERN_INFO
2356 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2357 dev_name(dev), start, end);
2358
2359 if (end < start) {
2360 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2361 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2362 dmi_get_system_info(DMI_BIOS_VENDOR),
2363 dmi_get_system_info(DMI_BIOS_VERSION),
2364 dmi_get_system_info(DMI_PRODUCT_VERSION));
2365 ret = -EIO;
2366 goto error;
2367 }
2368
2369 if (end >> agaw_to_width(domain->agaw)) {
2370 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2371 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2372 agaw_to_width(domain->agaw),
2373 dmi_get_system_info(DMI_BIOS_VENDOR),
2374 dmi_get_system_info(DMI_BIOS_VERSION),
2375 dmi_get_system_info(DMI_PRODUCT_VERSION));
2376 ret = -EIO;
2377 goto error;
2378 }
2379
2380 ret = iommu_domain_identity_map(domain, start, end);
2381 if (ret)
2382 goto error;
2383
2384 /* context entry init */
2385 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
2386 if (ret)
2387 goto error;
2388
2389 return 0;
2390
2391 error:
2392 domain_exit(domain);
2393 return ret;
2394 }
2395
2396 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2397 struct device *dev)
2398 {
2399 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2400 return 0;
2401 return iommu_prepare_identity_map(dev, rmrr->base_address,
2402 rmrr->end_address);
2403 }
2404
2405 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2406 static inline void iommu_prepare_isa(void)
2407 {
2408 struct pci_dev *pdev;
2409 int ret;
2410
2411 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2412 if (!pdev)
2413 return;
2414
2415 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
2416 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2417
2418 if (ret)
2419 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2420 "floppy might not work\n");
2421
2422 pci_dev_put(pdev);
2423 }
2424 #else
2425 static inline void iommu_prepare_isa(void)
2426 {
2427 return;
2428 }
2429 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2430
2431 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2432
2433 static int __init si_domain_init(int hw)
2434 {
2435 struct dmar_drhd_unit *drhd;
2436 struct intel_iommu *iommu;
2437 int nid, ret = 0;
2438 bool first = true;
2439
2440 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2441 if (!si_domain)
2442 return -EFAULT;
2443
2444 for_each_active_iommu(iommu, drhd) {
2445 ret = iommu_attach_domain(si_domain, iommu);
2446 if (ret < 0) {
2447 domain_exit(si_domain);
2448 return -EFAULT;
2449 } else if (first) {
2450 si_domain->id = ret;
2451 first = false;
2452 } else if (si_domain->id != ret) {
2453 domain_exit(si_domain);
2454 return -EFAULT;
2455 }
2456 domain_attach_iommu(si_domain, iommu);
2457 }
2458
2459 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2460 domain_exit(si_domain);
2461 return -EFAULT;
2462 }
2463
2464 pr_debug("IOMMU: identity mapping domain is domain %d\n",
2465 si_domain->id);
2466
2467 if (hw)
2468 return 0;
2469
2470 for_each_online_node(nid) {
2471 unsigned long start_pfn, end_pfn;
2472 int i;
2473
2474 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2475 ret = iommu_domain_identity_map(si_domain,
2476 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2477 if (ret)
2478 return ret;
2479 }
2480 }
2481
2482 return 0;
2483 }
2484
2485 static int identity_mapping(struct device *dev)
2486 {
2487 struct device_domain_info *info;
2488
2489 if (likely(!iommu_identity_mapping))
2490 return 0;
2491
2492 info = dev->archdata.iommu;
2493 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2494 return (info->domain == si_domain);
2495
2496 return 0;
2497 }
2498
2499 static int domain_add_dev_info(struct dmar_domain *domain,
2500 struct device *dev, int translation)
2501 {
2502 struct dmar_domain *ndomain;
2503 struct intel_iommu *iommu;
2504 u8 bus, devfn;
2505 int ret;
2506
2507 iommu = device_to_iommu(dev, &bus, &devfn);
2508 if (!iommu)
2509 return -ENODEV;
2510
2511 ndomain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2512 if (ndomain != domain)
2513 return -EBUSY;
2514
2515 ret = domain_context_mapping(domain, dev, translation);
2516 if (ret) {
2517 domain_remove_one_dev_info(domain, dev);
2518 return ret;
2519 }
2520
2521 return 0;
2522 }
2523
2524 static bool device_has_rmrr(struct device *dev)
2525 {
2526 struct dmar_rmrr_unit *rmrr;
2527 struct device *tmp;
2528 int i;
2529
2530 rcu_read_lock();
2531 for_each_rmrr_units(rmrr) {
2532 /*
2533 * Return TRUE if this RMRR contains the device that
2534 * is passed in.
2535 */
2536 for_each_active_dev_scope(rmrr->devices,
2537 rmrr->devices_cnt, i, tmp)
2538 if (tmp == dev) {
2539 rcu_read_unlock();
2540 return true;
2541 }
2542 }
2543 rcu_read_unlock();
2544 return false;
2545 }
2546
2547 /*
2548 * There are a couple cases where we need to restrict the functionality of
2549 * devices associated with RMRRs. The first is when evaluating a device for
2550 * identity mapping because problems exist when devices are moved in and out
2551 * of domains and their respective RMRR information is lost. This means that
2552 * a device with associated RMRRs will never be in a "passthrough" domain.
2553 * The second is use of the device through the IOMMU API. This interface
2554 * expects to have full control of the IOVA space for the device. We cannot
2555 * satisfy both the requirement that RMRR access is maintained and have an
2556 * unencumbered IOVA space. We also have no ability to quiesce the device's
2557 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2558 * We therefore prevent devices associated with an RMRR from participating in
2559 * the IOMMU API, which eliminates them from device assignment.
2560 *
2561 * In both cases we assume that PCI USB devices with RMRRs have them largely
2562 * for historical reasons and that the RMRR space is not actively used post
2563 * boot. This exclusion may change if vendors begin to abuse it.
2564 */
2565 static bool device_is_rmrr_locked(struct device *dev)
2566 {
2567 if (!device_has_rmrr(dev))
2568 return false;
2569
2570 if (dev_is_pci(dev)) {
2571 struct pci_dev *pdev = to_pci_dev(dev);
2572
2573 if ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
2574 return false;
2575 }
2576
2577 return true;
2578 }
2579
2580 static int iommu_should_identity_map(struct device *dev, int startup)
2581 {
2582
2583 if (dev_is_pci(dev)) {
2584 struct pci_dev *pdev = to_pci_dev(dev);
2585
2586 if (device_is_rmrr_locked(dev))
2587 return 0;
2588
2589 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2590 return 1;
2591
2592 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2593 return 1;
2594
2595 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2596 return 0;
2597
2598 /*
2599 * We want to start off with all devices in the 1:1 domain, and
2600 * take them out later if we find they can't access all of memory.
2601 *
2602 * However, we can't do this for PCI devices behind bridges,
2603 * because all PCI devices behind the same bridge will end up
2604 * with the same source-id on their transactions.
2605 *
2606 * Practically speaking, we can't change things around for these
2607 * devices at run-time, because we can't be sure there'll be no
2608 * DMA transactions in flight for any of their siblings.
2609 *
2610 * So PCI devices (unless they're on the root bus) as well as
2611 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2612 * the 1:1 domain, just in _case_ one of their siblings turns out
2613 * not to be able to map all of memory.
2614 */
2615 if (!pci_is_pcie(pdev)) {
2616 if (!pci_is_root_bus(pdev->bus))
2617 return 0;
2618 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2619 return 0;
2620 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2621 return 0;
2622 } else {
2623 if (device_has_rmrr(dev))
2624 return 0;
2625 }
2626
2627 /*
2628 * At boot time, we don't yet know if devices will be 64-bit capable.
2629 * Assume that they will — if they turn out not to be, then we can
2630 * take them out of the 1:1 domain later.
2631 */
2632 if (!startup) {
2633 /*
2634 * If the device's dma_mask is less than the system's memory
2635 * size then this is not a candidate for identity mapping.
2636 */
2637 u64 dma_mask = *dev->dma_mask;
2638
2639 if (dev->coherent_dma_mask &&
2640 dev->coherent_dma_mask < dma_mask)
2641 dma_mask = dev->coherent_dma_mask;
2642
2643 return dma_mask >= dma_get_required_mask(dev);
2644 }
2645
2646 return 1;
2647 }
2648
2649 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2650 {
2651 int ret;
2652
2653 if (!iommu_should_identity_map(dev, 1))
2654 return 0;
2655
2656 ret = domain_add_dev_info(si_domain, dev,
2657 hw ? CONTEXT_TT_PASS_THROUGH :
2658 CONTEXT_TT_MULTI_LEVEL);
2659 if (!ret)
2660 pr_info("IOMMU: %s identity mapping for device %s\n",
2661 hw ? "hardware" : "software", dev_name(dev));
2662 else if (ret == -ENODEV)
2663 /* device not associated with an iommu */
2664 ret = 0;
2665
2666 return ret;
2667 }
2668
2669
2670 static int __init iommu_prepare_static_identity_mapping(int hw)
2671 {
2672 struct pci_dev *pdev = NULL;
2673 struct dmar_drhd_unit *drhd;
2674 struct intel_iommu *iommu;
2675 struct device *dev;
2676 int i;
2677 int ret = 0;
2678
2679 ret = si_domain_init(hw);
2680 if (ret)
2681 return -EFAULT;
2682
2683 for_each_pci_dev(pdev) {
2684 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2685 if (ret)
2686 return ret;
2687 }
2688
2689 for_each_active_iommu(iommu, drhd)
2690 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2691 struct acpi_device_physical_node *pn;
2692 struct acpi_device *adev;
2693
2694 if (dev->bus != &acpi_bus_type)
2695 continue;
2696
2697 adev= to_acpi_device(dev);
2698 mutex_lock(&adev->physical_node_lock);
2699 list_for_each_entry(pn, &adev->physical_node_list, node) {
2700 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2701 if (ret)
2702 break;
2703 }
2704 mutex_unlock(&adev->physical_node_lock);
2705 if (ret)
2706 return ret;
2707 }
2708
2709 return 0;
2710 }
2711
2712 static void intel_iommu_init_qi(struct intel_iommu *iommu)
2713 {
2714 /*
2715 * Start from the sane iommu hardware state.
2716 * If the queued invalidation is already initialized by us
2717 * (for example, while enabling interrupt-remapping) then
2718 * we got the things already rolling from a sane state.
2719 */
2720 if (!iommu->qi) {
2721 /*
2722 * Clear any previous faults.
2723 */
2724 dmar_fault(-1, iommu);
2725 /*
2726 * Disable queued invalidation if supported and already enabled
2727 * before OS handover.
2728 */
2729 dmar_disable_qi(iommu);
2730 }
2731
2732 if (dmar_enable_qi(iommu)) {
2733 /*
2734 * Queued Invalidate not enabled, use Register Based Invalidate
2735 */
2736 iommu->flush.flush_context = __iommu_flush_context;
2737 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2738 pr_info("IOMMU: %s using Register based invalidation\n",
2739 iommu->name);
2740 } else {
2741 iommu->flush.flush_context = qi_flush_context;
2742 iommu->flush.flush_iotlb = qi_flush_iotlb;
2743 pr_info("IOMMU: %s using Queued invalidation\n", iommu->name);
2744 }
2745 }
2746
2747 static int __init init_dmars(void)
2748 {
2749 struct dmar_drhd_unit *drhd;
2750 struct dmar_rmrr_unit *rmrr;
2751 struct device *dev;
2752 struct intel_iommu *iommu;
2753 int i, ret;
2754
2755 /*
2756 * for each drhd
2757 * allocate root
2758 * initialize and program root entry to not present
2759 * endfor
2760 */
2761 for_each_drhd_unit(drhd) {
2762 /*
2763 * lock not needed as this is only incremented in the single
2764 * threaded kernel __init code path all other access are read
2765 * only
2766 */
2767 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
2768 g_num_of_iommus++;
2769 continue;
2770 }
2771 printk_once(KERN_ERR "intel-iommu: exceeded %d IOMMUs\n",
2772 DMAR_UNITS_SUPPORTED);
2773 }
2774
2775 /* Preallocate enough resources for IOMMU hot-addition */
2776 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
2777 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
2778
2779 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2780 GFP_KERNEL);
2781 if (!g_iommus) {
2782 printk(KERN_ERR "Allocating global iommu array failed\n");
2783 ret = -ENOMEM;
2784 goto error;
2785 }
2786
2787 deferred_flush = kzalloc(g_num_of_iommus *
2788 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2789 if (!deferred_flush) {
2790 ret = -ENOMEM;
2791 goto free_g_iommus;
2792 }
2793
2794 for_each_active_iommu(iommu, drhd) {
2795 g_iommus[iommu->seq_id] = iommu;
2796
2797 ret = iommu_init_domains(iommu);
2798 if (ret)
2799 goto free_iommu;
2800
2801 /*
2802 * TBD:
2803 * we could share the same root & context tables
2804 * among all IOMMU's. Need to Split it later.
2805 */
2806 ret = iommu_alloc_root_entry(iommu);
2807 if (ret)
2808 goto free_iommu;
2809 if (!ecap_pass_through(iommu->ecap))
2810 hw_pass_through = 0;
2811 }
2812
2813 for_each_active_iommu(iommu, drhd)
2814 intel_iommu_init_qi(iommu);
2815
2816 if (iommu_pass_through)
2817 iommu_identity_mapping |= IDENTMAP_ALL;
2818
2819 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
2820 iommu_identity_mapping |= IDENTMAP_GFX;
2821 #endif
2822
2823 check_tylersburg_isoch();
2824
2825 /*
2826 * If pass through is not set or not enabled, setup context entries for
2827 * identity mappings for rmrr, gfx, and isa and may fall back to static
2828 * identity mapping if iommu_identity_mapping is set.
2829 */
2830 if (iommu_identity_mapping) {
2831 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
2832 if (ret) {
2833 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2834 goto free_iommu;
2835 }
2836 }
2837 /*
2838 * For each rmrr
2839 * for each dev attached to rmrr
2840 * do
2841 * locate drhd for dev, alloc domain for dev
2842 * allocate free domain
2843 * allocate page table entries for rmrr
2844 * if context not allocated for bus
2845 * allocate and init context
2846 * set present in root table for this bus
2847 * init context with domain, translation etc
2848 * endfor
2849 * endfor
2850 */
2851 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2852 for_each_rmrr_units(rmrr) {
2853 /* some BIOS lists non-exist devices in DMAR table. */
2854 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
2855 i, dev) {
2856 ret = iommu_prepare_rmrr_dev(rmrr, dev);
2857 if (ret)
2858 printk(KERN_ERR
2859 "IOMMU: mapping reserved region failed\n");
2860 }
2861 }
2862
2863 iommu_prepare_isa();
2864
2865 /*
2866 * for each drhd
2867 * enable fault log
2868 * global invalidate context cache
2869 * global invalidate iotlb
2870 * enable translation
2871 */
2872 for_each_iommu(iommu, drhd) {
2873 if (drhd->ignored) {
2874 /*
2875 * we always have to disable PMRs or DMA may fail on
2876 * this device
2877 */
2878 if (force_on)
2879 iommu_disable_protect_mem_regions(iommu);
2880 continue;
2881 }
2882
2883 iommu_flush_write_buffer(iommu);
2884
2885 ret = dmar_set_interrupt(iommu);
2886 if (ret)
2887 goto free_iommu;
2888
2889 iommu_set_root_entry(iommu);
2890
2891 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
2892 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2893 iommu_enable_translation(iommu);
2894 iommu_disable_protect_mem_regions(iommu);
2895 }
2896
2897 return 0;
2898
2899 free_iommu:
2900 for_each_active_iommu(iommu, drhd) {
2901 disable_dmar_iommu(iommu);
2902 free_dmar_iommu(iommu);
2903 }
2904 kfree(deferred_flush);
2905 free_g_iommus:
2906 kfree(g_iommus);
2907 error:
2908 return ret;
2909 }
2910
2911 /* This takes a number of _MM_ pages, not VTD pages */
2912 static struct iova *intel_alloc_iova(struct device *dev,
2913 struct dmar_domain *domain,
2914 unsigned long nrpages, uint64_t dma_mask)
2915 {
2916 struct iova *iova = NULL;
2917
2918 /* Restrict dma_mask to the width that the iommu can handle */
2919 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2920
2921 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
2922 /*
2923 * First try to allocate an io virtual address in
2924 * DMA_BIT_MASK(32) and if that fails then try allocating
2925 * from higher range
2926 */
2927 iova = alloc_iova(&domain->iovad, nrpages,
2928 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2929 if (iova)
2930 return iova;
2931 }
2932 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2933 if (unlikely(!iova)) {
2934 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2935 nrpages, dev_name(dev));
2936 return NULL;
2937 }
2938
2939 return iova;
2940 }
2941
2942 static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
2943 {
2944 struct dmar_domain *domain;
2945 int ret;
2946
2947 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2948 if (!domain) {
2949 printk(KERN_ERR "Allocating domain for %s failed",
2950 dev_name(dev));
2951 return NULL;
2952 }
2953
2954 /* make sure context mapping is ok */
2955 if (unlikely(!domain_context_mapped(dev))) {
2956 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
2957 if (ret) {
2958 printk(KERN_ERR "Domain context map for %s failed",
2959 dev_name(dev));
2960 return NULL;
2961 }
2962 }
2963
2964 return domain;
2965 }
2966
2967 static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
2968 {
2969 struct device_domain_info *info;
2970
2971 /* No lock here, assumes no domain exit in normal case */
2972 info = dev->archdata.iommu;
2973 if (likely(info))
2974 return info->domain;
2975
2976 return __get_valid_domain_for_dev(dev);
2977 }
2978
2979 static int iommu_dummy(struct device *dev)
2980 {
2981 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2982 }
2983
2984 /* Check if the dev needs to go through non-identity map and unmap process.*/
2985 static int iommu_no_mapping(struct device *dev)
2986 {
2987 int found;
2988
2989 if (iommu_dummy(dev))
2990 return 1;
2991
2992 if (!iommu_identity_mapping)
2993 return 0;
2994
2995 found = identity_mapping(dev);
2996 if (found) {
2997 if (iommu_should_identity_map(dev, 0))
2998 return 1;
2999 else {
3000 /*
3001 * 32 bit DMA is removed from si_domain and fall back
3002 * to non-identity mapping.
3003 */
3004 domain_remove_one_dev_info(si_domain, dev);
3005 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
3006 dev_name(dev));
3007 return 0;
3008 }
3009 } else {
3010 /*
3011 * In case of a detached 64 bit DMA device from vm, the device
3012 * is put into si_domain for identity mapping.
3013 */
3014 if (iommu_should_identity_map(dev, 0)) {
3015 int ret;
3016 ret = domain_add_dev_info(si_domain, dev,
3017 hw_pass_through ?
3018 CONTEXT_TT_PASS_THROUGH :
3019 CONTEXT_TT_MULTI_LEVEL);
3020 if (!ret) {
3021 printk(KERN_INFO "64bit %s uses identity mapping\n",
3022 dev_name(dev));
3023 return 1;
3024 }
3025 }
3026 }
3027
3028 return 0;
3029 }
3030
3031 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3032 size_t size, int dir, u64 dma_mask)
3033 {
3034 struct dmar_domain *domain;
3035 phys_addr_t start_paddr;
3036 struct iova *iova;
3037 int prot = 0;
3038 int ret;
3039 struct intel_iommu *iommu;
3040 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3041
3042 BUG_ON(dir == DMA_NONE);
3043
3044 if (iommu_no_mapping(dev))
3045 return paddr;
3046
3047 domain = get_valid_domain_for_dev(dev);
3048 if (!domain)
3049 return 0;
3050
3051 iommu = domain_get_iommu(domain);
3052 size = aligned_nrpages(paddr, size);
3053
3054 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3055 if (!iova)
3056 goto error;
3057
3058 /*
3059 * Check if DMAR supports zero-length reads on write only
3060 * mappings..
3061 */
3062 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3063 !cap_zlr(iommu->cap))
3064 prot |= DMA_PTE_READ;
3065 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3066 prot |= DMA_PTE_WRITE;
3067 /*
3068 * paddr - (paddr + size) might be partial page, we should map the whole
3069 * page. Note: if two part of one page are separately mapped, we
3070 * might have two guest_addr mapping to the same host paddr, but this
3071 * is not a big problem
3072 */
3073 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
3074 mm_to_dma_pfn(paddr_pfn), size, prot);
3075 if (ret)
3076 goto error;
3077
3078 /* it's a non-present to present mapping. Only flush if caching mode */
3079 if (cap_caching_mode(iommu->cap))
3080 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 0, 1);
3081 else
3082 iommu_flush_write_buffer(iommu);
3083
3084 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
3085 start_paddr += paddr & ~PAGE_MASK;
3086 return start_paddr;
3087
3088 error:
3089 if (iova)
3090 __free_iova(&domain->iovad, iova);
3091 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
3092 dev_name(dev), size, (unsigned long long)paddr, dir);
3093 return 0;
3094 }
3095
3096 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3097 unsigned long offset, size_t size,
3098 enum dma_data_direction dir,
3099 struct dma_attrs *attrs)
3100 {
3101 return __intel_map_single(dev, page_to_phys(page) + offset, size,
3102 dir, *dev->dma_mask);
3103 }
3104
3105 static void flush_unmaps(void)
3106 {
3107 int i, j;
3108
3109 timer_on = 0;
3110
3111 /* just flush them all */
3112 for (i = 0; i < g_num_of_iommus; i++) {
3113 struct intel_iommu *iommu = g_iommus[i];
3114 if (!iommu)
3115 continue;
3116
3117 if (!deferred_flush[i].next)
3118 continue;
3119
3120 /* In caching mode, global flushes turn emulation expensive */
3121 if (!cap_caching_mode(iommu->cap))
3122 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3123 DMA_TLB_GLOBAL_FLUSH);
3124 for (j = 0; j < deferred_flush[i].next; j++) {
3125 unsigned long mask;
3126 struct iova *iova = deferred_flush[i].iova[j];
3127 struct dmar_domain *domain = deferred_flush[i].domain[j];
3128
3129 /* On real hardware multiple invalidations are expensive */
3130 if (cap_caching_mode(iommu->cap))
3131 iommu_flush_iotlb_psi(iommu, domain->id,
3132 iova->pfn_lo, iova_size(iova),
3133 !deferred_flush[i].freelist[j], 0);
3134 else {
3135 mask = ilog2(mm_to_dma_pfn(iova_size(iova)));
3136 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3137 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3138 }
3139 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
3140 if (deferred_flush[i].freelist[j])
3141 dma_free_pagelist(deferred_flush[i].freelist[j]);
3142 }
3143 deferred_flush[i].next = 0;
3144 }
3145
3146 list_size = 0;
3147 }
3148
3149 static void flush_unmaps_timeout(unsigned long data)
3150 {
3151 unsigned long flags;
3152
3153 spin_lock_irqsave(&async_umap_flush_lock, flags);
3154 flush_unmaps();
3155 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3156 }
3157
3158 static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
3159 {
3160 unsigned long flags;
3161 int next, iommu_id;
3162 struct intel_iommu *iommu;
3163
3164 spin_lock_irqsave(&async_umap_flush_lock, flags);
3165 if (list_size == HIGH_WATER_MARK)
3166 flush_unmaps();
3167
3168 iommu = domain_get_iommu(dom);
3169 iommu_id = iommu->seq_id;
3170
3171 next = deferred_flush[iommu_id].next;
3172 deferred_flush[iommu_id].domain[next] = dom;
3173 deferred_flush[iommu_id].iova[next] = iova;
3174 deferred_flush[iommu_id].freelist[next] = freelist;
3175 deferred_flush[iommu_id].next++;
3176
3177 if (!timer_on) {
3178 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3179 timer_on = 1;
3180 }
3181 list_size++;
3182 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3183 }
3184
3185 static void intel_unmap(struct device *dev, dma_addr_t dev_addr)
3186 {
3187 struct dmar_domain *domain;
3188 unsigned long start_pfn, last_pfn;
3189 struct iova *iova;
3190 struct intel_iommu *iommu;
3191 struct page *freelist;
3192
3193 if (iommu_no_mapping(dev))
3194 return;
3195
3196 domain = find_domain(dev);
3197 BUG_ON(!domain);
3198
3199 iommu = domain_get_iommu(domain);
3200
3201 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
3202 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3203 (unsigned long long)dev_addr))
3204 return;
3205
3206 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3207 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
3208
3209 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3210 dev_name(dev), start_pfn, last_pfn);
3211
3212 freelist = domain_unmap(domain, start_pfn, last_pfn);
3213
3214 if (intel_iommu_strict) {
3215 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
3216 last_pfn - start_pfn + 1, !freelist, 0);
3217 /* free iova */
3218 __free_iova(&domain->iovad, iova);
3219 dma_free_pagelist(freelist);
3220 } else {
3221 add_unmap(domain, iova, freelist);
3222 /*
3223 * queue up the release of the unmap to save the 1/6th of the
3224 * cpu used up by the iotlb flush operation...
3225 */
3226 }
3227 }
3228
3229 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3230 size_t size, enum dma_data_direction dir,
3231 struct dma_attrs *attrs)
3232 {
3233 intel_unmap(dev, dev_addr);
3234 }
3235
3236 static void *intel_alloc_coherent(struct device *dev, size_t size,
3237 dma_addr_t *dma_handle, gfp_t flags,
3238 struct dma_attrs *attrs)
3239 {
3240 struct page *page = NULL;
3241 int order;
3242
3243 size = PAGE_ALIGN(size);
3244 order = get_order(size);
3245
3246 if (!iommu_no_mapping(dev))
3247 flags &= ~(GFP_DMA | GFP_DMA32);
3248 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3249 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3250 flags |= GFP_DMA;
3251 else
3252 flags |= GFP_DMA32;
3253 }
3254
3255 if (flags & __GFP_WAIT) {
3256 unsigned int count = size >> PAGE_SHIFT;
3257
3258 page = dma_alloc_from_contiguous(dev, count, order);
3259 if (page && iommu_no_mapping(dev) &&
3260 page_to_phys(page) + size > dev->coherent_dma_mask) {
3261 dma_release_from_contiguous(dev, page, count);
3262 page = NULL;
3263 }
3264 }
3265
3266 if (!page)
3267 page = alloc_pages(flags, order);
3268 if (!page)
3269 return NULL;
3270 memset(page_address(page), 0, size);
3271
3272 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3273 DMA_BIDIRECTIONAL,
3274 dev->coherent_dma_mask);
3275 if (*dma_handle)
3276 return page_address(page);
3277 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3278 __free_pages(page, order);
3279
3280 return NULL;
3281 }
3282
3283 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3284 dma_addr_t dma_handle, struct dma_attrs *attrs)
3285 {
3286 int order;
3287 struct page *page = virt_to_page(vaddr);
3288
3289 size = PAGE_ALIGN(size);
3290 order = get_order(size);
3291
3292 intel_unmap(dev, dma_handle);
3293 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3294 __free_pages(page, order);
3295 }
3296
3297 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3298 int nelems, enum dma_data_direction dir,
3299 struct dma_attrs *attrs)
3300 {
3301 intel_unmap(dev, sglist[0].dma_address);
3302 }
3303
3304 static int intel_nontranslate_map_sg(struct device *hddev,
3305 struct scatterlist *sglist, int nelems, int dir)
3306 {
3307 int i;
3308 struct scatterlist *sg;
3309
3310 for_each_sg(sglist, sg, nelems, i) {
3311 BUG_ON(!sg_page(sg));
3312 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
3313 sg->dma_length = sg->length;
3314 }
3315 return nelems;
3316 }
3317
3318 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3319 enum dma_data_direction dir, struct dma_attrs *attrs)
3320 {
3321 int i;
3322 struct dmar_domain *domain;
3323 size_t size = 0;
3324 int prot = 0;
3325 struct iova *iova = NULL;
3326 int ret;
3327 struct scatterlist *sg;
3328 unsigned long start_vpfn;
3329 struct intel_iommu *iommu;
3330
3331 BUG_ON(dir == DMA_NONE);
3332 if (iommu_no_mapping(dev))
3333 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3334
3335 domain = get_valid_domain_for_dev(dev);
3336 if (!domain)
3337 return 0;
3338
3339 iommu = domain_get_iommu(domain);
3340
3341 for_each_sg(sglist, sg, nelems, i)
3342 size += aligned_nrpages(sg->offset, sg->length);
3343
3344 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3345 *dev->dma_mask);
3346 if (!iova) {
3347 sglist->dma_length = 0;
3348 return 0;
3349 }
3350
3351 /*
3352 * Check if DMAR supports zero-length reads on write only
3353 * mappings..
3354 */
3355 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3356 !cap_zlr(iommu->cap))
3357 prot |= DMA_PTE_READ;
3358 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3359 prot |= DMA_PTE_WRITE;
3360
3361 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
3362
3363 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3364 if (unlikely(ret)) {
3365 dma_pte_free_pagetable(domain, start_vpfn,
3366 start_vpfn + size - 1);
3367 __free_iova(&domain->iovad, iova);
3368 return 0;
3369 }
3370
3371 /* it's a non-present to present mapping. Only flush if caching mode */
3372 if (cap_caching_mode(iommu->cap))
3373 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 0, 1);
3374 else
3375 iommu_flush_write_buffer(iommu);
3376
3377 return nelems;
3378 }
3379
3380 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3381 {
3382 return !dma_addr;
3383 }
3384
3385 struct dma_map_ops intel_dma_ops = {
3386 .alloc = intel_alloc_coherent,
3387 .free = intel_free_coherent,
3388 .map_sg = intel_map_sg,
3389 .unmap_sg = intel_unmap_sg,
3390 .map_page = intel_map_page,
3391 .unmap_page = intel_unmap_page,
3392 .mapping_error = intel_mapping_error,
3393 };
3394
3395 static inline int iommu_domain_cache_init(void)
3396 {
3397 int ret = 0;
3398
3399 iommu_domain_cache = kmem_cache_create("iommu_domain",
3400 sizeof(struct dmar_domain),
3401 0,
3402 SLAB_HWCACHE_ALIGN,
3403
3404 NULL);
3405 if (!iommu_domain_cache) {
3406 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
3407 ret = -ENOMEM;
3408 }
3409
3410 return ret;
3411 }
3412
3413 static inline int iommu_devinfo_cache_init(void)
3414 {
3415 int ret = 0;
3416
3417 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3418 sizeof(struct device_domain_info),
3419 0,
3420 SLAB_HWCACHE_ALIGN,
3421 NULL);
3422 if (!iommu_devinfo_cache) {
3423 printk(KERN_ERR "Couldn't create devinfo cache\n");
3424 ret = -ENOMEM;
3425 }
3426
3427 return ret;
3428 }
3429
3430 static inline int iommu_iova_cache_init(void)
3431 {
3432 int ret = 0;
3433
3434 iommu_iova_cache = kmem_cache_create("iommu_iova",
3435 sizeof(struct iova),
3436 0,
3437 SLAB_HWCACHE_ALIGN,
3438 NULL);
3439 if (!iommu_iova_cache) {
3440 printk(KERN_ERR "Couldn't create iova cache\n");
3441 ret = -ENOMEM;
3442 }
3443
3444 return ret;
3445 }
3446
3447 static int __init iommu_init_mempool(void)
3448 {
3449 int ret;
3450 ret = iommu_iova_cache_init();
3451 if (ret)
3452 return ret;
3453
3454 ret = iommu_domain_cache_init();
3455 if (ret)
3456 goto domain_error;
3457
3458 ret = iommu_devinfo_cache_init();
3459 if (!ret)
3460 return ret;
3461
3462 kmem_cache_destroy(iommu_domain_cache);
3463 domain_error:
3464 kmem_cache_destroy(iommu_iova_cache);
3465
3466 return -ENOMEM;
3467 }
3468
3469 static void __init iommu_exit_mempool(void)
3470 {
3471 kmem_cache_destroy(iommu_devinfo_cache);
3472 kmem_cache_destroy(iommu_domain_cache);
3473 kmem_cache_destroy(iommu_iova_cache);
3474
3475 }
3476
3477 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3478 {
3479 struct dmar_drhd_unit *drhd;
3480 u32 vtbar;
3481 int rc;
3482
3483 /* We know that this device on this chipset has its own IOMMU.
3484 * If we find it under a different IOMMU, then the BIOS is lying
3485 * to us. Hope that the IOMMU for this device is actually
3486 * disabled, and it needs no translation...
3487 */
3488 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3489 if (rc) {
3490 /* "can't" happen */
3491 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3492 return;
3493 }
3494 vtbar &= 0xffff0000;
3495
3496 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3497 drhd = dmar_find_matched_drhd_unit(pdev);
3498 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3499 TAINT_FIRMWARE_WORKAROUND,
3500 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3501 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3502 }
3503 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3504
3505 static void __init init_no_remapping_devices(void)
3506 {
3507 struct dmar_drhd_unit *drhd;
3508 struct device *dev;
3509 int i;
3510
3511 for_each_drhd_unit(drhd) {
3512 if (!drhd->include_all) {
3513 for_each_active_dev_scope(drhd->devices,
3514 drhd->devices_cnt, i, dev)
3515 break;
3516 /* ignore DMAR unit if no devices exist */
3517 if (i == drhd->devices_cnt)
3518 drhd->ignored = 1;
3519 }
3520 }
3521
3522 for_each_active_drhd_unit(drhd) {
3523 if (drhd->include_all)
3524 continue;
3525
3526 for_each_active_dev_scope(drhd->devices,
3527 drhd->devices_cnt, i, dev)
3528 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
3529 break;
3530 if (i < drhd->devices_cnt)
3531 continue;
3532
3533 /* This IOMMU has *only* gfx devices. Either bypass it or
3534 set the gfx_mapped flag, as appropriate */
3535 if (dmar_map_gfx) {
3536 intel_iommu_gfx_mapped = 1;
3537 } else {
3538 drhd->ignored = 1;
3539 for_each_active_dev_scope(drhd->devices,
3540 drhd->devices_cnt, i, dev)
3541 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3542 }
3543 }
3544 }
3545
3546 #ifdef CONFIG_SUSPEND
3547 static int init_iommu_hw(void)
3548 {
3549 struct dmar_drhd_unit *drhd;
3550 struct intel_iommu *iommu = NULL;
3551
3552 for_each_active_iommu(iommu, drhd)
3553 if (iommu->qi)
3554 dmar_reenable_qi(iommu);
3555
3556 for_each_iommu(iommu, drhd) {
3557 if (drhd->ignored) {
3558 /*
3559 * we always have to disable PMRs or DMA may fail on
3560 * this device
3561 */
3562 if (force_on)
3563 iommu_disable_protect_mem_regions(iommu);
3564 continue;
3565 }
3566
3567 iommu_flush_write_buffer(iommu);
3568
3569 iommu_set_root_entry(iommu);
3570
3571 iommu->flush.flush_context(iommu, 0, 0, 0,
3572 DMA_CCMD_GLOBAL_INVL);
3573 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3574 iommu_enable_translation(iommu);
3575 iommu_disable_protect_mem_regions(iommu);
3576 }
3577
3578 return 0;
3579 }
3580
3581 static void iommu_flush_all(void)
3582 {
3583 struct dmar_drhd_unit *drhd;
3584 struct intel_iommu *iommu;
3585
3586 for_each_active_iommu(iommu, drhd) {
3587 iommu->flush.flush_context(iommu, 0, 0, 0,
3588 DMA_CCMD_GLOBAL_INVL);
3589 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3590 DMA_TLB_GLOBAL_FLUSH);
3591 }
3592 }
3593
3594 static int iommu_suspend(void)
3595 {
3596 struct dmar_drhd_unit *drhd;
3597 struct intel_iommu *iommu = NULL;
3598 unsigned long flag;
3599
3600 for_each_active_iommu(iommu, drhd) {
3601 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3602 GFP_ATOMIC);
3603 if (!iommu->iommu_state)
3604 goto nomem;
3605 }
3606
3607 iommu_flush_all();
3608
3609 for_each_active_iommu(iommu, drhd) {
3610 iommu_disable_translation(iommu);
3611
3612 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3613
3614 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3615 readl(iommu->reg + DMAR_FECTL_REG);
3616 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3617 readl(iommu->reg + DMAR_FEDATA_REG);
3618 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3619 readl(iommu->reg + DMAR_FEADDR_REG);
3620 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3621 readl(iommu->reg + DMAR_FEUADDR_REG);
3622
3623 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3624 }
3625 return 0;
3626
3627 nomem:
3628 for_each_active_iommu(iommu, drhd)
3629 kfree(iommu->iommu_state);
3630
3631 return -ENOMEM;
3632 }
3633
3634 static void iommu_resume(void)
3635 {
3636 struct dmar_drhd_unit *drhd;
3637 struct intel_iommu *iommu = NULL;
3638 unsigned long flag;
3639
3640 if (init_iommu_hw()) {
3641 if (force_on)
3642 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3643 else
3644 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3645 return;
3646 }
3647
3648 for_each_active_iommu(iommu, drhd) {
3649
3650 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3651
3652 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3653 iommu->reg + DMAR_FECTL_REG);
3654 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3655 iommu->reg + DMAR_FEDATA_REG);
3656 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3657 iommu->reg + DMAR_FEADDR_REG);
3658 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3659 iommu->reg + DMAR_FEUADDR_REG);
3660
3661 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3662 }
3663
3664 for_each_active_iommu(iommu, drhd)
3665 kfree(iommu->iommu_state);
3666 }
3667
3668 static struct syscore_ops iommu_syscore_ops = {
3669 .resume = iommu_resume,
3670 .suspend = iommu_suspend,
3671 };
3672
3673 static void __init init_iommu_pm_ops(void)
3674 {
3675 register_syscore_ops(&iommu_syscore_ops);
3676 }
3677
3678 #else
3679 static inline void init_iommu_pm_ops(void) {}
3680 #endif /* CONFIG_PM */
3681
3682
3683 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
3684 {
3685 struct acpi_dmar_reserved_memory *rmrr;
3686 struct dmar_rmrr_unit *rmrru;
3687
3688 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3689 if (!rmrru)
3690 return -ENOMEM;
3691
3692 rmrru->hdr = header;
3693 rmrr = (struct acpi_dmar_reserved_memory *)header;
3694 rmrru->base_address = rmrr->base_address;
3695 rmrru->end_address = rmrr->end_address;
3696 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
3697 ((void *)rmrr) + rmrr->header.length,
3698 &rmrru->devices_cnt);
3699 if (rmrru->devices_cnt && rmrru->devices == NULL) {
3700 kfree(rmrru);
3701 return -ENOMEM;
3702 }
3703
3704 list_add(&rmrru->list, &dmar_rmrr_units);
3705
3706 return 0;
3707 }
3708
3709 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
3710 {
3711 struct dmar_atsr_unit *atsru;
3712 struct acpi_dmar_atsr *tmp;
3713
3714 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
3715 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
3716 if (atsr->segment != tmp->segment)
3717 continue;
3718 if (atsr->header.length != tmp->header.length)
3719 continue;
3720 if (memcmp(atsr, tmp, atsr->header.length) == 0)
3721 return atsru;
3722 }
3723
3724 return NULL;
3725 }
3726
3727 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
3728 {
3729 struct acpi_dmar_atsr *atsr;
3730 struct dmar_atsr_unit *atsru;
3731
3732 if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
3733 return 0;
3734
3735 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3736 atsru = dmar_find_atsr(atsr);
3737 if (atsru)
3738 return 0;
3739
3740 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
3741 if (!atsru)
3742 return -ENOMEM;
3743
3744 /*
3745 * If memory is allocated from slab by ACPI _DSM method, we need to
3746 * copy the memory content because the memory buffer will be freed
3747 * on return.
3748 */
3749 atsru->hdr = (void *)(atsru + 1);
3750 memcpy(atsru->hdr, hdr, hdr->length);
3751 atsru->include_all = atsr->flags & 0x1;
3752 if (!atsru->include_all) {
3753 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
3754 (void *)atsr + atsr->header.length,
3755 &atsru->devices_cnt);
3756 if (atsru->devices_cnt && atsru->devices == NULL) {
3757 kfree(atsru);
3758 return -ENOMEM;
3759 }
3760 }
3761
3762 list_add_rcu(&atsru->list, &dmar_atsr_units);
3763
3764 return 0;
3765 }
3766
3767 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
3768 {
3769 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
3770 kfree(atsru);
3771 }
3772
3773 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
3774 {
3775 struct acpi_dmar_atsr *atsr;
3776 struct dmar_atsr_unit *atsru;
3777
3778 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3779 atsru = dmar_find_atsr(atsr);
3780 if (atsru) {
3781 list_del_rcu(&atsru->list);
3782 synchronize_rcu();
3783 intel_iommu_free_atsr(atsru);
3784 }
3785
3786 return 0;
3787 }
3788
3789 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
3790 {
3791 int i;
3792 struct device *dev;
3793 struct acpi_dmar_atsr *atsr;
3794 struct dmar_atsr_unit *atsru;
3795
3796 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3797 atsru = dmar_find_atsr(atsr);
3798 if (!atsru)
3799 return 0;
3800
3801 if (!atsru->include_all && atsru->devices && atsru->devices_cnt)
3802 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
3803 i, dev)
3804 return -EBUSY;
3805
3806 return 0;
3807 }
3808
3809 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
3810 {
3811 int sp, ret = 0;
3812 struct intel_iommu *iommu = dmaru->iommu;
3813
3814 if (g_iommus[iommu->seq_id])
3815 return 0;
3816
3817 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
3818 pr_warn("IOMMU: %s doesn't support hardware pass through.\n",
3819 iommu->name);
3820 return -ENXIO;
3821 }
3822 if (!ecap_sc_support(iommu->ecap) &&
3823 domain_update_iommu_snooping(iommu)) {
3824 pr_warn("IOMMU: %s doesn't support snooping.\n",
3825 iommu->name);
3826 return -ENXIO;
3827 }
3828 sp = domain_update_iommu_superpage(iommu) - 1;
3829 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
3830 pr_warn("IOMMU: %s doesn't support large page.\n",
3831 iommu->name);
3832 return -ENXIO;
3833 }
3834
3835 /*
3836 * Disable translation if already enabled prior to OS handover.
3837 */
3838 if (iommu->gcmd & DMA_GCMD_TE)
3839 iommu_disable_translation(iommu);
3840
3841 g_iommus[iommu->seq_id] = iommu;
3842 ret = iommu_init_domains(iommu);
3843 if (ret == 0)
3844 ret = iommu_alloc_root_entry(iommu);
3845 if (ret)
3846 goto out;
3847
3848 if (dmaru->ignored) {
3849 /*
3850 * we always have to disable PMRs or DMA may fail on this device
3851 */
3852 if (force_on)
3853 iommu_disable_protect_mem_regions(iommu);
3854 return 0;
3855 }
3856
3857 intel_iommu_init_qi(iommu);
3858 iommu_flush_write_buffer(iommu);
3859 ret = dmar_set_interrupt(iommu);
3860 if (ret)
3861 goto disable_iommu;
3862
3863 iommu_set_root_entry(iommu);
3864 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3865 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3866 iommu_enable_translation(iommu);
3867
3868 if (si_domain) {
3869 ret = iommu_attach_domain(si_domain, iommu);
3870 if (ret < 0 || si_domain->id != ret)
3871 goto disable_iommu;
3872 domain_attach_iommu(si_domain, iommu);
3873 }
3874
3875 iommu_disable_protect_mem_regions(iommu);
3876 return 0;
3877
3878 disable_iommu:
3879 disable_dmar_iommu(iommu);
3880 out:
3881 free_dmar_iommu(iommu);
3882 return ret;
3883 }
3884
3885 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
3886 {
3887 int ret = 0;
3888 struct intel_iommu *iommu = dmaru->iommu;
3889
3890 if (!intel_iommu_enabled)
3891 return 0;
3892 if (iommu == NULL)
3893 return -EINVAL;
3894
3895 if (insert) {
3896 ret = intel_iommu_add(dmaru);
3897 } else {
3898 disable_dmar_iommu(iommu);
3899 free_dmar_iommu(iommu);
3900 }
3901
3902 return ret;
3903 }
3904
3905 static void intel_iommu_free_dmars(void)
3906 {
3907 struct dmar_rmrr_unit *rmrru, *rmrr_n;
3908 struct dmar_atsr_unit *atsru, *atsr_n;
3909
3910 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
3911 list_del(&rmrru->list);
3912 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
3913 kfree(rmrru);
3914 }
3915
3916 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
3917 list_del(&atsru->list);
3918 intel_iommu_free_atsr(atsru);
3919 }
3920 }
3921
3922 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
3923 {
3924 int i, ret = 1;
3925 struct pci_bus *bus;
3926 struct pci_dev *bridge = NULL;
3927 struct device *tmp;
3928 struct acpi_dmar_atsr *atsr;
3929 struct dmar_atsr_unit *atsru;
3930
3931 dev = pci_physfn(dev);
3932 for (bus = dev->bus; bus; bus = bus->parent) {
3933 bridge = bus->self;
3934 if (!bridge || !pci_is_pcie(bridge) ||
3935 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
3936 return 0;
3937 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
3938 break;
3939 }
3940 if (!bridge)
3941 return 0;
3942
3943 rcu_read_lock();
3944 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
3945 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3946 if (atsr->segment != pci_domain_nr(dev->bus))
3947 continue;
3948
3949 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
3950 if (tmp == &bridge->dev)
3951 goto out;
3952
3953 if (atsru->include_all)
3954 goto out;
3955 }
3956 ret = 0;
3957 out:
3958 rcu_read_unlock();
3959
3960 return ret;
3961 }
3962
3963 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
3964 {
3965 int ret = 0;
3966 struct dmar_rmrr_unit *rmrru;
3967 struct dmar_atsr_unit *atsru;
3968 struct acpi_dmar_atsr *atsr;
3969 struct acpi_dmar_reserved_memory *rmrr;
3970
3971 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
3972 return 0;
3973
3974 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
3975 rmrr = container_of(rmrru->hdr,
3976 struct acpi_dmar_reserved_memory, header);
3977 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3978 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
3979 ((void *)rmrr) + rmrr->header.length,
3980 rmrr->segment, rmrru->devices,
3981 rmrru->devices_cnt);
3982 if(ret < 0)
3983 return ret;
3984 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
3985 dmar_remove_dev_scope(info, rmrr->segment,
3986 rmrru->devices, rmrru->devices_cnt);
3987 }
3988 }
3989
3990 list_for_each_entry(atsru, &dmar_atsr_units, list) {
3991 if (atsru->include_all)
3992 continue;
3993
3994 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3995 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3996 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
3997 (void *)atsr + atsr->header.length,
3998 atsr->segment, atsru->devices,
3999 atsru->devices_cnt);
4000 if (ret > 0)
4001 break;
4002 else if(ret < 0)
4003 return ret;
4004 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
4005 if (dmar_remove_dev_scope(info, atsr->segment,
4006 atsru->devices, atsru->devices_cnt))
4007 break;
4008 }
4009 }
4010
4011 return 0;
4012 }
4013
4014 /*
4015 * Here we only respond to action of unbound device from driver.
4016 *
4017 * Added device is not attached to its DMAR domain here yet. That will happen
4018 * when mapping the device to iova.
4019 */
4020 static int device_notifier(struct notifier_block *nb,
4021 unsigned long action, void *data)
4022 {
4023 struct device *dev = data;
4024 struct dmar_domain *domain;
4025
4026 if (iommu_dummy(dev))
4027 return 0;
4028
4029 if (action != BUS_NOTIFY_REMOVED_DEVICE)
4030 return 0;
4031
4032 /*
4033 * If the device is still attached to a device driver we can't
4034 * tear down the domain yet as DMA mappings may still be in use.
4035 * Wait for the BUS_NOTIFY_UNBOUND_DRIVER event to do that.
4036 */
4037 if (action == BUS_NOTIFY_DEL_DEVICE && dev->driver != NULL)
4038 return 0;
4039
4040 domain = find_domain(dev);
4041 if (!domain)
4042 return 0;
4043
4044 down_read(&dmar_global_lock);
4045 domain_remove_one_dev_info(domain, dev);
4046 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4047 domain_exit(domain);
4048 up_read(&dmar_global_lock);
4049
4050 return 0;
4051 }
4052
4053 static struct notifier_block device_nb = {
4054 .notifier_call = device_notifier,
4055 };
4056
4057 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4058 unsigned long val, void *v)
4059 {
4060 struct memory_notify *mhp = v;
4061 unsigned long long start, end;
4062 unsigned long start_vpfn, last_vpfn;
4063
4064 switch (val) {
4065 case MEM_GOING_ONLINE:
4066 start = mhp->start_pfn << PAGE_SHIFT;
4067 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4068 if (iommu_domain_identity_map(si_domain, start, end)) {
4069 pr_warn("dmar: failed to build identity map for [%llx-%llx]\n",
4070 start, end);
4071 return NOTIFY_BAD;
4072 }
4073 break;
4074
4075 case MEM_OFFLINE:
4076 case MEM_CANCEL_ONLINE:
4077 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4078 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4079 while (start_vpfn <= last_vpfn) {
4080 struct iova *iova;
4081 struct dmar_drhd_unit *drhd;
4082 struct intel_iommu *iommu;
4083 struct page *freelist;
4084
4085 iova = find_iova(&si_domain->iovad, start_vpfn);
4086 if (iova == NULL) {
4087 pr_debug("dmar: failed get IOVA for PFN %lx\n",
4088 start_vpfn);
4089 break;
4090 }
4091
4092 iova = split_and_remove_iova(&si_domain->iovad, iova,
4093 start_vpfn, last_vpfn);
4094 if (iova == NULL) {
4095 pr_warn("dmar: failed to split IOVA PFN [%lx-%lx]\n",
4096 start_vpfn, last_vpfn);
4097 return NOTIFY_BAD;
4098 }
4099
4100 freelist = domain_unmap(si_domain, iova->pfn_lo,
4101 iova->pfn_hi);
4102
4103 rcu_read_lock();
4104 for_each_active_iommu(iommu, drhd)
4105 iommu_flush_iotlb_psi(iommu, si_domain->id,
4106 iova->pfn_lo, iova_size(iova),
4107 !freelist, 0);
4108 rcu_read_unlock();
4109 dma_free_pagelist(freelist);
4110
4111 start_vpfn = iova->pfn_hi + 1;
4112 free_iova_mem(iova);
4113 }
4114 break;
4115 }
4116
4117 return NOTIFY_OK;
4118 }
4119
4120 static struct notifier_block intel_iommu_memory_nb = {
4121 .notifier_call = intel_iommu_memory_notifier,
4122 .priority = 0
4123 };
4124
4125
4126 static ssize_t intel_iommu_show_version(struct device *dev,
4127 struct device_attribute *attr,
4128 char *buf)
4129 {
4130 struct intel_iommu *iommu = dev_get_drvdata(dev);
4131 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4132 return sprintf(buf, "%d:%d\n",
4133 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4134 }
4135 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4136
4137 static ssize_t intel_iommu_show_address(struct device *dev,
4138 struct device_attribute *attr,
4139 char *buf)
4140 {
4141 struct intel_iommu *iommu = dev_get_drvdata(dev);
4142 return sprintf(buf, "%llx\n", iommu->reg_phys);
4143 }
4144 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4145
4146 static ssize_t intel_iommu_show_cap(struct device *dev,
4147 struct device_attribute *attr,
4148 char *buf)
4149 {
4150 struct intel_iommu *iommu = dev_get_drvdata(dev);
4151 return sprintf(buf, "%llx\n", iommu->cap);
4152 }
4153 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4154
4155 static ssize_t intel_iommu_show_ecap(struct device *dev,
4156 struct device_attribute *attr,
4157 char *buf)
4158 {
4159 struct intel_iommu *iommu = dev_get_drvdata(dev);
4160 return sprintf(buf, "%llx\n", iommu->ecap);
4161 }
4162 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4163
4164 static struct attribute *intel_iommu_attrs[] = {
4165 &dev_attr_version.attr,
4166 &dev_attr_address.attr,
4167 &dev_attr_cap.attr,
4168 &dev_attr_ecap.attr,
4169 NULL,
4170 };
4171
4172 static struct attribute_group intel_iommu_group = {
4173 .name = "intel-iommu",
4174 .attrs = intel_iommu_attrs,
4175 };
4176
4177 const struct attribute_group *intel_iommu_groups[] = {
4178 &intel_iommu_group,
4179 NULL,
4180 };
4181
4182 int __init intel_iommu_init(void)
4183 {
4184 int ret = -ENODEV;
4185 struct dmar_drhd_unit *drhd;
4186 struct intel_iommu *iommu;
4187
4188 /* VT-d is required for a TXT/tboot launch, so enforce that */
4189 force_on = tboot_force_iommu();
4190
4191 if (iommu_init_mempool()) {
4192 if (force_on)
4193 panic("tboot: Failed to initialize iommu memory\n");
4194 return -ENOMEM;
4195 }
4196
4197 down_write(&dmar_global_lock);
4198 if (dmar_table_init()) {
4199 if (force_on)
4200 panic("tboot: Failed to initialize DMAR table\n");
4201 goto out_free_dmar;
4202 }
4203
4204 /*
4205 * Disable translation if already enabled prior to OS handover.
4206 */
4207 for_each_active_iommu(iommu, drhd)
4208 if (iommu->gcmd & DMA_GCMD_TE)
4209 iommu_disable_translation(iommu);
4210
4211 if (dmar_dev_scope_init() < 0) {
4212 if (force_on)
4213 panic("tboot: Failed to initialize DMAR device scope\n");
4214 goto out_free_dmar;
4215 }
4216
4217 if (no_iommu || dmar_disabled)
4218 goto out_free_dmar;
4219
4220 if (list_empty(&dmar_rmrr_units))
4221 printk(KERN_INFO "DMAR: No RMRR found\n");
4222
4223 if (list_empty(&dmar_atsr_units))
4224 printk(KERN_INFO "DMAR: No ATSR found\n");
4225
4226 if (dmar_init_reserved_ranges()) {
4227 if (force_on)
4228 panic("tboot: Failed to reserve iommu ranges\n");
4229 goto out_free_reserved_range;
4230 }
4231
4232 init_no_remapping_devices();
4233
4234 ret = init_dmars();
4235 if (ret) {
4236 if (force_on)
4237 panic("tboot: Failed to initialize DMARs\n");
4238 printk(KERN_ERR "IOMMU: dmar init failed\n");
4239 goto out_free_reserved_range;
4240 }
4241 up_write(&dmar_global_lock);
4242 printk(KERN_INFO
4243 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
4244
4245 init_timer(&unmap_timer);
4246 #ifdef CONFIG_SWIOTLB
4247 swiotlb = 0;
4248 #endif
4249 dma_ops = &intel_dma_ops;
4250
4251 init_iommu_pm_ops();
4252
4253 for_each_active_iommu(iommu, drhd)
4254 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4255 intel_iommu_groups,
4256 iommu->name);
4257
4258 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4259 bus_register_notifier(&pci_bus_type, &device_nb);
4260 if (si_domain && !hw_pass_through)
4261 register_memory_notifier(&intel_iommu_memory_nb);
4262
4263 intel_iommu_enabled = 1;
4264
4265 return 0;
4266
4267 out_free_reserved_range:
4268 put_iova_domain(&reserved_iova_list);
4269 out_free_dmar:
4270 intel_iommu_free_dmars();
4271 up_write(&dmar_global_lock);
4272 iommu_exit_mempool();
4273 return ret;
4274 }
4275
4276 static int iommu_detach_dev_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4277 {
4278 struct intel_iommu *iommu = opaque;
4279
4280 iommu_detach_dev(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4281 return 0;
4282 }
4283
4284 /*
4285 * NB - intel-iommu lacks any sort of reference counting for the users of
4286 * dependent devices. If multiple endpoints have intersecting dependent
4287 * devices, unbinding the driver from any one of them will possibly leave
4288 * the others unable to operate.
4289 */
4290 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
4291 struct device *dev)
4292 {
4293 if (!iommu || !dev || !dev_is_pci(dev))
4294 return;
4295
4296 pci_for_each_dma_alias(to_pci_dev(dev), &iommu_detach_dev_cb, iommu);
4297 }
4298
4299 static void domain_remove_one_dev_info(struct dmar_domain *domain,
4300 struct device *dev)
4301 {
4302 struct device_domain_info *info, *tmp;
4303 struct intel_iommu *iommu;
4304 unsigned long flags;
4305 int found = 0;
4306 u8 bus, devfn;
4307
4308 iommu = device_to_iommu(dev, &bus, &devfn);
4309 if (!iommu)
4310 return;
4311
4312 spin_lock_irqsave(&device_domain_lock, flags);
4313 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
4314 if (info->iommu == iommu && info->bus == bus &&
4315 info->devfn == devfn) {
4316 unlink_domain_info(info);
4317 spin_unlock_irqrestore(&device_domain_lock, flags);
4318
4319 iommu_disable_dev_iotlb(info);
4320 iommu_detach_dev(iommu, info->bus, info->devfn);
4321 iommu_detach_dependent_devices(iommu, dev);
4322 free_devinfo_mem(info);
4323
4324 spin_lock_irqsave(&device_domain_lock, flags);
4325
4326 if (found)
4327 break;
4328 else
4329 continue;
4330 }
4331
4332 /* if there is no other devices under the same iommu
4333 * owned by this domain, clear this iommu in iommu_bmp
4334 * update iommu count and coherency
4335 */
4336 if (info->iommu == iommu)
4337 found = 1;
4338 }
4339
4340 spin_unlock_irqrestore(&device_domain_lock, flags);
4341
4342 if (found == 0) {
4343 domain_detach_iommu(domain, iommu);
4344 if (!domain_type_is_vm_or_si(domain))
4345 iommu_detach_domain(domain, iommu);
4346 }
4347 }
4348
4349 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4350 {
4351 int adjust_width;
4352
4353 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
4354 domain_reserve_special_ranges(domain);
4355
4356 /* calculate AGAW */
4357 domain->gaw = guest_width;
4358 adjust_width = guestwidth_to_adjustwidth(guest_width);
4359 domain->agaw = width_to_agaw(adjust_width);
4360
4361 domain->iommu_coherency = 0;
4362 domain->iommu_snooping = 0;
4363 domain->iommu_superpage = 0;
4364 domain->max_addr = 0;
4365
4366 /* always allocate the top pgd */
4367 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
4368 if (!domain->pgd)
4369 return -ENOMEM;
4370 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4371 return 0;
4372 }
4373
4374 static int intel_iommu_domain_init(struct iommu_domain *domain)
4375 {
4376 struct dmar_domain *dmar_domain;
4377
4378 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
4379 if (!dmar_domain) {
4380 printk(KERN_ERR
4381 "intel_iommu_domain_init: dmar_domain == NULL\n");
4382 return -ENOMEM;
4383 }
4384 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4385 printk(KERN_ERR
4386 "intel_iommu_domain_init() failed\n");
4387 domain_exit(dmar_domain);
4388 return -ENOMEM;
4389 }
4390 domain_update_iommu_cap(dmar_domain);
4391 domain->priv = dmar_domain;
4392
4393 domain->geometry.aperture_start = 0;
4394 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4395 domain->geometry.force_aperture = true;
4396
4397 return 0;
4398 }
4399
4400 static void intel_iommu_domain_destroy(struct iommu_domain *domain)
4401 {
4402 struct dmar_domain *dmar_domain = domain->priv;
4403
4404 domain->priv = NULL;
4405 domain_exit(dmar_domain);
4406 }
4407
4408 static int intel_iommu_attach_device(struct iommu_domain *domain,
4409 struct device *dev)
4410 {
4411 struct dmar_domain *dmar_domain = domain->priv;
4412 struct intel_iommu *iommu;
4413 int addr_width;
4414 u8 bus, devfn;
4415
4416 if (device_is_rmrr_locked(dev)) {
4417 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
4418 return -EPERM;
4419 }
4420
4421 /* normally dev is not mapped */
4422 if (unlikely(domain_context_mapped(dev))) {
4423 struct dmar_domain *old_domain;
4424
4425 old_domain = find_domain(dev);
4426 if (old_domain) {
4427 if (domain_type_is_vm_or_si(dmar_domain))
4428 domain_remove_one_dev_info(old_domain, dev);
4429 else
4430 domain_remove_dev_info(old_domain);
4431 }
4432 }
4433
4434 iommu = device_to_iommu(dev, &bus, &devfn);
4435 if (!iommu)
4436 return -ENODEV;
4437
4438 /* check if this iommu agaw is sufficient for max mapped address */
4439 addr_width = agaw_to_width(iommu->agaw);
4440 if (addr_width > cap_mgaw(iommu->cap))
4441 addr_width = cap_mgaw(iommu->cap);
4442
4443 if (dmar_domain->max_addr > (1LL << addr_width)) {
4444 printk(KERN_ERR "%s: iommu width (%d) is not "
4445 "sufficient for the mapped address (%llx)\n",
4446 __func__, addr_width, dmar_domain->max_addr);
4447 return -EFAULT;
4448 }
4449 dmar_domain->gaw = addr_width;
4450
4451 /*
4452 * Knock out extra levels of page tables if necessary
4453 */
4454 while (iommu->agaw < dmar_domain->agaw) {
4455 struct dma_pte *pte;
4456
4457 pte = dmar_domain->pgd;
4458 if (dma_pte_present(pte)) {
4459 dmar_domain->pgd = (struct dma_pte *)
4460 phys_to_virt(dma_pte_addr(pte));
4461 free_pgtable_page(pte);
4462 }
4463 dmar_domain->agaw--;
4464 }
4465
4466 return domain_add_dev_info(dmar_domain, dev, CONTEXT_TT_MULTI_LEVEL);
4467 }
4468
4469 static void intel_iommu_detach_device(struct iommu_domain *domain,
4470 struct device *dev)
4471 {
4472 struct dmar_domain *dmar_domain = domain->priv;
4473
4474 domain_remove_one_dev_info(dmar_domain, dev);
4475 }
4476
4477 static int intel_iommu_map(struct iommu_domain *domain,
4478 unsigned long iova, phys_addr_t hpa,
4479 size_t size, int iommu_prot)
4480 {
4481 struct dmar_domain *dmar_domain = domain->priv;
4482 u64 max_addr;
4483 int prot = 0;
4484 int ret;
4485
4486 if (iommu_prot & IOMMU_READ)
4487 prot |= DMA_PTE_READ;
4488 if (iommu_prot & IOMMU_WRITE)
4489 prot |= DMA_PTE_WRITE;
4490 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4491 prot |= DMA_PTE_SNP;
4492
4493 max_addr = iova + size;
4494 if (dmar_domain->max_addr < max_addr) {
4495 u64 end;
4496
4497 /* check if minimum agaw is sufficient for mapped address */
4498 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
4499 if (end < max_addr) {
4500 printk(KERN_ERR "%s: iommu width (%d) is not "
4501 "sufficient for the mapped address (%llx)\n",
4502 __func__, dmar_domain->gaw, max_addr);
4503 return -EFAULT;
4504 }
4505 dmar_domain->max_addr = max_addr;
4506 }
4507 /* Round up size to next multiple of PAGE_SIZE, if it and
4508 the low bits of hpa would take us onto the next page */
4509 size = aligned_nrpages(hpa, size);
4510 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4511 hpa >> VTD_PAGE_SHIFT, size, prot);
4512 return ret;
4513 }
4514
4515 static size_t intel_iommu_unmap(struct iommu_domain *domain,
4516 unsigned long iova, size_t size)
4517 {
4518 struct dmar_domain *dmar_domain = domain->priv;
4519 struct page *freelist = NULL;
4520 struct intel_iommu *iommu;
4521 unsigned long start_pfn, last_pfn;
4522 unsigned int npages;
4523 int iommu_id, num, ndomains, level = 0;
4524
4525 /* Cope with horrid API which requires us to unmap more than the
4526 size argument if it happens to be a large-page mapping. */
4527 if (!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level))
4528 BUG();
4529
4530 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4531 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4532
4533 start_pfn = iova >> VTD_PAGE_SHIFT;
4534 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4535
4536 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4537
4538 npages = last_pfn - start_pfn + 1;
4539
4540 for_each_set_bit(iommu_id, dmar_domain->iommu_bmp, g_num_of_iommus) {
4541 iommu = g_iommus[iommu_id];
4542
4543 /*
4544 * find bit position of dmar_domain
4545 */
4546 ndomains = cap_ndoms(iommu->cap);
4547 for_each_set_bit(num, iommu->domain_ids, ndomains) {
4548 if (iommu->domains[num] == dmar_domain)
4549 iommu_flush_iotlb_psi(iommu, num, start_pfn,
4550 npages, !freelist, 0);
4551 }
4552
4553 }
4554
4555 dma_free_pagelist(freelist);
4556
4557 if (dmar_domain->max_addr == iova + size)
4558 dmar_domain->max_addr = iova;
4559
4560 return size;
4561 }
4562
4563 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
4564 dma_addr_t iova)
4565 {
4566 struct dmar_domain *dmar_domain = domain->priv;
4567 struct dma_pte *pte;
4568 int level = 0;
4569 u64 phys = 0;
4570
4571 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
4572 if (pte)
4573 phys = dma_pte_addr(pte);
4574
4575 return phys;
4576 }
4577
4578 static bool intel_iommu_capable(enum iommu_cap cap)
4579 {
4580 if (cap == IOMMU_CAP_CACHE_COHERENCY)
4581 return domain_update_iommu_snooping(NULL) == 1;
4582 if (cap == IOMMU_CAP_INTR_REMAP)
4583 return irq_remapping_enabled == 1;
4584
4585 return false;
4586 }
4587
4588 static int intel_iommu_add_device(struct device *dev)
4589 {
4590 struct intel_iommu *iommu;
4591 struct iommu_group *group;
4592 u8 bus, devfn;
4593
4594 iommu = device_to_iommu(dev, &bus, &devfn);
4595 if (!iommu)
4596 return -ENODEV;
4597
4598 iommu_device_link(iommu->iommu_dev, dev);
4599
4600 group = iommu_group_get_for_dev(dev);
4601
4602 if (IS_ERR(group))
4603 return PTR_ERR(group);
4604
4605 iommu_group_put(group);
4606 return 0;
4607 }
4608
4609 static void intel_iommu_remove_device(struct device *dev)
4610 {
4611 struct intel_iommu *iommu;
4612 u8 bus, devfn;
4613
4614 iommu = device_to_iommu(dev, &bus, &devfn);
4615 if (!iommu)
4616 return;
4617
4618 iommu_group_remove_device(dev);
4619
4620 iommu_device_unlink(iommu->iommu_dev, dev);
4621 }
4622
4623 static const struct iommu_ops intel_iommu_ops = {
4624 .capable = intel_iommu_capable,
4625 .domain_init = intel_iommu_domain_init,
4626 .domain_destroy = intel_iommu_domain_destroy,
4627 .attach_dev = intel_iommu_attach_device,
4628 .detach_dev = intel_iommu_detach_device,
4629 .map = intel_iommu_map,
4630 .unmap = intel_iommu_unmap,
4631 .map_sg = default_iommu_map_sg,
4632 .iova_to_phys = intel_iommu_iova_to_phys,
4633 .add_device = intel_iommu_add_device,
4634 .remove_device = intel_iommu_remove_device,
4635 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
4636 };
4637
4638 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4639 {
4640 /* G4x/GM45 integrated gfx dmar support is totally busted. */
4641 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
4642 dmar_map_gfx = 0;
4643 }
4644
4645 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4646 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4647 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4648 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4649 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4650 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4651 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4652
4653 static void quirk_iommu_rwbf(struct pci_dev *dev)
4654 {
4655 /*
4656 * Mobile 4 Series Chipset neglects to set RWBF capability,
4657 * but needs it. Same seems to hold for the desktop versions.
4658 */
4659 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
4660 rwbf_quirk = 1;
4661 }
4662
4663 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
4664 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4665 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4666 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4667 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4668 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4669 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
4670
4671 #define GGC 0x52
4672 #define GGC_MEMORY_SIZE_MASK (0xf << 8)
4673 #define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4674 #define GGC_MEMORY_SIZE_1M (0x1 << 8)
4675 #define GGC_MEMORY_SIZE_2M (0x3 << 8)
4676 #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4677 #define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4678 #define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4679 #define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4680
4681 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
4682 {
4683 unsigned short ggc;
4684
4685 if (pci_read_config_word(dev, GGC, &ggc))
4686 return;
4687
4688 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
4689 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4690 dmar_map_gfx = 0;
4691 } else if (dmar_map_gfx) {
4692 /* we have to ensure the gfx device is idle before we flush */
4693 printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");
4694 intel_iommu_strict = 1;
4695 }
4696 }
4697 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4698 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4699 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4700 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4701
4702 /* On Tylersburg chipsets, some BIOSes have been known to enable the
4703 ISOCH DMAR unit for the Azalia sound device, but not give it any
4704 TLB entries, which causes it to deadlock. Check for that. We do
4705 this in a function called from init_dmars(), instead of in a PCI
4706 quirk, because we don't want to print the obnoxious "BIOS broken"
4707 message if VT-d is actually disabled.
4708 */
4709 static void __init check_tylersburg_isoch(void)
4710 {
4711 struct pci_dev *pdev;
4712 uint32_t vtisochctrl;
4713
4714 /* If there's no Azalia in the system anyway, forget it. */
4715 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4716 if (!pdev)
4717 return;
4718 pci_dev_put(pdev);
4719
4720 /* System Management Registers. Might be hidden, in which case
4721 we can't do the sanity check. But that's OK, because the
4722 known-broken BIOSes _don't_ actually hide it, so far. */
4723 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4724 if (!pdev)
4725 return;
4726
4727 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4728 pci_dev_put(pdev);
4729 return;
4730 }
4731
4732 pci_dev_put(pdev);
4733
4734 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4735 if (vtisochctrl & 1)
4736 return;
4737
4738 /* Drop all bits other than the number of TLB entries */
4739 vtisochctrl &= 0x1c;
4740
4741 /* If we have the recommended number of TLB entries (16), fine. */
4742 if (vtisochctrl == 0x10)
4743 return;
4744
4745 /* Zero TLB entries? You get to ride the short bus to school. */
4746 if (!vtisochctrl) {
4747 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4748 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4749 dmi_get_system_info(DMI_BIOS_VENDOR),
4750 dmi_get_system_info(DMI_BIOS_VERSION),
4751 dmi_get_system_info(DMI_PRODUCT_VERSION));
4752 iommu_identity_mapping |= IDENTMAP_AZALIA;
4753 return;
4754 }
4755
4756 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4757 vtisochctrl);
4758 }
Linux with added FPC-III support