?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
init from v2.6.32.60
[mach-moxart.git] / drivers / pci / intel-iommu.c
blob5b680dfd871e8e65e55efa3d33189e59bb7897f5
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 * Author: Fenghua Yu <fenghua.yu@intel.com>
22 */
23
24 #include <linux/init.h>
25 #include <linux/bitmap.h>
26 #include <linux/debugfs.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/timer.h>
36 #include <linux/iova.h>
37 #include <linux/iommu.h>
38 #include <linux/intel-iommu.h>
39 #include <linux/sysdev.h>
40 #include <linux/tboot.h>
41 #include <linux/dmi.h>
42 #include <asm/cacheflush.h>
43 #include <asm/iommu.h>
44 #include "pci.h"
45
46 #define ROOT_SIZE VTD_PAGE_SIZE
47 #define CONTEXT_SIZE VTD_PAGE_SIZE
48
49 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
50 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
51 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
52
53 #define IOAPIC_RANGE_START (0xfee00000)
54 #define IOAPIC_RANGE_END (0xfeefffff)
55 #define IOVA_START_ADDR (0x1000)
56
57 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
58
59 #define MAX_AGAW_WIDTH 64
60
61 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
62 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
63
64 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
65 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
66 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
67 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
68 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
69
70 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
71 #define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
72 #define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
73
74 /* page table handling */
75 #define LEVEL_STRIDE (9)
76 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
77
78 static inline int agaw_to_level(int agaw)
79 {
80 return agaw + 2;
81 }
82
83 static inline int agaw_to_width(int agaw)
84 {
85 return 30 + agaw * LEVEL_STRIDE;
86 }
87
88 static inline int width_to_agaw(int width)
89 {
90 return (width - 30) / LEVEL_STRIDE;
91 }
92
93 static inline unsigned int level_to_offset_bits(int level)
94 {
95 return (level - 1) * LEVEL_STRIDE;
96 }
97
98 static inline int pfn_level_offset(unsigned long pfn, int level)
99 {
100 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
101 }
102
103 static inline unsigned long level_mask(int level)
104 {
105 return -1UL << level_to_offset_bits(level);
106 }
107
108 static inline unsigned long level_size(int level)
109 {
110 return 1UL << level_to_offset_bits(level);
111 }
112
113 static inline unsigned long align_to_level(unsigned long pfn, int level)
114 {
115 return (pfn + level_size(level) - 1) & level_mask(level);
116 }
117
118 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
119 are never going to work. */
120 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
121 {
122 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
123 }
124
125 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
126 {
127 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
128 }
129 static inline unsigned long page_to_dma_pfn(struct page *pg)
130 {
131 return mm_to_dma_pfn(page_to_pfn(pg));
132 }
133 static inline unsigned long virt_to_dma_pfn(void *p)
134 {
135 return page_to_dma_pfn(virt_to_page(p));
136 }
137
138 /* global iommu list, set NULL for ignored DMAR units */
139 static struct intel_iommu **g_iommus;
140
141 static void __init check_tylersburg_isoch(void);
142 static int rwbf_quirk;
143
144 /*
145 * 0: Present
146 * 1-11: Reserved
147 * 12-63: Context Ptr (12 - (haw-1))
148 * 64-127: Reserved
149 */
150 struct root_entry {
151 u64 val;
152 u64 rsvd1;
153 };
154 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
155 static inline bool root_present(struct root_entry *root)
156 {
157 return (root->val & 1);
158 }
159 static inline void set_root_present(struct root_entry *root)
160 {
161 root->val |= 1;
162 }
163 static inline void set_root_value(struct root_entry *root, unsigned long value)
164 {
165 root->val |= value & VTD_PAGE_MASK;
166 }
167
168 static inline struct context_entry *
169 get_context_addr_from_root(struct root_entry *root)
170 {
171 return (struct context_entry *)
172 (root_present(root)?phys_to_virt(
173 root->val & VTD_PAGE_MASK) :
174 NULL);
175 }
176
177 /*
178 * low 64 bits:
179 * 0: present
180 * 1: fault processing disable
181 * 2-3: translation type
182 * 12-63: address space root
183 * high 64 bits:
184 * 0-2: address width
185 * 3-6: aval
186 * 8-23: domain id
187 */
188 struct context_entry {
189 u64 lo;
190 u64 hi;
191 };
192
193 static inline bool context_present(struct context_entry *context)
194 {
195 return (context->lo & 1);
196 }
197 static inline void context_set_present(struct context_entry *context)
198 {
199 context->lo |= 1;
200 }
201
202 static inline void context_set_fault_enable(struct context_entry *context)
203 {
204 context->lo &= (((u64)-1) << 2) | 1;
205 }
206
207 static inline void context_set_translation_type(struct context_entry *context,
208 unsigned long value)
209 {
210 context->lo &= (((u64)-1) << 4) | 3;
211 context->lo |= (value & 3) << 2;
212 }
213
214 static inline void context_set_address_root(struct context_entry *context,
215 unsigned long value)
216 {
217 context->lo |= value & VTD_PAGE_MASK;
218 }
219
220 static inline void context_set_address_width(struct context_entry *context,
221 unsigned long value)
222 {
223 context->hi |= value & 7;
224 }
225
226 static inline void context_set_domain_id(struct context_entry *context,
227 unsigned long value)
228 {
229 context->hi |= (value & ((1 << 16) - 1)) << 8;
230 }
231
232 static inline void context_clear_entry(struct context_entry *context)
233 {
234 context->lo = 0;
235 context->hi = 0;
236 }
237
238 /*
239 * 0: readable
240 * 1: writable
241 * 2-6: reserved
242 * 7: super page
243 * 8-10: available
244 * 11: snoop behavior
245 * 12-63: Host physcial address
246 */
247 struct dma_pte {
248 u64 val;
249 };
250
251 static inline void dma_clear_pte(struct dma_pte *pte)
252 {
253 pte->val = 0;
254 }
255
256 static inline void dma_set_pte_readable(struct dma_pte *pte)
257 {
258 pte->val |= DMA_PTE_READ;
259 }
260
261 static inline void dma_set_pte_writable(struct dma_pte *pte)
262 {
263 pte->val |= DMA_PTE_WRITE;
264 }
265
266 static inline void dma_set_pte_snp(struct dma_pte *pte)
267 {
268 pte->val |= DMA_PTE_SNP;
269 }
270
271 static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
272 {
273 pte->val = (pte->val & ~3) | (prot & 3);
274 }
275
276 static inline u64 dma_pte_addr(struct dma_pte *pte)
277 {
278 #ifdef CONFIG_64BIT
279 return pte->val & VTD_PAGE_MASK;
280 #else
281 /* Must have a full atomic 64-bit read */
282 return __cmpxchg64(pte, 0ULL, 0ULL) & VTD_PAGE_MASK;
283 #endif
284 }
285
286 static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
287 {
288 pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
289 }
290
291 static inline bool dma_pte_present(struct dma_pte *pte)
292 {
293 return (pte->val & 3) != 0;
294 }
295
296 static inline int first_pte_in_page(struct dma_pte *pte)
297 {
298 return !((unsigned long)pte & ~VTD_PAGE_MASK);
299 }
300
301 /*
302 * This domain is a statically identity mapping domain.
303 * 1. This domain creats a static 1:1 mapping to all usable memory.
304 * 2. It maps to each iommu if successful.
305 * 3. Each iommu mapps to this domain if successful.
306 */
307 static struct dmar_domain *si_domain;
308 static int hw_pass_through = 1;
309
310 /* devices under the same p2p bridge are owned in one domain */
311 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
312
313 /* domain represents a virtual machine, more than one devices
314 * across iommus may be owned in one domain, e.g. kvm guest.
315 */
316 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
317
318 /* si_domain contains mulitple devices */
319 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)
320
321 struct dmar_domain {
322 int id; /* domain id */
323 unsigned long iommu_bmp; /* bitmap of iommus this domain uses*/
324
325 struct list_head devices; /* all devices' list */
326 struct iova_domain iovad; /* iova's that belong to this domain */
327
328 struct dma_pte *pgd; /* virtual address */
329 int gaw; /* max guest address width */
330
331 /* adjusted guest address width, 0 is level 2 30-bit */
332 int agaw;
333
334 int flags; /* flags to find out type of domain */
335
336 int iommu_coherency;/* indicate coherency of iommu access */
337 int iommu_snooping; /* indicate snooping control feature*/
338 int iommu_count; /* reference count of iommu */
339 spinlock_t iommu_lock; /* protect iommu set in domain */
340 u64 max_addr; /* maximum mapped address */
341 };
342
343 /* PCI domain-device relationship */
344 struct device_domain_info {
345 struct list_head link; /* link to domain siblings */
346 struct list_head global; /* link to global list */
347 int segment; /* PCI domain */
348 u8 bus; /* PCI bus number */
349 u8 devfn; /* PCI devfn number */
350 struct pci_dev *dev; /* it's NULL for PCIE-to-PCI bridge */
351 struct intel_iommu *iommu; /* IOMMU used by this device */
352 struct dmar_domain *domain; /* pointer to domain */
353 };
354
355 static void flush_unmaps_timeout(unsigned long data);
356
357 DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
358
359 #define HIGH_WATER_MARK 250
360 struct deferred_flush_tables {
361 int next;
362 struct iova *iova[HIGH_WATER_MARK];
363 struct dmar_domain *domain[HIGH_WATER_MARK];
364 };
365
366 static struct deferred_flush_tables *deferred_flush;
367
368 /* bitmap for indexing intel_iommus */
369 static int g_num_of_iommus;
370
371 static DEFINE_SPINLOCK(async_umap_flush_lock);
372 static LIST_HEAD(unmaps_to_do);
373
374 static int timer_on;
375 static long list_size;
376
377 static void domain_remove_dev_info(struct dmar_domain *domain);
378
379 #ifdef CONFIG_DMAR_DEFAULT_ON
380 int dmar_disabled = 0;
381 #else
382 int dmar_disabled = 1;
383 #endif /*CONFIG_DMAR_DEFAULT_ON*/
384
385 static int dmar_map_gfx = 1;
386 static int dmar_forcedac;
387 static int intel_iommu_strict;
388
389 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
390 static DEFINE_SPINLOCK(device_domain_lock);
391 static LIST_HEAD(device_domain_list);
392
393 static struct iommu_ops intel_iommu_ops;
394
395 static int __init intel_iommu_setup(char *str)
396 {
397 if (!str)
398 return -EINVAL;
399 while (*str) {
400 if (!strncmp(str, "on", 2)) {
401 dmar_disabled = 0;
402 printk(KERN_INFO "Intel-IOMMU: enabled\n");
403 } else if (!strncmp(str, "off", 3)) {
404 dmar_disabled = 1;
405 printk(KERN_INFO "Intel-IOMMU: disabled\n");
406 } else if (!strncmp(str, "igfx_off", 8)) {
407 dmar_map_gfx = 0;
408 printk(KERN_INFO
409 "Intel-IOMMU: disable GFX device mapping\n");
410 } else if (!strncmp(str, "forcedac", 8)) {
411 printk(KERN_INFO
412 "Intel-IOMMU: Forcing DAC for PCI devices\n");
413 dmar_forcedac = 1;
414 } else if (!strncmp(str, "strict", 6)) {
415 printk(KERN_INFO
416 "Intel-IOMMU: disable batched IOTLB flush\n");
417 intel_iommu_strict = 1;
418 }
419
420 str += strcspn(str, ",");
421 while (*str == ',')
422 str++;
423 }
424 return 0;
425 }
426 __setup("intel_iommu=", intel_iommu_setup);
427
428 static struct kmem_cache *iommu_domain_cache;
429 static struct kmem_cache *iommu_devinfo_cache;
430 static struct kmem_cache *iommu_iova_cache;
431
432 static inline void *iommu_kmem_cache_alloc(struct kmem_cache *cachep)
433 {
434 unsigned int flags;
435 void *vaddr;
436
437 /* trying to avoid low memory issues */
438 flags = current->flags & PF_MEMALLOC;
439 current->flags |= PF_MEMALLOC;
440 vaddr = kmem_cache_alloc(cachep, GFP_ATOMIC);
441 current->flags &= (~PF_MEMALLOC | flags);
442 return vaddr;
443 }
444
445
446 static inline void *alloc_pgtable_page(void)
447 {
448 unsigned int flags;
449 void *vaddr;
450
451 /* trying to avoid low memory issues */
452 flags = current->flags & PF_MEMALLOC;
453 current->flags |= PF_MEMALLOC;
454 vaddr = (void *)get_zeroed_page(GFP_ATOMIC);
455 current->flags &= (~PF_MEMALLOC | flags);
456 return vaddr;
457 }
458
459 static inline void free_pgtable_page(void *vaddr)
460 {
461 free_page((unsigned long)vaddr);
462 }
463
464 static inline void *alloc_domain_mem(void)
465 {
466 return iommu_kmem_cache_alloc(iommu_domain_cache);
467 }
468
469 static void free_domain_mem(void *vaddr)
470 {
471 kmem_cache_free(iommu_domain_cache, vaddr);
472 }
473
474 static inline void * alloc_devinfo_mem(void)
475 {
476 return iommu_kmem_cache_alloc(iommu_devinfo_cache);
477 }
478
479 static inline void free_devinfo_mem(void *vaddr)
480 {
481 kmem_cache_free(iommu_devinfo_cache, vaddr);
482 }
483
484 struct iova *alloc_iova_mem(void)
485 {
486 return iommu_kmem_cache_alloc(iommu_iova_cache);
487 }
488
489 void free_iova_mem(struct iova *iova)
490 {
491 kmem_cache_free(iommu_iova_cache, iova);
492 }
493
494
495 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
496 {
497 unsigned long sagaw;
498 int agaw = -1;
499
500 sagaw = cap_sagaw(iommu->cap);
501 for (agaw = width_to_agaw(max_gaw);
502 agaw >= 0; agaw--) {
503 if (test_bit(agaw, &sagaw))
504 break;
505 }
506
507 return agaw;
508 }
509
510 /*
511 * Calculate max SAGAW for each iommu.
512 */
513 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
514 {
515 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
516 }
517
518 /*
519 * calculate agaw for each iommu.
520 * "SAGAW" may be different across iommus, use a default agaw, and
521 * get a supported less agaw for iommus that don't support the default agaw.
522 */
523 int iommu_calculate_agaw(struct intel_iommu *iommu)
524 {
525 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
526 }
527
528 /* This functionin only returns single iommu in a domain */
529 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
530 {
531 int iommu_id;
532
533 /* si_domain and vm domain should not get here. */
534 BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
535 BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
536
537 iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
538 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
539 return NULL;
540
541 return g_iommus[iommu_id];
542 }
543
544 static void domain_update_iommu_coherency(struct dmar_domain *domain)
545 {
546 int i;
547
548 domain->iommu_coherency = 1;
549
550 i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
551 for (; i < g_num_of_iommus; ) {
552 if (!ecap_coherent(g_iommus[i]->ecap)) {
553 domain->iommu_coherency = 0;
554 break;
555 }
556 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
557 }
558 }
559
560 static void domain_update_iommu_snooping(struct dmar_domain *domain)
561 {
562 int i;
563
564 domain->iommu_snooping = 1;
565
566 i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
567 for (; i < g_num_of_iommus; ) {
568 if (!ecap_sc_support(g_iommus[i]->ecap)) {
569 domain->iommu_snooping = 0;
570 break;
571 }
572 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
573 }
574 }
575
576 /* Some capabilities may be different across iommus */
577 static void domain_update_iommu_cap(struct dmar_domain *domain)
578 {
579 domain_update_iommu_coherency(domain);
580 domain_update_iommu_snooping(domain);
581 }
582
583 static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
584 {
585 struct dmar_drhd_unit *drhd = NULL;
586 int i;
587
588 for_each_drhd_unit(drhd) {
589 if (drhd->ignored)
590 continue;
591 if (segment != drhd->segment)
592 continue;
593
594 for (i = 0; i < drhd->devices_cnt; i++) {
595 if (drhd->devices[i] &&
596 drhd->devices[i]->bus->number == bus &&
597 drhd->devices[i]->devfn == devfn)
598 return drhd->iommu;
599 if (drhd->devices[i] &&
600 drhd->devices[i]->subordinate &&
601 drhd->devices[i]->subordinate->number <= bus &&
602 drhd->devices[i]->subordinate->subordinate >= bus)
603 return drhd->iommu;
604 }
605
606 if (drhd->include_all)
607 return drhd->iommu;
608 }
609
610 return NULL;
611 }
612
613 static void domain_flush_cache(struct dmar_domain *domain,
614 void *addr, int size)
615 {
616 if (!domain->iommu_coherency)
617 clflush_cache_range(addr, size);
618 }
619
620 /* Gets context entry for a given bus and devfn */
621 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
622 u8 bus, u8 devfn)
623 {
624 struct root_entry *root;
625 struct context_entry *context;
626 unsigned long phy_addr;
627 unsigned long flags;
628
629 spin_lock_irqsave(&iommu->lock, flags);
630 root = &iommu->root_entry[bus];
631 context = get_context_addr_from_root(root);
632 if (!context) {
633 context = (struct context_entry *)alloc_pgtable_page();
634 if (!context) {
635 spin_unlock_irqrestore(&iommu->lock, flags);
636 return NULL;
637 }
638 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
639 phy_addr = virt_to_phys((void *)context);
640 set_root_value(root, phy_addr);
641 set_root_present(root);
642 __iommu_flush_cache(iommu, root, sizeof(*root));
643 }
644 spin_unlock_irqrestore(&iommu->lock, flags);
645 return &context[devfn];
646 }
647
648 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
649 {
650 struct root_entry *root;
651 struct context_entry *context;
652 int ret;
653 unsigned long flags;
654
655 spin_lock_irqsave(&iommu->lock, flags);
656 root = &iommu->root_entry[bus];
657 context = get_context_addr_from_root(root);
658 if (!context) {
659 ret = 0;
660 goto out;
661 }
662 ret = context_present(&context[devfn]);
663 out:
664 spin_unlock_irqrestore(&iommu->lock, flags);
665 return ret;
666 }
667
668 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
669 {
670 struct root_entry *root;
671 struct context_entry *context;
672 unsigned long flags;
673
674 spin_lock_irqsave(&iommu->lock, flags);
675 root = &iommu->root_entry[bus];
676 context = get_context_addr_from_root(root);
677 if (context) {
678 context_clear_entry(&context[devfn]);
679 __iommu_flush_cache(iommu, &context[devfn], \
680 sizeof(*context));
681 }
682 spin_unlock_irqrestore(&iommu->lock, flags);
683 }
684
685 static void free_context_table(struct intel_iommu *iommu)
686 {
687 struct root_entry *root;
688 int i;
689 unsigned long flags;
690 struct context_entry *context;
691
692 spin_lock_irqsave(&iommu->lock, flags);
693 if (!iommu->root_entry) {
694 goto out;
695 }
696 for (i = 0; i < ROOT_ENTRY_NR; i++) {
697 root = &iommu->root_entry[i];
698 context = get_context_addr_from_root(root);
699 if (context)
700 free_pgtable_page(context);
701 }
702 free_pgtable_page(iommu->root_entry);
703 iommu->root_entry = NULL;
704 out:
705 spin_unlock_irqrestore(&iommu->lock, flags);
706 }
707
708 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
709 unsigned long pfn)
710 {
711 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
712 struct dma_pte *parent, *pte = NULL;
713 int level = agaw_to_level(domain->agaw);
714 int offset;
715
716 BUG_ON(!domain->pgd);
717 BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width);
718 parent = domain->pgd;
719
720 while (level > 0) {
721 void *tmp_page;
722
723 offset = pfn_level_offset(pfn, level);
724 pte = &parent[offset];
725 if (level == 1)
726 break;
727
728 if (!dma_pte_present(pte)) {
729 uint64_t pteval;
730
731 tmp_page = alloc_pgtable_page();
732
733 if (!tmp_page)
734 return NULL;
735
736 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
737 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
738 if (cmpxchg64(&pte->val, 0ULL, pteval)) {
739 /* Someone else set it while we were thinking; use theirs. */
740 free_pgtable_page(tmp_page);
741 } else {
742 dma_pte_addr(pte);
743 domain_flush_cache(domain, pte, sizeof(*pte));
744 }
745 }
746 parent = phys_to_virt(dma_pte_addr(pte));
747 level--;
748 }
749
750 return pte;
751 }
752
753 /* return address's pte at specific level */
754 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
755 unsigned long pfn,
756 int level)
757 {
758 struct dma_pte *parent, *pte = NULL;
759 int total = agaw_to_level(domain->agaw);
760 int offset;
761
762 parent = domain->pgd;
763 while (level <= total) {
764 offset = pfn_level_offset(pfn, total);
765 pte = &parent[offset];
766 if (level == total)
767 return pte;
768
769 if (!dma_pte_present(pte))
770 break;
771 parent = phys_to_virt(dma_pte_addr(pte));
772 total--;
773 }
774 return NULL;
775 }
776
777 /* clear last level pte, a tlb flush should be followed */
778 static void dma_pte_clear_range(struct dmar_domain *domain,
779 unsigned long start_pfn,
780 unsigned long last_pfn)
781 {
782 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
783 struct dma_pte *first_pte, *pte;
784
785 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
786 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
787 BUG_ON(start_pfn > last_pfn);
788
789 /* we don't need lock here; nobody else touches the iova range */
790 do {
791 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1);
792 if (!pte) {
793 start_pfn = align_to_level(start_pfn + 1, 2);
794 continue;
795 }
796 do {
797 dma_clear_pte(pte);
798 start_pfn++;
799 pte++;
800 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
801
802 domain_flush_cache(domain, first_pte,
803 (void *)pte - (void *)first_pte);
804
805 } while (start_pfn && start_pfn <= last_pfn);
806 }
807
808 /* free page table pages. last level pte should already be cleared */
809 static void dma_pte_free_pagetable(struct dmar_domain *domain,
810 unsigned long start_pfn,
811 unsigned long last_pfn)
812 {
813 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
814 struct dma_pte *first_pte, *pte;
815 int total = agaw_to_level(domain->agaw);
816 int level;
817 unsigned long tmp;
818
819 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
820 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
821 BUG_ON(start_pfn > last_pfn);
822
823 /* We don't need lock here; nobody else touches the iova range */
824 level = 2;
825 while (level <= total) {
826 tmp = align_to_level(start_pfn, level);
827
828 /* If we can't even clear one PTE at this level, we're done */
829 if (tmp + level_size(level) - 1 > last_pfn)
830 return;
831
832 do {
833 first_pte = pte = dma_pfn_level_pte(domain, tmp, level);
834 if (!pte) {
835 tmp = align_to_level(tmp + 1, level + 1);
836 continue;
837 }
838 do {
839 if (dma_pte_present(pte)) {
840 free_pgtable_page(phys_to_virt(dma_pte_addr(pte)));
841 dma_clear_pte(pte);
842 }
843 pte++;
844 tmp += level_size(level);
845 } while (!first_pte_in_page(pte) &&
846 tmp + level_size(level) - 1 <= last_pfn);
847
848 domain_flush_cache(domain, first_pte,
849 (void *)pte - (void *)first_pte);
850
851 } while (tmp && tmp + level_size(level) - 1 <= last_pfn);
852 level++;
853 }
854 /* free pgd */
855 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
856 free_pgtable_page(domain->pgd);
857 domain->pgd = NULL;
858 }
859 }
860
861 /* iommu handling */
862 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
863 {
864 struct root_entry *root;
865 unsigned long flags;
866
867 root = (struct root_entry *)alloc_pgtable_page();
868 if (!root)
869 return -ENOMEM;
870
871 __iommu_flush_cache(iommu, root, ROOT_SIZE);
872
873 spin_lock_irqsave(&iommu->lock, flags);
874 iommu->root_entry = root;
875 spin_unlock_irqrestore(&iommu->lock, flags);
876
877 return 0;
878 }
879
880 static void iommu_set_root_entry(struct intel_iommu *iommu)
881 {
882 void *addr;
883 u32 sts;
884 unsigned long flag;
885
886 addr = iommu->root_entry;
887
888 spin_lock_irqsave(&iommu->register_lock, flag);
889 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
890
891 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
892
893 /* Make sure hardware complete it */
894 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
895 readl, (sts & DMA_GSTS_RTPS), sts);
896
897 spin_unlock_irqrestore(&iommu->register_lock, flag);
898 }
899
900 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
901 {
902 u32 val;
903 unsigned long flag;
904
905 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
906 return;
907
908 spin_lock_irqsave(&iommu->register_lock, flag);
909 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
910
911 /* Make sure hardware complete it */
912 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
913 readl, (!(val & DMA_GSTS_WBFS)), val);
914
915 spin_unlock_irqrestore(&iommu->register_lock, flag);
916 }
917
918 /* return value determine if we need a write buffer flush */
919 static void __iommu_flush_context(struct intel_iommu *iommu,
920 u16 did, u16 source_id, u8 function_mask,
921 u64 type)
922 {
923 u64 val = 0;
924 unsigned long flag;
925
926 switch (type) {
927 case DMA_CCMD_GLOBAL_INVL:
928 val = DMA_CCMD_GLOBAL_INVL;
929 break;
930 case DMA_CCMD_DOMAIN_INVL:
931 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
932 break;
933 case DMA_CCMD_DEVICE_INVL:
934 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
935 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
936 break;
937 default:
938 BUG();
939 }
940 val |= DMA_CCMD_ICC;
941
942 spin_lock_irqsave(&iommu->register_lock, flag);
943 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
944
945 /* Make sure hardware complete it */
946 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
947 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
948
949 spin_unlock_irqrestore(&iommu->register_lock, flag);
950 }
951
952 /* return value determine if we need a write buffer flush */
953 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
954 u64 addr, unsigned int size_order, u64 type)
955 {
956 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
957 u64 val = 0, val_iva = 0;
958 unsigned long flag;
959
960 switch (type) {
961 case DMA_TLB_GLOBAL_FLUSH:
962 /* global flush doesn't need set IVA_REG */
963 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
964 break;
965 case DMA_TLB_DSI_FLUSH:
966 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
967 break;
968 case DMA_TLB_PSI_FLUSH:
969 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
970 /* Note: always flush non-leaf currently */
971 val_iva = size_order | addr;
972 break;
973 default:
974 BUG();
975 }
976 /* Note: set drain read/write */
977 #if 0
978 /*
979 * This is probably to be super secure.. Looks like we can
980 * ignore it without any impact.
981 */
982 if (cap_read_drain(iommu->cap))
983 val |= DMA_TLB_READ_DRAIN;
984 #endif
985 if (cap_write_drain(iommu->cap))
986 val |= DMA_TLB_WRITE_DRAIN;
987
988 spin_lock_irqsave(&iommu->register_lock, flag);
989 /* Note: Only uses first TLB reg currently */
990 if (val_iva)
991 dmar_writeq(iommu->reg + tlb_offset, val_iva);
992 dmar_writeq(iommu->reg + tlb_offset + 8, val);
993
994 /* Make sure hardware complete it */
995 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
996 dmar_readq, (!(val & DMA_TLB_IVT)), val);
997
998 spin_unlock_irqrestore(&iommu->register_lock, flag);
999
1000 /* check IOTLB invalidation granularity */
1001 if (DMA_TLB_IAIG(val) == 0)
1002 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1003 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1004 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
1005 (unsigned long long)DMA_TLB_IIRG(type),
1006 (unsigned long long)DMA_TLB_IAIG(val));
1007 }
1008
1009 static struct device_domain_info *iommu_support_dev_iotlb(
1010 struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
1011 {
1012 int found = 0;
1013 unsigned long flags;
1014 struct device_domain_info *info;
1015 struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
1016
1017 if (!ecap_dev_iotlb_support(iommu->ecap))
1018 return NULL;
1019
1020 if (!iommu->qi)
1021 return NULL;
1022
1023 spin_lock_irqsave(&device_domain_lock, flags);
1024 list_for_each_entry(info, &domain->devices, link)
1025 if (info->bus == bus && info->devfn == devfn) {
1026 found = 1;
1027 break;
1028 }
1029 spin_unlock_irqrestore(&device_domain_lock, flags);
1030
1031 if (!found || !info->dev)
1032 return NULL;
1033
1034 if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1035 return NULL;
1036
1037 if (!dmar_find_matched_atsr_unit(info->dev))
1038 return NULL;
1039
1040 info->iommu = iommu;
1041
1042 return info;
1043 }
1044
1045 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1046 {
1047 if (!info)
1048 return;
1049
1050 pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1051 }
1052
1053 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1054 {
1055 if (!info->dev || !pci_ats_enabled(info->dev))
1056 return;
1057
1058 pci_disable_ats(info->dev);
1059 }
1060
1061 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1062 u64 addr, unsigned mask)
1063 {
1064 u16 sid, qdep;
1065 unsigned long flags;
1066 struct device_domain_info *info;
1067
1068 spin_lock_irqsave(&device_domain_lock, flags);
1069 list_for_each_entry(info, &domain->devices, link) {
1070 if (!info->dev || !pci_ats_enabled(info->dev))
1071 continue;
1072
1073 sid = info->bus << 8 | info->devfn;
1074 qdep = pci_ats_queue_depth(info->dev);
1075 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1076 }
1077 spin_unlock_irqrestore(&device_domain_lock, flags);
1078 }
1079
1080 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1081 unsigned long pfn, unsigned int pages)
1082 {
1083 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1084 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1085
1086 BUG_ON(pages == 0);
1087
1088 /*
1089 * Fallback to domain selective flush if no PSI support or the size is
1090 * too big.
1091 * PSI requires page size to be 2 ^ x, and the base address is naturally
1092 * aligned to the size
1093 */
1094 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1095 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1096 DMA_TLB_DSI_FLUSH);
1097 else
1098 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1099 DMA_TLB_PSI_FLUSH);
1100
1101 /*
1102 * In caching mode, domain ID 0 is reserved for non-present to present
1103 * mapping flush. Device IOTLB doesn't need to be flushed in this case.
1104 */
1105 if (!cap_caching_mode(iommu->cap) || did)
1106 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
1107 }
1108
1109 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1110 {
1111 u32 pmen;
1112 unsigned long flags;
1113
1114 spin_lock_irqsave(&iommu->register_lock, flags);
1115 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1116 pmen &= ~DMA_PMEN_EPM;
1117 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1118
1119 /* wait for the protected region status bit to clear */
1120 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1121 readl, !(pmen & DMA_PMEN_PRS), pmen);
1122
1123 spin_unlock_irqrestore(&iommu->register_lock, flags);
1124 }
1125
1126 static int iommu_enable_translation(struct intel_iommu *iommu)
1127 {
1128 u32 sts;
1129 unsigned long flags;
1130
1131 spin_lock_irqsave(&iommu->register_lock, flags);
1132 iommu->gcmd |= DMA_GCMD_TE;
1133 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1134
1135 /* Make sure hardware complete it */
1136 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1137 readl, (sts & DMA_GSTS_TES), sts);
1138
1139 spin_unlock_irqrestore(&iommu->register_lock, flags);
1140 return 0;
1141 }
1142
1143 static int iommu_disable_translation(struct intel_iommu *iommu)
1144 {
1145 u32 sts;
1146 unsigned long flag;
1147
1148 spin_lock_irqsave(&iommu->register_lock, flag);
1149 iommu->gcmd &= ~DMA_GCMD_TE;
1150 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1151
1152 /* Make sure hardware complete it */
1153 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1154 readl, (!(sts & DMA_GSTS_TES)), sts);
1155
1156 spin_unlock_irqrestore(&iommu->register_lock, flag);
1157 return 0;
1158 }
1159
1160
1161 static int iommu_init_domains(struct intel_iommu *iommu)
1162 {
1163 unsigned long ndomains;
1164 unsigned long nlongs;
1165
1166 ndomains = cap_ndoms(iommu->cap);
1167 pr_debug("Number of Domains supportd <%ld>\n", ndomains);
1168 nlongs = BITS_TO_LONGS(ndomains);
1169
1170 spin_lock_init(&iommu->lock);
1171
1172 /* TBD: there might be 64K domains,
1173 * consider other allocation for future chip
1174 */
1175 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1176 if (!iommu->domain_ids) {
1177 printk(KERN_ERR "Allocating domain id array failed\n");
1178 return -ENOMEM;
1179 }
1180 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1181 GFP_KERNEL);
1182 if (!iommu->domains) {
1183 printk(KERN_ERR "Allocating domain array failed\n");
1184 return -ENOMEM;
1185 }
1186
1187 /*
1188 * if Caching mode is set, then invalid translations are tagged
1189 * with domainid 0. Hence we need to pre-allocate it.
1190 */
1191 if (cap_caching_mode(iommu->cap))
1192 set_bit(0, iommu->domain_ids);
1193 return 0;
1194 }
1195
1196
1197 static void domain_exit(struct dmar_domain *domain);
1198 static void vm_domain_exit(struct dmar_domain *domain);
1199
1200 void free_dmar_iommu(struct intel_iommu *iommu)
1201 {
1202 struct dmar_domain *domain;
1203 int i;
1204 unsigned long flags;
1205
1206 if ((iommu->domains) && (iommu->domain_ids)) {
1207 i = find_first_bit(iommu->domain_ids, cap_ndoms(iommu->cap));
1208 for (; i < cap_ndoms(iommu->cap); ) {
1209 domain = iommu->domains[i];
1210 clear_bit(i, iommu->domain_ids);
1211
1212 spin_lock_irqsave(&domain->iommu_lock, flags);
1213 if (--domain->iommu_count == 0) {
1214 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1215 vm_domain_exit(domain);
1216 else
1217 domain_exit(domain);
1218 }
1219 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1220
1221 i = find_next_bit(iommu->domain_ids,
1222 cap_ndoms(iommu->cap), i+1);
1223 }
1224 }
1225
1226 if (iommu->gcmd & DMA_GCMD_TE)
1227 iommu_disable_translation(iommu);
1228
1229 if (iommu->irq) {
1230 set_irq_data(iommu->irq, NULL);
1231 /* This will mask the irq */
1232 free_irq(iommu->irq, iommu);
1233 destroy_irq(iommu->irq);
1234 }
1235
1236 kfree(iommu->domains);
1237 kfree(iommu->domain_ids);
1238
1239 g_iommus[iommu->seq_id] = NULL;
1240
1241 /* if all iommus are freed, free g_iommus */
1242 for (i = 0; i < g_num_of_iommus; i++) {
1243 if (g_iommus[i])
1244 break;
1245 }
1246
1247 if (i == g_num_of_iommus)
1248 kfree(g_iommus);
1249
1250 /* free context mapping */
1251 free_context_table(iommu);
1252 }
1253
1254 static struct dmar_domain *alloc_domain(void)
1255 {
1256 struct dmar_domain *domain;
1257
1258 domain = alloc_domain_mem();
1259 if (!domain)
1260 return NULL;
1261
1262 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
1263 domain->flags = 0;
1264
1265 return domain;
1266 }
1267
1268 static int iommu_attach_domain(struct dmar_domain *domain,
1269 struct intel_iommu *iommu)
1270 {
1271 int num;
1272 unsigned long ndomains;
1273 unsigned long flags;
1274
1275 ndomains = cap_ndoms(iommu->cap);
1276
1277 spin_lock_irqsave(&iommu->lock, flags);
1278
1279 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1280 if (num >= ndomains) {
1281 spin_unlock_irqrestore(&iommu->lock, flags);
1282 printk(KERN_ERR "IOMMU: no free domain ids\n");
1283 return -ENOMEM;
1284 }
1285
1286 domain->id = num;
1287 set_bit(num, iommu->domain_ids);
1288 set_bit(iommu->seq_id, &domain->iommu_bmp);
1289 iommu->domains[num] = domain;
1290 spin_unlock_irqrestore(&iommu->lock, flags);
1291
1292 return 0;
1293 }
1294
1295 static void iommu_detach_domain(struct dmar_domain *domain,
1296 struct intel_iommu *iommu)
1297 {
1298 unsigned long flags;
1299 int num, ndomains;
1300 int found = 0;
1301
1302 spin_lock_irqsave(&iommu->lock, flags);
1303 ndomains = cap_ndoms(iommu->cap);
1304 num = find_first_bit(iommu->domain_ids, ndomains);
1305 for (; num < ndomains; ) {
1306 if (iommu->domains[num] == domain) {
1307 found = 1;
1308 break;
1309 }
1310 num = find_next_bit(iommu->domain_ids,
1311 cap_ndoms(iommu->cap), num+1);
1312 }
1313
1314 if (found) {
1315 clear_bit(num, iommu->domain_ids);
1316 clear_bit(iommu->seq_id, &domain->iommu_bmp);
1317 iommu->domains[num] = NULL;
1318 }
1319 spin_unlock_irqrestore(&iommu->lock, flags);
1320 }
1321
1322 static struct iova_domain reserved_iova_list;
1323 static struct lock_class_key reserved_rbtree_key;
1324
1325 static void dmar_init_reserved_ranges(void)
1326 {
1327 struct pci_dev *pdev = NULL;
1328 struct iova *iova;
1329 int i;
1330
1331 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1332
1333 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1334 &reserved_rbtree_key);
1335
1336 /* IOAPIC ranges shouldn't be accessed by DMA */
1337 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1338 IOVA_PFN(IOAPIC_RANGE_END));
1339 if (!iova)
1340 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1341
1342 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1343 for_each_pci_dev(pdev) {
1344 struct resource *r;
1345
1346 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1347 r = &pdev->resource[i];
1348 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1349 continue;
1350 iova = reserve_iova(&reserved_iova_list,
1351 IOVA_PFN(r->start),
1352 IOVA_PFN(r->end));
1353 if (!iova)
1354 printk(KERN_ERR "Reserve iova failed\n");
1355 }
1356 }
1357
1358 }
1359
1360 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1361 {
1362 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1363 }
1364
1365 static inline int guestwidth_to_adjustwidth(int gaw)
1366 {
1367 int agaw;
1368 int r = (gaw - 12) % 9;
1369
1370 if (r == 0)
1371 agaw = gaw;
1372 else
1373 agaw = gaw + 9 - r;
1374 if (agaw > 64)
1375 agaw = 64;
1376 return agaw;
1377 }
1378
1379 static int domain_init(struct dmar_domain *domain, int guest_width)
1380 {
1381 struct intel_iommu *iommu;
1382 int adjust_width, agaw;
1383 unsigned long sagaw;
1384
1385 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1386 spin_lock_init(&domain->iommu_lock);
1387
1388 domain_reserve_special_ranges(domain);
1389
1390 /* calculate AGAW */
1391 iommu = domain_get_iommu(domain);
1392 if (guest_width > cap_mgaw(iommu->cap))
1393 guest_width = cap_mgaw(iommu->cap);
1394 domain->gaw = guest_width;
1395 adjust_width = guestwidth_to_adjustwidth(guest_width);
1396 agaw = width_to_agaw(adjust_width);
1397 sagaw = cap_sagaw(iommu->cap);
1398 if (!test_bit(agaw, &sagaw)) {
1399 /* hardware doesn't support it, choose a bigger one */
1400 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1401 agaw = find_next_bit(&sagaw, 5, agaw);
1402 if (agaw >= 5)
1403 return -ENODEV;
1404 }
1405 domain->agaw = agaw;
1406 INIT_LIST_HEAD(&domain->devices);
1407
1408 if (ecap_coherent(iommu->ecap))
1409 domain->iommu_coherency = 1;
1410 else
1411 domain->iommu_coherency = 0;
1412
1413 if (ecap_sc_support(iommu->ecap))
1414 domain->iommu_snooping = 1;
1415 else
1416 domain->iommu_snooping = 0;
1417
1418 domain->iommu_count = 1;
1419
1420 /* always allocate the top pgd */
1421 domain->pgd = (struct dma_pte *)alloc_pgtable_page();
1422 if (!domain->pgd)
1423 return -ENOMEM;
1424 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1425 return 0;
1426 }
1427
1428 static void domain_exit(struct dmar_domain *domain)
1429 {
1430 struct dmar_drhd_unit *drhd;
1431 struct intel_iommu *iommu;
1432
1433 /* Domain 0 is reserved, so dont process it */
1434 if (!domain)
1435 return;
1436
1437 domain_remove_dev_info(domain);
1438 /* destroy iovas */
1439 put_iova_domain(&domain->iovad);
1440
1441 /* clear ptes */
1442 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1443
1444 /* free page tables */
1445 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1446
1447 for_each_active_iommu(iommu, drhd)
1448 if (test_bit(iommu->seq_id, &domain->iommu_bmp))
1449 iommu_detach_domain(domain, iommu);
1450
1451 free_domain_mem(domain);
1452 }
1453
1454 static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1455 u8 bus, u8 devfn, int translation)
1456 {
1457 struct context_entry *context;
1458 unsigned long flags;
1459 struct intel_iommu *iommu;
1460 struct dma_pte *pgd;
1461 unsigned long num;
1462 unsigned long ndomains;
1463 int id;
1464 int agaw;
1465 struct device_domain_info *info = NULL;
1466
1467 pr_debug("Set context mapping for %02x:%02x.%d\n",
1468 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1469
1470 BUG_ON(!domain->pgd);
1471 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1472 translation != CONTEXT_TT_MULTI_LEVEL);
1473
1474 iommu = device_to_iommu(segment, bus, devfn);
1475 if (!iommu)
1476 return -ENODEV;
1477
1478 context = device_to_context_entry(iommu, bus, devfn);
1479 if (!context)
1480 return -ENOMEM;
1481 spin_lock_irqsave(&iommu->lock, flags);
1482 if (context_present(context)) {
1483 spin_unlock_irqrestore(&iommu->lock, flags);
1484 return 0;
1485 }
1486
1487 id = domain->id;
1488 pgd = domain->pgd;
1489
1490 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1491 domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
1492 int found = 0;
1493
1494 /* find an available domain id for this device in iommu */
1495 ndomains = cap_ndoms(iommu->cap);
1496 num = find_first_bit(iommu->domain_ids, ndomains);
1497 for (; num < ndomains; ) {
1498 if (iommu->domains[num] == domain) {
1499 id = num;
1500 found = 1;
1501 break;
1502 }
1503 num = find_next_bit(iommu->domain_ids,
1504 cap_ndoms(iommu->cap), num+1);
1505 }
1506
1507 if (found == 0) {
1508 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1509 if (num >= ndomains) {
1510 spin_unlock_irqrestore(&iommu->lock, flags);
1511 printk(KERN_ERR "IOMMU: no free domain ids\n");
1512 return -EFAULT;
1513 }
1514
1515 set_bit(num, iommu->domain_ids);
1516 iommu->domains[num] = domain;
1517 id = num;
1518 }
1519
1520 /* Skip top levels of page tables for
1521 * iommu which has less agaw than default.
1522 * Unnecessary for PT mode.
1523 */
1524 if (translation != CONTEXT_TT_PASS_THROUGH) {
1525 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1526 pgd = phys_to_virt(dma_pte_addr(pgd));
1527 if (!dma_pte_present(pgd)) {
1528 spin_unlock_irqrestore(&iommu->lock, flags);
1529 return -ENOMEM;
1530 }
1531 }
1532 }
1533 }
1534
1535 context_set_domain_id(context, id);
1536
1537 if (translation != CONTEXT_TT_PASS_THROUGH) {
1538 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1539 translation = info ? CONTEXT_TT_DEV_IOTLB :
1540 CONTEXT_TT_MULTI_LEVEL;
1541 }
1542 /*
1543 * In pass through mode, AW must be programmed to indicate the largest
1544 * AGAW value supported by hardware. And ASR is ignored by hardware.
1545 */
1546 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
1547 context_set_address_width(context, iommu->msagaw);
1548 else {
1549 context_set_address_root(context, virt_to_phys(pgd));
1550 context_set_address_width(context, iommu->agaw);
1551 }
1552
1553 context_set_translation_type(context, translation);
1554 context_set_fault_enable(context);
1555 context_set_present(context);
1556 domain_flush_cache(domain, context, sizeof(*context));
1557
1558 /*
1559 * It's a non-present to present mapping. If hardware doesn't cache
1560 * non-present entry we only need to flush the write-buffer. If the
1561 * _does_ cache non-present entries, then it does so in the special
1562 * domain #0, which we have to flush:
1563 */
1564 if (cap_caching_mode(iommu->cap)) {
1565 iommu->flush.flush_context(iommu, 0,
1566 (((u16)bus) << 8) | devfn,
1567 DMA_CCMD_MASK_NOBIT,
1568 DMA_CCMD_DEVICE_INVL);
1569 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_DSI_FLUSH);
1570 } else {
1571 iommu_flush_write_buffer(iommu);
1572 }
1573 iommu_enable_dev_iotlb(info);
1574 spin_unlock_irqrestore(&iommu->lock, flags);
1575
1576 spin_lock_irqsave(&domain->iommu_lock, flags);
1577 if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {
1578 domain->iommu_count++;
1579 domain_update_iommu_cap(domain);
1580 }
1581 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1582 return 0;
1583 }
1584
1585 static int
1586 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1587 int translation)
1588 {
1589 int ret;
1590 struct pci_dev *tmp, *parent;
1591
1592 ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
1593 pdev->bus->number, pdev->devfn,
1594 translation);
1595 if (ret)
1596 return ret;
1597
1598 /* dependent device mapping */
1599 tmp = pci_find_upstream_pcie_bridge(pdev);
1600 if (!tmp)
1601 return 0;
1602 /* Secondary interface's bus number and devfn 0 */
1603 parent = pdev->bus->self;
1604 while (parent != tmp) {
1605 ret = domain_context_mapping_one(domain,
1606 pci_domain_nr(parent->bus),
1607 parent->bus->number,
1608 parent->devfn, translation);
1609 if (ret)
1610 return ret;
1611 parent = parent->bus->self;
1612 }
1613 if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
1614 return domain_context_mapping_one(domain,
1615 pci_domain_nr(tmp->subordinate),
1616 tmp->subordinate->number, 0,
1617 translation);
1618 else /* this is a legacy PCI bridge */
1619 return domain_context_mapping_one(domain,
1620 pci_domain_nr(tmp->bus),
1621 tmp->bus->number,
1622 tmp->devfn,
1623 translation);
1624 }
1625
1626 static int domain_context_mapped(struct pci_dev *pdev)
1627 {
1628 int ret;
1629 struct pci_dev *tmp, *parent;
1630 struct intel_iommu *iommu;
1631
1632 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1633 pdev->devfn);
1634 if (!iommu)
1635 return -ENODEV;
1636
1637 ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
1638 if (!ret)
1639 return ret;
1640 /* dependent device mapping */
1641 tmp = pci_find_upstream_pcie_bridge(pdev);
1642 if (!tmp)
1643 return ret;
1644 /* Secondary interface's bus number and devfn 0 */
1645 parent = pdev->bus->self;
1646 while (parent != tmp) {
1647 ret = device_context_mapped(iommu, parent->bus->number,
1648 parent->devfn);
1649 if (!ret)
1650 return ret;
1651 parent = parent->bus->self;
1652 }
1653 if (tmp->is_pcie)
1654 return device_context_mapped(iommu, tmp->subordinate->number,
1655 0);
1656 else
1657 return device_context_mapped(iommu, tmp->bus->number,
1658 tmp->devfn);
1659 }
1660
1661 /* Returns a number of VTD pages, but aligned to MM page size */
1662 static inline unsigned long aligned_nrpages(unsigned long host_addr,
1663 size_t size)
1664 {
1665 host_addr &= ~PAGE_MASK;
1666 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1667 }
1668
1669 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1670 struct scatterlist *sg, unsigned long phys_pfn,
1671 unsigned long nr_pages, int prot)
1672 {
1673 struct dma_pte *first_pte = NULL, *pte = NULL;
1674 phys_addr_t uninitialized_var(pteval);
1675 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1676 unsigned long sg_res;
1677
1678 BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1679
1680 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1681 return -EINVAL;
1682
1683 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1684
1685 if (sg)
1686 sg_res = 0;
1687 else {
1688 sg_res = nr_pages + 1;
1689 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1690 }
1691
1692 while (nr_pages--) {
1693 uint64_t tmp;
1694
1695 if (!sg_res) {
1696 sg_res = aligned_nrpages(sg->offset, sg->length);
1697 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1698 sg->dma_length = sg->length;
1699 pteval = page_to_phys(sg_page(sg)) | prot;
1700 }
1701 if (!pte) {
1702 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn);
1703 if (!pte)
1704 return -ENOMEM;
1705 }
1706 /* We don't need lock here, nobody else
1707 * touches the iova range
1708 */
1709 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
1710 if (tmp) {
1711 static int dumps = 5;
1712 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1713 iov_pfn, tmp, (unsigned long long)pteval);
1714 if (dumps) {
1715 dumps--;
1716 debug_dma_dump_mappings(NULL);
1717 }
1718 WARN_ON(1);
1719 }
1720 pte++;
1721 if (!nr_pages || first_pte_in_page(pte)) {
1722 domain_flush_cache(domain, first_pte,
1723 (void *)pte - (void *)first_pte);
1724 pte = NULL;
1725 }
1726 iov_pfn++;
1727 pteval += VTD_PAGE_SIZE;
1728 sg_res--;
1729 if (!sg_res)
1730 sg = sg_next(sg);
1731 }
1732 return 0;
1733 }
1734
1735 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1736 struct scatterlist *sg, unsigned long nr_pages,
1737 int prot)
1738 {
1739 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
1740 }
1741
1742 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1743 unsigned long phys_pfn, unsigned long nr_pages,
1744 int prot)
1745 {
1746 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
1747 }
1748
1749 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
1750 {
1751 if (!iommu)
1752 return;
1753
1754 clear_context_table(iommu, bus, devfn);
1755 iommu->flush.flush_context(iommu, 0, 0, 0,
1756 DMA_CCMD_GLOBAL_INVL);
1757 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
1758 }
1759
1760 static void domain_remove_dev_info(struct dmar_domain *domain)
1761 {
1762 struct device_domain_info *info;
1763 unsigned long flags;
1764 struct intel_iommu *iommu;
1765
1766 spin_lock_irqsave(&device_domain_lock, flags);
1767 while (!list_empty(&domain->devices)) {
1768 info = list_entry(domain->devices.next,
1769 struct device_domain_info, link);
1770 list_del(&info->link);
1771 list_del(&info->global);
1772 if (info->dev)
1773 info->dev->dev.archdata.iommu = NULL;
1774 spin_unlock_irqrestore(&device_domain_lock, flags);
1775
1776 iommu_disable_dev_iotlb(info);
1777 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
1778 iommu_detach_dev(iommu, info->bus, info->devfn);
1779 free_devinfo_mem(info);
1780
1781 spin_lock_irqsave(&device_domain_lock, flags);
1782 }
1783 spin_unlock_irqrestore(&device_domain_lock, flags);
1784 }
1785
1786 /*
1787 * find_domain
1788 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1789 */
1790 static struct dmar_domain *
1791 find_domain(struct pci_dev *pdev)
1792 {
1793 struct device_domain_info *info;
1794
1795 /* No lock here, assumes no domain exit in normal case */
1796 info = pdev->dev.archdata.iommu;
1797 if (info)
1798 return info->domain;
1799 return NULL;
1800 }
1801
1802 /* domain is initialized */
1803 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1804 {
1805 struct dmar_domain *domain, *found = NULL;
1806 struct intel_iommu *iommu;
1807 struct dmar_drhd_unit *drhd;
1808 struct device_domain_info *info, *tmp;
1809 struct pci_dev *dev_tmp;
1810 unsigned long flags;
1811 int bus = 0, devfn = 0;
1812 int segment;
1813 int ret;
1814
1815 domain = find_domain(pdev);
1816 if (domain)
1817 return domain;
1818
1819 segment = pci_domain_nr(pdev->bus);
1820
1821 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1822 if (dev_tmp) {
1823 if (dev_tmp->is_pcie) {
1824 bus = dev_tmp->subordinate->number;
1825 devfn = 0;
1826 } else {
1827 bus = dev_tmp->bus->number;
1828 devfn = dev_tmp->devfn;
1829 }
1830 spin_lock_irqsave(&device_domain_lock, flags);
1831 list_for_each_entry(info, &device_domain_list, global) {
1832 if (info->segment == segment &&
1833 info->bus == bus && info->devfn == devfn) {
1834 found = info->domain;
1835 break;
1836 }
1837 }
1838 spin_unlock_irqrestore(&device_domain_lock, flags);
1839 /* pcie-pci bridge already has a domain, uses it */
1840 if (found) {
1841 domain = found;
1842 goto found_domain;
1843 }
1844 }
1845
1846 domain = alloc_domain();
1847 if (!domain)
1848 goto error;
1849
1850 /* Allocate new domain for the device */
1851 drhd = dmar_find_matched_drhd_unit(pdev);
1852 if (!drhd) {
1853 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1854 pci_name(pdev));
1855 return NULL;
1856 }
1857 iommu = drhd->iommu;
1858
1859 ret = iommu_attach_domain(domain, iommu);
1860 if (ret) {
1861 free_domain_mem(domain);
1862 goto error;
1863 }
1864
1865 if (domain_init(domain, gaw)) {
1866 domain_exit(domain);
1867 goto error;
1868 }
1869
1870 /* register pcie-to-pci device */
1871 if (dev_tmp) {
1872 info = alloc_devinfo_mem();
1873 if (!info) {
1874 domain_exit(domain);
1875 goto error;
1876 }
1877 info->segment = segment;
1878 info->bus = bus;
1879 info->devfn = devfn;
1880 info->dev = NULL;
1881 info->domain = domain;
1882 /* This domain is shared by devices under p2p bridge */
1883 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
1884
1885 /* pcie-to-pci bridge already has a domain, uses it */
1886 found = NULL;
1887 spin_lock_irqsave(&device_domain_lock, flags);
1888 list_for_each_entry(tmp, &device_domain_list, global) {
1889 if (tmp->segment == segment &&
1890 tmp->bus == bus && tmp->devfn == devfn) {
1891 found = tmp->domain;
1892 break;
1893 }
1894 }
1895 if (found) {
1896 free_devinfo_mem(info);
1897 domain_exit(domain);
1898 domain = found;
1899 } else {
1900 list_add(&info->link, &domain->devices);
1901 list_add(&info->global, &device_domain_list);
1902 }
1903 spin_unlock_irqrestore(&device_domain_lock, flags);
1904 }
1905
1906 found_domain:
1907 info = alloc_devinfo_mem();
1908 if (!info)
1909 goto error;
1910 info->segment = segment;
1911 info->bus = pdev->bus->number;
1912 info->devfn = pdev->devfn;
1913 info->dev = pdev;
1914 info->domain = domain;
1915 spin_lock_irqsave(&device_domain_lock, flags);
1916 /* somebody is fast */
1917 found = find_domain(pdev);
1918 if (found != NULL) {
1919 spin_unlock_irqrestore(&device_domain_lock, flags);
1920 if (found != domain) {
1921 domain_exit(domain);
1922 domain = found;
1923 }
1924 free_devinfo_mem(info);
1925 return domain;
1926 }
1927 list_add(&info->link, &domain->devices);
1928 list_add(&info->global, &device_domain_list);
1929 pdev->dev.archdata.iommu = info;
1930 spin_unlock_irqrestore(&device_domain_lock, flags);
1931 return domain;
1932 error:
1933 /* recheck it here, maybe others set it */
1934 return find_domain(pdev);
1935 }
1936
1937 static int iommu_identity_mapping;
1938 #define IDENTMAP_ALL 1
1939 #define IDENTMAP_GFX 2
1940 #define IDENTMAP_AZALIA 4
1941
1942 static int iommu_domain_identity_map(struct dmar_domain *domain,
1943 unsigned long long start,
1944 unsigned long long end)
1945 {
1946 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
1947 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
1948
1949 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
1950 dma_to_mm_pfn(last_vpfn))) {
1951 printk(KERN_ERR "IOMMU: reserve iova failed\n");
1952 return -ENOMEM;
1953 }
1954
1955 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
1956 start, end, domain->id);
1957 /*
1958 * RMRR range might have overlap with physical memory range,
1959 * clear it first
1960 */
1961 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
1962
1963 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
1964 last_vpfn - first_vpfn + 1,
1965 DMA_PTE_READ|DMA_PTE_WRITE);
1966 }
1967
1968 static int iommu_prepare_identity_map(struct pci_dev *pdev,
1969 unsigned long long start,
1970 unsigned long long end)
1971 {
1972 struct dmar_domain *domain;
1973 int ret;
1974
1975 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
1976 if (!domain)
1977 return -ENOMEM;
1978
1979 /* For _hardware_ passthrough, don't bother. But for software
1980 passthrough, we do it anyway -- it may indicate a memory
1981 range which is reserved in E820, so which didn't get set
1982 up to start with in si_domain */
1983 if (domain == si_domain && hw_pass_through) {
1984 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
1985 pci_name(pdev), start, end);
1986 return 0;
1987 }
1988
1989 printk(KERN_INFO
1990 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1991 pci_name(pdev), start, end);
1992
1993 if (end < start) {
1994 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
1995 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
1996 dmi_get_system_info(DMI_BIOS_VENDOR),
1997 dmi_get_system_info(DMI_BIOS_VERSION),
1998 dmi_get_system_info(DMI_PRODUCT_VERSION));
1999 ret = -EIO;
2000 goto error;
2001 }
2002
2003 if (end >> agaw_to_width(domain->agaw)) {
2004 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2005 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2006 agaw_to_width(domain->agaw),
2007 dmi_get_system_info(DMI_BIOS_VENDOR),
2008 dmi_get_system_info(DMI_BIOS_VERSION),
2009 dmi_get_system_info(DMI_PRODUCT_VERSION));
2010 ret = -EIO;
2011 goto error;
2012 }
2013
2014 ret = iommu_domain_identity_map(domain, start, end);
2015 if (ret)
2016 goto error;
2017
2018 /* context entry init */
2019 ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
2020 if (ret)
2021 goto error;
2022
2023 return 0;
2024
2025 error:
2026 domain_exit(domain);
2027 return ret;
2028 }
2029
2030 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2031 struct pci_dev *pdev)
2032 {
2033 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2034 return 0;
2035 return iommu_prepare_identity_map(pdev, rmrr->base_address,
2036 rmrr->end_address + 1);
2037 }
2038
2039 #ifdef CONFIG_DMAR_FLOPPY_WA
2040 static inline void iommu_prepare_isa(void)
2041 {
2042 struct pci_dev *pdev;
2043 int ret;
2044
2045 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2046 if (!pdev)
2047 return;
2048
2049 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
2050 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024);
2051
2052 if (ret)
2053 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2054 "floppy might not work\n");
2055
2056 }
2057 #else
2058 static inline void iommu_prepare_isa(void)
2059 {
2060 return;
2061 }
2062 #endif /* !CONFIG_DMAR_FLPY_WA */
2063
2064 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2065
2066 static int __init si_domain_work_fn(unsigned long start_pfn,
2067 unsigned long end_pfn, void *datax)
2068 {
2069 int *ret = datax;
2070
2071 *ret = iommu_domain_identity_map(si_domain,
2072 (uint64_t)start_pfn << PAGE_SHIFT,
2073 (uint64_t)end_pfn << PAGE_SHIFT);
2074 return *ret;
2075
2076 }
2077
2078 static int __init si_domain_init(int hw)
2079 {
2080 struct dmar_drhd_unit *drhd;
2081 struct intel_iommu *iommu;
2082 int nid, ret = 0;
2083
2084 si_domain = alloc_domain();
2085 if (!si_domain)
2086 return -EFAULT;
2087
2088 pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
2089
2090 for_each_active_iommu(iommu, drhd) {
2091 ret = iommu_attach_domain(si_domain, iommu);
2092 if (ret) {
2093 domain_exit(si_domain);
2094 return -EFAULT;
2095 }
2096 }
2097
2098 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2099 domain_exit(si_domain);
2100 return -EFAULT;
2101 }
2102
2103 si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2104
2105 if (hw)
2106 return 0;
2107
2108 for_each_online_node(nid) {
2109 work_with_active_regions(nid, si_domain_work_fn, &ret);
2110 if (ret)
2111 return ret;
2112 }
2113
2114 return 0;
2115 }
2116
2117 static void domain_remove_one_dev_info(struct dmar_domain *domain,
2118 struct pci_dev *pdev);
2119 static int identity_mapping(struct pci_dev *pdev)
2120 {
2121 struct device_domain_info *info;
2122
2123 if (likely(!iommu_identity_mapping))
2124 return 0;
2125
2126
2127 list_for_each_entry(info, &si_domain->devices, link)
2128 if (info->dev == pdev)
2129 return 1;
2130 return 0;
2131 }
2132
2133 static int domain_add_dev_info(struct dmar_domain *domain,
2134 struct pci_dev *pdev,
2135 int translation)
2136 {
2137 struct device_domain_info *info;
2138 unsigned long flags;
2139 int ret;
2140
2141 info = alloc_devinfo_mem();
2142 if (!info)
2143 return -ENOMEM;
2144
2145 ret = domain_context_mapping(domain, pdev, translation);
2146 if (ret) {
2147 free_devinfo_mem(info);
2148 return ret;
2149 }
2150
2151 info->segment = pci_domain_nr(pdev->bus);
2152 info->bus = pdev->bus->number;
2153 info->devfn = pdev->devfn;
2154 info->dev = pdev;
2155 info->domain = domain;
2156
2157 spin_lock_irqsave(&device_domain_lock, flags);
2158 list_add(&info->link, &domain->devices);
2159 list_add(&info->global, &device_domain_list);
2160 pdev->dev.archdata.iommu = info;
2161 spin_unlock_irqrestore(&device_domain_lock, flags);
2162
2163 return 0;
2164 }
2165
2166 static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
2167 {
2168 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2169 return 1;
2170
2171 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2172 return 1;
2173
2174 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2175 return 0;
2176
2177 /*
2178 * We want to start off with all devices in the 1:1 domain, and
2179 * take them out later if we find they can't access all of memory.
2180 *
2181 * However, we can't do this for PCI devices behind bridges,
2182 * because all PCI devices behind the same bridge will end up
2183 * with the same source-id on their transactions.
2184 *
2185 * Practically speaking, we can't change things around for these
2186 * devices at run-time, because we can't be sure there'll be no
2187 * DMA transactions in flight for any of their siblings.
2188 *
2189 * So PCI devices (unless they're on the root bus) as well as
2190 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2191 * the 1:1 domain, just in _case_ one of their siblings turns out
2192 * not to be able to map all of memory.
2193 */
2194 if (!pdev->is_pcie) {
2195 if (!pci_is_root_bus(pdev->bus))
2196 return 0;
2197 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2198 return 0;
2199 } else if (pdev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
2200 return 0;
2201
2202 /*
2203 * At boot time, we don't yet know if devices will be 64-bit capable.
2204 * Assume that they will -- if they turn out not to be, then we can
2205 * take them out of the 1:1 domain later.
2206 */
2207 if (!startup)
2208 return pdev->dma_mask > DMA_BIT_MASK(32);
2209
2210 return 1;
2211 }
2212
2213 static int __init iommu_prepare_static_identity_mapping(int hw)
2214 {
2215 struct pci_dev *pdev = NULL;
2216 int ret;
2217
2218 ret = si_domain_init(hw);
2219 if (ret)
2220 return -EFAULT;
2221
2222 for_each_pci_dev(pdev) {
2223 if (iommu_should_identity_map(pdev, 1)) {
2224 printk(KERN_INFO "IOMMU: %s identity mapping for device %s\n",
2225 hw ? "hardware" : "software", pci_name(pdev));
2226
2227 ret = domain_add_dev_info(si_domain, pdev,
2228 hw ? CONTEXT_TT_PASS_THROUGH :
2229 CONTEXT_TT_MULTI_LEVEL);
2230 if (ret)
2231 return ret;
2232 }
2233 }
2234
2235 return 0;
2236 }
2237
2238 int __init init_dmars(void)
2239 {
2240 struct dmar_drhd_unit *drhd;
2241 struct dmar_rmrr_unit *rmrr;
2242 struct pci_dev *pdev;
2243 struct intel_iommu *iommu;
2244 int i, ret;
2245
2246 /*
2247 * for each drhd
2248 * allocate root
2249 * initialize and program root entry to not present
2250 * endfor
2251 */
2252 for_each_drhd_unit(drhd) {
2253 g_num_of_iommus++;
2254 /*
2255 * lock not needed as this is only incremented in the single
2256 * threaded kernel __init code path all other access are read
2257 * only
2258 */
2259 }
2260
2261 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2262 GFP_KERNEL);
2263 if (!g_iommus) {
2264 printk(KERN_ERR "Allocating global iommu array failed\n");
2265 ret = -ENOMEM;
2266 goto error;
2267 }
2268
2269 deferred_flush = kzalloc(g_num_of_iommus *
2270 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2271 if (!deferred_flush) {
2272 ret = -ENOMEM;
2273 goto error;
2274 }
2275
2276 for_each_drhd_unit(drhd) {
2277 if (drhd->ignored)
2278 continue;
2279
2280 iommu = drhd->iommu;
2281 g_iommus[iommu->seq_id] = iommu;
2282
2283 ret = iommu_init_domains(iommu);
2284 if (ret)
2285 goto error;
2286
2287 /*
2288 * TBD:
2289 * we could share the same root & context tables
2290 * amoung all IOMMU's. Need to Split it later.
2291 */
2292 ret = iommu_alloc_root_entry(iommu);
2293 if (ret) {
2294 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2295 goto error;
2296 }
2297 if (!ecap_pass_through(iommu->ecap))
2298 hw_pass_through = 0;
2299 }
2300
2301 /*
2302 * Start from the sane iommu hardware state.
2303 */
2304 for_each_drhd_unit(drhd) {
2305 if (drhd->ignored)
2306 continue;
2307
2308 iommu = drhd->iommu;
2309
2310 /*
2311 * If the queued invalidation is already initialized by us
2312 * (for example, while enabling interrupt-remapping) then
2313 * we got the things already rolling from a sane state.
2314 */
2315 if (iommu->qi)
2316 continue;
2317
2318 /*
2319 * Clear any previous faults.
2320 */
2321 dmar_fault(-1, iommu);
2322 /*
2323 * Disable queued invalidation if supported and already enabled
2324 * before OS handover.
2325 */
2326 dmar_disable_qi(iommu);
2327 }
2328
2329 for_each_drhd_unit(drhd) {
2330 if (drhd->ignored)
2331 continue;
2332
2333 iommu = drhd->iommu;
2334
2335 if (dmar_enable_qi(iommu)) {
2336 /*
2337 * Queued Invalidate not enabled, use Register Based
2338 * Invalidate
2339 */
2340 iommu->flush.flush_context = __iommu_flush_context;
2341 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2342 printk(KERN_INFO "IOMMU 0x%Lx: using Register based "
2343 "invalidation\n",
2344 (unsigned long long)drhd->reg_base_addr);
2345 } else {
2346 iommu->flush.flush_context = qi_flush_context;
2347 iommu->flush.flush_iotlb = qi_flush_iotlb;
2348 printk(KERN_INFO "IOMMU 0x%Lx: using Queued "
2349 "invalidation\n",
2350 (unsigned long long)drhd->reg_base_addr);
2351 }
2352 }
2353
2354 if (iommu_pass_through)
2355 iommu_identity_mapping |= IDENTMAP_ALL;
2356
2357 #ifdef CONFIG_DMAR_BROKEN_GFX_WA
2358 iommu_identity_mapping |= IDENTMAP_GFX;
2359 #endif
2360
2361 check_tylersburg_isoch();
2362
2363 /*
2364 * If pass through is not set or not enabled, setup context entries for
2365 * identity mappings for rmrr, gfx, and isa and may fall back to static
2366 * identity mapping if iommu_identity_mapping is set.
2367 */
2368 if (iommu_identity_mapping) {
2369 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
2370 if (ret) {
2371 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2372 goto error;
2373 }
2374 }
2375 /*
2376 * For each rmrr
2377 * for each dev attached to rmrr
2378 * do
2379 * locate drhd for dev, alloc domain for dev
2380 * allocate free domain
2381 * allocate page table entries for rmrr
2382 * if context not allocated for bus
2383 * allocate and init context
2384 * set present in root table for this bus
2385 * init context with domain, translation etc
2386 * endfor
2387 * endfor
2388 */
2389 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2390 for_each_rmrr_units(rmrr) {
2391 for (i = 0; i < rmrr->devices_cnt; i++) {
2392 pdev = rmrr->devices[i];
2393 /*
2394 * some BIOS lists non-exist devices in DMAR
2395 * table.
2396 */
2397 if (!pdev)
2398 continue;
2399 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2400 if (ret)
2401 printk(KERN_ERR
2402 "IOMMU: mapping reserved region failed\n");
2403 }
2404 }
2405
2406 iommu_prepare_isa();
2407
2408 /*
2409 * for each drhd
2410 * enable fault log
2411 * global invalidate context cache
2412 * global invalidate iotlb
2413 * enable translation
2414 */
2415 for_each_drhd_unit(drhd) {
2416 if (drhd->ignored)
2417 continue;
2418 iommu = drhd->iommu;
2419
2420 iommu_flush_write_buffer(iommu);
2421
2422 ret = dmar_set_interrupt(iommu);
2423 if (ret)
2424 goto error;
2425
2426 iommu_set_root_entry(iommu);
2427
2428 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
2429 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2430
2431 ret = iommu_enable_translation(iommu);
2432 if (ret)
2433 goto error;
2434
2435 iommu_disable_protect_mem_regions(iommu);
2436 }
2437
2438 return 0;
2439 error:
2440 for_each_drhd_unit(drhd) {
2441 if (drhd->ignored)
2442 continue;
2443 iommu = drhd->iommu;
2444 free_iommu(iommu);
2445 }
2446 kfree(g_iommus);
2447 return ret;
2448 }
2449
2450 /* This takes a number of _MM_ pages, not VTD pages */
2451 static struct iova *intel_alloc_iova(struct device *dev,
2452 struct dmar_domain *domain,
2453 unsigned long nrpages, uint64_t dma_mask)
2454 {
2455 struct pci_dev *pdev = to_pci_dev(dev);
2456 struct iova *iova = NULL;
2457
2458 /* Restrict dma_mask to the width that the iommu can handle */
2459 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2460
2461 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
2462 /*
2463 * First try to allocate an io virtual address in
2464 * DMA_BIT_MASK(32) and if that fails then try allocating
2465 * from higher range
2466 */
2467 iova = alloc_iova(&domain->iovad, nrpages,
2468 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2469 if (iova)
2470 return iova;
2471 }
2472 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2473 if (unlikely(!iova)) {
2474 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2475 nrpages, pci_name(pdev));
2476 return NULL;
2477 }
2478
2479 return iova;
2480 }
2481
2482 static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
2483 {
2484 struct dmar_domain *domain;
2485 int ret;
2486
2487 domain = get_domain_for_dev(pdev,
2488 DEFAULT_DOMAIN_ADDRESS_WIDTH);
2489 if (!domain) {
2490 printk(KERN_ERR
2491 "Allocating domain for %s failed", pci_name(pdev));
2492 return NULL;
2493 }
2494
2495 /* make sure context mapping is ok */
2496 if (unlikely(!domain_context_mapped(pdev))) {
2497 ret = domain_context_mapping(domain, pdev,
2498 CONTEXT_TT_MULTI_LEVEL);
2499 if (ret) {
2500 printk(KERN_ERR
2501 "Domain context map for %s failed",
2502 pci_name(pdev));
2503 return NULL;
2504 }
2505 }
2506
2507 return domain;
2508 }
2509
2510 static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
2511 {
2512 struct device_domain_info *info;
2513
2514 /* No lock here, assumes no domain exit in normal case */
2515 info = dev->dev.archdata.iommu;
2516 if (likely(info))
2517 return info->domain;
2518
2519 return __get_valid_domain_for_dev(dev);
2520 }
2521
2522 static int iommu_dummy(struct pci_dev *pdev)
2523 {
2524 return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2525 }
2526
2527 /* Check if the pdev needs to go through non-identity map and unmap process.*/
2528 static int iommu_no_mapping(struct device *dev)
2529 {
2530 struct pci_dev *pdev;
2531 int found;
2532
2533 if (unlikely(dev->bus != &pci_bus_type))
2534 return 1;
2535
2536 pdev = to_pci_dev(dev);
2537 if (iommu_dummy(pdev))
2538 return 1;
2539
2540 if (!iommu_identity_mapping)
2541 return 0;
2542
2543 found = identity_mapping(pdev);
2544 if (found) {
2545 if (iommu_should_identity_map(pdev, 0))
2546 return 1;
2547 else {
2548 /*
2549 * 32 bit DMA is removed from si_domain and fall back
2550 * to non-identity mapping.
2551 */
2552 domain_remove_one_dev_info(si_domain, pdev);
2553 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2554 pci_name(pdev));
2555 return 0;
2556 }
2557 } else {
2558 /*
2559 * In case of a detached 64 bit DMA device from vm, the device
2560 * is put into si_domain for identity mapping.
2561 */
2562 if (iommu_should_identity_map(pdev, 0)) {
2563 int ret;
2564 ret = domain_add_dev_info(si_domain, pdev,
2565 hw_pass_through ?
2566 CONTEXT_TT_PASS_THROUGH :
2567 CONTEXT_TT_MULTI_LEVEL);
2568 if (!ret) {
2569 printk(KERN_INFO "64bit %s uses identity mapping\n",
2570 pci_name(pdev));
2571 return 1;
2572 }
2573 }
2574 }
2575
2576 return 0;
2577 }
2578
2579 static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2580 size_t size, int dir, u64 dma_mask)
2581 {
2582 struct pci_dev *pdev = to_pci_dev(hwdev);
2583 struct dmar_domain *domain;
2584 phys_addr_t start_paddr;
2585 struct iova *iova;
2586 int prot = 0;
2587 int ret;
2588 struct intel_iommu *iommu;
2589 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
2590
2591 BUG_ON(dir == DMA_NONE);
2592
2593 if (iommu_no_mapping(hwdev))
2594 return paddr;
2595
2596 domain = get_valid_domain_for_dev(pdev);
2597 if (!domain)
2598 return 0;
2599
2600 iommu = domain_get_iommu(domain);
2601 size = aligned_nrpages(paddr, size);
2602
2603 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
2604 pdev->dma_mask);
2605 if (!iova)
2606 goto error;
2607
2608 /*
2609 * Check if DMAR supports zero-length reads on write only
2610 * mappings..
2611 */
2612 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2613 !cap_zlr(iommu->cap))
2614 prot |= DMA_PTE_READ;
2615 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2616 prot |= DMA_PTE_WRITE;
2617 /*
2618 * paddr - (paddr + size) might be partial page, we should map the whole
2619 * page. Note: if two part of one page are separately mapped, we
2620 * might have two guest_addr mapping to the same host paddr, but this
2621 * is not a big problem
2622 */
2623 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
2624 mm_to_dma_pfn(paddr_pfn), size, prot);
2625 if (ret)
2626 goto error;
2627
2628 /* it's a non-present to present mapping. Only flush if caching mode */
2629 if (cap_caching_mode(iommu->cap))
2630 iommu_flush_iotlb_psi(iommu, 0, mm_to_dma_pfn(iova->pfn_lo), size);
2631 else
2632 iommu_flush_write_buffer(iommu);
2633
2634 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2635 start_paddr += paddr & ~PAGE_MASK;
2636 return start_paddr;
2637
2638 error:
2639 if (iova)
2640 __free_iova(&domain->iovad, iova);
2641 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
2642 pci_name(pdev), size, (unsigned long long)paddr, dir);
2643 return 0;
2644 }
2645
2646 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2647 unsigned long offset, size_t size,
2648 enum dma_data_direction dir,
2649 struct dma_attrs *attrs)
2650 {
2651 return __intel_map_single(dev, page_to_phys(page) + offset, size,
2652 dir, to_pci_dev(dev)->dma_mask);
2653 }
2654
2655 static void flush_unmaps(void)
2656 {
2657 int i, j;
2658
2659 timer_on = 0;
2660
2661 /* just flush them all */
2662 for (i = 0; i < g_num_of_iommus; i++) {
2663 struct intel_iommu *iommu = g_iommus[i];
2664 if (!iommu)
2665 continue;
2666
2667 if (!deferred_flush[i].next)
2668 continue;
2669
2670 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2671 DMA_TLB_GLOBAL_FLUSH);
2672 for (j = 0; j < deferred_flush[i].next; j++) {
2673 unsigned long mask;
2674 struct iova *iova = deferred_flush[i].iova[j];
2675
2676 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
2677 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
2678 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
2679 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
2680 }
2681 deferred_flush[i].next = 0;
2682 }
2683
2684 list_size = 0;
2685 }
2686
2687 static void flush_unmaps_timeout(unsigned long data)
2688 {
2689 unsigned long flags;
2690
2691 spin_lock_irqsave(&async_umap_flush_lock, flags);
2692 flush_unmaps();
2693 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2694 }
2695
2696 static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2697 {
2698 unsigned long flags;
2699 int next, iommu_id;
2700 struct intel_iommu *iommu;
2701
2702 spin_lock_irqsave(&async_umap_flush_lock, flags);
2703 if (list_size == HIGH_WATER_MARK)
2704 flush_unmaps();
2705
2706 iommu = domain_get_iommu(dom);
2707 iommu_id = iommu->seq_id;
2708
2709 next = deferred_flush[iommu_id].next;
2710 deferred_flush[iommu_id].domain[next] = dom;
2711 deferred_flush[iommu_id].iova[next] = iova;
2712 deferred_flush[iommu_id].next++;
2713
2714 if (!timer_on) {
2715 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2716 timer_on = 1;
2717 }
2718 list_size++;
2719 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2720 }
2721
2722 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2723 size_t size, enum dma_data_direction dir,
2724 struct dma_attrs *attrs)
2725 {
2726 struct pci_dev *pdev = to_pci_dev(dev);
2727 struct dmar_domain *domain;
2728 unsigned long start_pfn, last_pfn;
2729 struct iova *iova;
2730 struct intel_iommu *iommu;
2731
2732 if (iommu_no_mapping(dev))
2733 return;
2734
2735 domain = find_domain(pdev);
2736 BUG_ON(!domain);
2737
2738 iommu = domain_get_iommu(domain);
2739
2740 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
2741 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
2742 (unsigned long long)dev_addr))
2743 return;
2744
2745 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2746 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2747
2748 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2749 pci_name(pdev), start_pfn, last_pfn);
2750
2751 /* clear the whole page */
2752 dma_pte_clear_range(domain, start_pfn, last_pfn);
2753
2754 /* free page tables */
2755 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2756
2757 if (intel_iommu_strict) {
2758 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2759 last_pfn - start_pfn + 1);
2760 /* free iova */
2761 __free_iova(&domain->iovad, iova);
2762 } else {
2763 add_unmap(domain, iova);
2764 /*
2765 * queue up the release of the unmap to save the 1/6th of the
2766 * cpu used up by the iotlb flush operation...
2767 */
2768 }
2769 }
2770
2771 static void *intel_alloc_coherent(struct device *hwdev, size_t size,
2772 dma_addr_t *dma_handle, gfp_t flags)
2773 {
2774 void *vaddr;
2775 int order;
2776
2777 size = PAGE_ALIGN(size);
2778 order = get_order(size);
2779
2780 if (!iommu_no_mapping(hwdev))
2781 flags &= ~(GFP_DMA | GFP_DMA32);
2782 else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
2783 if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
2784 flags |= GFP_DMA;
2785 else
2786 flags |= GFP_DMA32;
2787 }
2788
2789 vaddr = (void *)__get_free_pages(flags, order);
2790 if (!vaddr)
2791 return NULL;
2792 memset(vaddr, 0, size);
2793
2794 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2795 DMA_BIDIRECTIONAL,
2796 hwdev->coherent_dma_mask);
2797 if (*dma_handle)
2798 return vaddr;
2799 free_pages((unsigned long)vaddr, order);
2800 return NULL;
2801 }
2802
2803 static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2804 dma_addr_t dma_handle)
2805 {
2806 int order;
2807
2808 size = PAGE_ALIGN(size);
2809 order = get_order(size);
2810
2811 intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
2812 free_pages((unsigned long)vaddr, order);
2813 }
2814
2815 static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2816 int nelems, enum dma_data_direction dir,
2817 struct dma_attrs *attrs)
2818 {
2819 struct pci_dev *pdev = to_pci_dev(hwdev);
2820 struct dmar_domain *domain;
2821 unsigned long start_pfn, last_pfn;
2822 struct iova *iova;
2823 struct intel_iommu *iommu;
2824
2825 if (iommu_no_mapping(hwdev))
2826 return;
2827
2828 domain = find_domain(pdev);
2829 BUG_ON(!domain);
2830
2831 iommu = domain_get_iommu(domain);
2832
2833 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
2834 if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
2835 (unsigned long long)sglist[0].dma_address))
2836 return;
2837
2838 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2839 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2840
2841 /* clear the whole page */
2842 dma_pte_clear_range(domain, start_pfn, last_pfn);
2843
2844 /* free page tables */
2845 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2846
2847 if (intel_iommu_strict) {
2848 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2849 last_pfn - start_pfn + 1);
2850 /* free iova */
2851 __free_iova(&domain->iovad, iova);
2852 } else {
2853 add_unmap(domain, iova);
2854 /*
2855 * queue up the release of the unmap to save the 1/6th of the
2856 * cpu used up by the iotlb flush operation...
2857 */
2858 }
2859 }
2860
2861 static int intel_nontranslate_map_sg(struct device *hddev,
2862 struct scatterlist *sglist, int nelems, int dir)
2863 {
2864 int i;
2865 struct scatterlist *sg;
2866
2867 for_each_sg(sglist, sg, nelems, i) {
2868 BUG_ON(!sg_page(sg));
2869 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
2870 sg->dma_length = sg->length;
2871 }
2872 return nelems;
2873 }
2874
2875 static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
2876 enum dma_data_direction dir, struct dma_attrs *attrs)
2877 {
2878 int i;
2879 struct pci_dev *pdev = to_pci_dev(hwdev);
2880 struct dmar_domain *domain;
2881 size_t size = 0;
2882 int prot = 0;
2883 size_t offset_pfn = 0;
2884 struct iova *iova = NULL;
2885 int ret;
2886 struct scatterlist *sg;
2887 unsigned long start_vpfn;
2888 struct intel_iommu *iommu;
2889
2890 BUG_ON(dir == DMA_NONE);
2891 if (iommu_no_mapping(hwdev))
2892 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
2893
2894 domain = get_valid_domain_for_dev(pdev);
2895 if (!domain)
2896 return 0;
2897
2898 iommu = domain_get_iommu(domain);
2899
2900 for_each_sg(sglist, sg, nelems, i)
2901 size += aligned_nrpages(sg->offset, sg->length);
2902
2903 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
2904 pdev->dma_mask);
2905 if (!iova) {
2906 sglist->dma_length = 0;
2907 return 0;
2908 }
2909
2910 /*
2911 * Check if DMAR supports zero-length reads on write only
2912 * mappings..
2913 */
2914 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2915 !cap_zlr(iommu->cap))
2916 prot |= DMA_PTE_READ;
2917 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2918 prot |= DMA_PTE_WRITE;
2919
2920 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
2921
2922 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
2923 if (unlikely(ret)) {
2924 /* clear the page */
2925 dma_pte_clear_range(domain, start_vpfn,
2926 start_vpfn + size - 1);
2927 /* free page tables */
2928 dma_pte_free_pagetable(domain, start_vpfn,
2929 start_vpfn + size - 1);
2930 /* free iova */
2931 __free_iova(&domain->iovad, iova);
2932 return 0;
2933 }
2934
2935 /* it's a non-present to present mapping. Only flush if caching mode */
2936 if (cap_caching_mode(iommu->cap))
2937 iommu_flush_iotlb_psi(iommu, 0, start_vpfn, offset_pfn);
2938 else
2939 iommu_flush_write_buffer(iommu);
2940
2941 return nelems;
2942 }
2943
2944 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
2945 {
2946 return !dma_addr;
2947 }
2948
2949 struct dma_map_ops intel_dma_ops = {
2950 .alloc_coherent = intel_alloc_coherent,
2951 .free_coherent = intel_free_coherent,
2952 .map_sg = intel_map_sg,
2953 .unmap_sg = intel_unmap_sg,
2954 .map_page = intel_map_page,
2955 .unmap_page = intel_unmap_page,
2956 .mapping_error = intel_mapping_error,
2957 };
2958
2959 static inline int iommu_domain_cache_init(void)
2960 {
2961 int ret = 0;
2962
2963 iommu_domain_cache = kmem_cache_create("iommu_domain",
2964 sizeof(struct dmar_domain),
2965 0,
2966 SLAB_HWCACHE_ALIGN,
2967
2968 NULL);
2969 if (!iommu_domain_cache) {
2970 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
2971 ret = -ENOMEM;
2972 }
2973
2974 return ret;
2975 }
2976
2977 static inline int iommu_devinfo_cache_init(void)
2978 {
2979 int ret = 0;
2980
2981 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
2982 sizeof(struct device_domain_info),
2983 0,
2984 SLAB_HWCACHE_ALIGN,
2985 NULL);
2986 if (!iommu_devinfo_cache) {
2987 printk(KERN_ERR "Couldn't create devinfo cache\n");
2988 ret = -ENOMEM;
2989 }
2990
2991 return ret;
2992 }
2993
2994 static inline int iommu_iova_cache_init(void)
2995 {
2996 int ret = 0;
2997
2998 iommu_iova_cache = kmem_cache_create("iommu_iova",
2999 sizeof(struct iova),
3000 0,
3001 SLAB_HWCACHE_ALIGN,
3002 NULL);
3003 if (!iommu_iova_cache) {
3004 printk(KERN_ERR "Couldn't create iova cache\n");
3005 ret = -ENOMEM;
3006 }
3007
3008 return ret;
3009 }
3010
3011 static int __init iommu_init_mempool(void)
3012 {
3013 int ret;
3014 ret = iommu_iova_cache_init();
3015 if (ret)
3016 return ret;
3017
3018 ret = iommu_domain_cache_init();
3019 if (ret)
3020 goto domain_error;
3021
3022 ret = iommu_devinfo_cache_init();
3023 if (!ret)
3024 return ret;
3025
3026 kmem_cache_destroy(iommu_domain_cache);
3027 domain_error:
3028 kmem_cache_destroy(iommu_iova_cache);
3029
3030 return -ENOMEM;
3031 }
3032
3033 static void __init iommu_exit_mempool(void)
3034 {
3035 kmem_cache_destroy(iommu_devinfo_cache);
3036 kmem_cache_destroy(iommu_domain_cache);
3037 kmem_cache_destroy(iommu_iova_cache);
3038
3039 }
3040
3041 static void __init init_no_remapping_devices(void)
3042 {
3043 struct dmar_drhd_unit *drhd;
3044
3045 for_each_drhd_unit(drhd) {
3046 if (!drhd->include_all) {
3047 int i;
3048 for (i = 0; i < drhd->devices_cnt; i++)
3049 if (drhd->devices[i] != NULL)
3050 break;
3051 /* ignore DMAR unit if no pci devices exist */
3052 if (i == drhd->devices_cnt)
3053 drhd->ignored = 1;
3054 }
3055 }
3056
3057 if (dmar_map_gfx)
3058 return;
3059
3060 for_each_drhd_unit(drhd) {
3061 int i;
3062 if (drhd->ignored || drhd->include_all)
3063 continue;
3064
3065 for (i = 0; i < drhd->devices_cnt; i++)
3066 if (drhd->devices[i] &&
3067 !IS_GFX_DEVICE(drhd->devices[i]))
3068 break;
3069
3070 if (i < drhd->devices_cnt)
3071 continue;
3072
3073 /* bypass IOMMU if it is just for gfx devices */
3074 drhd->ignored = 1;
3075 for (i = 0; i < drhd->devices_cnt; i++) {
3076 if (!drhd->devices[i])
3077 continue;
3078 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3079 }
3080 }
3081 }
3082
3083 #ifdef CONFIG_SUSPEND
3084 static int init_iommu_hw(void)
3085 {
3086 struct dmar_drhd_unit *drhd;
3087 struct intel_iommu *iommu = NULL;
3088
3089 for_each_active_iommu(iommu, drhd)
3090 if (iommu->qi)
3091 dmar_reenable_qi(iommu);
3092
3093 for_each_active_iommu(iommu, drhd) {
3094 iommu_flush_write_buffer(iommu);
3095
3096 iommu_set_root_entry(iommu);
3097
3098 iommu->flush.flush_context(iommu, 0, 0, 0,
3099 DMA_CCMD_GLOBAL_INVL);
3100 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3101 DMA_TLB_GLOBAL_FLUSH);
3102 iommu_enable_translation(iommu);
3103 iommu_disable_protect_mem_regions(iommu);
3104 }
3105
3106 return 0;
3107 }
3108
3109 static void iommu_flush_all(void)
3110 {
3111 struct dmar_drhd_unit *drhd;
3112 struct intel_iommu *iommu;
3113
3114 for_each_active_iommu(iommu, drhd) {
3115 iommu->flush.flush_context(iommu, 0, 0, 0,
3116 DMA_CCMD_GLOBAL_INVL);
3117 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3118 DMA_TLB_GLOBAL_FLUSH);
3119 }
3120 }
3121
3122 static int iommu_suspend(struct sys_device *dev, pm_message_t state)
3123 {
3124 struct dmar_drhd_unit *drhd;
3125 struct intel_iommu *iommu = NULL;
3126 unsigned long flag;
3127
3128 for_each_active_iommu(iommu, drhd) {
3129 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3130 GFP_ATOMIC);
3131 if (!iommu->iommu_state)
3132 goto nomem;
3133 }
3134
3135 iommu_flush_all();
3136
3137 for_each_active_iommu(iommu, drhd) {
3138 iommu_disable_translation(iommu);
3139
3140 spin_lock_irqsave(&iommu->register_lock, flag);
3141
3142 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3143 readl(iommu->reg + DMAR_FECTL_REG);
3144 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3145 readl(iommu->reg + DMAR_FEDATA_REG);
3146 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3147 readl(iommu->reg + DMAR_FEADDR_REG);
3148 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3149 readl(iommu->reg + DMAR_FEUADDR_REG);
3150
3151 spin_unlock_irqrestore(&iommu->register_lock, flag);
3152 }
3153 return 0;
3154
3155 nomem:
3156 for_each_active_iommu(iommu, drhd)
3157 kfree(iommu->iommu_state);
3158
3159 return -ENOMEM;
3160 }
3161
3162 static int iommu_resume(struct sys_device *dev)
3163 {
3164 struct dmar_drhd_unit *drhd;
3165 struct intel_iommu *iommu = NULL;
3166 unsigned long flag;
3167
3168 if (init_iommu_hw()) {
3169 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3170 return -EIO;
3171 }
3172
3173 for_each_active_iommu(iommu, drhd) {
3174
3175 spin_lock_irqsave(&iommu->register_lock, flag);
3176
3177 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3178 iommu->reg + DMAR_FECTL_REG);
3179 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3180 iommu->reg + DMAR_FEDATA_REG);
3181 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3182 iommu->reg + DMAR_FEADDR_REG);
3183 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3184 iommu->reg + DMAR_FEUADDR_REG);
3185
3186 spin_unlock_irqrestore(&iommu->register_lock, flag);
3187 }
3188
3189 for_each_active_iommu(iommu, drhd)
3190 kfree(iommu->iommu_state);
3191
3192 return 0;
3193 }
3194
3195 static struct sysdev_class iommu_sysclass = {
3196 .name = "iommu",
3197 .resume = iommu_resume,
3198 .suspend = iommu_suspend,
3199 };
3200
3201 static struct sys_device device_iommu = {
3202 .cls = &iommu_sysclass,
3203 };
3204
3205 static int __init init_iommu_sysfs(void)
3206 {
3207 int error;
3208
3209 error = sysdev_class_register(&iommu_sysclass);
3210 if (error)
3211 return error;
3212
3213 error = sysdev_register(&device_iommu);
3214 if (error)
3215 sysdev_class_unregister(&iommu_sysclass);
3216
3217 return error;
3218 }
3219
3220 #else
3221 static int __init init_iommu_sysfs(void)
3222 {
3223 return 0;
3224 }
3225 #endif /* CONFIG_PM */
3226
3227 /*
3228 * Here we only respond to action of unbound device from driver.
3229 *
3230 * Added device is not attached to its DMAR domain here yet. That will happen
3231 * when mapping the device to iova.
3232 */
3233 static int device_notifier(struct notifier_block *nb,
3234 unsigned long action, void *data)
3235 {
3236 struct device *dev = data;
3237 struct pci_dev *pdev = to_pci_dev(dev);
3238 struct dmar_domain *domain;
3239
3240 if (iommu_no_mapping(dev))
3241 return 0;
3242
3243 domain = find_domain(pdev);
3244 if (!domain)
3245 return 0;
3246
3247 if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through) {
3248 domain_remove_one_dev_info(domain, pdev);
3249
3250 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3251 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
3252 list_empty(&domain->devices))
3253 domain_exit(domain);
3254 }
3255
3256 return 0;
3257 }
3258
3259 static struct notifier_block device_nb = {
3260 .notifier_call = device_notifier,
3261 };
3262
3263 int __init intel_iommu_init(void)
3264 {
3265 int ret = 0;
3266 int force_on = 0;
3267
3268 /* VT-d is required for a TXT/tboot launch, so enforce that */
3269 force_on = tboot_force_iommu();
3270
3271 if (dmar_table_init()) {
3272 if (force_on)
3273 panic("tboot: Failed to initialize DMAR table\n");
3274 return -ENODEV;
3275 }
3276
3277 if (dmar_dev_scope_init()) {
3278 if (force_on)
3279 panic("tboot: Failed to initialize DMAR device scope\n");
3280 return -ENODEV;
3281 }
3282
3283 /*
3284 * Check the need for DMA-remapping initialization now.
3285 * Above initialization will also be used by Interrupt-remapping.
3286 */
3287 if (no_iommu || swiotlb || dmar_disabled)
3288 return -ENODEV;
3289
3290 iommu_init_mempool();
3291 dmar_init_reserved_ranges();
3292
3293 init_no_remapping_devices();
3294
3295 ret = init_dmars();
3296 if (ret) {
3297 if (force_on)
3298 panic("tboot: Failed to initialize DMARs\n");
3299 printk(KERN_ERR "IOMMU: dmar init failed\n");
3300 put_iova_domain(&reserved_iova_list);
3301 iommu_exit_mempool();
3302 return ret;
3303 }
3304 printk(KERN_INFO
3305 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3306
3307 init_timer(&unmap_timer);
3308 force_iommu = 1;
3309 dma_ops = &intel_dma_ops;
3310
3311 init_iommu_sysfs();
3312
3313 register_iommu(&intel_iommu_ops);
3314
3315 bus_register_notifier(&pci_bus_type, &device_nb);
3316
3317 return 0;
3318 }
3319
3320 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3321 struct pci_dev *pdev)
3322 {
3323 struct pci_dev *tmp, *parent;
3324
3325 if (!iommu || !pdev)
3326 return;
3327
3328 /* dependent device detach */
3329 tmp = pci_find_upstream_pcie_bridge(pdev);
3330 /* Secondary interface's bus number and devfn 0 */
3331 if (tmp) {
3332 parent = pdev->bus->self;
3333 while (parent != tmp) {
3334 iommu_detach_dev(iommu, parent->bus->number,
3335 parent->devfn);
3336 parent = parent->bus->self;
3337 }
3338 if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
3339 iommu_detach_dev(iommu,
3340 tmp->subordinate->number, 0);
3341 else /* this is a legacy PCI bridge */
3342 iommu_detach_dev(iommu, tmp->bus->number,
3343 tmp->devfn);
3344 }
3345 }
3346
3347 static void domain_remove_one_dev_info(struct dmar_domain *domain,
3348 struct pci_dev *pdev)
3349 {
3350 struct device_domain_info *info;
3351 struct intel_iommu *iommu;
3352 unsigned long flags;
3353 int found = 0;
3354 struct list_head *entry, *tmp;
3355
3356 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3357 pdev->devfn);
3358 if (!iommu)
3359 return;
3360
3361 spin_lock_irqsave(&device_domain_lock, flags);
3362 list_for_each_safe(entry, tmp, &domain->devices) {
3363 info = list_entry(entry, struct device_domain_info, link);
3364 /* No need to compare PCI domain; it has to be the same */
3365 if (info->bus == pdev->bus->number &&
3366 info->devfn == pdev->devfn) {
3367 list_del(&info->link);
3368 list_del(&info->global);
3369 if (info->dev)
3370 info->dev->dev.archdata.iommu = NULL;
3371 spin_unlock_irqrestore(&device_domain_lock, flags);
3372
3373 iommu_disable_dev_iotlb(info);
3374 iommu_detach_dev(iommu, info->bus, info->devfn);
3375 iommu_detach_dependent_devices(iommu, pdev);
3376 free_devinfo_mem(info);
3377
3378 spin_lock_irqsave(&device_domain_lock, flags);
3379
3380 if (found)
3381 break;
3382 else
3383 continue;
3384 }
3385
3386 /* if there is no other devices under the same iommu
3387 * owned by this domain, clear this iommu in iommu_bmp
3388 * update iommu count and coherency
3389 */
3390 if (iommu == device_to_iommu(info->segment, info->bus,
3391 info->devfn))
3392 found = 1;
3393 }
3394
3395 if (found == 0) {
3396 unsigned long tmp_flags;
3397 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
3398 clear_bit(iommu->seq_id, &domain->iommu_bmp);
3399 domain->iommu_count--;
3400 domain_update_iommu_cap(domain);
3401 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
3402
3403 spin_lock_irqsave(&iommu->lock, tmp_flags);
3404 clear_bit(domain->id, iommu->domain_ids);
3405 iommu->domains[domain->id] = NULL;
3406 spin_unlock_irqrestore(&iommu->lock, tmp_flags);
3407 }
3408
3409 spin_unlock_irqrestore(&device_domain_lock, flags);
3410 }
3411
3412 static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3413 {
3414 struct device_domain_info *info;
3415 struct intel_iommu *iommu;
3416 unsigned long flags1, flags2;
3417
3418 spin_lock_irqsave(&device_domain_lock, flags1);
3419 while (!list_empty(&domain->devices)) {
3420 info = list_entry(domain->devices.next,
3421 struct device_domain_info, link);
3422 list_del(&info->link);
3423 list_del(&info->global);
3424 if (info->dev)
3425 info->dev->dev.archdata.iommu = NULL;
3426
3427 spin_unlock_irqrestore(&device_domain_lock, flags1);
3428
3429 iommu_disable_dev_iotlb(info);
3430 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
3431 iommu_detach_dev(iommu, info->bus, info->devfn);
3432 iommu_detach_dependent_devices(iommu, info->dev);
3433
3434 /* clear this iommu in iommu_bmp, update iommu count
3435 * and capabilities
3436 */
3437 spin_lock_irqsave(&domain->iommu_lock, flags2);
3438 if (test_and_clear_bit(iommu->seq_id,
3439 &domain->iommu_bmp)) {
3440 domain->iommu_count--;
3441 domain_update_iommu_cap(domain);
3442 }
3443 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3444
3445 free_devinfo_mem(info);
3446 spin_lock_irqsave(&device_domain_lock, flags1);
3447 }
3448 spin_unlock_irqrestore(&device_domain_lock, flags1);
3449 }
3450
3451 /* domain id for virtual machine, it won't be set in context */
3452 static unsigned long vm_domid;
3453
3454 static int vm_domain_min_agaw(struct dmar_domain *domain)
3455 {
3456 int i;
3457 int min_agaw = domain->agaw;
3458
3459 i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
3460 for (; i < g_num_of_iommus; ) {
3461 if (min_agaw > g_iommus[i]->agaw)
3462 min_agaw = g_iommus[i]->agaw;
3463
3464 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
3465 }
3466
3467 return min_agaw;
3468 }
3469
3470 static struct dmar_domain *iommu_alloc_vm_domain(void)
3471 {
3472 struct dmar_domain *domain;
3473
3474 domain = alloc_domain_mem();
3475 if (!domain)
3476 return NULL;
3477
3478 domain->id = vm_domid++;
3479 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
3480 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3481
3482 return domain;
3483 }
3484
3485 static int md_domain_init(struct dmar_domain *domain, int guest_width)
3486 {
3487 int adjust_width;
3488
3489 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
3490 spin_lock_init(&domain->iommu_lock);
3491
3492 domain_reserve_special_ranges(domain);
3493
3494 /* calculate AGAW */
3495 domain->gaw = guest_width;
3496 adjust_width = guestwidth_to_adjustwidth(guest_width);
3497 domain->agaw = width_to_agaw(adjust_width);
3498
3499 INIT_LIST_HEAD(&domain->devices);
3500
3501 domain->iommu_count = 0;
3502 domain->iommu_coherency = 0;
3503 domain->iommu_snooping = 0;
3504 domain->max_addr = 0;
3505
3506 /* always allocate the top pgd */
3507 domain->pgd = (struct dma_pte *)alloc_pgtable_page();
3508 if (!domain->pgd)
3509 return -ENOMEM;
3510 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3511 return 0;
3512 }
3513
3514 static void iommu_free_vm_domain(struct dmar_domain *domain)
3515 {
3516 unsigned long flags;
3517 struct dmar_drhd_unit *drhd;
3518 struct intel_iommu *iommu;
3519 unsigned long i;
3520 unsigned long ndomains;
3521
3522 for_each_drhd_unit(drhd) {
3523 if (drhd->ignored)
3524 continue;
3525 iommu = drhd->iommu;
3526
3527 ndomains = cap_ndoms(iommu->cap);
3528 i = find_first_bit(iommu->domain_ids, ndomains);
3529 for (; i < ndomains; ) {
3530 if (iommu->domains[i] == domain) {
3531 spin_lock_irqsave(&iommu->lock, flags);
3532 clear_bit(i, iommu->domain_ids);
3533 iommu->domains[i] = NULL;
3534 spin_unlock_irqrestore(&iommu->lock, flags);
3535 break;
3536 }
3537 i = find_next_bit(iommu->domain_ids, ndomains, i+1);
3538 }
3539 }
3540 }
3541
3542 static void vm_domain_exit(struct dmar_domain *domain)
3543 {
3544 /* Domain 0 is reserved, so dont process it */
3545 if (!domain)
3546 return;
3547
3548 vm_domain_remove_all_dev_info(domain);
3549 /* destroy iovas */
3550 put_iova_domain(&domain->iovad);
3551
3552 /* clear ptes */
3553 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3554
3555 /* free page tables */
3556 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3557
3558 iommu_free_vm_domain(domain);
3559 free_domain_mem(domain);
3560 }
3561
3562 static int intel_iommu_domain_init(struct iommu_domain *domain)
3563 {
3564 struct dmar_domain *dmar_domain;
3565
3566 dmar_domain = iommu_alloc_vm_domain();
3567 if (!dmar_domain) {
3568 printk(KERN_ERR
3569 "intel_iommu_domain_init: dmar_domain == NULL\n");
3570 return -ENOMEM;
3571 }
3572 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
3573 printk(KERN_ERR
3574 "intel_iommu_domain_init() failed\n");
3575 vm_domain_exit(dmar_domain);
3576 return -ENOMEM;
3577 }
3578 domain->priv = dmar_domain;
3579
3580 return 0;
3581 }
3582
3583 static void intel_iommu_domain_destroy(struct iommu_domain *domain)
3584 {
3585 struct dmar_domain *dmar_domain = domain->priv;
3586
3587 domain->priv = NULL;
3588 vm_domain_exit(dmar_domain);
3589 }
3590
3591 static int intel_iommu_attach_device(struct iommu_domain *domain,
3592 struct device *dev)
3593 {
3594 struct dmar_domain *dmar_domain = domain->priv;
3595 struct pci_dev *pdev = to_pci_dev(dev);
3596 struct intel_iommu *iommu;
3597 int addr_width;
3598 u64 end;
3599
3600 /* normally pdev is not mapped */
3601 if (unlikely(domain_context_mapped(pdev))) {
3602 struct dmar_domain *old_domain;
3603
3604 old_domain = find_domain(pdev);
3605 if (old_domain) {
3606 if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
3607 dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
3608 domain_remove_one_dev_info(old_domain, pdev);
3609 else
3610 domain_remove_dev_info(old_domain);
3611 }
3612 }
3613
3614 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3615 pdev->devfn);
3616 if (!iommu)
3617 return -ENODEV;
3618
3619 /* check if this iommu agaw is sufficient for max mapped address */
3620 addr_width = agaw_to_width(iommu->agaw);
3621 end = DOMAIN_MAX_ADDR(addr_width);
3622 end = end & VTD_PAGE_MASK;
3623 if (end < dmar_domain->max_addr) {
3624 printk(KERN_ERR "%s: iommu agaw (%d) is not "
3625 "sufficient for the mapped address (%llx)\n",
3626 __func__, iommu->agaw, dmar_domain->max_addr);
3627 return -EFAULT;
3628 }
3629
3630 return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
3631 }
3632
3633 static void intel_iommu_detach_device(struct iommu_domain *domain,
3634 struct device *dev)
3635 {
3636 struct dmar_domain *dmar_domain = domain->priv;
3637 struct pci_dev *pdev = to_pci_dev(dev);
3638
3639 domain_remove_one_dev_info(dmar_domain, pdev);
3640 }
3641
3642 static int intel_iommu_map_range(struct iommu_domain *domain,
3643 unsigned long iova, phys_addr_t hpa,
3644 size_t size, int iommu_prot)
3645 {
3646 struct dmar_domain *dmar_domain = domain->priv;
3647 u64 max_addr;
3648 int addr_width;
3649 int prot = 0;
3650 int ret;
3651
3652 if (iommu_prot & IOMMU_READ)
3653 prot |= DMA_PTE_READ;
3654 if (iommu_prot & IOMMU_WRITE)
3655 prot |= DMA_PTE_WRITE;
3656 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
3657 prot |= DMA_PTE_SNP;
3658
3659 max_addr = iova + size;
3660 if (dmar_domain->max_addr < max_addr) {
3661 int min_agaw;
3662 u64 end;
3663
3664 /* check if minimum agaw is sufficient for mapped address */
3665 min_agaw = vm_domain_min_agaw(dmar_domain);
3666 addr_width = agaw_to_width(min_agaw);
3667 end = DOMAIN_MAX_ADDR(addr_width);
3668 end = end & VTD_PAGE_MASK;
3669 if (end < max_addr) {
3670 printk(KERN_ERR "%s: iommu agaw (%d) is not "
3671 "sufficient for the mapped address (%llx)\n",
3672 __func__, min_agaw, max_addr);
3673 return -EFAULT;
3674 }
3675 dmar_domain->max_addr = max_addr;
3676 }
3677 /* Round up size to next multiple of PAGE_SIZE, if it and
3678 the low bits of hpa would take us onto the next page */
3679 size = aligned_nrpages(hpa, size);
3680 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
3681 hpa >> VTD_PAGE_SHIFT, size, prot);
3682 return ret;
3683 }
3684
3685 static void intel_iommu_unmap_range(struct iommu_domain *domain,
3686 unsigned long iova, size_t size)
3687 {
3688 struct dmar_domain *dmar_domain = domain->priv;
3689
3690 if (!size)
3691 return;
3692
3693 dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
3694 (iova + size - 1) >> VTD_PAGE_SHIFT);
3695
3696 if (dmar_domain->max_addr == iova + size)
3697 dmar_domain->max_addr = iova;
3698 }
3699
3700 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
3701 unsigned long iova)
3702 {
3703 struct dmar_domain *dmar_domain = domain->priv;
3704 struct dma_pte *pte;
3705 u64 phys = 0;
3706
3707 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT);
3708 if (pte)
3709 phys = dma_pte_addr(pte);
3710
3711 return phys;
3712 }
3713
3714 static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
3715 unsigned long cap)
3716 {
3717 struct dmar_domain *dmar_domain = domain->priv;
3718
3719 if (cap == IOMMU_CAP_CACHE_COHERENCY)
3720 return dmar_domain->iommu_snooping;
3721
3722 return 0;
3723 }
3724
3725 static struct iommu_ops intel_iommu_ops = {
3726 .domain_init = intel_iommu_domain_init,
3727 .domain_destroy = intel_iommu_domain_destroy,
3728 .attach_dev = intel_iommu_attach_device,
3729 .detach_dev = intel_iommu_detach_device,
3730 .map = intel_iommu_map_range,
3731 .unmap = intel_iommu_unmap_range,
3732 .iova_to_phys = intel_iommu_iova_to_phys,
3733 .domain_has_cap = intel_iommu_domain_has_cap,
3734 };
3735
3736 static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
3737 {
3738 /*
3739 * Mobile 4 Series Chipset neglects to set RWBF capability,
3740 * but needs it:
3741 */
3742 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
3743 rwbf_quirk = 1;
3744
3745 /* https://bugzilla.redhat.com/show_bug.cgi?id=538163 */
3746 if (dev->revision == 0x07) {
3747 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
3748 dmar_map_gfx = 0;
3749 }
3750 }
3751
3752 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
3753
3754 /* On Tylersburg chipsets, some BIOSes have been known to enable the
3755 ISOCH DMAR unit for the Azalia sound device, but not give it any
3756 TLB entries, which causes it to deadlock. Check for that. We do
3757 this in a function called from init_dmars(), instead of in a PCI
3758 quirk, because we don't want to print the obnoxious "BIOS broken"
3759 message if VT-d is actually disabled.
3760 */
3761 static void __init check_tylersburg_isoch(void)
3762 {
3763 struct pci_dev *pdev;
3764 uint32_t vtisochctrl;
3765
3766 /* If there's no Azalia in the system anyway, forget it. */
3767 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
3768 if (!pdev)
3769 return;
3770 pci_dev_put(pdev);
3771
3772 /* System Management Registers. Might be hidden, in which case
3773 we can't do the sanity check. But that's OK, because the
3774 known-broken BIOSes _don't_ actually hide it, so far. */
3775 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
3776 if (!pdev)
3777 return;
3778
3779 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
3780 pci_dev_put(pdev);
3781 return;
3782 }
3783
3784 pci_dev_put(pdev);
3785
3786 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
3787 if (vtisochctrl & 1)
3788 return;
3789
3790 /* Drop all bits other than the number of TLB entries */
3791 vtisochctrl &= 0x1c;
3792
3793 /* If we have the recommended number of TLB entries (16), fine. */
3794 if (vtisochctrl == 0x10)
3795 return;
3796
3797 /* Zero TLB entries? You get to ride the short bus to school. */
3798 if (!vtisochctrl) {
3799 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
3800 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
3801 dmi_get_system_info(DMI_BIOS_VENDOR),
3802 dmi_get_system_info(DMI_BIOS_VERSION),
3803 dmi_get_system_info(DMI_PRODUCT_VERSION));
3804 iommu_identity_mapping |= IDENTMAP_AZALIA;
3805 return;
3806 }
3807
3808 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
3809 vtisochctrl);
3810 }
Linux MOXA ART SoC ARM port