?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge commit 'refs/merge-requests/1' of git://gitorious.org/linux-on-wince-htc/linux_...
[htc-linux.git] / drivers / pci / intel-iommu.c
blob2498602151e6cb081189d81ac62eb2850c52abfb
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 * Author: Fenghua Yu <fenghua.yu@intel.com>
22 */
23
24 #include <linux/init.h>
25 #include <linux/bitmap.h>
26 #include <linux/debugfs.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/timer.h>
36 #include <linux/iova.h>
37 #include <linux/iommu.h>
38 #include <linux/intel-iommu.h>
39 #include <linux/sysdev.h>
40 #include <linux/tboot.h>
41 #include <linux/dmi.h>
42 #include <asm/cacheflush.h>
43 #include <asm/iommu.h>
44 #include "pci.h"
45
46 #define ROOT_SIZE VTD_PAGE_SIZE
47 #define CONTEXT_SIZE VTD_PAGE_SIZE
48
49 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
50 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
51 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
52
53 #define IOAPIC_RANGE_START (0xfee00000)
54 #define IOAPIC_RANGE_END (0xfeefffff)
55 #define IOVA_START_ADDR (0x1000)
56
57 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
58
59 #define MAX_AGAW_WIDTH 64
60
61 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
62 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
63
64 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
65 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
66 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
67 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
68 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
69
70 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
71 #define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
72 #define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
73
74
75 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
76 are never going to work. */
77 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
78 {
79 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
80 }
81
82 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
83 {
84 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
85 }
86 static inline unsigned long page_to_dma_pfn(struct page *pg)
87 {
88 return mm_to_dma_pfn(page_to_pfn(pg));
89 }
90 static inline unsigned long virt_to_dma_pfn(void *p)
91 {
92 return page_to_dma_pfn(virt_to_page(p));
93 }
94
95 /* global iommu list, set NULL for ignored DMAR units */
96 static struct intel_iommu **g_iommus;
97
98 static void __init check_tylersburg_isoch(void);
99 static int rwbf_quirk;
100
101 /*
102 * 0: Present
103 * 1-11: Reserved
104 * 12-63: Context Ptr (12 - (haw-1))
105 * 64-127: Reserved
106 */
107 struct root_entry {
108 u64 val;
109 u64 rsvd1;
110 };
111 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
112 static inline bool root_present(struct root_entry *root)
113 {
114 return (root->val & 1);
115 }
116 static inline void set_root_present(struct root_entry *root)
117 {
118 root->val |= 1;
119 }
120 static inline void set_root_value(struct root_entry *root, unsigned long value)
121 {
122 root->val |= value & VTD_PAGE_MASK;
123 }
124
125 static inline struct context_entry *
126 get_context_addr_from_root(struct root_entry *root)
127 {
128 return (struct context_entry *)
129 (root_present(root)?phys_to_virt(
130 root->val & VTD_PAGE_MASK) :
131 NULL);
132 }
133
134 /*
135 * low 64 bits:
136 * 0: present
137 * 1: fault processing disable
138 * 2-3: translation type
139 * 12-63: address space root
140 * high 64 bits:
141 * 0-2: address width
142 * 3-6: aval
143 * 8-23: domain id
144 */
145 struct context_entry {
146 u64 lo;
147 u64 hi;
148 };
149
150 static inline bool context_present(struct context_entry *context)
151 {
152 return (context->lo & 1);
153 }
154 static inline void context_set_present(struct context_entry *context)
155 {
156 context->lo |= 1;
157 }
158
159 static inline void context_set_fault_enable(struct context_entry *context)
160 {
161 context->lo &= (((u64)-1) << 2) | 1;
162 }
163
164 static inline void context_set_translation_type(struct context_entry *context,
165 unsigned long value)
166 {
167 context->lo &= (((u64)-1) << 4) | 3;
168 context->lo |= (value & 3) << 2;
169 }
170
171 static inline void context_set_address_root(struct context_entry *context,
172 unsigned long value)
173 {
174 context->lo |= value & VTD_PAGE_MASK;
175 }
176
177 static inline void context_set_address_width(struct context_entry *context,
178 unsigned long value)
179 {
180 context->hi |= value & 7;
181 }
182
183 static inline void context_set_domain_id(struct context_entry *context,
184 unsigned long value)
185 {
186 context->hi |= (value & ((1 << 16) - 1)) << 8;
187 }
188
189 static inline void context_clear_entry(struct context_entry *context)
190 {
191 context->lo = 0;
192 context->hi = 0;
193 }
194
195 /*
196 * 0: readable
197 * 1: writable
198 * 2-6: reserved
199 * 7: super page
200 * 8-10: available
201 * 11: snoop behavior
202 * 12-63: Host physcial address
203 */
204 struct dma_pte {
205 u64 val;
206 };
207
208 static inline void dma_clear_pte(struct dma_pte *pte)
209 {
210 pte->val = 0;
211 }
212
213 static inline void dma_set_pte_readable(struct dma_pte *pte)
214 {
215 pte->val |= DMA_PTE_READ;
216 }
217
218 static inline void dma_set_pte_writable(struct dma_pte *pte)
219 {
220 pte->val |= DMA_PTE_WRITE;
221 }
222
223 static inline void dma_set_pte_snp(struct dma_pte *pte)
224 {
225 pte->val |= DMA_PTE_SNP;
226 }
227
228 static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
229 {
230 pte->val = (pte->val & ~3) | (prot & 3);
231 }
232
233 static inline u64 dma_pte_addr(struct dma_pte *pte)
234 {
235 #ifdef CONFIG_64BIT
236 return pte->val & VTD_PAGE_MASK;
237 #else
238 /* Must have a full atomic 64-bit read */
239 return __cmpxchg64(pte, 0ULL, 0ULL) & VTD_PAGE_MASK;
240 #endif
241 }
242
243 static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
244 {
245 pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
246 }
247
248 static inline bool dma_pte_present(struct dma_pte *pte)
249 {
250 return (pte->val & 3) != 0;
251 }
252
253 static inline int first_pte_in_page(struct dma_pte *pte)
254 {
255 return !((unsigned long)pte & ~VTD_PAGE_MASK);
256 }
257
258 /*
259 * This domain is a statically identity mapping domain.
260 * 1. This domain creats a static 1:1 mapping to all usable memory.
261 * 2. It maps to each iommu if successful.
262 * 3. Each iommu mapps to this domain if successful.
263 */
264 static struct dmar_domain *si_domain;
265 static int hw_pass_through = 1;
266
267 /* devices under the same p2p bridge are owned in one domain */
268 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
269
270 /* domain represents a virtual machine, more than one devices
271 * across iommus may be owned in one domain, e.g. kvm guest.
272 */
273 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
274
275 /* si_domain contains mulitple devices */
276 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)
277
278 struct dmar_domain {
279 int id; /* domain id */
280 unsigned long iommu_bmp; /* bitmap of iommus this domain uses*/
281
282 struct list_head devices; /* all devices' list */
283 struct iova_domain iovad; /* iova's that belong to this domain */
284
285 struct dma_pte *pgd; /* virtual address */
286 int gaw; /* max guest address width */
287
288 /* adjusted guest address width, 0 is level 2 30-bit */
289 int agaw;
290
291 int flags; /* flags to find out type of domain */
292
293 int iommu_coherency;/* indicate coherency of iommu access */
294 int iommu_snooping; /* indicate snooping control feature*/
295 int iommu_count; /* reference count of iommu */
296 spinlock_t iommu_lock; /* protect iommu set in domain */
297 u64 max_addr; /* maximum mapped address */
298 };
299
300 /* PCI domain-device relationship */
301 struct device_domain_info {
302 struct list_head link; /* link to domain siblings */
303 struct list_head global; /* link to global list */
304 int segment; /* PCI domain */
305 u8 bus; /* PCI bus number */
306 u8 devfn; /* PCI devfn number */
307 struct pci_dev *dev; /* it's NULL for PCIE-to-PCI bridge */
308 struct intel_iommu *iommu; /* IOMMU used by this device */
309 struct dmar_domain *domain; /* pointer to domain */
310 };
311
312 static void flush_unmaps_timeout(unsigned long data);
313
314 DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
315
316 #define HIGH_WATER_MARK 250
317 struct deferred_flush_tables {
318 int next;
319 struct iova *iova[HIGH_WATER_MARK];
320 struct dmar_domain *domain[HIGH_WATER_MARK];
321 };
322
323 static struct deferred_flush_tables *deferred_flush;
324
325 /* bitmap for indexing intel_iommus */
326 static int g_num_of_iommus;
327
328 static DEFINE_SPINLOCK(async_umap_flush_lock);
329 static LIST_HEAD(unmaps_to_do);
330
331 static int timer_on;
332 static long list_size;
333
334 static void domain_remove_dev_info(struct dmar_domain *domain);
335
336 #ifdef CONFIG_DMAR_DEFAULT_ON
337 int dmar_disabled = 0;
338 #else
339 int dmar_disabled = 1;
340 #endif /*CONFIG_DMAR_DEFAULT_ON*/
341
342 static int __initdata dmar_map_gfx = 1;
343 static int dmar_forcedac;
344 static int intel_iommu_strict;
345
346 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
347 static DEFINE_SPINLOCK(device_domain_lock);
348 static LIST_HEAD(device_domain_list);
349
350 static struct iommu_ops intel_iommu_ops;
351
352 static int __init intel_iommu_setup(char *str)
353 {
354 if (!str)
355 return -EINVAL;
356 while (*str) {
357 if (!strncmp(str, "on", 2)) {
358 dmar_disabled = 0;
359 printk(KERN_INFO "Intel-IOMMU: enabled\n");
360 } else if (!strncmp(str, "off", 3)) {
361 dmar_disabled = 1;
362 printk(KERN_INFO "Intel-IOMMU: disabled\n");
363 } else if (!strncmp(str, "igfx_off", 8)) {
364 dmar_map_gfx = 0;
365 printk(KERN_INFO
366 "Intel-IOMMU: disable GFX device mapping\n");
367 } else if (!strncmp(str, "forcedac", 8)) {
368 printk(KERN_INFO
369 "Intel-IOMMU: Forcing DAC for PCI devices\n");
370 dmar_forcedac = 1;
371 } else if (!strncmp(str, "strict", 6)) {
372 printk(KERN_INFO
373 "Intel-IOMMU: disable batched IOTLB flush\n");
374 intel_iommu_strict = 1;
375 }
376
377 str += strcspn(str, ",");
378 while (*str == ',')
379 str++;
380 }
381 return 0;
382 }
383 __setup("intel_iommu=", intel_iommu_setup);
384
385 static struct kmem_cache *iommu_domain_cache;
386 static struct kmem_cache *iommu_devinfo_cache;
387 static struct kmem_cache *iommu_iova_cache;
388
389 static inline void *iommu_kmem_cache_alloc(struct kmem_cache *cachep)
390 {
391 unsigned int flags;
392 void *vaddr;
393
394 /* trying to avoid low memory issues */
395 flags = current->flags & PF_MEMALLOC;
396 current->flags |= PF_MEMALLOC;
397 vaddr = kmem_cache_alloc(cachep, GFP_ATOMIC);
398 current->flags &= (~PF_MEMALLOC | flags);
399 return vaddr;
400 }
401
402
403 static inline void *alloc_pgtable_page(void)
404 {
405 unsigned int flags;
406 void *vaddr;
407
408 /* trying to avoid low memory issues */
409 flags = current->flags & PF_MEMALLOC;
410 current->flags |= PF_MEMALLOC;
411 vaddr = (void *)get_zeroed_page(GFP_ATOMIC);
412 current->flags &= (~PF_MEMALLOC | flags);
413 return vaddr;
414 }
415
416 static inline void free_pgtable_page(void *vaddr)
417 {
418 free_page((unsigned long)vaddr);
419 }
420
421 static inline void *alloc_domain_mem(void)
422 {
423 return iommu_kmem_cache_alloc(iommu_domain_cache);
424 }
425
426 static void free_domain_mem(void *vaddr)
427 {
428 kmem_cache_free(iommu_domain_cache, vaddr);
429 }
430
431 static inline void * alloc_devinfo_mem(void)
432 {
433 return iommu_kmem_cache_alloc(iommu_devinfo_cache);
434 }
435
436 static inline void free_devinfo_mem(void *vaddr)
437 {
438 kmem_cache_free(iommu_devinfo_cache, vaddr);
439 }
440
441 struct iova *alloc_iova_mem(void)
442 {
443 return iommu_kmem_cache_alloc(iommu_iova_cache);
444 }
445
446 void free_iova_mem(struct iova *iova)
447 {
448 kmem_cache_free(iommu_iova_cache, iova);
449 }
450
451
452 static inline int width_to_agaw(int width);
453
454 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
455 {
456 unsigned long sagaw;
457 int agaw = -1;
458
459 sagaw = cap_sagaw(iommu->cap);
460 for (agaw = width_to_agaw(max_gaw);
461 agaw >= 0; agaw--) {
462 if (test_bit(agaw, &sagaw))
463 break;
464 }
465
466 return agaw;
467 }
468
469 /*
470 * Calculate max SAGAW for each iommu.
471 */
472 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
473 {
474 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
475 }
476
477 /*
478 * calculate agaw for each iommu.
479 * "SAGAW" may be different across iommus, use a default agaw, and
480 * get a supported less agaw for iommus that don't support the default agaw.
481 */
482 int iommu_calculate_agaw(struct intel_iommu *iommu)
483 {
484 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
485 }
486
487 /* This functionin only returns single iommu in a domain */
488 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
489 {
490 int iommu_id;
491
492 /* si_domain and vm domain should not get here. */
493 BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
494 BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
495
496 iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
497 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
498 return NULL;
499
500 return g_iommus[iommu_id];
501 }
502
503 static void domain_update_iommu_coherency(struct dmar_domain *domain)
504 {
505 int i;
506
507 domain->iommu_coherency = 1;
508
509 i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
510 for (; i < g_num_of_iommus; ) {
511 if (!ecap_coherent(g_iommus[i]->ecap)) {
512 domain->iommu_coherency = 0;
513 break;
514 }
515 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
516 }
517 }
518
519 static void domain_update_iommu_snooping(struct dmar_domain *domain)
520 {
521 int i;
522
523 domain->iommu_snooping = 1;
524
525 i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
526 for (; i < g_num_of_iommus; ) {
527 if (!ecap_sc_support(g_iommus[i]->ecap)) {
528 domain->iommu_snooping = 0;
529 break;
530 }
531 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
532 }
533 }
534
535 /* Some capabilities may be different across iommus */
536 static void domain_update_iommu_cap(struct dmar_domain *domain)
537 {
538 domain_update_iommu_coherency(domain);
539 domain_update_iommu_snooping(domain);
540 }
541
542 static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
543 {
544 struct dmar_drhd_unit *drhd = NULL;
545 int i;
546
547 for_each_drhd_unit(drhd) {
548 if (drhd->ignored)
549 continue;
550 if (segment != drhd->segment)
551 continue;
552
553 for (i = 0; i < drhd->devices_cnt; i++) {
554 if (drhd->devices[i] &&
555 drhd->devices[i]->bus->number == bus &&
556 drhd->devices[i]->devfn == devfn)
557 return drhd->iommu;
558 if (drhd->devices[i] &&
559 drhd->devices[i]->subordinate &&
560 drhd->devices[i]->subordinate->number <= bus &&
561 drhd->devices[i]->subordinate->subordinate >= bus)
562 return drhd->iommu;
563 }
564
565 if (drhd->include_all)
566 return drhd->iommu;
567 }
568
569 return NULL;
570 }
571
572 static void domain_flush_cache(struct dmar_domain *domain,
573 void *addr, int size)
574 {
575 if (!domain->iommu_coherency)
576 clflush_cache_range(addr, size);
577 }
578
579 /* Gets context entry for a given bus and devfn */
580 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
581 u8 bus, u8 devfn)
582 {
583 struct root_entry *root;
584 struct context_entry *context;
585 unsigned long phy_addr;
586 unsigned long flags;
587
588 spin_lock_irqsave(&iommu->lock, flags);
589 root = &iommu->root_entry[bus];
590 context = get_context_addr_from_root(root);
591 if (!context) {
592 context = (struct context_entry *)alloc_pgtable_page();
593 if (!context) {
594 spin_unlock_irqrestore(&iommu->lock, flags);
595 return NULL;
596 }
597 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
598 phy_addr = virt_to_phys((void *)context);
599 set_root_value(root, phy_addr);
600 set_root_present(root);
601 __iommu_flush_cache(iommu, root, sizeof(*root));
602 }
603 spin_unlock_irqrestore(&iommu->lock, flags);
604 return &context[devfn];
605 }
606
607 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
608 {
609 struct root_entry *root;
610 struct context_entry *context;
611 int ret;
612 unsigned long flags;
613
614 spin_lock_irqsave(&iommu->lock, flags);
615 root = &iommu->root_entry[bus];
616 context = get_context_addr_from_root(root);
617 if (!context) {
618 ret = 0;
619 goto out;
620 }
621 ret = context_present(&context[devfn]);
622 out:
623 spin_unlock_irqrestore(&iommu->lock, flags);
624 return ret;
625 }
626
627 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
628 {
629 struct root_entry *root;
630 struct context_entry *context;
631 unsigned long flags;
632
633 spin_lock_irqsave(&iommu->lock, flags);
634 root = &iommu->root_entry[bus];
635 context = get_context_addr_from_root(root);
636 if (context) {
637 context_clear_entry(&context[devfn]);
638 __iommu_flush_cache(iommu, &context[devfn], \
639 sizeof(*context));
640 }
641 spin_unlock_irqrestore(&iommu->lock, flags);
642 }
643
644 static void free_context_table(struct intel_iommu *iommu)
645 {
646 struct root_entry *root;
647 int i;
648 unsigned long flags;
649 struct context_entry *context;
650
651 spin_lock_irqsave(&iommu->lock, flags);
652 if (!iommu->root_entry) {
653 goto out;
654 }
655 for (i = 0; i < ROOT_ENTRY_NR; i++) {
656 root = &iommu->root_entry[i];
657 context = get_context_addr_from_root(root);
658 if (context)
659 free_pgtable_page(context);
660 }
661 free_pgtable_page(iommu->root_entry);
662 iommu->root_entry = NULL;
663 out:
664 spin_unlock_irqrestore(&iommu->lock, flags);
665 }
666
667 /* page table handling */
668 #define LEVEL_STRIDE (9)
669 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
670
671 static inline int agaw_to_level(int agaw)
672 {
673 return agaw + 2;
674 }
675
676 static inline int agaw_to_width(int agaw)
677 {
678 return 30 + agaw * LEVEL_STRIDE;
679
680 }
681
682 static inline int width_to_agaw(int width)
683 {
684 return (width - 30) / LEVEL_STRIDE;
685 }
686
687 static inline unsigned int level_to_offset_bits(int level)
688 {
689 return (level - 1) * LEVEL_STRIDE;
690 }
691
692 static inline int pfn_level_offset(unsigned long pfn, int level)
693 {
694 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
695 }
696
697 static inline unsigned long level_mask(int level)
698 {
699 return -1UL << level_to_offset_bits(level);
700 }
701
702 static inline unsigned long level_size(int level)
703 {
704 return 1UL << level_to_offset_bits(level);
705 }
706
707 static inline unsigned long align_to_level(unsigned long pfn, int level)
708 {
709 return (pfn + level_size(level) - 1) & level_mask(level);
710 }
711
712 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
713 unsigned long pfn)
714 {
715 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
716 struct dma_pte *parent, *pte = NULL;
717 int level = agaw_to_level(domain->agaw);
718 int offset;
719
720 BUG_ON(!domain->pgd);
721 BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width);
722 parent = domain->pgd;
723
724 while (level > 0) {
725 void *tmp_page;
726
727 offset = pfn_level_offset(pfn, level);
728 pte = &parent[offset];
729 if (level == 1)
730 break;
731
732 if (!dma_pte_present(pte)) {
733 uint64_t pteval;
734
735 tmp_page = alloc_pgtable_page();
736
737 if (!tmp_page)
738 return NULL;
739
740 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
741 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
742 if (cmpxchg64(&pte->val, 0ULL, pteval)) {
743 /* Someone else set it while we were thinking; use theirs. */
744 free_pgtable_page(tmp_page);
745 } else {
746 dma_pte_addr(pte);
747 domain_flush_cache(domain, pte, sizeof(*pte));
748 }
749 }
750 parent = phys_to_virt(dma_pte_addr(pte));
751 level--;
752 }
753
754 return pte;
755 }
756
757 /* return address's pte at specific level */
758 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
759 unsigned long pfn,
760 int level)
761 {
762 struct dma_pte *parent, *pte = NULL;
763 int total = agaw_to_level(domain->agaw);
764 int offset;
765
766 parent = domain->pgd;
767 while (level <= total) {
768 offset = pfn_level_offset(pfn, total);
769 pte = &parent[offset];
770 if (level == total)
771 return pte;
772
773 if (!dma_pte_present(pte))
774 break;
775 parent = phys_to_virt(dma_pte_addr(pte));
776 total--;
777 }
778 return NULL;
779 }
780
781 /* clear last level pte, a tlb flush should be followed */
782 static void dma_pte_clear_range(struct dmar_domain *domain,
783 unsigned long start_pfn,
784 unsigned long last_pfn)
785 {
786 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
787 struct dma_pte *first_pte, *pte;
788
789 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
790 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
791 BUG_ON(start_pfn > last_pfn);
792
793 /* we don't need lock here; nobody else touches the iova range */
794 do {
795 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1);
796 if (!pte) {
797 start_pfn = align_to_level(start_pfn + 1, 2);
798 continue;
799 }
800 do {
801 dma_clear_pte(pte);
802 start_pfn++;
803 pte++;
804 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
805
806 domain_flush_cache(domain, first_pte,
807 (void *)pte - (void *)first_pte);
808
809 } while (start_pfn && start_pfn <= last_pfn);
810 }
811
812 /* free page table pages. last level pte should already be cleared */
813 static void dma_pte_free_pagetable(struct dmar_domain *domain,
814 unsigned long start_pfn,
815 unsigned long last_pfn)
816 {
817 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
818 struct dma_pte *first_pte, *pte;
819 int total = agaw_to_level(domain->agaw);
820 int level;
821 unsigned long tmp;
822
823 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
824 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
825 BUG_ON(start_pfn > last_pfn);
826
827 /* We don't need lock here; nobody else touches the iova range */
828 level = 2;
829 while (level <= total) {
830 tmp = align_to_level(start_pfn, level);
831
832 /* If we can't even clear one PTE at this level, we're done */
833 if (tmp + level_size(level) - 1 > last_pfn)
834 return;
835
836 do {
837 first_pte = pte = dma_pfn_level_pte(domain, tmp, level);
838 if (!pte) {
839 tmp = align_to_level(tmp + 1, level + 1);
840 continue;
841 }
842 do {
843 if (dma_pte_present(pte)) {
844 free_pgtable_page(phys_to_virt(dma_pte_addr(pte)));
845 dma_clear_pte(pte);
846 }
847 pte++;
848 tmp += level_size(level);
849 } while (!first_pte_in_page(pte) &&
850 tmp + level_size(level) - 1 <= last_pfn);
851
852 domain_flush_cache(domain, first_pte,
853 (void *)pte - (void *)first_pte);
854
855 } while (tmp && tmp + level_size(level) - 1 <= last_pfn);
856 level++;
857 }
858 /* free pgd */
859 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
860 free_pgtable_page(domain->pgd);
861 domain->pgd = NULL;
862 }
863 }
864
865 /* iommu handling */
866 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
867 {
868 struct root_entry *root;
869 unsigned long flags;
870
871 root = (struct root_entry *)alloc_pgtable_page();
872 if (!root)
873 return -ENOMEM;
874
875 __iommu_flush_cache(iommu, root, ROOT_SIZE);
876
877 spin_lock_irqsave(&iommu->lock, flags);
878 iommu->root_entry = root;
879 spin_unlock_irqrestore(&iommu->lock, flags);
880
881 return 0;
882 }
883
884 static void iommu_set_root_entry(struct intel_iommu *iommu)
885 {
886 void *addr;
887 u32 sts;
888 unsigned long flag;
889
890 addr = iommu->root_entry;
891
892 spin_lock_irqsave(&iommu->register_lock, flag);
893 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
894
895 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
896
897 /* Make sure hardware complete it */
898 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
899 readl, (sts & DMA_GSTS_RTPS), sts);
900
901 spin_unlock_irqrestore(&iommu->register_lock, flag);
902 }
903
904 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
905 {
906 u32 val;
907 unsigned long flag;
908
909 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
910 return;
911
912 spin_lock_irqsave(&iommu->register_lock, flag);
913 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
914
915 /* Make sure hardware complete it */
916 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
917 readl, (!(val & DMA_GSTS_WBFS)), val);
918
919 spin_unlock_irqrestore(&iommu->register_lock, flag);
920 }
921
922 /* return value determine if we need a write buffer flush */
923 static void __iommu_flush_context(struct intel_iommu *iommu,
924 u16 did, u16 source_id, u8 function_mask,
925 u64 type)
926 {
927 u64 val = 0;
928 unsigned long flag;
929
930 switch (type) {
931 case DMA_CCMD_GLOBAL_INVL:
932 val = DMA_CCMD_GLOBAL_INVL;
933 break;
934 case DMA_CCMD_DOMAIN_INVL:
935 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
936 break;
937 case DMA_CCMD_DEVICE_INVL:
938 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
939 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
940 break;
941 default:
942 BUG();
943 }
944 val |= DMA_CCMD_ICC;
945
946 spin_lock_irqsave(&iommu->register_lock, flag);
947 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
948
949 /* Make sure hardware complete it */
950 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
951 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
952
953 spin_unlock_irqrestore(&iommu->register_lock, flag);
954 }
955
956 /* return value determine if we need a write buffer flush */
957 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
958 u64 addr, unsigned int size_order, u64 type)
959 {
960 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
961 u64 val = 0, val_iva = 0;
962 unsigned long flag;
963
964 switch (type) {
965 case DMA_TLB_GLOBAL_FLUSH:
966 /* global flush doesn't need set IVA_REG */
967 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
968 break;
969 case DMA_TLB_DSI_FLUSH:
970 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
971 break;
972 case DMA_TLB_PSI_FLUSH:
973 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
974 /* Note: always flush non-leaf currently */
975 val_iva = size_order | addr;
976 break;
977 default:
978 BUG();
979 }
980 /* Note: set drain read/write */
981 #if 0
982 /*
983 * This is probably to be super secure.. Looks like we can
984 * ignore it without any impact.
985 */
986 if (cap_read_drain(iommu->cap))
987 val |= DMA_TLB_READ_DRAIN;
988 #endif
989 if (cap_write_drain(iommu->cap))
990 val |= DMA_TLB_WRITE_DRAIN;
991
992 spin_lock_irqsave(&iommu->register_lock, flag);
993 /* Note: Only uses first TLB reg currently */
994 if (val_iva)
995 dmar_writeq(iommu->reg + tlb_offset, val_iva);
996 dmar_writeq(iommu->reg + tlb_offset + 8, val);
997
998 /* Make sure hardware complete it */
999 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1000 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1001
1002 spin_unlock_irqrestore(&iommu->register_lock, flag);
1003
1004 /* check IOTLB invalidation granularity */
1005 if (DMA_TLB_IAIG(val) == 0)
1006 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1007 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1008 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
1009 (unsigned long long)DMA_TLB_IIRG(type),
1010 (unsigned long long)DMA_TLB_IAIG(val));
1011 }
1012
1013 static struct device_domain_info *iommu_support_dev_iotlb(
1014 struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
1015 {
1016 int found = 0;
1017 unsigned long flags;
1018 struct device_domain_info *info;
1019 struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
1020
1021 if (!ecap_dev_iotlb_support(iommu->ecap))
1022 return NULL;
1023
1024 if (!iommu->qi)
1025 return NULL;
1026
1027 spin_lock_irqsave(&device_domain_lock, flags);
1028 list_for_each_entry(info, &domain->devices, link)
1029 if (info->bus == bus && info->devfn == devfn) {
1030 found = 1;
1031 break;
1032 }
1033 spin_unlock_irqrestore(&device_domain_lock, flags);
1034
1035 if (!found || !info->dev)
1036 return NULL;
1037
1038 if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1039 return NULL;
1040
1041 if (!dmar_find_matched_atsr_unit(info->dev))
1042 return NULL;
1043
1044 info->iommu = iommu;
1045
1046 return info;
1047 }
1048
1049 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1050 {
1051 if (!info)
1052 return;
1053
1054 pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1055 }
1056
1057 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1058 {
1059 if (!info->dev || !pci_ats_enabled(info->dev))
1060 return;
1061
1062 pci_disable_ats(info->dev);
1063 }
1064
1065 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1066 u64 addr, unsigned mask)
1067 {
1068 u16 sid, qdep;
1069 unsigned long flags;
1070 struct device_domain_info *info;
1071
1072 spin_lock_irqsave(&device_domain_lock, flags);
1073 list_for_each_entry(info, &domain->devices, link) {
1074 if (!info->dev || !pci_ats_enabled(info->dev))
1075 continue;
1076
1077 sid = info->bus << 8 | info->devfn;
1078 qdep = pci_ats_queue_depth(info->dev);
1079 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1080 }
1081 spin_unlock_irqrestore(&device_domain_lock, flags);
1082 }
1083
1084 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1085 unsigned long pfn, unsigned int pages)
1086 {
1087 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1088 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1089
1090 BUG_ON(pages == 0);
1091
1092 /*
1093 * Fallback to domain selective flush if no PSI support or the size is
1094 * too big.
1095 * PSI requires page size to be 2 ^ x, and the base address is naturally
1096 * aligned to the size
1097 */
1098 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1099 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1100 DMA_TLB_DSI_FLUSH);
1101 else
1102 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1103 DMA_TLB_PSI_FLUSH);
1104
1105 /*
1106 * In caching mode, domain ID 0 is reserved for non-present to present
1107 * mapping flush. Device IOTLB doesn't need to be flushed in this case.
1108 */
1109 if (!cap_caching_mode(iommu->cap) || did)
1110 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
1111 }
1112
1113 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1114 {
1115 u32 pmen;
1116 unsigned long flags;
1117
1118 spin_lock_irqsave(&iommu->register_lock, flags);
1119 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1120 pmen &= ~DMA_PMEN_EPM;
1121 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1122
1123 /* wait for the protected region status bit to clear */
1124 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1125 readl, !(pmen & DMA_PMEN_PRS), pmen);
1126
1127 spin_unlock_irqrestore(&iommu->register_lock, flags);
1128 }
1129
1130 static int iommu_enable_translation(struct intel_iommu *iommu)
1131 {
1132 u32 sts;
1133 unsigned long flags;
1134
1135 spin_lock_irqsave(&iommu->register_lock, flags);
1136 iommu->gcmd |= DMA_GCMD_TE;
1137 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1138
1139 /* Make sure hardware complete it */
1140 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1141 readl, (sts & DMA_GSTS_TES), sts);
1142
1143 spin_unlock_irqrestore(&iommu->register_lock, flags);
1144 return 0;
1145 }
1146
1147 static int iommu_disable_translation(struct intel_iommu *iommu)
1148 {
1149 u32 sts;
1150 unsigned long flag;
1151
1152 spin_lock_irqsave(&iommu->register_lock, flag);
1153 iommu->gcmd &= ~DMA_GCMD_TE;
1154 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1155
1156 /* Make sure hardware complete it */
1157 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1158 readl, (!(sts & DMA_GSTS_TES)), sts);
1159
1160 spin_unlock_irqrestore(&iommu->register_lock, flag);
1161 return 0;
1162 }
1163
1164
1165 static int iommu_init_domains(struct intel_iommu *iommu)
1166 {
1167 unsigned long ndomains;
1168 unsigned long nlongs;
1169
1170 ndomains = cap_ndoms(iommu->cap);
1171 pr_debug("Number of Domains supportd <%ld>\n", ndomains);
1172 nlongs = BITS_TO_LONGS(ndomains);
1173
1174 spin_lock_init(&iommu->lock);
1175
1176 /* TBD: there might be 64K domains,
1177 * consider other allocation for future chip
1178 */
1179 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1180 if (!iommu->domain_ids) {
1181 printk(KERN_ERR "Allocating domain id array failed\n");
1182 return -ENOMEM;
1183 }
1184 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1185 GFP_KERNEL);
1186 if (!iommu->domains) {
1187 printk(KERN_ERR "Allocating domain array failed\n");
1188 return -ENOMEM;
1189 }
1190
1191 /*
1192 * if Caching mode is set, then invalid translations are tagged
1193 * with domainid 0. Hence we need to pre-allocate it.
1194 */
1195 if (cap_caching_mode(iommu->cap))
1196 set_bit(0, iommu->domain_ids);
1197 return 0;
1198 }
1199
1200
1201 static void domain_exit(struct dmar_domain *domain);
1202 static void vm_domain_exit(struct dmar_domain *domain);
1203
1204 void free_dmar_iommu(struct intel_iommu *iommu)
1205 {
1206 struct dmar_domain *domain;
1207 int i;
1208 unsigned long flags;
1209
1210 if ((iommu->domains) && (iommu->domain_ids)) {
1211 i = find_first_bit(iommu->domain_ids, cap_ndoms(iommu->cap));
1212 for (; i < cap_ndoms(iommu->cap); ) {
1213 domain = iommu->domains[i];
1214 clear_bit(i, iommu->domain_ids);
1215
1216 spin_lock_irqsave(&domain->iommu_lock, flags);
1217 if (--domain->iommu_count == 0) {
1218 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1219 vm_domain_exit(domain);
1220 else
1221 domain_exit(domain);
1222 }
1223 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1224
1225 i = find_next_bit(iommu->domain_ids,
1226 cap_ndoms(iommu->cap), i+1);
1227 }
1228 }
1229
1230 if (iommu->gcmd & DMA_GCMD_TE)
1231 iommu_disable_translation(iommu);
1232
1233 if (iommu->irq) {
1234 set_irq_data(iommu->irq, NULL);
1235 /* This will mask the irq */
1236 free_irq(iommu->irq, iommu);
1237 destroy_irq(iommu->irq);
1238 }
1239
1240 kfree(iommu->domains);
1241 kfree(iommu->domain_ids);
1242
1243 g_iommus[iommu->seq_id] = NULL;
1244
1245 /* if all iommus are freed, free g_iommus */
1246 for (i = 0; i < g_num_of_iommus; i++) {
1247 if (g_iommus[i])
1248 break;
1249 }
1250
1251 if (i == g_num_of_iommus)
1252 kfree(g_iommus);
1253
1254 /* free context mapping */
1255 free_context_table(iommu);
1256 }
1257
1258 static struct dmar_domain *alloc_domain(void)
1259 {
1260 struct dmar_domain *domain;
1261
1262 domain = alloc_domain_mem();
1263 if (!domain)
1264 return NULL;
1265
1266 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
1267 domain->flags = 0;
1268
1269 return domain;
1270 }
1271
1272 static int iommu_attach_domain(struct dmar_domain *domain,
1273 struct intel_iommu *iommu)
1274 {
1275 int num;
1276 unsigned long ndomains;
1277 unsigned long flags;
1278
1279 ndomains = cap_ndoms(iommu->cap);
1280
1281 spin_lock_irqsave(&iommu->lock, flags);
1282
1283 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1284 if (num >= ndomains) {
1285 spin_unlock_irqrestore(&iommu->lock, flags);
1286 printk(KERN_ERR "IOMMU: no free domain ids\n");
1287 return -ENOMEM;
1288 }
1289
1290 domain->id = num;
1291 set_bit(num, iommu->domain_ids);
1292 set_bit(iommu->seq_id, &domain->iommu_bmp);
1293 iommu->domains[num] = domain;
1294 spin_unlock_irqrestore(&iommu->lock, flags);
1295
1296 return 0;
1297 }
1298
1299 static void iommu_detach_domain(struct dmar_domain *domain,
1300 struct intel_iommu *iommu)
1301 {
1302 unsigned long flags;
1303 int num, ndomains;
1304 int found = 0;
1305
1306 spin_lock_irqsave(&iommu->lock, flags);
1307 ndomains = cap_ndoms(iommu->cap);
1308 num = find_first_bit(iommu->domain_ids, ndomains);
1309 for (; num < ndomains; ) {
1310 if (iommu->domains[num] == domain) {
1311 found = 1;
1312 break;
1313 }
1314 num = find_next_bit(iommu->domain_ids,
1315 cap_ndoms(iommu->cap), num+1);
1316 }
1317
1318 if (found) {
1319 clear_bit(num, iommu->domain_ids);
1320 clear_bit(iommu->seq_id, &domain->iommu_bmp);
1321 iommu->domains[num] = NULL;
1322 }
1323 spin_unlock_irqrestore(&iommu->lock, flags);
1324 }
1325
1326 static struct iova_domain reserved_iova_list;
1327 static struct lock_class_key reserved_rbtree_key;
1328
1329 static void dmar_init_reserved_ranges(void)
1330 {
1331 struct pci_dev *pdev = NULL;
1332 struct iova *iova;
1333 int i;
1334
1335 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1336
1337 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1338 &reserved_rbtree_key);
1339
1340 /* IOAPIC ranges shouldn't be accessed by DMA */
1341 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1342 IOVA_PFN(IOAPIC_RANGE_END));
1343 if (!iova)
1344 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1345
1346 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1347 for_each_pci_dev(pdev) {
1348 struct resource *r;
1349
1350 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1351 r = &pdev->resource[i];
1352 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1353 continue;
1354 iova = reserve_iova(&reserved_iova_list,
1355 IOVA_PFN(r->start),
1356 IOVA_PFN(r->end));
1357 if (!iova)
1358 printk(KERN_ERR "Reserve iova failed\n");
1359 }
1360 }
1361
1362 }
1363
1364 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1365 {
1366 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1367 }
1368
1369 static inline int guestwidth_to_adjustwidth(int gaw)
1370 {
1371 int agaw;
1372 int r = (gaw - 12) % 9;
1373
1374 if (r == 0)
1375 agaw = gaw;
1376 else
1377 agaw = gaw + 9 - r;
1378 if (agaw > 64)
1379 agaw = 64;
1380 return agaw;
1381 }
1382
1383 static int domain_init(struct dmar_domain *domain, int guest_width)
1384 {
1385 struct intel_iommu *iommu;
1386 int adjust_width, agaw;
1387 unsigned long sagaw;
1388
1389 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1390 spin_lock_init(&domain->iommu_lock);
1391
1392 domain_reserve_special_ranges(domain);
1393
1394 /* calculate AGAW */
1395 iommu = domain_get_iommu(domain);
1396 if (guest_width > cap_mgaw(iommu->cap))
1397 guest_width = cap_mgaw(iommu->cap);
1398 domain->gaw = guest_width;
1399 adjust_width = guestwidth_to_adjustwidth(guest_width);
1400 agaw = width_to_agaw(adjust_width);
1401 sagaw = cap_sagaw(iommu->cap);
1402 if (!test_bit(agaw, &sagaw)) {
1403 /* hardware doesn't support it, choose a bigger one */
1404 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1405 agaw = find_next_bit(&sagaw, 5, agaw);
1406 if (agaw >= 5)
1407 return -ENODEV;
1408 }
1409 domain->agaw = agaw;
1410 INIT_LIST_HEAD(&domain->devices);
1411
1412 if (ecap_coherent(iommu->ecap))
1413 domain->iommu_coherency = 1;
1414 else
1415 domain->iommu_coherency = 0;
1416
1417 if (ecap_sc_support(iommu->ecap))
1418 domain->iommu_snooping = 1;
1419 else
1420 domain->iommu_snooping = 0;
1421
1422 domain->iommu_count = 1;
1423
1424 /* always allocate the top pgd */
1425 domain->pgd = (struct dma_pte *)alloc_pgtable_page();
1426 if (!domain->pgd)
1427 return -ENOMEM;
1428 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1429 return 0;
1430 }
1431
1432 static void domain_exit(struct dmar_domain *domain)
1433 {
1434 struct dmar_drhd_unit *drhd;
1435 struct intel_iommu *iommu;
1436
1437 /* Domain 0 is reserved, so dont process it */
1438 if (!domain)
1439 return;
1440
1441 domain_remove_dev_info(domain);
1442 /* destroy iovas */
1443 put_iova_domain(&domain->iovad);
1444
1445 /* clear ptes */
1446 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1447
1448 /* free page tables */
1449 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1450
1451 for_each_active_iommu(iommu, drhd)
1452 if (test_bit(iommu->seq_id, &domain->iommu_bmp))
1453 iommu_detach_domain(domain, iommu);
1454
1455 free_domain_mem(domain);
1456 }
1457
1458 static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1459 u8 bus, u8 devfn, int translation)
1460 {
1461 struct context_entry *context;
1462 unsigned long flags;
1463 struct intel_iommu *iommu;
1464 struct dma_pte *pgd;
1465 unsigned long num;
1466 unsigned long ndomains;
1467 int id;
1468 int agaw;
1469 struct device_domain_info *info = NULL;
1470
1471 pr_debug("Set context mapping for %02x:%02x.%d\n",
1472 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1473
1474 BUG_ON(!domain->pgd);
1475 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1476 translation != CONTEXT_TT_MULTI_LEVEL);
1477
1478 iommu = device_to_iommu(segment, bus, devfn);
1479 if (!iommu)
1480 return -ENODEV;
1481
1482 context = device_to_context_entry(iommu, bus, devfn);
1483 if (!context)
1484 return -ENOMEM;
1485 spin_lock_irqsave(&iommu->lock, flags);
1486 if (context_present(context)) {
1487 spin_unlock_irqrestore(&iommu->lock, flags);
1488 return 0;
1489 }
1490
1491 id = domain->id;
1492 pgd = domain->pgd;
1493
1494 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1495 domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
1496 int found = 0;
1497
1498 /* find an available domain id for this device in iommu */
1499 ndomains = cap_ndoms(iommu->cap);
1500 num = find_first_bit(iommu->domain_ids, ndomains);
1501 for (; num < ndomains; ) {
1502 if (iommu->domains[num] == domain) {
1503 id = num;
1504 found = 1;
1505 break;
1506 }
1507 num = find_next_bit(iommu->domain_ids,
1508 cap_ndoms(iommu->cap), num+1);
1509 }
1510
1511 if (found == 0) {
1512 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1513 if (num >= ndomains) {
1514 spin_unlock_irqrestore(&iommu->lock, flags);
1515 printk(KERN_ERR "IOMMU: no free domain ids\n");
1516 return -EFAULT;
1517 }
1518
1519 set_bit(num, iommu->domain_ids);
1520 iommu->domains[num] = domain;
1521 id = num;
1522 }
1523
1524 /* Skip top levels of page tables for
1525 * iommu which has less agaw than default.
1526 * Unnecessary for PT mode.
1527 */
1528 if (translation != CONTEXT_TT_PASS_THROUGH) {
1529 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1530 pgd = phys_to_virt(dma_pte_addr(pgd));
1531 if (!dma_pte_present(pgd)) {
1532 spin_unlock_irqrestore(&iommu->lock, flags);
1533 return -ENOMEM;
1534 }
1535 }
1536 }
1537 }
1538
1539 context_set_domain_id(context, id);
1540
1541 if (translation != CONTEXT_TT_PASS_THROUGH) {
1542 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1543 translation = info ? CONTEXT_TT_DEV_IOTLB :
1544 CONTEXT_TT_MULTI_LEVEL;
1545 }
1546 /*
1547 * In pass through mode, AW must be programmed to indicate the largest
1548 * AGAW value supported by hardware. And ASR is ignored by hardware.
1549 */
1550 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
1551 context_set_address_width(context, iommu->msagaw);
1552 else {
1553 context_set_address_root(context, virt_to_phys(pgd));
1554 context_set_address_width(context, iommu->agaw);
1555 }
1556
1557 context_set_translation_type(context, translation);
1558 context_set_fault_enable(context);
1559 context_set_present(context);
1560 domain_flush_cache(domain, context, sizeof(*context));
1561
1562 /*
1563 * It's a non-present to present mapping. If hardware doesn't cache
1564 * non-present entry we only need to flush the write-buffer. If the
1565 * _does_ cache non-present entries, then it does so in the special
1566 * domain #0, which we have to flush:
1567 */
1568 if (cap_caching_mode(iommu->cap)) {
1569 iommu->flush.flush_context(iommu, 0,
1570 (((u16)bus) << 8) | devfn,
1571 DMA_CCMD_MASK_NOBIT,
1572 DMA_CCMD_DEVICE_INVL);
1573 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_DSI_FLUSH);
1574 } else {
1575 iommu_flush_write_buffer(iommu);
1576 }
1577 iommu_enable_dev_iotlb(info);
1578 spin_unlock_irqrestore(&iommu->lock, flags);
1579
1580 spin_lock_irqsave(&domain->iommu_lock, flags);
1581 if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {
1582 domain->iommu_count++;
1583 domain_update_iommu_cap(domain);
1584 }
1585 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1586 return 0;
1587 }
1588
1589 static int
1590 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1591 int translation)
1592 {
1593 int ret;
1594 struct pci_dev *tmp, *parent;
1595
1596 ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
1597 pdev->bus->number, pdev->devfn,
1598 translation);
1599 if (ret)
1600 return ret;
1601
1602 /* dependent device mapping */
1603 tmp = pci_find_upstream_pcie_bridge(pdev);
1604 if (!tmp)
1605 return 0;
1606 /* Secondary interface's bus number and devfn 0 */
1607 parent = pdev->bus->self;
1608 while (parent != tmp) {
1609 ret = domain_context_mapping_one(domain,
1610 pci_domain_nr(parent->bus),
1611 parent->bus->number,
1612 parent->devfn, translation);
1613 if (ret)
1614 return ret;
1615 parent = parent->bus->self;
1616 }
1617 if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
1618 return domain_context_mapping_one(domain,
1619 pci_domain_nr(tmp->subordinate),
1620 tmp->subordinate->number, 0,
1621 translation);
1622 else /* this is a legacy PCI bridge */
1623 return domain_context_mapping_one(domain,
1624 pci_domain_nr(tmp->bus),
1625 tmp->bus->number,
1626 tmp->devfn,
1627 translation);
1628 }
1629
1630 static int domain_context_mapped(struct pci_dev *pdev)
1631 {
1632 int ret;
1633 struct pci_dev *tmp, *parent;
1634 struct intel_iommu *iommu;
1635
1636 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1637 pdev->devfn);
1638 if (!iommu)
1639 return -ENODEV;
1640
1641 ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
1642 if (!ret)
1643 return ret;
1644 /* dependent device mapping */
1645 tmp = pci_find_upstream_pcie_bridge(pdev);
1646 if (!tmp)
1647 return ret;
1648 /* Secondary interface's bus number and devfn 0 */
1649 parent = pdev->bus->self;
1650 while (parent != tmp) {
1651 ret = device_context_mapped(iommu, parent->bus->number,
1652 parent->devfn);
1653 if (!ret)
1654 return ret;
1655 parent = parent->bus->self;
1656 }
1657 if (tmp->is_pcie)
1658 return device_context_mapped(iommu, tmp->subordinate->number,
1659 0);
1660 else
1661 return device_context_mapped(iommu, tmp->bus->number,
1662 tmp->devfn);
1663 }
1664
1665 /* Returns a number of VTD pages, but aligned to MM page size */
1666 static inline unsigned long aligned_nrpages(unsigned long host_addr,
1667 size_t size)
1668 {
1669 host_addr &= ~PAGE_MASK;
1670 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1671 }
1672
1673 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1674 struct scatterlist *sg, unsigned long phys_pfn,
1675 unsigned long nr_pages, int prot)
1676 {
1677 struct dma_pte *first_pte = NULL, *pte = NULL;
1678 phys_addr_t uninitialized_var(pteval);
1679 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1680 unsigned long sg_res;
1681
1682 BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1683
1684 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1685 return -EINVAL;
1686
1687 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1688
1689 if (sg)
1690 sg_res = 0;
1691 else {
1692 sg_res = nr_pages + 1;
1693 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1694 }
1695
1696 while (nr_pages--) {
1697 uint64_t tmp;
1698
1699 if (!sg_res) {
1700 sg_res = aligned_nrpages(sg->offset, sg->length);
1701 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1702 sg->dma_length = sg->length;
1703 pteval = page_to_phys(sg_page(sg)) | prot;
1704 }
1705 if (!pte) {
1706 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn);
1707 if (!pte)
1708 return -ENOMEM;
1709 }
1710 /* We don't need lock here, nobody else
1711 * touches the iova range
1712 */
1713 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
1714 if (tmp) {
1715 static int dumps = 5;
1716 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1717 iov_pfn, tmp, (unsigned long long)pteval);
1718 if (dumps) {
1719 dumps--;
1720 debug_dma_dump_mappings(NULL);
1721 }
1722 WARN_ON(1);
1723 }
1724 pte++;
1725 if (!nr_pages || first_pte_in_page(pte)) {
1726 domain_flush_cache(domain, first_pte,
1727 (void *)pte - (void *)first_pte);
1728 pte = NULL;
1729 }
1730 iov_pfn++;
1731 pteval += VTD_PAGE_SIZE;
1732 sg_res--;
1733 if (!sg_res)
1734 sg = sg_next(sg);
1735 }
1736 return 0;
1737 }
1738
1739 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1740 struct scatterlist *sg, unsigned long nr_pages,
1741 int prot)
1742 {
1743 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
1744 }
1745
1746 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1747 unsigned long phys_pfn, unsigned long nr_pages,
1748 int prot)
1749 {
1750 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
1751 }
1752
1753 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
1754 {
1755 if (!iommu)
1756 return;
1757
1758 clear_context_table(iommu, bus, devfn);
1759 iommu->flush.flush_context(iommu, 0, 0, 0,
1760 DMA_CCMD_GLOBAL_INVL);
1761 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
1762 }
1763
1764 static void domain_remove_dev_info(struct dmar_domain *domain)
1765 {
1766 struct device_domain_info *info;
1767 unsigned long flags;
1768 struct intel_iommu *iommu;
1769
1770 spin_lock_irqsave(&device_domain_lock, flags);
1771 while (!list_empty(&domain->devices)) {
1772 info = list_entry(domain->devices.next,
1773 struct device_domain_info, link);
1774 list_del(&info->link);
1775 list_del(&info->global);
1776 if (info->dev)
1777 info->dev->dev.archdata.iommu = NULL;
1778 spin_unlock_irqrestore(&device_domain_lock, flags);
1779
1780 iommu_disable_dev_iotlb(info);
1781 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
1782 iommu_detach_dev(iommu, info->bus, info->devfn);
1783 free_devinfo_mem(info);
1784
1785 spin_lock_irqsave(&device_domain_lock, flags);
1786 }
1787 spin_unlock_irqrestore(&device_domain_lock, flags);
1788 }
1789
1790 /*
1791 * find_domain
1792 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1793 */
1794 static struct dmar_domain *
1795 find_domain(struct pci_dev *pdev)
1796 {
1797 struct device_domain_info *info;
1798
1799 /* No lock here, assumes no domain exit in normal case */
1800 info = pdev->dev.archdata.iommu;
1801 if (info)
1802 return info->domain;
1803 return NULL;
1804 }
1805
1806 /* domain is initialized */
1807 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1808 {
1809 struct dmar_domain *domain, *found = NULL;
1810 struct intel_iommu *iommu;
1811 struct dmar_drhd_unit *drhd;
1812 struct device_domain_info *info, *tmp;
1813 struct pci_dev *dev_tmp;
1814 unsigned long flags;
1815 int bus = 0, devfn = 0;
1816 int segment;
1817 int ret;
1818
1819 domain = find_domain(pdev);
1820 if (domain)
1821 return domain;
1822
1823 segment = pci_domain_nr(pdev->bus);
1824
1825 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1826 if (dev_tmp) {
1827 if (dev_tmp->is_pcie) {
1828 bus = dev_tmp->subordinate->number;
1829 devfn = 0;
1830 } else {
1831 bus = dev_tmp->bus->number;
1832 devfn = dev_tmp->devfn;
1833 }
1834 spin_lock_irqsave(&device_domain_lock, flags);
1835 list_for_each_entry(info, &device_domain_list, global) {
1836 if (info->segment == segment &&
1837 info->bus == bus && info->devfn == devfn) {
1838 found = info->domain;
1839 break;
1840 }
1841 }
1842 spin_unlock_irqrestore(&device_domain_lock, flags);
1843 /* pcie-pci bridge already has a domain, uses it */
1844 if (found) {
1845 domain = found;
1846 goto found_domain;
1847 }
1848 }
1849
1850 domain = alloc_domain();
1851 if (!domain)
1852 goto error;
1853
1854 /* Allocate new domain for the device */
1855 drhd = dmar_find_matched_drhd_unit(pdev);
1856 if (!drhd) {
1857 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1858 pci_name(pdev));
1859 return NULL;
1860 }
1861 iommu = drhd->iommu;
1862
1863 ret = iommu_attach_domain(domain, iommu);
1864 if (ret) {
1865 domain_exit(domain);
1866 goto error;
1867 }
1868
1869 if (domain_init(domain, gaw)) {
1870 domain_exit(domain);
1871 goto error;
1872 }
1873
1874 /* register pcie-to-pci device */
1875 if (dev_tmp) {
1876 info = alloc_devinfo_mem();
1877 if (!info) {
1878 domain_exit(domain);
1879 goto error;
1880 }
1881 info->segment = segment;
1882 info->bus = bus;
1883 info->devfn = devfn;
1884 info->dev = NULL;
1885 info->domain = domain;
1886 /* This domain is shared by devices under p2p bridge */
1887 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
1888
1889 /* pcie-to-pci bridge already has a domain, uses it */
1890 found = NULL;
1891 spin_lock_irqsave(&device_domain_lock, flags);
1892 list_for_each_entry(tmp, &device_domain_list, global) {
1893 if (tmp->segment == segment &&
1894 tmp->bus == bus && tmp->devfn == devfn) {
1895 found = tmp->domain;
1896 break;
1897 }
1898 }
1899 if (found) {
1900 free_devinfo_mem(info);
1901 domain_exit(domain);
1902 domain = found;
1903 } else {
1904 list_add(&info->link, &domain->devices);
1905 list_add(&info->global, &device_domain_list);
1906 }
1907 spin_unlock_irqrestore(&device_domain_lock, flags);
1908 }
1909
1910 found_domain:
1911 info = alloc_devinfo_mem();
1912 if (!info)
1913 goto error;
1914 info->segment = segment;
1915 info->bus = pdev->bus->number;
1916 info->devfn = pdev->devfn;
1917 info->dev = pdev;
1918 info->domain = domain;
1919 spin_lock_irqsave(&device_domain_lock, flags);
1920 /* somebody is fast */
1921 found = find_domain(pdev);
1922 if (found != NULL) {
1923 spin_unlock_irqrestore(&device_domain_lock, flags);
1924 if (found != domain) {
1925 domain_exit(domain);
1926 domain = found;
1927 }
1928 free_devinfo_mem(info);
1929 return domain;
1930 }
1931 list_add(&info->link, &domain->devices);
1932 list_add(&info->global, &device_domain_list);
1933 pdev->dev.archdata.iommu = info;
1934 spin_unlock_irqrestore(&device_domain_lock, flags);
1935 return domain;
1936 error:
1937 /* recheck it here, maybe others set it */
1938 return find_domain(pdev);
1939 }
1940
1941 static int iommu_identity_mapping;
1942 #define IDENTMAP_ALL 1
1943 #define IDENTMAP_GFX 2
1944 #define IDENTMAP_AZALIA 4
1945
1946 static int iommu_domain_identity_map(struct dmar_domain *domain,
1947 unsigned long long start,
1948 unsigned long long end)
1949 {
1950 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
1951 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
1952
1953 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
1954 dma_to_mm_pfn(last_vpfn))) {
1955 printk(KERN_ERR "IOMMU: reserve iova failed\n");
1956 return -ENOMEM;
1957 }
1958
1959 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
1960 start, end, domain->id);
1961 /*
1962 * RMRR range might have overlap with physical memory range,
1963 * clear it first
1964 */
1965 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
1966
1967 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
1968 last_vpfn - first_vpfn + 1,
1969 DMA_PTE_READ|DMA_PTE_WRITE);
1970 }
1971
1972 static int iommu_prepare_identity_map(struct pci_dev *pdev,
1973 unsigned long long start,
1974 unsigned long long end)
1975 {
1976 struct dmar_domain *domain;
1977 int ret;
1978
1979 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
1980 if (!domain)
1981 return -ENOMEM;
1982
1983 /* For _hardware_ passthrough, don't bother. But for software
1984 passthrough, we do it anyway -- it may indicate a memory
1985 range which is reserved in E820, so which didn't get set
1986 up to start with in si_domain */
1987 if (domain == si_domain && hw_pass_through) {
1988 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
1989 pci_name(pdev), start, end);
1990 return 0;
1991 }
1992
1993 printk(KERN_INFO
1994 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1995 pci_name(pdev), start, end);
1996
1997 if (end < start) {
1998 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
1999 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2000 dmi_get_system_info(DMI_BIOS_VENDOR),
2001 dmi_get_system_info(DMI_BIOS_VERSION),
2002 dmi_get_system_info(DMI_PRODUCT_VERSION));
2003 ret = -EIO;
2004 goto error;
2005 }
2006
2007 if (end >> agaw_to_width(domain->agaw)) {
2008 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2009 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2010 agaw_to_width(domain->agaw),
2011 dmi_get_system_info(DMI_BIOS_VENDOR),
2012 dmi_get_system_info(DMI_BIOS_VERSION),
2013 dmi_get_system_info(DMI_PRODUCT_VERSION));
2014 ret = -EIO;
2015 goto error;
2016 }
2017
2018 ret = iommu_domain_identity_map(domain, start, end);
2019 if (ret)
2020 goto error;
2021
2022 /* context entry init */
2023 ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
2024 if (ret)
2025 goto error;
2026
2027 return 0;
2028
2029 error:
2030 domain_exit(domain);
2031 return ret;
2032 }
2033
2034 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2035 struct pci_dev *pdev)
2036 {
2037 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2038 return 0;
2039 return iommu_prepare_identity_map(pdev, rmrr->base_address,
2040 rmrr->end_address + 1);
2041 }
2042
2043 #ifdef CONFIG_DMAR_FLOPPY_WA
2044 static inline void iommu_prepare_isa(void)
2045 {
2046 struct pci_dev *pdev;
2047 int ret;
2048
2049 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2050 if (!pdev)
2051 return;
2052
2053 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
2054 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024);
2055
2056 if (ret)
2057 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2058 "floppy might not work\n");
2059
2060 }
2061 #else
2062 static inline void iommu_prepare_isa(void)
2063 {
2064 return;
2065 }
2066 #endif /* !CONFIG_DMAR_FLPY_WA */
2067
2068 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2069
2070 static int __init si_domain_work_fn(unsigned long start_pfn,
2071 unsigned long end_pfn, void *datax)
2072 {
2073 int *ret = datax;
2074
2075 *ret = iommu_domain_identity_map(si_domain,
2076 (uint64_t)start_pfn << PAGE_SHIFT,
2077 (uint64_t)end_pfn << PAGE_SHIFT);
2078 return *ret;
2079
2080 }
2081
2082 static int __init si_domain_init(int hw)
2083 {
2084 struct dmar_drhd_unit *drhd;
2085 struct intel_iommu *iommu;
2086 int nid, ret = 0;
2087
2088 si_domain = alloc_domain();
2089 if (!si_domain)
2090 return -EFAULT;
2091
2092 pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
2093
2094 for_each_active_iommu(iommu, drhd) {
2095 ret = iommu_attach_domain(si_domain, iommu);
2096 if (ret) {
2097 domain_exit(si_domain);
2098 return -EFAULT;
2099 }
2100 }
2101
2102 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2103 domain_exit(si_domain);
2104 return -EFAULT;
2105 }
2106
2107 si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2108
2109 if (hw)
2110 return 0;
2111
2112 for_each_online_node(nid) {
2113 work_with_active_regions(nid, si_domain_work_fn, &ret);
2114 if (ret)
2115 return ret;
2116 }
2117
2118 return 0;
2119 }
2120
2121 static void domain_remove_one_dev_info(struct dmar_domain *domain,
2122 struct pci_dev *pdev);
2123 static int identity_mapping(struct pci_dev *pdev)
2124 {
2125 struct device_domain_info *info;
2126
2127 if (likely(!iommu_identity_mapping))
2128 return 0;
2129
2130
2131 list_for_each_entry(info, &si_domain->devices, link)
2132 if (info->dev == pdev)
2133 return 1;
2134 return 0;
2135 }
2136
2137 static int domain_add_dev_info(struct dmar_domain *domain,
2138 struct pci_dev *pdev,
2139 int translation)
2140 {
2141 struct device_domain_info *info;
2142 unsigned long flags;
2143 int ret;
2144
2145 info = alloc_devinfo_mem();
2146 if (!info)
2147 return -ENOMEM;
2148
2149 ret = domain_context_mapping(domain, pdev, translation);
2150 if (ret) {
2151 free_devinfo_mem(info);
2152 return ret;
2153 }
2154
2155 info->segment = pci_domain_nr(pdev->bus);
2156 info->bus = pdev->bus->number;
2157 info->devfn = pdev->devfn;
2158 info->dev = pdev;
2159 info->domain = domain;
2160
2161 spin_lock_irqsave(&device_domain_lock, flags);
2162 list_add(&info->link, &domain->devices);
2163 list_add(&info->global, &device_domain_list);
2164 pdev->dev.archdata.iommu = info;
2165 spin_unlock_irqrestore(&device_domain_lock, flags);
2166
2167 return 0;
2168 }
2169
2170 static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
2171 {
2172 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2173 return 1;
2174
2175 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2176 return 1;
2177
2178 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2179 return 0;
2180
2181 /*
2182 * We want to start off with all devices in the 1:1 domain, and
2183 * take them out later if we find they can't access all of memory.
2184 *
2185 * However, we can't do this for PCI devices behind bridges,
2186 * because all PCI devices behind the same bridge will end up
2187 * with the same source-id on their transactions.
2188 *
2189 * Practically speaking, we can't change things around for these
2190 * devices at run-time, because we can't be sure there'll be no
2191 * DMA transactions in flight for any of their siblings.
2192 *
2193 * So PCI devices (unless they're on the root bus) as well as
2194 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2195 * the 1:1 domain, just in _case_ one of their siblings turns out
2196 * not to be able to map all of memory.
2197 */
2198 if (!pdev->is_pcie) {
2199 if (!pci_is_root_bus(pdev->bus))
2200 return 0;
2201 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2202 return 0;
2203 } else if (pdev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
2204 return 0;
2205
2206 /*
2207 * At boot time, we don't yet know if devices will be 64-bit capable.
2208 * Assume that they will -- if they turn out not to be, then we can
2209 * take them out of the 1:1 domain later.
2210 */
2211 if (!startup)
2212 return pdev->dma_mask > DMA_BIT_MASK(32);
2213
2214 return 1;
2215 }
2216
2217 static int __init iommu_prepare_static_identity_mapping(int hw)
2218 {
2219 struct pci_dev *pdev = NULL;
2220 int ret;
2221
2222 ret = si_domain_init(hw);
2223 if (ret)
2224 return -EFAULT;
2225
2226 for_each_pci_dev(pdev) {
2227 if (iommu_should_identity_map(pdev, 1)) {
2228 printk(KERN_INFO "IOMMU: %s identity mapping for device %s\n",
2229 hw ? "hardware" : "software", pci_name(pdev));
2230
2231 ret = domain_add_dev_info(si_domain, pdev,
2232 hw ? CONTEXT_TT_PASS_THROUGH :
2233 CONTEXT_TT_MULTI_LEVEL);
2234 if (ret)
2235 return ret;
2236 }
2237 }
2238
2239 return 0;
2240 }
2241
2242 int __init init_dmars(void)
2243 {
2244 struct dmar_drhd_unit *drhd;
2245 struct dmar_rmrr_unit *rmrr;
2246 struct pci_dev *pdev;
2247 struct intel_iommu *iommu;
2248 int i, ret;
2249
2250 /*
2251 * for each drhd
2252 * allocate root
2253 * initialize and program root entry to not present
2254 * endfor
2255 */
2256 for_each_drhd_unit(drhd) {
2257 g_num_of_iommus++;
2258 /*
2259 * lock not needed as this is only incremented in the single
2260 * threaded kernel __init code path all other access are read
2261 * only
2262 */
2263 }
2264
2265 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2266 GFP_KERNEL);
2267 if (!g_iommus) {
2268 printk(KERN_ERR "Allocating global iommu array failed\n");
2269 ret = -ENOMEM;
2270 goto error;
2271 }
2272
2273 deferred_flush = kzalloc(g_num_of_iommus *
2274 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2275 if (!deferred_flush) {
2276 ret = -ENOMEM;
2277 goto error;
2278 }
2279
2280 for_each_drhd_unit(drhd) {
2281 if (drhd->ignored)
2282 continue;
2283
2284 iommu = drhd->iommu;
2285 g_iommus[iommu->seq_id] = iommu;
2286
2287 ret = iommu_init_domains(iommu);
2288 if (ret)
2289 goto error;
2290
2291 /*
2292 * TBD:
2293 * we could share the same root & context tables
2294 * amoung all IOMMU's. Need to Split it later.
2295 */
2296 ret = iommu_alloc_root_entry(iommu);
2297 if (ret) {
2298 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2299 goto error;
2300 }
2301 if (!ecap_pass_through(iommu->ecap))
2302 hw_pass_through = 0;
2303 }
2304
2305 /*
2306 * Start from the sane iommu hardware state.
2307 */
2308 for_each_drhd_unit(drhd) {
2309 if (drhd->ignored)
2310 continue;
2311
2312 iommu = drhd->iommu;
2313
2314 /*
2315 * If the queued invalidation is already initialized by us
2316 * (for example, while enabling interrupt-remapping) then
2317 * we got the things already rolling from a sane state.
2318 */
2319 if (iommu->qi)
2320 continue;
2321
2322 /*
2323 * Clear any previous faults.
2324 */
2325 dmar_fault(-1, iommu);
2326 /*
2327 * Disable queued invalidation if supported and already enabled
2328 * before OS handover.
2329 */
2330 dmar_disable_qi(iommu);
2331 }
2332
2333 for_each_drhd_unit(drhd) {
2334 if (drhd->ignored)
2335 continue;
2336
2337 iommu = drhd->iommu;
2338
2339 if (dmar_enable_qi(iommu)) {
2340 /*
2341 * Queued Invalidate not enabled, use Register Based
2342 * Invalidate
2343 */
2344 iommu->flush.flush_context = __iommu_flush_context;
2345 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2346 printk(KERN_INFO "IOMMU 0x%Lx: using Register based "
2347 "invalidation\n",
2348 (unsigned long long)drhd->reg_base_addr);
2349 } else {
2350 iommu->flush.flush_context = qi_flush_context;
2351 iommu->flush.flush_iotlb = qi_flush_iotlb;
2352 printk(KERN_INFO "IOMMU 0x%Lx: using Queued "
2353 "invalidation\n",
2354 (unsigned long long)drhd->reg_base_addr);
2355 }
2356 }
2357
2358 if (iommu_pass_through)
2359 iommu_identity_mapping |= IDENTMAP_ALL;
2360
2361 #ifdef CONFIG_DMAR_BROKEN_GFX_WA
2362 iommu_identity_mapping |= IDENTMAP_GFX;
2363 #endif
2364
2365 check_tylersburg_isoch();
2366
2367 /*
2368 * If pass through is not set or not enabled, setup context entries for
2369 * identity mappings for rmrr, gfx, and isa and may fall back to static
2370 * identity mapping if iommu_identity_mapping is set.
2371 */
2372 if (iommu_identity_mapping) {
2373 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
2374 if (ret) {
2375 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2376 goto error;
2377 }
2378 }
2379 /*
2380 * For each rmrr
2381 * for each dev attached to rmrr
2382 * do
2383 * locate drhd for dev, alloc domain for dev
2384 * allocate free domain
2385 * allocate page table entries for rmrr
2386 * if context not allocated for bus
2387 * allocate and init context
2388 * set present in root table for this bus
2389 * init context with domain, translation etc
2390 * endfor
2391 * endfor
2392 */
2393 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2394 for_each_rmrr_units(rmrr) {
2395 for (i = 0; i < rmrr->devices_cnt; i++) {
2396 pdev = rmrr->devices[i];
2397 /*
2398 * some BIOS lists non-exist devices in DMAR
2399 * table.
2400 */
2401 if (!pdev)
2402 continue;
2403 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2404 if (ret)
2405 printk(KERN_ERR
2406 "IOMMU: mapping reserved region failed\n");
2407 }
2408 }
2409
2410 iommu_prepare_isa();
2411
2412 /*
2413 * for each drhd
2414 * enable fault log
2415 * global invalidate context cache
2416 * global invalidate iotlb
2417 * enable translation
2418 */
2419 for_each_drhd_unit(drhd) {
2420 if (drhd->ignored)
2421 continue;
2422 iommu = drhd->iommu;
2423
2424 iommu_flush_write_buffer(iommu);
2425
2426 ret = dmar_set_interrupt(iommu);
2427 if (ret)
2428 goto error;
2429
2430 iommu_set_root_entry(iommu);
2431
2432 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
2433 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2434
2435 ret = iommu_enable_translation(iommu);
2436 if (ret)
2437 goto error;
2438
2439 iommu_disable_protect_mem_regions(iommu);
2440 }
2441
2442 return 0;
2443 error:
2444 for_each_drhd_unit(drhd) {
2445 if (drhd->ignored)
2446 continue;
2447 iommu = drhd->iommu;
2448 free_iommu(iommu);
2449 }
2450 kfree(g_iommus);
2451 return ret;
2452 }
2453
2454 /* This takes a number of _MM_ pages, not VTD pages */
2455 static struct iova *intel_alloc_iova(struct device *dev,
2456 struct dmar_domain *domain,
2457 unsigned long nrpages, uint64_t dma_mask)
2458 {
2459 struct pci_dev *pdev = to_pci_dev(dev);
2460 struct iova *iova = NULL;
2461
2462 /* Restrict dma_mask to the width that the iommu can handle */
2463 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2464
2465 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
2466 /*
2467 * First try to allocate an io virtual address in
2468 * DMA_BIT_MASK(32) and if that fails then try allocating
2469 * from higher range
2470 */
2471 iova = alloc_iova(&domain->iovad, nrpages,
2472 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2473 if (iova)
2474 return iova;
2475 }
2476 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2477 if (unlikely(!iova)) {
2478 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2479 nrpages, pci_name(pdev));
2480 return NULL;
2481 }
2482
2483 return iova;
2484 }
2485
2486 static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
2487 {
2488 struct dmar_domain *domain;
2489 int ret;
2490
2491 domain = get_domain_for_dev(pdev,
2492 DEFAULT_DOMAIN_ADDRESS_WIDTH);
2493 if (!domain) {
2494 printk(KERN_ERR
2495 "Allocating domain for %s failed", pci_name(pdev));
2496 return NULL;
2497 }
2498
2499 /* make sure context mapping is ok */
2500 if (unlikely(!domain_context_mapped(pdev))) {
2501 ret = domain_context_mapping(domain, pdev,
2502 CONTEXT_TT_MULTI_LEVEL);
2503 if (ret) {
2504 printk(KERN_ERR
2505 "Domain context map for %s failed",
2506 pci_name(pdev));
2507 return NULL;
2508 }
2509 }
2510
2511 return domain;
2512 }
2513
2514 static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
2515 {
2516 struct device_domain_info *info;
2517
2518 /* No lock here, assumes no domain exit in normal case */
2519 info = dev->dev.archdata.iommu;
2520 if (likely(info))
2521 return info->domain;
2522
2523 return __get_valid_domain_for_dev(dev);
2524 }
2525
2526 static int iommu_dummy(struct pci_dev *pdev)
2527 {
2528 return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2529 }
2530
2531 /* Check if the pdev needs to go through non-identity map and unmap process.*/
2532 static int iommu_no_mapping(struct device *dev)
2533 {
2534 struct pci_dev *pdev;
2535 int found;
2536
2537 if (unlikely(dev->bus != &pci_bus_type))
2538 return 1;
2539
2540 pdev = to_pci_dev(dev);
2541 if (iommu_dummy(pdev))
2542 return 1;
2543
2544 if (!iommu_identity_mapping)
2545 return 0;
2546
2547 found = identity_mapping(pdev);
2548 if (found) {
2549 if (iommu_should_identity_map(pdev, 0))
2550 return 1;
2551 else {
2552 /*
2553 * 32 bit DMA is removed from si_domain and fall back
2554 * to non-identity mapping.
2555 */
2556 domain_remove_one_dev_info(si_domain, pdev);
2557 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2558 pci_name(pdev));
2559 return 0;
2560 }
2561 } else {
2562 /*
2563 * In case of a detached 64 bit DMA device from vm, the device
2564 * is put into si_domain for identity mapping.
2565 */
2566 if (iommu_should_identity_map(pdev, 0)) {
2567 int ret;
2568 ret = domain_add_dev_info(si_domain, pdev,
2569 hw_pass_through ?
2570 CONTEXT_TT_PASS_THROUGH :
2571 CONTEXT_TT_MULTI_LEVEL);
2572 if (!ret) {
2573 printk(KERN_INFO "64bit %s uses identity mapping\n",
2574 pci_name(pdev));
2575 return 1;
2576 }
2577 }
2578 }
2579
2580 return 0;
2581 }
2582
2583 static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2584 size_t size, int dir, u64 dma_mask)
2585 {
2586 struct pci_dev *pdev = to_pci_dev(hwdev);
2587 struct dmar_domain *domain;
2588 phys_addr_t start_paddr;
2589 struct iova *iova;
2590 int prot = 0;
2591 int ret;
2592 struct intel_iommu *iommu;
2593 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
2594
2595 BUG_ON(dir == DMA_NONE);
2596
2597 if (iommu_no_mapping(hwdev))
2598 return paddr;
2599
2600 domain = get_valid_domain_for_dev(pdev);
2601 if (!domain)
2602 return 0;
2603
2604 iommu = domain_get_iommu(domain);
2605 size = aligned_nrpages(paddr, size);
2606
2607 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
2608 pdev->dma_mask);
2609 if (!iova)
2610 goto error;
2611
2612 /*
2613 * Check if DMAR supports zero-length reads on write only
2614 * mappings..
2615 */
2616 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2617 !cap_zlr(iommu->cap))
2618 prot |= DMA_PTE_READ;
2619 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2620 prot |= DMA_PTE_WRITE;
2621 /*
2622 * paddr - (paddr + size) might be partial page, we should map the whole
2623 * page. Note: if two part of one page are separately mapped, we
2624 * might have two guest_addr mapping to the same host paddr, but this
2625 * is not a big problem
2626 */
2627 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
2628 mm_to_dma_pfn(paddr_pfn), size, prot);
2629 if (ret)
2630 goto error;
2631
2632 /* it's a non-present to present mapping. Only flush if caching mode */
2633 if (cap_caching_mode(iommu->cap))
2634 iommu_flush_iotlb_psi(iommu, 0, mm_to_dma_pfn(iova->pfn_lo), size);
2635 else
2636 iommu_flush_write_buffer(iommu);
2637
2638 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2639 start_paddr += paddr & ~PAGE_MASK;
2640 return start_paddr;
2641
2642 error:
2643 if (iova)
2644 __free_iova(&domain->iovad, iova);
2645 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
2646 pci_name(pdev), size, (unsigned long long)paddr, dir);
2647 return 0;
2648 }
2649
2650 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2651 unsigned long offset, size_t size,
2652 enum dma_data_direction dir,
2653 struct dma_attrs *attrs)
2654 {
2655 return __intel_map_single(dev, page_to_phys(page) + offset, size,
2656 dir, to_pci_dev(dev)->dma_mask);
2657 }
2658
2659 static void flush_unmaps(void)
2660 {
2661 int i, j;
2662
2663 timer_on = 0;
2664
2665 /* just flush them all */
2666 for (i = 0; i < g_num_of_iommus; i++) {
2667 struct intel_iommu *iommu = g_iommus[i];
2668 if (!iommu)
2669 continue;
2670
2671 if (!deferred_flush[i].next)
2672 continue;
2673
2674 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2675 DMA_TLB_GLOBAL_FLUSH);
2676 for (j = 0; j < deferred_flush[i].next; j++) {
2677 unsigned long mask;
2678 struct iova *iova = deferred_flush[i].iova[j];
2679
2680 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
2681 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
2682 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
2683 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
2684 }
2685 deferred_flush[i].next = 0;
2686 }
2687
2688 list_size = 0;
2689 }
2690
2691 static void flush_unmaps_timeout(unsigned long data)
2692 {
2693 unsigned long flags;
2694
2695 spin_lock_irqsave(&async_umap_flush_lock, flags);
2696 flush_unmaps();
2697 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2698 }
2699
2700 static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2701 {
2702 unsigned long flags;
2703 int next, iommu_id;
2704 struct intel_iommu *iommu;
2705
2706 spin_lock_irqsave(&async_umap_flush_lock, flags);
2707 if (list_size == HIGH_WATER_MARK)
2708 flush_unmaps();
2709
2710 iommu = domain_get_iommu(dom);
2711 iommu_id = iommu->seq_id;
2712
2713 next = deferred_flush[iommu_id].next;
2714 deferred_flush[iommu_id].domain[next] = dom;
2715 deferred_flush[iommu_id].iova[next] = iova;
2716 deferred_flush[iommu_id].next++;
2717
2718 if (!timer_on) {
2719 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2720 timer_on = 1;
2721 }
2722 list_size++;
2723 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2724 }
2725
2726 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2727 size_t size, enum dma_data_direction dir,
2728 struct dma_attrs *attrs)
2729 {
2730 struct pci_dev *pdev = to_pci_dev(dev);
2731 struct dmar_domain *domain;
2732 unsigned long start_pfn, last_pfn;
2733 struct iova *iova;
2734 struct intel_iommu *iommu;
2735
2736 if (iommu_no_mapping(dev))
2737 return;
2738
2739 domain = find_domain(pdev);
2740 BUG_ON(!domain);
2741
2742 iommu = domain_get_iommu(domain);
2743
2744 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
2745 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
2746 (unsigned long long)dev_addr))
2747 return;
2748
2749 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2750 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2751
2752 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2753 pci_name(pdev), start_pfn, last_pfn);
2754
2755 /* clear the whole page */
2756 dma_pte_clear_range(domain, start_pfn, last_pfn);
2757
2758 /* free page tables */
2759 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2760
2761 if (intel_iommu_strict) {
2762 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2763 last_pfn - start_pfn + 1);
2764 /* free iova */
2765 __free_iova(&domain->iovad, iova);
2766 } else {
2767 add_unmap(domain, iova);
2768 /*
2769 * queue up the release of the unmap to save the 1/6th of the
2770 * cpu used up by the iotlb flush operation...
2771 */
2772 }
2773 }
2774
2775 static void *intel_alloc_coherent(struct device *hwdev, size_t size,
2776 dma_addr_t *dma_handle, gfp_t flags)
2777 {
2778 void *vaddr;
2779 int order;
2780
2781 size = PAGE_ALIGN(size);
2782 order = get_order(size);
2783
2784 if (!iommu_no_mapping(hwdev))
2785 flags &= ~(GFP_DMA | GFP_DMA32);
2786 else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
2787 if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
2788 flags |= GFP_DMA;
2789 else
2790 flags |= GFP_DMA32;
2791 }
2792
2793 vaddr = (void *)__get_free_pages(flags, order);
2794 if (!vaddr)
2795 return NULL;
2796 memset(vaddr, 0, size);
2797
2798 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2799 DMA_BIDIRECTIONAL,
2800 hwdev->coherent_dma_mask);
2801 if (*dma_handle)
2802 return vaddr;
2803 free_pages((unsigned long)vaddr, order);
2804 return NULL;
2805 }
2806
2807 static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2808 dma_addr_t dma_handle)
2809 {
2810 int order;
2811
2812 size = PAGE_ALIGN(size);
2813 order = get_order(size);
2814
2815 intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
2816 free_pages((unsigned long)vaddr, order);
2817 }
2818
2819 static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2820 int nelems, enum dma_data_direction dir,
2821 struct dma_attrs *attrs)
2822 {
2823 struct pci_dev *pdev = to_pci_dev(hwdev);
2824 struct dmar_domain *domain;
2825 unsigned long start_pfn, last_pfn;
2826 struct iova *iova;
2827 struct intel_iommu *iommu;
2828
2829 if (iommu_no_mapping(hwdev))
2830 return;
2831
2832 domain = find_domain(pdev);
2833 BUG_ON(!domain);
2834
2835 iommu = domain_get_iommu(domain);
2836
2837 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
2838 if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
2839 (unsigned long long)sglist[0].dma_address))
2840 return;
2841
2842 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2843 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2844
2845 /* clear the whole page */
2846 dma_pte_clear_range(domain, start_pfn, last_pfn);
2847
2848 /* free page tables */
2849 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2850
2851 if (intel_iommu_strict) {
2852 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2853 last_pfn - start_pfn + 1);
2854 /* free iova */
2855 __free_iova(&domain->iovad, iova);
2856 } else {
2857 add_unmap(domain, iova);
2858 /*
2859 * queue up the release of the unmap to save the 1/6th of the
2860 * cpu used up by the iotlb flush operation...
2861 */
2862 }
2863 }
2864
2865 static int intel_nontranslate_map_sg(struct device *hddev,
2866 struct scatterlist *sglist, int nelems, int dir)
2867 {
2868 int i;
2869 struct scatterlist *sg;
2870
2871 for_each_sg(sglist, sg, nelems, i) {
2872 BUG_ON(!sg_page(sg));
2873 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
2874 sg->dma_length = sg->length;
2875 }
2876 return nelems;
2877 }
2878
2879 static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
2880 enum dma_data_direction dir, struct dma_attrs *attrs)
2881 {
2882 int i;
2883 struct pci_dev *pdev = to_pci_dev(hwdev);
2884 struct dmar_domain *domain;
2885 size_t size = 0;
2886 int prot = 0;
2887 size_t offset_pfn = 0;
2888 struct iova *iova = NULL;
2889 int ret;
2890 struct scatterlist *sg;
2891 unsigned long start_vpfn;
2892 struct intel_iommu *iommu;
2893
2894 BUG_ON(dir == DMA_NONE);
2895 if (iommu_no_mapping(hwdev))
2896 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
2897
2898 domain = get_valid_domain_for_dev(pdev);
2899 if (!domain)
2900 return 0;
2901
2902 iommu = domain_get_iommu(domain);
2903
2904 for_each_sg(sglist, sg, nelems, i)
2905 size += aligned_nrpages(sg->offset, sg->length);
2906
2907 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
2908 pdev->dma_mask);
2909 if (!iova) {
2910 sglist->dma_length = 0;
2911 return 0;
2912 }
2913
2914 /*
2915 * Check if DMAR supports zero-length reads on write only
2916 * mappings..
2917 */
2918 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2919 !cap_zlr(iommu->cap))
2920 prot |= DMA_PTE_READ;
2921 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2922 prot |= DMA_PTE_WRITE;
2923
2924 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
2925
2926 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
2927 if (unlikely(ret)) {
2928 /* clear the page */
2929 dma_pte_clear_range(domain, start_vpfn,
2930 start_vpfn + size - 1);
2931 /* free page tables */
2932 dma_pte_free_pagetable(domain, start_vpfn,
2933 start_vpfn + size - 1);
2934 /* free iova */
2935 __free_iova(&domain->iovad, iova);
2936 return 0;
2937 }
2938
2939 /* it's a non-present to present mapping. Only flush if caching mode */
2940 if (cap_caching_mode(iommu->cap))
2941 iommu_flush_iotlb_psi(iommu, 0, start_vpfn, offset_pfn);
2942 else
2943 iommu_flush_write_buffer(iommu);
2944
2945 return nelems;
2946 }
2947
2948 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
2949 {
2950 return !dma_addr;
2951 }
2952
2953 struct dma_map_ops intel_dma_ops = {
2954 .alloc_coherent = intel_alloc_coherent,
2955 .free_coherent = intel_free_coherent,
2956 .map_sg = intel_map_sg,
2957 .unmap_sg = intel_unmap_sg,
2958 .map_page = intel_map_page,
2959 .unmap_page = intel_unmap_page,
2960 .mapping_error = intel_mapping_error,
2961 };
2962
2963 static inline int iommu_domain_cache_init(void)
2964 {
2965 int ret = 0;
2966
2967 iommu_domain_cache = kmem_cache_create("iommu_domain",
2968 sizeof(struct dmar_domain),
2969 0,
2970 SLAB_HWCACHE_ALIGN,
2971
2972 NULL);
2973 if (!iommu_domain_cache) {
2974 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
2975 ret = -ENOMEM;
2976 }
2977
2978 return ret;
2979 }
2980
2981 static inline int iommu_devinfo_cache_init(void)
2982 {
2983 int ret = 0;
2984
2985 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
2986 sizeof(struct device_domain_info),
2987 0,
2988 SLAB_HWCACHE_ALIGN,
2989 NULL);
2990 if (!iommu_devinfo_cache) {
2991 printk(KERN_ERR "Couldn't create devinfo cache\n");
2992 ret = -ENOMEM;
2993 }
2994
2995 return ret;
2996 }
2997
2998 static inline int iommu_iova_cache_init(void)
2999 {
3000 int ret = 0;
3001
3002 iommu_iova_cache = kmem_cache_create("iommu_iova",
3003 sizeof(struct iova),
3004 0,
3005 SLAB_HWCACHE_ALIGN,
3006 NULL);
3007 if (!iommu_iova_cache) {
3008 printk(KERN_ERR "Couldn't create iova cache\n");
3009 ret = -ENOMEM;
3010 }
3011
3012 return ret;
3013 }
3014
3015 static int __init iommu_init_mempool(void)
3016 {
3017 int ret;
3018 ret = iommu_iova_cache_init();
3019 if (ret)
3020 return ret;
3021
3022 ret = iommu_domain_cache_init();
3023 if (ret)
3024 goto domain_error;
3025
3026 ret = iommu_devinfo_cache_init();
3027 if (!ret)
3028 return ret;
3029
3030 kmem_cache_destroy(iommu_domain_cache);
3031 domain_error:
3032 kmem_cache_destroy(iommu_iova_cache);
3033
3034 return -ENOMEM;
3035 }
3036
3037 static void __init iommu_exit_mempool(void)
3038 {
3039 kmem_cache_destroy(iommu_devinfo_cache);
3040 kmem_cache_destroy(iommu_domain_cache);
3041 kmem_cache_destroy(iommu_iova_cache);
3042
3043 }
3044
3045 static void __init init_no_remapping_devices(void)
3046 {
3047 struct dmar_drhd_unit *drhd;
3048
3049 for_each_drhd_unit(drhd) {
3050 if (!drhd->include_all) {
3051 int i;
3052 for (i = 0; i < drhd->devices_cnt; i++)
3053 if (drhd->devices[i] != NULL)
3054 break;
3055 /* ignore DMAR unit if no pci devices exist */
3056 if (i == drhd->devices_cnt)
3057 drhd->ignored = 1;
3058 }
3059 }
3060
3061 if (dmar_map_gfx)
3062 return;
3063
3064 for_each_drhd_unit(drhd) {
3065 int i;
3066 if (drhd->ignored || drhd->include_all)
3067 continue;
3068
3069 for (i = 0; i < drhd->devices_cnt; i++)
3070 if (drhd->devices[i] &&
3071 !IS_GFX_DEVICE(drhd->devices[i]))
3072 break;
3073
3074 if (i < drhd->devices_cnt)
3075 continue;
3076
3077 /* bypass IOMMU if it is just for gfx devices */
3078 drhd->ignored = 1;
3079 for (i = 0; i < drhd->devices_cnt; i++) {
3080 if (!drhd->devices[i])
3081 continue;
3082 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3083 }
3084 }
3085 }
3086
3087 #ifdef CONFIG_SUSPEND
3088 static int init_iommu_hw(void)
3089 {
3090 struct dmar_drhd_unit *drhd;
3091 struct intel_iommu *iommu = NULL;
3092
3093 for_each_active_iommu(iommu, drhd)
3094 if (iommu->qi)
3095 dmar_reenable_qi(iommu);
3096
3097 for_each_active_iommu(iommu, drhd) {
3098 iommu_flush_write_buffer(iommu);
3099
3100 iommu_set_root_entry(iommu);
3101
3102 iommu->flush.flush_context(iommu, 0, 0, 0,
3103 DMA_CCMD_GLOBAL_INVL);
3104 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3105 DMA_TLB_GLOBAL_FLUSH);
3106 iommu_enable_translation(iommu);
3107 iommu_disable_protect_mem_regions(iommu);
3108 }
3109
3110 return 0;
3111 }
3112
3113 static void iommu_flush_all(void)
3114 {
3115 struct dmar_drhd_unit *drhd;
3116 struct intel_iommu *iommu;
3117
3118 for_each_active_iommu(iommu, drhd) {
3119 iommu->flush.flush_context(iommu, 0, 0, 0,
3120 DMA_CCMD_GLOBAL_INVL);
3121 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3122 DMA_TLB_GLOBAL_FLUSH);
3123 }
3124 }
3125
3126 static int iommu_suspend(struct sys_device *dev, pm_message_t state)
3127 {
3128 struct dmar_drhd_unit *drhd;
3129 struct intel_iommu *iommu = NULL;
3130 unsigned long flag;
3131
3132 for_each_active_iommu(iommu, drhd) {
3133 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3134 GFP_ATOMIC);
3135 if (!iommu->iommu_state)
3136 goto nomem;
3137 }
3138
3139 iommu_flush_all();
3140
3141 for_each_active_iommu(iommu, drhd) {
3142 iommu_disable_translation(iommu);
3143
3144 spin_lock_irqsave(&iommu->register_lock, flag);
3145
3146 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3147 readl(iommu->reg + DMAR_FECTL_REG);
3148 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3149 readl(iommu->reg + DMAR_FEDATA_REG);
3150 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3151 readl(iommu->reg + DMAR_FEADDR_REG);
3152 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3153 readl(iommu->reg + DMAR_FEUADDR_REG);
3154
3155 spin_unlock_irqrestore(&iommu->register_lock, flag);
3156 }
3157 return 0;
3158
3159 nomem:
3160 for_each_active_iommu(iommu, drhd)
3161 kfree(iommu->iommu_state);
3162
3163 return -ENOMEM;
3164 }
3165
3166 static int iommu_resume(struct sys_device *dev)
3167 {
3168 struct dmar_drhd_unit *drhd;
3169 struct intel_iommu *iommu = NULL;
3170 unsigned long flag;
3171
3172 if (init_iommu_hw()) {
3173 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3174 return -EIO;
3175 }
3176
3177 for_each_active_iommu(iommu, drhd) {
3178
3179 spin_lock_irqsave(&iommu->register_lock, flag);
3180
3181 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3182 iommu->reg + DMAR_FECTL_REG);
3183 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3184 iommu->reg + DMAR_FEDATA_REG);
3185 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3186 iommu->reg + DMAR_FEADDR_REG);
3187 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3188 iommu->reg + DMAR_FEUADDR_REG);
3189
3190 spin_unlock_irqrestore(&iommu->register_lock, flag);
3191 }
3192
3193 for_each_active_iommu(iommu, drhd)
3194 kfree(iommu->iommu_state);
3195
3196 return 0;
3197 }
3198
3199 static struct sysdev_class iommu_sysclass = {
3200 .name = "iommu",
3201 .resume = iommu_resume,
3202 .suspend = iommu_suspend,
3203 };
3204
3205 static struct sys_device device_iommu = {
3206 .cls = &iommu_sysclass,
3207 };
3208
3209 static int __init init_iommu_sysfs(void)
3210 {
3211 int error;
3212
3213 error = sysdev_class_register(&iommu_sysclass);
3214 if (error)
3215 return error;
3216
3217 error = sysdev_register(&device_iommu);
3218 if (error)
3219 sysdev_class_unregister(&iommu_sysclass);
3220
3221 return error;
3222 }
3223
3224 #else
3225 static int __init init_iommu_sysfs(void)
3226 {
3227 return 0;
3228 }
3229 #endif /* CONFIG_PM */
3230
3231 /*
3232 * Here we only respond to action of unbound device from driver.
3233 *
3234 * Added device is not attached to its DMAR domain here yet. That will happen
3235 * when mapping the device to iova.
3236 */
3237 static int device_notifier(struct notifier_block *nb,
3238 unsigned long action, void *data)
3239 {
3240 struct device *dev = data;
3241 struct pci_dev *pdev = to_pci_dev(dev);
3242 struct dmar_domain *domain;
3243
3244 if (iommu_no_mapping(dev))
3245 return 0;
3246
3247 domain = find_domain(pdev);
3248 if (!domain)
3249 return 0;
3250
3251 if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through)
3252 domain_remove_one_dev_info(domain, pdev);
3253
3254 return 0;
3255 }
3256
3257 static struct notifier_block device_nb = {
3258 .notifier_call = device_notifier,
3259 };
3260
3261 int __init intel_iommu_init(void)
3262 {
3263 int ret = 0;
3264 int force_on = 0;
3265
3266 /* VT-d is required for a TXT/tboot launch, so enforce that */
3267 force_on = tboot_force_iommu();
3268
3269 if (dmar_table_init()) {
3270 if (force_on)
3271 panic("tboot: Failed to initialize DMAR table\n");
3272 return -ENODEV;
3273 }
3274
3275 if (dmar_dev_scope_init()) {
3276 if (force_on)
3277 panic("tboot: Failed to initialize DMAR device scope\n");
3278 return -ENODEV;
3279 }
3280
3281 /*
3282 * Check the need for DMA-remapping initialization now.
3283 * Above initialization will also be used by Interrupt-remapping.
3284 */
3285 if (no_iommu || swiotlb || dmar_disabled)
3286 return -ENODEV;
3287
3288 iommu_init_mempool();
3289 dmar_init_reserved_ranges();
3290
3291 init_no_remapping_devices();
3292
3293 ret = init_dmars();
3294 if (ret) {
3295 if (force_on)
3296 panic("tboot: Failed to initialize DMARs\n");
3297 printk(KERN_ERR "IOMMU: dmar init failed\n");
3298 put_iova_domain(&reserved_iova_list);
3299 iommu_exit_mempool();
3300 return ret;
3301 }
3302 printk(KERN_INFO
3303 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3304
3305 init_timer(&unmap_timer);
3306 force_iommu = 1;
3307 dma_ops = &intel_dma_ops;
3308
3309 init_iommu_sysfs();
3310
3311 register_iommu(&intel_iommu_ops);
3312
3313 bus_register_notifier(&pci_bus_type, &device_nb);
3314
3315 return 0;
3316 }
3317
3318 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3319 struct pci_dev *pdev)
3320 {
3321 struct pci_dev *tmp, *parent;
3322
3323 if (!iommu || !pdev)
3324 return;
3325
3326 /* dependent device detach */
3327 tmp = pci_find_upstream_pcie_bridge(pdev);
3328 /* Secondary interface's bus number and devfn 0 */
3329 if (tmp) {
3330 parent = pdev->bus->self;
3331 while (parent != tmp) {
3332 iommu_detach_dev(iommu, parent->bus->number,
3333 parent->devfn);
3334 parent = parent->bus->self;
3335 }
3336 if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
3337 iommu_detach_dev(iommu,
3338 tmp->subordinate->number, 0);
3339 else /* this is a legacy PCI bridge */
3340 iommu_detach_dev(iommu, tmp->bus->number,
3341 tmp->devfn);
3342 }
3343 }
3344
3345 static void domain_remove_one_dev_info(struct dmar_domain *domain,
3346 struct pci_dev *pdev)
3347 {
3348 struct device_domain_info *info;
3349 struct intel_iommu *iommu;
3350 unsigned long flags;
3351 int found = 0;
3352 struct list_head *entry, *tmp;
3353
3354 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3355 pdev->devfn);
3356 if (!iommu)
3357 return;
3358
3359 spin_lock_irqsave(&device_domain_lock, flags);
3360 list_for_each_safe(entry, tmp, &domain->devices) {
3361 info = list_entry(entry, struct device_domain_info, link);
3362 /* No need to compare PCI domain; it has to be the same */
3363 if (info->bus == pdev->bus->number &&
3364 info->devfn == pdev->devfn) {
3365 list_del(&info->link);
3366 list_del(&info->global);
3367 if (info->dev)
3368 info->dev->dev.archdata.iommu = NULL;
3369 spin_unlock_irqrestore(&device_domain_lock, flags);
3370
3371 iommu_disable_dev_iotlb(info);
3372 iommu_detach_dev(iommu, info->bus, info->devfn);
3373 iommu_detach_dependent_devices(iommu, pdev);
3374 free_devinfo_mem(info);
3375
3376 spin_lock_irqsave(&device_domain_lock, flags);
3377
3378 if (found)
3379 break;
3380 else
3381 continue;
3382 }
3383
3384 /* if there is no other devices under the same iommu
3385 * owned by this domain, clear this iommu in iommu_bmp
3386 * update iommu count and coherency
3387 */
3388 if (iommu == device_to_iommu(info->segment, info->bus,
3389 info->devfn))
3390 found = 1;
3391 }
3392
3393 if (found == 0) {
3394 unsigned long tmp_flags;
3395 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
3396 clear_bit(iommu->seq_id, &domain->iommu_bmp);
3397 domain->iommu_count--;
3398 domain_update_iommu_cap(domain);
3399 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
3400 }
3401
3402 spin_unlock_irqrestore(&device_domain_lock, flags);
3403 }
3404
3405 static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3406 {
3407 struct device_domain_info *info;
3408 struct intel_iommu *iommu;
3409 unsigned long flags1, flags2;
3410
3411 spin_lock_irqsave(&device_domain_lock, flags1);
3412 while (!list_empty(&domain->devices)) {
3413 info = list_entry(domain->devices.next,
3414 struct device_domain_info, link);
3415 list_del(&info->link);
3416 list_del(&info->global);
3417 if (info->dev)
3418 info->dev->dev.archdata.iommu = NULL;
3419
3420 spin_unlock_irqrestore(&device_domain_lock, flags1);
3421
3422 iommu_disable_dev_iotlb(info);
3423 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
3424 iommu_detach_dev(iommu, info->bus, info->devfn);
3425 iommu_detach_dependent_devices(iommu, info->dev);
3426
3427 /* clear this iommu in iommu_bmp, update iommu count
3428 * and capabilities
3429 */
3430 spin_lock_irqsave(&domain->iommu_lock, flags2);
3431 if (test_and_clear_bit(iommu->seq_id,
3432 &domain->iommu_bmp)) {
3433 domain->iommu_count--;
3434 domain_update_iommu_cap(domain);
3435 }
3436 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3437
3438 free_devinfo_mem(info);
3439 spin_lock_irqsave(&device_domain_lock, flags1);
3440 }
3441 spin_unlock_irqrestore(&device_domain_lock, flags1);
3442 }
3443
3444 /* domain id for virtual machine, it won't be set in context */
3445 static unsigned long vm_domid;
3446
3447 static int vm_domain_min_agaw(struct dmar_domain *domain)
3448 {
3449 int i;
3450 int min_agaw = domain->agaw;
3451
3452 i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
3453 for (; i < g_num_of_iommus; ) {
3454 if (min_agaw > g_iommus[i]->agaw)
3455 min_agaw = g_iommus[i]->agaw;
3456
3457 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
3458 }
3459
3460 return min_agaw;
3461 }
3462
3463 static struct dmar_domain *iommu_alloc_vm_domain(void)
3464 {
3465 struct dmar_domain *domain;
3466
3467 domain = alloc_domain_mem();
3468 if (!domain)
3469 return NULL;
3470
3471 domain->id = vm_domid++;
3472 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
3473 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3474
3475 return domain;
3476 }
3477
3478 static int md_domain_init(struct dmar_domain *domain, int guest_width)
3479 {
3480 int adjust_width;
3481
3482 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
3483 spin_lock_init(&domain->iommu_lock);
3484
3485 domain_reserve_special_ranges(domain);
3486
3487 /* calculate AGAW */
3488 domain->gaw = guest_width;
3489 adjust_width = guestwidth_to_adjustwidth(guest_width);
3490 domain->agaw = width_to_agaw(adjust_width);
3491
3492 INIT_LIST_HEAD(&domain->devices);
3493
3494 domain->iommu_count = 0;
3495 domain->iommu_coherency = 0;
3496 domain->iommu_snooping = 0;
3497 domain->max_addr = 0;
3498
3499 /* always allocate the top pgd */
3500 domain->pgd = (struct dma_pte *)alloc_pgtable_page();
3501 if (!domain->pgd)
3502 return -ENOMEM;
3503 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3504 return 0;
3505 }
3506
3507 static void iommu_free_vm_domain(struct dmar_domain *domain)
3508 {
3509 unsigned long flags;
3510 struct dmar_drhd_unit *drhd;
3511 struct intel_iommu *iommu;
3512 unsigned long i;
3513 unsigned long ndomains;
3514
3515 for_each_drhd_unit(drhd) {
3516 if (drhd->ignored)
3517 continue;
3518 iommu = drhd->iommu;
3519
3520 ndomains = cap_ndoms(iommu->cap);
3521 i = find_first_bit(iommu->domain_ids, ndomains);
3522 for (; i < ndomains; ) {
3523 if (iommu->domains[i] == domain) {
3524 spin_lock_irqsave(&iommu->lock, flags);
3525 clear_bit(i, iommu->domain_ids);
3526 iommu->domains[i] = NULL;
3527 spin_unlock_irqrestore(&iommu->lock, flags);
3528 break;
3529 }
3530 i = find_next_bit(iommu->domain_ids, ndomains, i+1);
3531 }
3532 }
3533 }
3534
3535 static void vm_domain_exit(struct dmar_domain *domain)
3536 {
3537 /* Domain 0 is reserved, so dont process it */
3538 if (!domain)
3539 return;
3540
3541 vm_domain_remove_all_dev_info(domain);
3542 /* destroy iovas */
3543 put_iova_domain(&domain->iovad);
3544
3545 /* clear ptes */
3546 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3547
3548 /* free page tables */
3549 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3550
3551 iommu_free_vm_domain(domain);
3552 free_domain_mem(domain);
3553 }
3554
3555 static int intel_iommu_domain_init(struct iommu_domain *domain)
3556 {
3557 struct dmar_domain *dmar_domain;
3558
3559 dmar_domain = iommu_alloc_vm_domain();
3560 if (!dmar_domain) {
3561 printk(KERN_ERR
3562 "intel_iommu_domain_init: dmar_domain == NULL\n");
3563 return -ENOMEM;
3564 }
3565 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
3566 printk(KERN_ERR
3567 "intel_iommu_domain_init() failed\n");
3568 vm_domain_exit(dmar_domain);
3569 return -ENOMEM;
3570 }
3571 domain->priv = dmar_domain;
3572
3573 return 0;
3574 }
3575
3576 static void intel_iommu_domain_destroy(struct iommu_domain *domain)
3577 {
3578 struct dmar_domain *dmar_domain = domain->priv;
3579
3580 domain->priv = NULL;
3581 vm_domain_exit(dmar_domain);
3582 }
3583
3584 static int intel_iommu_attach_device(struct iommu_domain *domain,
3585 struct device *dev)
3586 {
3587 struct dmar_domain *dmar_domain = domain->priv;
3588 struct pci_dev *pdev = to_pci_dev(dev);
3589 struct intel_iommu *iommu;
3590 int addr_width;
3591 u64 end;
3592
3593 /* normally pdev is not mapped */
3594 if (unlikely(domain_context_mapped(pdev))) {
3595 struct dmar_domain *old_domain;
3596
3597 old_domain = find_domain(pdev);
3598 if (old_domain) {
3599 if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
3600 dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
3601 domain_remove_one_dev_info(old_domain, pdev);
3602 else
3603 domain_remove_dev_info(old_domain);
3604 }
3605 }
3606
3607 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3608 pdev->devfn);
3609 if (!iommu)
3610 return -ENODEV;
3611
3612 /* check if this iommu agaw is sufficient for max mapped address */
3613 addr_width = agaw_to_width(iommu->agaw);
3614 end = DOMAIN_MAX_ADDR(addr_width);
3615 end = end & VTD_PAGE_MASK;
3616 if (end < dmar_domain->max_addr) {
3617 printk(KERN_ERR "%s: iommu agaw (%d) is not "
3618 "sufficient for the mapped address (%llx)\n",
3619 __func__, iommu->agaw, dmar_domain->max_addr);
3620 return -EFAULT;
3621 }
3622
3623 return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
3624 }
3625
3626 static void intel_iommu_detach_device(struct iommu_domain *domain,
3627 struct device *dev)
3628 {
3629 struct dmar_domain *dmar_domain = domain->priv;
3630 struct pci_dev *pdev = to_pci_dev(dev);
3631
3632 domain_remove_one_dev_info(dmar_domain, pdev);
3633 }
3634
3635 static int intel_iommu_map_range(struct iommu_domain *domain,
3636 unsigned long iova, phys_addr_t hpa,
3637 size_t size, int iommu_prot)
3638 {
3639 struct dmar_domain *dmar_domain = domain->priv;
3640 u64 max_addr;
3641 int addr_width;
3642 int prot = 0;
3643 int ret;
3644
3645 if (iommu_prot & IOMMU_READ)
3646 prot |= DMA_PTE_READ;
3647 if (iommu_prot & IOMMU_WRITE)
3648 prot |= DMA_PTE_WRITE;
3649 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
3650 prot |= DMA_PTE_SNP;
3651
3652 max_addr = iova + size;
3653 if (dmar_domain->max_addr < max_addr) {
3654 int min_agaw;
3655 u64 end;
3656
3657 /* check if minimum agaw is sufficient for mapped address */
3658 min_agaw = vm_domain_min_agaw(dmar_domain);
3659 addr_width = agaw_to_width(min_agaw);
3660 end = DOMAIN_MAX_ADDR(addr_width);
3661 end = end & VTD_PAGE_MASK;
3662 if (end < max_addr) {
3663 printk(KERN_ERR "%s: iommu agaw (%d) is not "
3664 "sufficient for the mapped address (%llx)\n",
3665 __func__, min_agaw, max_addr);
3666 return -EFAULT;
3667 }
3668 dmar_domain->max_addr = max_addr;
3669 }
3670 /* Round up size to next multiple of PAGE_SIZE, if it and
3671 the low bits of hpa would take us onto the next page */
3672 size = aligned_nrpages(hpa, size);
3673 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
3674 hpa >> VTD_PAGE_SHIFT, size, prot);
3675 return ret;
3676 }
3677
3678 static void intel_iommu_unmap_range(struct iommu_domain *domain,
3679 unsigned long iova, size_t size)
3680 {
3681 struct dmar_domain *dmar_domain = domain->priv;
3682
3683 if (!size)
3684 return;
3685
3686 dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
3687 (iova + size - 1) >> VTD_PAGE_SHIFT);
3688
3689 if (dmar_domain->max_addr == iova + size)
3690 dmar_domain->max_addr = iova;
3691 }
3692
3693 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
3694 unsigned long iova)
3695 {
3696 struct dmar_domain *dmar_domain = domain->priv;
3697 struct dma_pte *pte;
3698 u64 phys = 0;
3699
3700 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT);
3701 if (pte)
3702 phys = dma_pte_addr(pte);
3703
3704 return phys;
3705 }
3706
3707 static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
3708 unsigned long cap)
3709 {
3710 struct dmar_domain *dmar_domain = domain->priv;
3711
3712 if (cap == IOMMU_CAP_CACHE_COHERENCY)
3713 return dmar_domain->iommu_snooping;
3714
3715 return 0;
3716 }
3717
3718 static struct iommu_ops intel_iommu_ops = {
3719 .domain_init = intel_iommu_domain_init,
3720 .domain_destroy = intel_iommu_domain_destroy,
3721 .attach_dev = intel_iommu_attach_device,
3722 .detach_dev = intel_iommu_detach_device,
3723 .map = intel_iommu_map_range,
3724 .unmap = intel_iommu_unmap_range,
3725 .iova_to_phys = intel_iommu_iova_to_phys,
3726 .domain_has_cap = intel_iommu_domain_has_cap,
3727 };
3728
3729 static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
3730 {
3731 /*
3732 * Mobile 4 Series Chipset neglects to set RWBF capability,
3733 * but needs it:
3734 */
3735 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
3736 rwbf_quirk = 1;
3737 }
3738
3739 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
3740
3741 /* On Tylersburg chipsets, some BIOSes have been known to enable the
3742 ISOCH DMAR unit for the Azalia sound device, but not give it any
3743 TLB entries, which causes it to deadlock. Check for that. We do
3744 this in a function called from init_dmars(), instead of in a PCI
3745 quirk, because we don't want to print the obnoxious "BIOS broken"
3746 message if VT-d is actually disabled.
3747 */
3748 static void __init check_tylersburg_isoch(void)
3749 {
3750 struct pci_dev *pdev;
3751 uint32_t vtisochctrl;
3752
3753 /* If there's no Azalia in the system anyway, forget it. */
3754 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
3755 if (!pdev)
3756 return;
3757 pci_dev_put(pdev);
3758
3759 /* System Management Registers. Might be hidden, in which case
3760 we can't do the sanity check. But that's OK, because the
3761 known-broken BIOSes _don't_ actually hide it, so far. */
3762 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
3763 if (!pdev)
3764 return;
3765
3766 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
3767 pci_dev_put(pdev);
3768 return;
3769 }
3770
3771 pci_dev_put(pdev);
3772
3773 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
3774 if (vtisochctrl & 1)
3775 return;
3776
3777 /* Drop all bits other than the number of TLB entries */
3778 vtisochctrl &= 0x1c;
3779
3780 /* If we have the recommended number of TLB entries (16), fine. */
3781 if (vtisochctrl == 0x10)
3782 return;
3783
3784 /* Zero TLB entries? You get to ride the short bus to school. */
3785 if (!vtisochctrl) {
3786 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
3787 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
3788 dmi_get_system_info(DMI_BIOS_VENDOR),
3789 dmi_get_system_info(DMI_BIOS_VERSION),
3790 dmi_get_system_info(DMI_PRODUCT_VERSION));
3791 iommu_identity_mapping |= IDENTMAP_AZALIA;
3792 return;
3793 }
3794
3795 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
3796 vtisochctrl);
3797 }
kernels