?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
repo init
[linux-rt-nao.git] / drivers / pci / intel-iommu.c
blobc90c1a48ca4259c2175531c3d45432296795c082
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 * Author: Fenghua Yu <fenghua.yu@intel.com>
22 */
23
24 #include <linux/init.h>
25 #include <linux/bitmap.h>
26 #include <linux/debugfs.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/timer.h>
36 #include <linux/iova.h>
37 #include <linux/iommu.h>
38 #include <linux/intel-iommu.h>
39 #include <asm/cacheflush.h>
40 #include <asm/iommu.h>
41 #include "pci.h"
42
43 #define ROOT_SIZE VTD_PAGE_SIZE
44 #define CONTEXT_SIZE VTD_PAGE_SIZE
45
46 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
47 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
48
49 #define IOAPIC_RANGE_START (0xfee00000)
50 #define IOAPIC_RANGE_END (0xfeefffff)
51 #define IOVA_START_ADDR (0x1000)
52
53 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
54
55 #define DOMAIN_MAX_ADDR(gaw) ((((u64)1) << gaw) - 1)
56
57 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
58 #define DMA_32BIT_PFN IOVA_PFN(DMA_32BIT_MASK)
59 #define DMA_64BIT_PFN IOVA_PFN(DMA_64BIT_MASK)
60
61 /* global iommu list, set NULL for ignored DMAR units */
62 static struct intel_iommu **g_iommus;
63
64 static int rwbf_quirk;
65
66 /*
67 * 0: Present
68 * 1-11: Reserved
69 * 12-63: Context Ptr (12 - (haw-1))
70 * 64-127: Reserved
71 */
72 struct root_entry {
73 u64 val;
74 u64 rsvd1;
75 };
76 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
77 static inline bool root_present(struct root_entry *root)
78 {
79 return (root->val & 1);
80 }
81 static inline void set_root_present(struct root_entry *root)
82 {
83 root->val |= 1;
84 }
85 static inline void set_root_value(struct root_entry *root, unsigned long value)
86 {
87 root->val |= value & VTD_PAGE_MASK;
88 }
89
90 static inline struct context_entry *
91 get_context_addr_from_root(struct root_entry *root)
92 {
93 return (struct context_entry *)
94 (root_present(root)?phys_to_virt(
95 root->val & VTD_PAGE_MASK) :
96 NULL);
97 }
98
99 /*
100 * low 64 bits:
101 * 0: present
102 * 1: fault processing disable
103 * 2-3: translation type
104 * 12-63: address space root
105 * high 64 bits:
106 * 0-2: address width
107 * 3-6: aval
108 * 8-23: domain id
109 */
110 struct context_entry {
111 u64 lo;
112 u64 hi;
113 };
114
115 static inline bool context_present(struct context_entry *context)
116 {
117 return (context->lo & 1);
118 }
119 static inline void context_set_present(struct context_entry *context)
120 {
121 context->lo |= 1;
122 }
123
124 static inline void context_set_fault_enable(struct context_entry *context)
125 {
126 context->lo &= (((u64)-1) << 2) | 1;
127 }
128
129 #define CONTEXT_TT_MULTI_LEVEL 0
130
131 static inline void context_set_translation_type(struct context_entry *context,
132 unsigned long value)
133 {
134 context->lo &= (((u64)-1) << 4) | 3;
135 context->lo |= (value & 3) << 2;
136 }
137
138 static inline void context_set_address_root(struct context_entry *context,
139 unsigned long value)
140 {
141 context->lo |= value & VTD_PAGE_MASK;
142 }
143
144 static inline void context_set_address_width(struct context_entry *context,
145 unsigned long value)
146 {
147 context->hi |= value & 7;
148 }
149
150 static inline void context_set_domain_id(struct context_entry *context,
151 unsigned long value)
152 {
153 context->hi |= (value & ((1 << 16) - 1)) << 8;
154 }
155
156 static inline void context_clear_entry(struct context_entry *context)
157 {
158 context->lo = 0;
159 context->hi = 0;
160 }
161
162 /*
163 * 0: readable
164 * 1: writable
165 * 2-6: reserved
166 * 7: super page
167 * 8-11: available
168 * 12-63: Host physcial address
169 */
170 struct dma_pte {
171 u64 val;
172 };
173
174 static inline void dma_clear_pte(struct dma_pte *pte)
175 {
176 pte->val = 0;
177 }
178
179 static inline void dma_set_pte_readable(struct dma_pte *pte)
180 {
181 pte->val |= DMA_PTE_READ;
182 }
183
184 static inline void dma_set_pte_writable(struct dma_pte *pte)
185 {
186 pte->val |= DMA_PTE_WRITE;
187 }
188
189 static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
190 {
191 pte->val = (pte->val & ~3) | (prot & 3);
192 }
193
194 static inline u64 dma_pte_addr(struct dma_pte *pte)
195 {
196 return (pte->val & VTD_PAGE_MASK);
197 }
198
199 static inline void dma_set_pte_addr(struct dma_pte *pte, u64 addr)
200 {
201 pte->val |= (addr & VTD_PAGE_MASK);
202 }
203
204 static inline bool dma_pte_present(struct dma_pte *pte)
205 {
206 return (pte->val & 3) != 0;
207 }
208
209 /* devices under the same p2p bridge are owned in one domain */
210 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
211
212 /* domain represents a virtual machine, more than one devices
213 * across iommus may be owned in one domain, e.g. kvm guest.
214 */
215 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
216
217 struct dmar_domain {
218 int id; /* domain id */
219 unsigned long iommu_bmp; /* bitmap of iommus this domain uses*/
220
221 struct list_head devices; /* all devices' list */
222 struct iova_domain iovad; /* iova's that belong to this domain */
223
224 struct dma_pte *pgd; /* virtual address */
225 spinlock_t mapping_lock; /* page table lock */
226 int gaw; /* max guest address width */
227
228 /* adjusted guest address width, 0 is level 2 30-bit */
229 int agaw;
230
231 int flags; /* flags to find out type of domain */
232
233 int iommu_coherency;/* indicate coherency of iommu access */
234 int iommu_count; /* reference count of iommu */
235 spinlock_t iommu_lock; /* protect iommu set in domain */
236 u64 max_addr; /* maximum mapped address */
237 };
238
239 /* PCI domain-device relationship */
240 struct device_domain_info {
241 struct list_head link; /* link to domain siblings */
242 struct list_head global; /* link to global list */
243 u8 bus; /* PCI bus numer */
244 u8 devfn; /* PCI devfn number */
245 struct pci_dev *dev; /* it's NULL for PCIE-to-PCI bridge */
246 struct dmar_domain *domain; /* pointer to domain */
247 };
248
249 static void flush_unmaps_timeout(unsigned long data);
250
251 DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
252
253 #define HIGH_WATER_MARK 250
254 struct deferred_flush_tables {
255 int next;
256 struct iova *iova[HIGH_WATER_MARK];
257 struct dmar_domain *domain[HIGH_WATER_MARK];
258 };
259
260 static struct deferred_flush_tables *deferred_flush;
261
262 /* bitmap for indexing intel_iommus */
263 static int g_num_of_iommus;
264
265 static DEFINE_SPINLOCK(async_umap_flush_lock);
266 static LIST_HEAD(unmaps_to_do);
267
268 static int timer_on;
269 static long list_size;
270
271 static void domain_remove_dev_info(struct dmar_domain *domain);
272
273 #ifdef CONFIG_DMAR_DEFAULT_ON
274 int dmar_disabled = 0;
275 #else
276 int dmar_disabled = 1;
277 #endif /*CONFIG_DMAR_DEFAULT_ON*/
278
279 static int __initdata dmar_map_gfx = 1;
280 static int dmar_forcedac;
281 static int intel_iommu_strict;
282
283 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
284 static DEFINE_SPINLOCK(device_domain_lock);
285 static LIST_HEAD(device_domain_list);
286
287 static struct iommu_ops intel_iommu_ops;
288
289 static int __init intel_iommu_setup(char *str)
290 {
291 if (!str)
292 return -EINVAL;
293 while (*str) {
294 if (!strncmp(str, "on", 2)) {
295 dmar_disabled = 0;
296 printk(KERN_INFO "Intel-IOMMU: enabled\n");
297 } else if (!strncmp(str, "off", 3)) {
298 dmar_disabled = 1;
299 printk(KERN_INFO "Intel-IOMMU: disabled\n");
300 } else if (!strncmp(str, "igfx_off", 8)) {
301 dmar_map_gfx = 0;
302 printk(KERN_INFO
303 "Intel-IOMMU: disable GFX device mapping\n");
304 } else if (!strncmp(str, "forcedac", 8)) {
305 printk(KERN_INFO
306 "Intel-IOMMU: Forcing DAC for PCI devices\n");
307 dmar_forcedac = 1;
308 } else if (!strncmp(str, "strict", 6)) {
309 printk(KERN_INFO
310 "Intel-IOMMU: disable batched IOTLB flush\n");
311 intel_iommu_strict = 1;
312 }
313
314 str += strcspn(str, ",");
315 while (*str == ',')
316 str++;
317 }
318 return 0;
319 }
320 __setup("intel_iommu=", intel_iommu_setup);
321
322 static struct kmem_cache *iommu_domain_cache;
323 static struct kmem_cache *iommu_devinfo_cache;
324 static struct kmem_cache *iommu_iova_cache;
325
326 static inline void *iommu_kmem_cache_alloc(struct kmem_cache *cachep)
327 {
328 unsigned int flags;
329 void *vaddr;
330
331 /* trying to avoid low memory issues */
332 flags = current->flags & PF_MEMALLOC;
333 current->flags |= PF_MEMALLOC;
334 vaddr = kmem_cache_alloc(cachep, GFP_ATOMIC);
335 current->flags &= (~PF_MEMALLOC | flags);
336 return vaddr;
337 }
338
339
340 static inline void *alloc_pgtable_page(void)
341 {
342 unsigned int flags;
343 void *vaddr;
344
345 /* trying to avoid low memory issues */
346 flags = current->flags & PF_MEMALLOC;
347 current->flags |= PF_MEMALLOC;
348 vaddr = (void *)get_zeroed_page(GFP_ATOMIC);
349 current->flags &= (~PF_MEMALLOC | flags);
350 return vaddr;
351 }
352
353 static inline void free_pgtable_page(void *vaddr)
354 {
355 free_page((unsigned long)vaddr);
356 }
357
358 static inline void *alloc_domain_mem(void)
359 {
360 return iommu_kmem_cache_alloc(iommu_domain_cache);
361 }
362
363 static void free_domain_mem(void *vaddr)
364 {
365 kmem_cache_free(iommu_domain_cache, vaddr);
366 }
367
368 static inline void * alloc_devinfo_mem(void)
369 {
370 return iommu_kmem_cache_alloc(iommu_devinfo_cache);
371 }
372
373 static inline void free_devinfo_mem(void *vaddr)
374 {
375 kmem_cache_free(iommu_devinfo_cache, vaddr);
376 }
377
378 struct iova *alloc_iova_mem(void)
379 {
380 return iommu_kmem_cache_alloc(iommu_iova_cache);
381 }
382
383 void free_iova_mem(struct iova *iova)
384 {
385 kmem_cache_free(iommu_iova_cache, iova);
386 }
387
388
389 static inline int width_to_agaw(int width);
390
391 /* calculate agaw for each iommu.
392 * "SAGAW" may be different across iommus, use a default agaw, and
393 * get a supported less agaw for iommus that don't support the default agaw.
394 */
395 int iommu_calculate_agaw(struct intel_iommu *iommu)
396 {
397 unsigned long sagaw;
398 int agaw = -1;
399
400 sagaw = cap_sagaw(iommu->cap);
401 for (agaw = width_to_agaw(DEFAULT_DOMAIN_ADDRESS_WIDTH);
402 agaw >= 0; agaw--) {
403 if (test_bit(agaw, &sagaw))
404 break;
405 }
406
407 return agaw;
408 }
409
410 /* in native case, each domain is related to only one iommu */
411 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
412 {
413 int iommu_id;
414
415 BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
416
417 iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
418 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
419 return NULL;
420
421 return g_iommus[iommu_id];
422 }
423
424 /* "Coherency" capability may be different across iommus */
425 static void domain_update_iommu_coherency(struct dmar_domain *domain)
426 {
427 int i;
428
429 domain->iommu_coherency = 1;
430
431 i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
432 for (; i < g_num_of_iommus; ) {
433 if (!ecap_coherent(g_iommus[i]->ecap)) {
434 domain->iommu_coherency = 0;
435 break;
436 }
437 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
438 }
439 }
440
441 static struct intel_iommu *device_to_iommu(u8 bus, u8 devfn)
442 {
443 struct dmar_drhd_unit *drhd = NULL;
444 int i;
445
446 for_each_drhd_unit(drhd) {
447 if (drhd->ignored)
448 continue;
449
450 for (i = 0; i < drhd->devices_cnt; i++) {
451 if (drhd->devices[i] &&
452 drhd->devices[i]->bus->number == bus &&
453 drhd->devices[i]->devfn == devfn)
454 return drhd->iommu;
455 if (drhd->devices[i] &&
456 drhd->devices[i]->subordinate &&
457 drhd->devices[i]->subordinate->number <= bus &&
458 drhd->devices[i]->subordinate->subordinate >= bus)
459 return drhd->iommu;
460 }
461
462 if (drhd->include_all)
463 return drhd->iommu;
464 }
465
466 return NULL;
467 }
468
469 static void domain_flush_cache(struct dmar_domain *domain,
470 void *addr, int size)
471 {
472 if (!domain->iommu_coherency)
473 clflush_cache_range(addr, size);
474 }
475
476 /* Gets context entry for a given bus and devfn */
477 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
478 u8 bus, u8 devfn)
479 {
480 struct root_entry *root;
481 struct context_entry *context;
482 unsigned long phy_addr;
483 unsigned long flags;
484
485 spin_lock_irqsave(&iommu->lock, flags);
486 root = &iommu->root_entry[bus];
487 context = get_context_addr_from_root(root);
488 if (!context) {
489 context = (struct context_entry *)alloc_pgtable_page();
490 if (!context) {
491 spin_unlock_irqrestore(&iommu->lock, flags);
492 return NULL;
493 }
494 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
495 phy_addr = virt_to_phys((void *)context);
496 set_root_value(root, phy_addr);
497 set_root_present(root);
498 __iommu_flush_cache(iommu, root, sizeof(*root));
499 }
500 spin_unlock_irqrestore(&iommu->lock, flags);
501 return &context[devfn];
502 }
503
504 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
505 {
506 struct root_entry *root;
507 struct context_entry *context;
508 int ret;
509 unsigned long flags;
510
511 spin_lock_irqsave(&iommu->lock, flags);
512 root = &iommu->root_entry[bus];
513 context = get_context_addr_from_root(root);
514 if (!context) {
515 ret = 0;
516 goto out;
517 }
518 ret = context_present(&context[devfn]);
519 out:
520 spin_unlock_irqrestore(&iommu->lock, flags);
521 return ret;
522 }
523
524 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
525 {
526 struct root_entry *root;
527 struct context_entry *context;
528 unsigned long flags;
529
530 spin_lock_irqsave(&iommu->lock, flags);
531 root = &iommu->root_entry[bus];
532 context = get_context_addr_from_root(root);
533 if (context) {
534 context_clear_entry(&context[devfn]);
535 __iommu_flush_cache(iommu, &context[devfn], \
536 sizeof(*context));
537 }
538 spin_unlock_irqrestore(&iommu->lock, flags);
539 }
540
541 static void free_context_table(struct intel_iommu *iommu)
542 {
543 struct root_entry *root;
544 int i;
545 unsigned long flags;
546 struct context_entry *context;
547
548 spin_lock_irqsave(&iommu->lock, flags);
549 if (!iommu->root_entry) {
550 goto out;
551 }
552 for (i = 0; i < ROOT_ENTRY_NR; i++) {
553 root = &iommu->root_entry[i];
554 context = get_context_addr_from_root(root);
555 if (context)
556 free_pgtable_page(context);
557 }
558 free_pgtable_page(iommu->root_entry);
559 iommu->root_entry = NULL;
560 out:
561 spin_unlock_irqrestore(&iommu->lock, flags);
562 }
563
564 /* page table handling */
565 #define LEVEL_STRIDE (9)
566 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
567
568 static inline int agaw_to_level(int agaw)
569 {
570 return agaw + 2;
571 }
572
573 static inline int agaw_to_width(int agaw)
574 {
575 return 30 + agaw * LEVEL_STRIDE;
576
577 }
578
579 static inline int width_to_agaw(int width)
580 {
581 return (width - 30) / LEVEL_STRIDE;
582 }
583
584 static inline unsigned int level_to_offset_bits(int level)
585 {
586 return (12 + (level - 1) * LEVEL_STRIDE);
587 }
588
589 static inline int address_level_offset(u64 addr, int level)
590 {
591 return ((addr >> level_to_offset_bits(level)) & LEVEL_MASK);
592 }
593
594 static inline u64 level_mask(int level)
595 {
596 return ((u64)-1 << level_to_offset_bits(level));
597 }
598
599 static inline u64 level_size(int level)
600 {
601 return ((u64)1 << level_to_offset_bits(level));
602 }
603
604 static inline u64 align_to_level(u64 addr, int level)
605 {
606 return ((addr + level_size(level) - 1) & level_mask(level));
607 }
608
609 static struct dma_pte * addr_to_dma_pte(struct dmar_domain *domain, u64 addr)
610 {
611 int addr_width = agaw_to_width(domain->agaw);
612 struct dma_pte *parent, *pte = NULL;
613 int level = agaw_to_level(domain->agaw);
614 int offset;
615 unsigned long flags;
616
617 BUG_ON(!domain->pgd);
618
619 addr &= (((u64)1) << addr_width) - 1;
620 parent = domain->pgd;
621
622 spin_lock_irqsave(&domain->mapping_lock, flags);
623 while (level > 0) {
624 void *tmp_page;
625
626 offset = address_level_offset(addr, level);
627 pte = &parent[offset];
628 if (level == 1)
629 break;
630
631 if (!dma_pte_present(pte)) {
632 tmp_page = alloc_pgtable_page();
633
634 if (!tmp_page) {
635 spin_unlock_irqrestore(&domain->mapping_lock,
636 flags);
637 return NULL;
638 }
639 domain_flush_cache(domain, tmp_page, PAGE_SIZE);
640 dma_set_pte_addr(pte, virt_to_phys(tmp_page));
641 /*
642 * high level table always sets r/w, last level page
643 * table control read/write
644 */
645 dma_set_pte_readable(pte);
646 dma_set_pte_writable(pte);
647 domain_flush_cache(domain, pte, sizeof(*pte));
648 }
649 parent = phys_to_virt(dma_pte_addr(pte));
650 level--;
651 }
652
653 spin_unlock_irqrestore(&domain->mapping_lock, flags);
654 return pte;
655 }
656
657 /* return address's pte at specific level */
658 static struct dma_pte *dma_addr_level_pte(struct dmar_domain *domain, u64 addr,
659 int level)
660 {
661 struct dma_pte *parent, *pte = NULL;
662 int total = agaw_to_level(domain->agaw);
663 int offset;
664
665 parent = domain->pgd;
666 while (level <= total) {
667 offset = address_level_offset(addr, total);
668 pte = &parent[offset];
669 if (level == total)
670 return pte;
671
672 if (!dma_pte_present(pte))
673 break;
674 parent = phys_to_virt(dma_pte_addr(pte));
675 total--;
676 }
677 return NULL;
678 }
679
680 /* clear one page's page table */
681 static void dma_pte_clear_one(struct dmar_domain *domain, u64 addr)
682 {
683 struct dma_pte *pte = NULL;
684
685 /* get last level pte */
686 pte = dma_addr_level_pte(domain, addr, 1);
687
688 if (pte) {
689 dma_clear_pte(pte);
690 domain_flush_cache(domain, pte, sizeof(*pte));
691 }
692 }
693
694 /* clear last level pte, a tlb flush should be followed */
695 static void dma_pte_clear_range(struct dmar_domain *domain, u64 start, u64 end)
696 {
697 int addr_width = agaw_to_width(domain->agaw);
698
699 start &= (((u64)1) << addr_width) - 1;
700 end &= (((u64)1) << addr_width) - 1;
701 /* in case it's partial page */
702 start = PAGE_ALIGN(start);
703 end &= PAGE_MASK;
704
705 /* we don't need lock here, nobody else touches the iova range */
706 while (start < end) {
707 dma_pte_clear_one(domain, start);
708 start += VTD_PAGE_SIZE;
709 }
710 }
711
712 /* free page table pages. last level pte should already be cleared */
713 static void dma_pte_free_pagetable(struct dmar_domain *domain,
714 u64 start, u64 end)
715 {
716 int addr_width = agaw_to_width(domain->agaw);
717 struct dma_pte *pte;
718 int total = agaw_to_level(domain->agaw);
719 int level;
720 u64 tmp;
721
722 start &= (((u64)1) << addr_width) - 1;
723 end &= (((u64)1) << addr_width) - 1;
724
725 /* we don't need lock here, nobody else touches the iova range */
726 level = 2;
727 while (level <= total) {
728 tmp = align_to_level(start, level);
729 if (tmp >= end || (tmp + level_size(level) > end))
730 return;
731
732 while (tmp < end) {
733 pte = dma_addr_level_pte(domain, tmp, level);
734 if (pte) {
735 free_pgtable_page(
736 phys_to_virt(dma_pte_addr(pte)));
737 dma_clear_pte(pte);
738 domain_flush_cache(domain, pte, sizeof(*pte));
739 }
740 tmp += level_size(level);
741 }
742 level++;
743 }
744 /* free pgd */
745 if (start == 0 && end >= ((((u64)1) << addr_width) - 1)) {
746 free_pgtable_page(domain->pgd);
747 domain->pgd = NULL;
748 }
749 }
750
751 /* iommu handling */
752 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
753 {
754 struct root_entry *root;
755 unsigned long flags;
756
757 root = (struct root_entry *)alloc_pgtable_page();
758 if (!root)
759 return -ENOMEM;
760
761 __iommu_flush_cache(iommu, root, ROOT_SIZE);
762
763 spin_lock_irqsave(&iommu->lock, flags);
764 iommu->root_entry = root;
765 spin_unlock_irqrestore(&iommu->lock, flags);
766
767 return 0;
768 }
769
770 static void iommu_set_root_entry(struct intel_iommu *iommu)
771 {
772 void *addr;
773 u32 cmd, sts;
774 unsigned long flag;
775
776 addr = iommu->root_entry;
777
778 spin_lock_irqsave(&iommu->register_lock, flag);
779 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
780
781 cmd = iommu->gcmd | DMA_GCMD_SRTP;
782 writel(cmd, iommu->reg + DMAR_GCMD_REG);
783
784 /* Make sure hardware complete it */
785 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
786 readl, (sts & DMA_GSTS_RTPS), sts);
787
788 spin_unlock_irqrestore(&iommu->register_lock, flag);
789 }
790
791 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
792 {
793 u32 val;
794 unsigned long flag;
795
796 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
797 return;
798 val = iommu->gcmd | DMA_GCMD_WBF;
799
800 spin_lock_irqsave(&iommu->register_lock, flag);
801 writel(val, iommu->reg + DMAR_GCMD_REG);
802
803 /* Make sure hardware complete it */
804 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
805 readl, (!(val & DMA_GSTS_WBFS)), val);
806
807 spin_unlock_irqrestore(&iommu->register_lock, flag);
808 }
809
810 /* return value determine if we need a write buffer flush */
811 static int __iommu_flush_context(struct intel_iommu *iommu,
812 u16 did, u16 source_id, u8 function_mask, u64 type,
813 int non_present_entry_flush)
814 {
815 u64 val = 0;
816 unsigned long flag;
817
818 /*
819 * In the non-present entry flush case, if hardware doesn't cache
820 * non-present entry we do nothing and if hardware cache non-present
821 * entry, we flush entries of domain 0 (the domain id is used to cache
822 * any non-present entries)
823 */
824 if (non_present_entry_flush) {
825 if (!cap_caching_mode(iommu->cap))
826 return 1;
827 else
828 did = 0;
829 }
830
831 switch (type) {
832 case DMA_CCMD_GLOBAL_INVL:
833 val = DMA_CCMD_GLOBAL_INVL;
834 break;
835 case DMA_CCMD_DOMAIN_INVL:
836 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
837 break;
838 case DMA_CCMD_DEVICE_INVL:
839 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
840 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
841 break;
842 default:
843 BUG();
844 }
845 val |= DMA_CCMD_ICC;
846
847 spin_lock_irqsave(&iommu->register_lock, flag);
848 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
849
850 /* Make sure hardware complete it */
851 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
852 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
853
854 spin_unlock_irqrestore(&iommu->register_lock, flag);
855
856 /* flush context entry will implicitly flush write buffer */
857 return 0;
858 }
859
860 /* return value determine if we need a write buffer flush */
861 static int __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
862 u64 addr, unsigned int size_order, u64 type,
863 int non_present_entry_flush)
864 {
865 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
866 u64 val = 0, val_iva = 0;
867 unsigned long flag;
868
869 /*
870 * In the non-present entry flush case, if hardware doesn't cache
871 * non-present entry we do nothing and if hardware cache non-present
872 * entry, we flush entries of domain 0 (the domain id is used to cache
873 * any non-present entries)
874 */
875 if (non_present_entry_flush) {
876 if (!cap_caching_mode(iommu->cap))
877 return 1;
878 else
879 did = 0;
880 }
881
882 switch (type) {
883 case DMA_TLB_GLOBAL_FLUSH:
884 /* global flush doesn't need set IVA_REG */
885 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
886 break;
887 case DMA_TLB_DSI_FLUSH:
888 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
889 break;
890 case DMA_TLB_PSI_FLUSH:
891 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
892 /* Note: always flush non-leaf currently */
893 val_iva = size_order | addr;
894 break;
895 default:
896 BUG();
897 }
898 /* Note: set drain read/write */
899 #if 0
900 /*
901 * This is probably to be super secure.. Looks like we can
902 * ignore it without any impact.
903 */
904 if (cap_read_drain(iommu->cap))
905 val |= DMA_TLB_READ_DRAIN;
906 #endif
907 if (cap_write_drain(iommu->cap))
908 val |= DMA_TLB_WRITE_DRAIN;
909
910 spin_lock_irqsave(&iommu->register_lock, flag);
911 /* Note: Only uses first TLB reg currently */
912 if (val_iva)
913 dmar_writeq(iommu->reg + tlb_offset, val_iva);
914 dmar_writeq(iommu->reg + tlb_offset + 8, val);
915
916 /* Make sure hardware complete it */
917 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
918 dmar_readq, (!(val & DMA_TLB_IVT)), val);
919
920 spin_unlock_irqrestore(&iommu->register_lock, flag);
921
922 /* check IOTLB invalidation granularity */
923 if (DMA_TLB_IAIG(val) == 0)
924 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
925 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
926 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
927 (unsigned long long)DMA_TLB_IIRG(type),
928 (unsigned long long)DMA_TLB_IAIG(val));
929 /* flush iotlb entry will implicitly flush write buffer */
930 return 0;
931 }
932
933 static int iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
934 u64 addr, unsigned int pages, int non_present_entry_flush)
935 {
936 unsigned int mask;
937
938 BUG_ON(addr & (~VTD_PAGE_MASK));
939 BUG_ON(pages == 0);
940
941 /* Fallback to domain selective flush if no PSI support */
942 if (!cap_pgsel_inv(iommu->cap))
943 return iommu->flush.flush_iotlb(iommu, did, 0, 0,
944 DMA_TLB_DSI_FLUSH,
945 non_present_entry_flush);
946
947 /*
948 * PSI requires page size to be 2 ^ x, and the base address is naturally
949 * aligned to the size
950 */
951 mask = ilog2(__roundup_pow_of_two(pages));
952 /* Fallback to domain selective flush if size is too big */
953 if (mask > cap_max_amask_val(iommu->cap))
954 return iommu->flush.flush_iotlb(iommu, did, 0, 0,
955 DMA_TLB_DSI_FLUSH, non_present_entry_flush);
956
957 return iommu->flush.flush_iotlb(iommu, did, addr, mask,
958 DMA_TLB_PSI_FLUSH,
959 non_present_entry_flush);
960 }
961
962 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
963 {
964 u32 pmen;
965 unsigned long flags;
966
967 spin_lock_irqsave(&iommu->register_lock, flags);
968 pmen = readl(iommu->reg + DMAR_PMEN_REG);
969 pmen &= ~DMA_PMEN_EPM;
970 writel(pmen, iommu->reg + DMAR_PMEN_REG);
971
972 /* wait for the protected region status bit to clear */
973 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
974 readl, !(pmen & DMA_PMEN_PRS), pmen);
975
976 spin_unlock_irqrestore(&iommu->register_lock, flags);
977 }
978
979 static int iommu_enable_translation(struct intel_iommu *iommu)
980 {
981 u32 sts;
982 unsigned long flags;
983
984 spin_lock_irqsave(&iommu->register_lock, flags);
985 writel(iommu->gcmd|DMA_GCMD_TE, iommu->reg + DMAR_GCMD_REG);
986
987 /* Make sure hardware complete it */
988 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
989 readl, (sts & DMA_GSTS_TES), sts);
990
991 iommu->gcmd |= DMA_GCMD_TE;
992 spin_unlock_irqrestore(&iommu->register_lock, flags);
993 return 0;
994 }
995
996 static int iommu_disable_translation(struct intel_iommu *iommu)
997 {
998 u32 sts;
999 unsigned long flag;
1000
1001 spin_lock_irqsave(&iommu->register_lock, flag);
1002 iommu->gcmd &= ~DMA_GCMD_TE;
1003 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1004
1005 /* Make sure hardware complete it */
1006 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1007 readl, (!(sts & DMA_GSTS_TES)), sts);
1008
1009 spin_unlock_irqrestore(&iommu->register_lock, flag);
1010 return 0;
1011 }
1012
1013
1014 static int iommu_init_domains(struct intel_iommu *iommu)
1015 {
1016 unsigned long ndomains;
1017 unsigned long nlongs;
1018
1019 ndomains = cap_ndoms(iommu->cap);
1020 pr_debug("Number of Domains supportd <%ld>\n", ndomains);
1021 nlongs = BITS_TO_LONGS(ndomains);
1022
1023 /* TBD: there might be 64K domains,
1024 * consider other allocation for future chip
1025 */
1026 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1027 if (!iommu->domain_ids) {
1028 printk(KERN_ERR "Allocating domain id array failed\n");
1029 return -ENOMEM;
1030 }
1031 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1032 GFP_KERNEL);
1033 if (!iommu->domains) {
1034 printk(KERN_ERR "Allocating domain array failed\n");
1035 kfree(iommu->domain_ids);
1036 return -ENOMEM;
1037 }
1038
1039 spin_lock_init(&iommu->lock);
1040
1041 /*
1042 * if Caching mode is set, then invalid translations are tagged
1043 * with domainid 0. Hence we need to pre-allocate it.
1044 */
1045 if (cap_caching_mode(iommu->cap))
1046 set_bit(0, iommu->domain_ids);
1047 return 0;
1048 }
1049
1050
1051 static void domain_exit(struct dmar_domain *domain);
1052 static void vm_domain_exit(struct dmar_domain *domain);
1053
1054 void free_dmar_iommu(struct intel_iommu *iommu)
1055 {
1056 struct dmar_domain *domain;
1057 int i;
1058 unsigned long flags;
1059
1060 i = find_first_bit(iommu->domain_ids, cap_ndoms(iommu->cap));
1061 for (; i < cap_ndoms(iommu->cap); ) {
1062 domain = iommu->domains[i];
1063 clear_bit(i, iommu->domain_ids);
1064
1065 spin_lock_irqsave(&domain->iommu_lock, flags);
1066 if (--domain->iommu_count == 0) {
1067 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1068 vm_domain_exit(domain);
1069 else
1070 domain_exit(domain);
1071 }
1072 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1073
1074 i = find_next_bit(iommu->domain_ids,
1075 cap_ndoms(iommu->cap), i+1);
1076 }
1077
1078 if (iommu->gcmd & DMA_GCMD_TE)
1079 iommu_disable_translation(iommu);
1080
1081 if (iommu->irq) {
1082 set_irq_data(iommu->irq, NULL);
1083 /* This will mask the irq */
1084 free_irq(iommu->irq, iommu);
1085 destroy_irq(iommu->irq);
1086 }
1087
1088 kfree(iommu->domains);
1089 kfree(iommu->domain_ids);
1090
1091 g_iommus[iommu->seq_id] = NULL;
1092
1093 /* if all iommus are freed, free g_iommus */
1094 for (i = 0; i < g_num_of_iommus; i++) {
1095 if (g_iommus[i])
1096 break;
1097 }
1098
1099 if (i == g_num_of_iommus)
1100 kfree(g_iommus);
1101
1102 /* free context mapping */
1103 free_context_table(iommu);
1104 }
1105
1106 static struct dmar_domain * iommu_alloc_domain(struct intel_iommu *iommu)
1107 {
1108 unsigned long num;
1109 unsigned long ndomains;
1110 struct dmar_domain *domain;
1111 unsigned long flags;
1112
1113 domain = alloc_domain_mem();
1114 if (!domain)
1115 return NULL;
1116
1117 ndomains = cap_ndoms(iommu->cap);
1118
1119 spin_lock_irqsave(&iommu->lock, flags);
1120 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1121 if (num >= ndomains) {
1122 spin_unlock_irqrestore(&iommu->lock, flags);
1123 free_domain_mem(domain);
1124 printk(KERN_ERR "IOMMU: no free domain ids\n");
1125 return NULL;
1126 }
1127
1128 set_bit(num, iommu->domain_ids);
1129 domain->id = num;
1130 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
1131 set_bit(iommu->seq_id, &domain->iommu_bmp);
1132 domain->flags = 0;
1133 iommu->domains[num] = domain;
1134 spin_unlock_irqrestore(&iommu->lock, flags);
1135
1136 return domain;
1137 }
1138
1139 static void iommu_free_domain(struct dmar_domain *domain)
1140 {
1141 unsigned long flags;
1142 struct intel_iommu *iommu;
1143
1144 iommu = domain_get_iommu(domain);
1145
1146 spin_lock_irqsave(&iommu->lock, flags);
1147 clear_bit(domain->id, iommu->domain_ids);
1148 spin_unlock_irqrestore(&iommu->lock, flags);
1149 }
1150
1151 static struct iova_domain reserved_iova_list;
1152 static struct lock_class_key reserved_alloc_key;
1153 static struct lock_class_key reserved_rbtree_key;
1154
1155 static void dmar_init_reserved_ranges(void)
1156 {
1157 struct pci_dev *pdev = NULL;
1158 struct iova *iova;
1159 int i;
1160 u64 addr, size;
1161
1162 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1163
1164 lockdep_set_class(&reserved_iova_list.iova_alloc_lock,
1165 &reserved_alloc_key);
1166 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1167 &reserved_rbtree_key);
1168
1169 /* IOAPIC ranges shouldn't be accessed by DMA */
1170 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1171 IOVA_PFN(IOAPIC_RANGE_END));
1172 if (!iova)
1173 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1174
1175 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1176 for_each_pci_dev(pdev) {
1177 struct resource *r;
1178
1179 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1180 r = &pdev->resource[i];
1181 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1182 continue;
1183 addr = r->start;
1184 addr &= PAGE_MASK;
1185 size = r->end - addr;
1186 size = PAGE_ALIGN(size);
1187 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(addr),
1188 IOVA_PFN(size + addr) - 1);
1189 if (!iova)
1190 printk(KERN_ERR "Reserve iova failed\n");
1191 }
1192 }
1193
1194 }
1195
1196 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1197 {
1198 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1199 }
1200
1201 static inline int guestwidth_to_adjustwidth(int gaw)
1202 {
1203 int agaw;
1204 int r = (gaw - 12) % 9;
1205
1206 if (r == 0)
1207 agaw = gaw;
1208 else
1209 agaw = gaw + 9 - r;
1210 if (agaw > 64)
1211 agaw = 64;
1212 return agaw;
1213 }
1214
1215 static int domain_init(struct dmar_domain *domain, int guest_width)
1216 {
1217 struct intel_iommu *iommu;
1218 int adjust_width, agaw;
1219 unsigned long sagaw;
1220
1221 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1222 spin_lock_init(&domain->mapping_lock);
1223 spin_lock_init(&domain->iommu_lock);
1224
1225 domain_reserve_special_ranges(domain);
1226
1227 /* calculate AGAW */
1228 iommu = domain_get_iommu(domain);
1229 if (guest_width > cap_mgaw(iommu->cap))
1230 guest_width = cap_mgaw(iommu->cap);
1231 domain->gaw = guest_width;
1232 adjust_width = guestwidth_to_adjustwidth(guest_width);
1233 agaw = width_to_agaw(adjust_width);
1234 sagaw = cap_sagaw(iommu->cap);
1235 if (!test_bit(agaw, &sagaw)) {
1236 /* hardware doesn't support it, choose a bigger one */
1237 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1238 agaw = find_next_bit(&sagaw, 5, agaw);
1239 if (agaw >= 5)
1240 return -ENODEV;
1241 }
1242 domain->agaw = agaw;
1243 INIT_LIST_HEAD(&domain->devices);
1244
1245 if (ecap_coherent(iommu->ecap))
1246 domain->iommu_coherency = 1;
1247 else
1248 domain->iommu_coherency = 0;
1249
1250 domain->iommu_count = 1;
1251
1252 /* always allocate the top pgd */
1253 domain->pgd = (struct dma_pte *)alloc_pgtable_page();
1254 if (!domain->pgd)
1255 return -ENOMEM;
1256 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1257 return 0;
1258 }
1259
1260 static void domain_exit(struct dmar_domain *domain)
1261 {
1262 u64 end;
1263
1264 /* Domain 0 is reserved, so dont process it */
1265 if (!domain)
1266 return;
1267
1268 domain_remove_dev_info(domain);
1269 /* destroy iovas */
1270 put_iova_domain(&domain->iovad);
1271 end = DOMAIN_MAX_ADDR(domain->gaw);
1272 end = end & (~PAGE_MASK);
1273
1274 /* clear ptes */
1275 dma_pte_clear_range(domain, 0, end);
1276
1277 /* free page tables */
1278 dma_pte_free_pagetable(domain, 0, end);
1279
1280 iommu_free_domain(domain);
1281 free_domain_mem(domain);
1282 }
1283
1284 static int domain_context_mapping_one(struct dmar_domain *domain,
1285 u8 bus, u8 devfn)
1286 {
1287 struct context_entry *context;
1288 unsigned long flags;
1289 struct intel_iommu *iommu;
1290 struct dma_pte *pgd;
1291 unsigned long num;
1292 unsigned long ndomains;
1293 int id;
1294 int agaw;
1295
1296 pr_debug("Set context mapping for %02x:%02x.%d\n",
1297 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1298 BUG_ON(!domain->pgd);
1299
1300 iommu = device_to_iommu(bus, devfn);
1301 if (!iommu)
1302 return -ENODEV;
1303
1304 context = device_to_context_entry(iommu, bus, devfn);
1305 if (!context)
1306 return -ENOMEM;
1307 spin_lock_irqsave(&iommu->lock, flags);
1308 if (context_present(context)) {
1309 spin_unlock_irqrestore(&iommu->lock, flags);
1310 return 0;
1311 }
1312
1313 id = domain->id;
1314 pgd = domain->pgd;
1315
1316 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) {
1317 int found = 0;
1318
1319 /* find an available domain id for this device in iommu */
1320 ndomains = cap_ndoms(iommu->cap);
1321 num = find_first_bit(iommu->domain_ids, ndomains);
1322 for (; num < ndomains; ) {
1323 if (iommu->domains[num] == domain) {
1324 id = num;
1325 found = 1;
1326 break;
1327 }
1328 num = find_next_bit(iommu->domain_ids,
1329 cap_ndoms(iommu->cap), num+1);
1330 }
1331
1332 if (found == 0) {
1333 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1334 if (num >= ndomains) {
1335 spin_unlock_irqrestore(&iommu->lock, flags);
1336 printk(KERN_ERR "IOMMU: no free domain ids\n");
1337 return -EFAULT;
1338 }
1339
1340 set_bit(num, iommu->domain_ids);
1341 iommu->domains[num] = domain;
1342 id = num;
1343 }
1344
1345 /* Skip top levels of page tables for
1346 * iommu which has less agaw than default.
1347 */
1348 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1349 pgd = phys_to_virt(dma_pte_addr(pgd));
1350 if (!dma_pte_present(pgd)) {
1351 spin_unlock_irqrestore(&iommu->lock, flags);
1352 return -ENOMEM;
1353 }
1354 }
1355 }
1356
1357 context_set_domain_id(context, id);
1358 context_set_address_width(context, iommu->agaw);
1359 context_set_address_root(context, virt_to_phys(pgd));
1360 context_set_translation_type(context, CONTEXT_TT_MULTI_LEVEL);
1361 context_set_fault_enable(context);
1362 context_set_present(context);
1363 domain_flush_cache(domain, context, sizeof(*context));
1364
1365 /* it's a non-present to present mapping */
1366 if (iommu->flush.flush_context(iommu, domain->id,
1367 (((u16)bus) << 8) | devfn, DMA_CCMD_MASK_NOBIT,
1368 DMA_CCMD_DEVICE_INVL, 1))
1369 iommu_flush_write_buffer(iommu);
1370 else
1371 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_DSI_FLUSH, 0);
1372
1373 spin_unlock_irqrestore(&iommu->lock, flags);
1374
1375 spin_lock_irqsave(&domain->iommu_lock, flags);
1376 if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {
1377 domain->iommu_count++;
1378 domain_update_iommu_coherency(domain);
1379 }
1380 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1381 return 0;
1382 }
1383
1384 static int
1385 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev)
1386 {
1387 int ret;
1388 struct pci_dev *tmp, *parent;
1389
1390 ret = domain_context_mapping_one(domain, pdev->bus->number,
1391 pdev->devfn);
1392 if (ret)
1393 return ret;
1394
1395 /* dependent device mapping */
1396 tmp = pci_find_upstream_pcie_bridge(pdev);
1397 if (!tmp)
1398 return 0;
1399 /* Secondary interface's bus number and devfn 0 */
1400 parent = pdev->bus->self;
1401 while (parent != tmp) {
1402 ret = domain_context_mapping_one(domain, parent->bus->number,
1403 parent->devfn);
1404 if (ret)
1405 return ret;
1406 parent = parent->bus->self;
1407 }
1408 if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
1409 return domain_context_mapping_one(domain,
1410 tmp->subordinate->number, 0);
1411 else /* this is a legacy PCI bridge */
1412 return domain_context_mapping_one(domain,
1413 tmp->bus->number, tmp->devfn);
1414 }
1415
1416 static int domain_context_mapped(struct pci_dev *pdev)
1417 {
1418 int ret;
1419 struct pci_dev *tmp, *parent;
1420 struct intel_iommu *iommu;
1421
1422 iommu = device_to_iommu(pdev->bus->number, pdev->devfn);
1423 if (!iommu)
1424 return -ENODEV;
1425
1426 ret = device_context_mapped(iommu,
1427 pdev->bus->number, pdev->devfn);
1428 if (!ret)
1429 return ret;
1430 /* dependent device mapping */
1431 tmp = pci_find_upstream_pcie_bridge(pdev);
1432 if (!tmp)
1433 return ret;
1434 /* Secondary interface's bus number and devfn 0 */
1435 parent = pdev->bus->self;
1436 while (parent != tmp) {
1437 ret = device_context_mapped(iommu, parent->bus->number,
1438 parent->devfn);
1439 if (!ret)
1440 return ret;
1441 parent = parent->bus->self;
1442 }
1443 if (tmp->is_pcie)
1444 return device_context_mapped(iommu,
1445 tmp->subordinate->number, 0);
1446 else
1447 return device_context_mapped(iommu,
1448 tmp->bus->number, tmp->devfn);
1449 }
1450
1451 static int
1452 domain_page_mapping(struct dmar_domain *domain, dma_addr_t iova,
1453 u64 hpa, size_t size, int prot)
1454 {
1455 u64 start_pfn, end_pfn;
1456 struct dma_pte *pte;
1457 int index;
1458 int addr_width = agaw_to_width(domain->agaw);
1459
1460 hpa &= (((u64)1) << addr_width) - 1;
1461
1462 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1463 return -EINVAL;
1464 iova &= PAGE_MASK;
1465 start_pfn = ((u64)hpa) >> VTD_PAGE_SHIFT;
1466 end_pfn = (VTD_PAGE_ALIGN(((u64)hpa) + size)) >> VTD_PAGE_SHIFT;
1467 index = 0;
1468 while (start_pfn < end_pfn) {
1469 pte = addr_to_dma_pte(domain, iova + VTD_PAGE_SIZE * index);
1470 if (!pte)
1471 return -ENOMEM;
1472 /* We don't need lock here, nobody else
1473 * touches the iova range
1474 */
1475 BUG_ON(dma_pte_addr(pte));
1476 dma_set_pte_addr(pte, start_pfn << VTD_PAGE_SHIFT);
1477 dma_set_pte_prot(pte, prot);
1478 domain_flush_cache(domain, pte, sizeof(*pte));
1479 start_pfn++;
1480 index++;
1481 }
1482 return 0;
1483 }
1484
1485 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
1486 {
1487 if (!iommu)
1488 return;
1489
1490 clear_context_table(iommu, bus, devfn);
1491 iommu->flush.flush_context(iommu, 0, 0, 0,
1492 DMA_CCMD_GLOBAL_INVL, 0);
1493 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1494 DMA_TLB_GLOBAL_FLUSH, 0);
1495 }
1496
1497 static void domain_remove_dev_info(struct dmar_domain *domain)
1498 {
1499 struct device_domain_info *info;
1500 unsigned long flags;
1501 struct intel_iommu *iommu;
1502
1503 spin_lock_irqsave(&device_domain_lock, flags);
1504 while (!list_empty(&domain->devices)) {
1505 info = list_entry(domain->devices.next,
1506 struct device_domain_info, link);
1507 list_del(&info->link);
1508 list_del(&info->global);
1509 if (info->dev)
1510 info->dev->dev.archdata.iommu = NULL;
1511 spin_unlock_irqrestore(&device_domain_lock, flags);
1512
1513 iommu = device_to_iommu(info->bus, info->devfn);
1514 iommu_detach_dev(iommu, info->bus, info->devfn);
1515 free_devinfo_mem(info);
1516
1517 spin_lock_irqsave(&device_domain_lock, flags);
1518 }
1519 spin_unlock_irqrestore(&device_domain_lock, flags);
1520 }
1521
1522 /*
1523 * find_domain
1524 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1525 */
1526 static struct dmar_domain *
1527 find_domain(struct pci_dev *pdev)
1528 {
1529 struct device_domain_info *info;
1530
1531 /* No lock here, assumes no domain exit in normal case */
1532 info = pdev->dev.archdata.iommu;
1533 if (info)
1534 return info->domain;
1535 return NULL;
1536 }
1537
1538 /* domain is initialized */
1539 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1540 {
1541 struct dmar_domain *domain, *found = NULL;
1542 struct intel_iommu *iommu;
1543 struct dmar_drhd_unit *drhd;
1544 struct device_domain_info *info, *tmp;
1545 struct pci_dev *dev_tmp;
1546 unsigned long flags;
1547 int bus = 0, devfn = 0;
1548
1549 domain = find_domain(pdev);
1550 if (domain)
1551 return domain;
1552
1553 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1554 if (dev_tmp) {
1555 if (dev_tmp->is_pcie) {
1556 bus = dev_tmp->subordinate->number;
1557 devfn = 0;
1558 } else {
1559 bus = dev_tmp->bus->number;
1560 devfn = dev_tmp->devfn;
1561 }
1562 spin_lock_irqsave(&device_domain_lock, flags);
1563 list_for_each_entry(info, &device_domain_list, global) {
1564 if (info->bus == bus && info->devfn == devfn) {
1565 found = info->domain;
1566 break;
1567 }
1568 }
1569 spin_unlock_irqrestore(&device_domain_lock, flags);
1570 /* pcie-pci bridge already has a domain, uses it */
1571 if (found) {
1572 domain = found;
1573 goto found_domain;
1574 }
1575 }
1576
1577 /* Allocate new domain for the device */
1578 drhd = dmar_find_matched_drhd_unit(pdev);
1579 if (!drhd) {
1580 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1581 pci_name(pdev));
1582 return NULL;
1583 }
1584 iommu = drhd->iommu;
1585
1586 domain = iommu_alloc_domain(iommu);
1587 if (!domain)
1588 goto error;
1589
1590 if (domain_init(domain, gaw)) {
1591 domain_exit(domain);
1592 goto error;
1593 }
1594
1595 /* register pcie-to-pci device */
1596 if (dev_tmp) {
1597 info = alloc_devinfo_mem();
1598 if (!info) {
1599 domain_exit(domain);
1600 goto error;
1601 }
1602 info->bus = bus;
1603 info->devfn = devfn;
1604 info->dev = NULL;
1605 info->domain = domain;
1606 /* This domain is shared by devices under p2p bridge */
1607 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
1608
1609 /* pcie-to-pci bridge already has a domain, uses it */
1610 found = NULL;
1611 spin_lock_irqsave(&device_domain_lock, flags);
1612 list_for_each_entry(tmp, &device_domain_list, global) {
1613 if (tmp->bus == bus && tmp->devfn == devfn) {
1614 found = tmp->domain;
1615 break;
1616 }
1617 }
1618 if (found) {
1619 free_devinfo_mem(info);
1620 domain_exit(domain);
1621 domain = found;
1622 } else {
1623 list_add(&info->link, &domain->devices);
1624 list_add(&info->global, &device_domain_list);
1625 }
1626 spin_unlock_irqrestore(&device_domain_lock, flags);
1627 }
1628
1629 found_domain:
1630 info = alloc_devinfo_mem();
1631 if (!info)
1632 goto error;
1633 info->bus = pdev->bus->number;
1634 info->devfn = pdev->devfn;
1635 info->dev = pdev;
1636 info->domain = domain;
1637 spin_lock_irqsave(&device_domain_lock, flags);
1638 /* somebody is fast */
1639 found = find_domain(pdev);
1640 if (found != NULL) {
1641 spin_unlock_irqrestore(&device_domain_lock, flags);
1642 if (found != domain) {
1643 domain_exit(domain);
1644 domain = found;
1645 }
1646 free_devinfo_mem(info);
1647 return domain;
1648 }
1649 list_add(&info->link, &domain->devices);
1650 list_add(&info->global, &device_domain_list);
1651 pdev->dev.archdata.iommu = info;
1652 spin_unlock_irqrestore(&device_domain_lock, flags);
1653 return domain;
1654 error:
1655 /* recheck it here, maybe others set it */
1656 return find_domain(pdev);
1657 }
1658
1659 static int iommu_prepare_identity_map(struct pci_dev *pdev,
1660 unsigned long long start,
1661 unsigned long long end)
1662 {
1663 struct dmar_domain *domain;
1664 unsigned long size;
1665 unsigned long long base;
1666 int ret;
1667
1668 printk(KERN_INFO
1669 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1670 pci_name(pdev), start, end);
1671 /* page table init */
1672 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
1673 if (!domain)
1674 return -ENOMEM;
1675
1676 /* The address might not be aligned */
1677 base = start & PAGE_MASK;
1678 size = end - base;
1679 size = PAGE_ALIGN(size);
1680 if (!reserve_iova(&domain->iovad, IOVA_PFN(base),
1681 IOVA_PFN(base + size) - 1)) {
1682 printk(KERN_ERR "IOMMU: reserve iova failed\n");
1683 ret = -ENOMEM;
1684 goto error;
1685 }
1686
1687 pr_debug("Mapping reserved region %lx@%llx for %s\n",
1688 size, base, pci_name(pdev));
1689 /*
1690 * RMRR range might have overlap with physical memory range,
1691 * clear it first
1692 */
1693 dma_pte_clear_range(domain, base, base + size);
1694
1695 ret = domain_page_mapping(domain, base, base, size,
1696 DMA_PTE_READ|DMA_PTE_WRITE);
1697 if (ret)
1698 goto error;
1699
1700 /* context entry init */
1701 ret = domain_context_mapping(domain, pdev);
1702 if (!ret)
1703 return 0;
1704 error:
1705 domain_exit(domain);
1706 return ret;
1707
1708 }
1709
1710 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
1711 struct pci_dev *pdev)
1712 {
1713 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
1714 return 0;
1715 return iommu_prepare_identity_map(pdev, rmrr->base_address,
1716 rmrr->end_address + 1);
1717 }
1718
1719 #ifdef CONFIG_DMAR_GFX_WA
1720 struct iommu_prepare_data {
1721 struct pci_dev *pdev;
1722 int ret;
1723 };
1724
1725 static int __init iommu_prepare_work_fn(unsigned long start_pfn,
1726 unsigned long end_pfn, void *datax)
1727 {
1728 struct iommu_prepare_data *data;
1729
1730 data = (struct iommu_prepare_data *)datax;
1731
1732 data->ret = iommu_prepare_identity_map(data->pdev,
1733 start_pfn<<PAGE_SHIFT, end_pfn<<PAGE_SHIFT);
1734 return data->ret;
1735
1736 }
1737
1738 static int __init iommu_prepare_with_active_regions(struct pci_dev *pdev)
1739 {
1740 int nid;
1741 struct iommu_prepare_data data;
1742
1743 data.pdev = pdev;
1744 data.ret = 0;
1745
1746 for_each_online_node(nid) {
1747 work_with_active_regions(nid, iommu_prepare_work_fn, &data);
1748 if (data.ret)
1749 return data.ret;
1750 }
1751 return data.ret;
1752 }
1753
1754 static void __init iommu_prepare_gfx_mapping(void)
1755 {
1756 struct pci_dev *pdev = NULL;
1757 int ret;
1758
1759 for_each_pci_dev(pdev) {
1760 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO ||
1761 !IS_GFX_DEVICE(pdev))
1762 continue;
1763 printk(KERN_INFO "IOMMU: gfx device %s 1-1 mapping\n",
1764 pci_name(pdev));
1765 ret = iommu_prepare_with_active_regions(pdev);
1766 if (ret)
1767 printk(KERN_ERR "IOMMU: mapping reserved region failed\n");
1768 }
1769 }
1770 #else /* !CONFIG_DMAR_GFX_WA */
1771 static inline void iommu_prepare_gfx_mapping(void)
1772 {
1773 return;
1774 }
1775 #endif
1776
1777 #ifdef CONFIG_DMAR_FLOPPY_WA
1778 static inline void iommu_prepare_isa(void)
1779 {
1780 struct pci_dev *pdev;
1781 int ret;
1782
1783 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
1784 if (!pdev)
1785 return;
1786
1787 printk(KERN_INFO "IOMMU: Prepare 0-16M unity mapping for LPC\n");
1788 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024);
1789
1790 if (ret)
1791 printk("IOMMU: Failed to create 0-64M identity map, "
1792 "floppy might not work\n");
1793
1794 }
1795 #else
1796 static inline void iommu_prepare_isa(void)
1797 {
1798 return;
1799 }
1800 #endif /* !CONFIG_DMAR_FLPY_WA */
1801
1802 static int __init init_dmars(void)
1803 {
1804 struct dmar_drhd_unit *drhd;
1805 struct dmar_rmrr_unit *rmrr;
1806 struct pci_dev *pdev;
1807 struct intel_iommu *iommu;
1808 int i, ret;
1809
1810 /*
1811 * for each drhd
1812 * allocate root
1813 * initialize and program root entry to not present
1814 * endfor
1815 */
1816 for_each_drhd_unit(drhd) {
1817 g_num_of_iommus++;
1818 /*
1819 * lock not needed as this is only incremented in the single
1820 * threaded kernel __init code path all other access are read
1821 * only
1822 */
1823 }
1824
1825 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
1826 GFP_KERNEL);
1827 if (!g_iommus) {
1828 printk(KERN_ERR "Allocating global iommu array failed\n");
1829 ret = -ENOMEM;
1830 goto error;
1831 }
1832
1833 deferred_flush = kzalloc(g_num_of_iommus *
1834 sizeof(struct deferred_flush_tables), GFP_KERNEL);
1835 if (!deferred_flush) {
1836 kfree(g_iommus);
1837 ret = -ENOMEM;
1838 goto error;
1839 }
1840
1841 for_each_drhd_unit(drhd) {
1842 if (drhd->ignored)
1843 continue;
1844
1845 iommu = drhd->iommu;
1846 g_iommus[iommu->seq_id] = iommu;
1847
1848 ret = iommu_init_domains(iommu);
1849 if (ret)
1850 goto error;
1851
1852 /*
1853 * TBD:
1854 * we could share the same root & context tables
1855 * amoung all IOMMU's. Need to Split it later.
1856 */
1857 ret = iommu_alloc_root_entry(iommu);
1858 if (ret) {
1859 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
1860 goto error;
1861 }
1862 }
1863
1864 /*
1865 * Start from the sane iommu hardware state.
1866 */
1867 for_each_drhd_unit(drhd) {
1868 if (drhd->ignored)
1869 continue;
1870
1871 iommu = drhd->iommu;
1872
1873 /*
1874 * If the queued invalidation is already initialized by us
1875 * (for example, while enabling interrupt-remapping) then
1876 * we got the things already rolling from a sane state.
1877 */
1878 if (iommu->qi)
1879 continue;
1880
1881 /*
1882 * Clear any previous faults.
1883 */
1884 dmar_fault(-1, iommu);
1885 /*
1886 * Disable queued invalidation if supported and already enabled
1887 * before OS handover.
1888 */
1889 dmar_disable_qi(iommu);
1890 }
1891
1892 for_each_drhd_unit(drhd) {
1893 if (drhd->ignored)
1894 continue;
1895
1896 iommu = drhd->iommu;
1897
1898 if (dmar_enable_qi(iommu)) {
1899 /*
1900 * Queued Invalidate not enabled, use Register Based
1901 * Invalidate
1902 */
1903 iommu->flush.flush_context = __iommu_flush_context;
1904 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
1905 printk(KERN_INFO "IOMMU 0x%Lx: using Register based "
1906 "invalidation\n",
1907 (unsigned long long)drhd->reg_base_addr);
1908 } else {
1909 iommu->flush.flush_context = qi_flush_context;
1910 iommu->flush.flush_iotlb = qi_flush_iotlb;
1911 printk(KERN_INFO "IOMMU 0x%Lx: using Queued "
1912 "invalidation\n",
1913 (unsigned long long)drhd->reg_base_addr);
1914 }
1915 }
1916
1917 /*
1918 * For each rmrr
1919 * for each dev attached to rmrr
1920 * do
1921 * locate drhd for dev, alloc domain for dev
1922 * allocate free domain
1923 * allocate page table entries for rmrr
1924 * if context not allocated for bus
1925 * allocate and init context
1926 * set present in root table for this bus
1927 * init context with domain, translation etc
1928 * endfor
1929 * endfor
1930 */
1931 for_each_rmrr_units(rmrr) {
1932 for (i = 0; i < rmrr->devices_cnt; i++) {
1933 pdev = rmrr->devices[i];
1934 /* some BIOS lists non-exist devices in DMAR table */
1935 if (!pdev)
1936 continue;
1937 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
1938 if (ret)
1939 printk(KERN_ERR
1940 "IOMMU: mapping reserved region failed\n");
1941 }
1942 }
1943
1944 iommu_prepare_gfx_mapping();
1945
1946 iommu_prepare_isa();
1947
1948 /*
1949 * for each drhd
1950 * enable fault log
1951 * global invalidate context cache
1952 * global invalidate iotlb
1953 * enable translation
1954 */
1955 for_each_drhd_unit(drhd) {
1956 if (drhd->ignored)
1957 continue;
1958 iommu = drhd->iommu;
1959
1960 iommu_flush_write_buffer(iommu);
1961
1962 ret = dmar_set_interrupt(iommu);
1963 if (ret)
1964 goto error;
1965
1966 iommu_set_root_entry(iommu);
1967
1968 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL,
1969 0);
1970 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH,
1971 0);
1972 iommu_disable_protect_mem_regions(iommu);
1973
1974 ret = iommu_enable_translation(iommu);
1975 if (ret)
1976 goto error;
1977 }
1978
1979 return 0;
1980 error:
1981 for_each_drhd_unit(drhd) {
1982 if (drhd->ignored)
1983 continue;
1984 iommu = drhd->iommu;
1985 free_iommu(iommu);
1986 }
1987 kfree(g_iommus);
1988 return ret;
1989 }
1990
1991 static inline u64 aligned_size(u64 host_addr, size_t size)
1992 {
1993 u64 addr;
1994 addr = (host_addr & (~PAGE_MASK)) + size;
1995 return PAGE_ALIGN(addr);
1996 }
1997
1998 struct iova *
1999 iommu_alloc_iova(struct dmar_domain *domain, size_t size, u64 end)
2000 {
2001 struct iova *piova;
2002
2003 /* Make sure it's in range */
2004 end = min_t(u64, DOMAIN_MAX_ADDR(domain->gaw), end);
2005 if (!size || (IOVA_START_ADDR + size > end))
2006 return NULL;
2007
2008 piova = alloc_iova(&domain->iovad,
2009 size >> PAGE_SHIFT, IOVA_PFN(end), 1);
2010 return piova;
2011 }
2012
2013 static struct iova *
2014 __intel_alloc_iova(struct device *dev, struct dmar_domain *domain,
2015 size_t size, u64 dma_mask)
2016 {
2017 struct pci_dev *pdev = to_pci_dev(dev);
2018 struct iova *iova = NULL;
2019
2020 if (dma_mask <= DMA_32BIT_MASK || dmar_forcedac)
2021 iova = iommu_alloc_iova(domain, size, dma_mask);
2022 else {
2023 /*
2024 * First try to allocate an io virtual address in
2025 * DMA_32BIT_MASK and if that fails then try allocating
2026 * from higher range
2027 */
2028 iova = iommu_alloc_iova(domain, size, DMA_32BIT_MASK);
2029 if (!iova)
2030 iova = iommu_alloc_iova(domain, size, dma_mask);
2031 }
2032
2033 if (!iova) {
2034 printk(KERN_ERR"Allocating iova for %s failed", pci_name(pdev));
2035 return NULL;
2036 }
2037
2038 return iova;
2039 }
2040
2041 static struct dmar_domain *
2042 get_valid_domain_for_dev(struct pci_dev *pdev)
2043 {
2044 struct dmar_domain *domain;
2045 int ret;
2046
2047 domain = get_domain_for_dev(pdev,
2048 DEFAULT_DOMAIN_ADDRESS_WIDTH);
2049 if (!domain) {
2050 printk(KERN_ERR
2051 "Allocating domain for %s failed", pci_name(pdev));
2052 return NULL;
2053 }
2054
2055 /* make sure context mapping is ok */
2056 if (unlikely(!domain_context_mapped(pdev))) {
2057 ret = domain_context_mapping(domain, pdev);
2058 if (ret) {
2059 printk(KERN_ERR
2060 "Domain context map for %s failed",
2061 pci_name(pdev));
2062 return NULL;
2063 }
2064 }
2065
2066 return domain;
2067 }
2068
2069 static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2070 size_t size, int dir, u64 dma_mask)
2071 {
2072 struct pci_dev *pdev = to_pci_dev(hwdev);
2073 struct dmar_domain *domain;
2074 phys_addr_t start_paddr;
2075 struct iova *iova;
2076 int prot = 0;
2077 int ret;
2078 struct intel_iommu *iommu;
2079
2080 BUG_ON(dir == DMA_NONE);
2081 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2082 return paddr;
2083
2084 domain = get_valid_domain_for_dev(pdev);
2085 if (!domain)
2086 return 0;
2087
2088 iommu = domain_get_iommu(domain);
2089 size = aligned_size((u64)paddr, size);
2090
2091 iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
2092 if (!iova)
2093 goto error;
2094
2095 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2096
2097 /*
2098 * Check if DMAR supports zero-length reads on write only
2099 * mappings..
2100 */
2101 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2102 !cap_zlr(iommu->cap))
2103 prot |= DMA_PTE_READ;
2104 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2105 prot |= DMA_PTE_WRITE;
2106 /*
2107 * paddr - (paddr + size) might be partial page, we should map the whole
2108 * page. Note: if two part of one page are separately mapped, we
2109 * might have two guest_addr mapping to the same host paddr, but this
2110 * is not a big problem
2111 */
2112 ret = domain_page_mapping(domain, start_paddr,
2113 ((u64)paddr) & PAGE_MASK, size, prot);
2114 if (ret)
2115 goto error;
2116
2117 /* it's a non-present to present mapping */
2118 ret = iommu_flush_iotlb_psi(iommu, domain->id,
2119 start_paddr, size >> VTD_PAGE_SHIFT, 1);
2120 if (ret)
2121 iommu_flush_write_buffer(iommu);
2122
2123 return start_paddr + ((u64)paddr & (~PAGE_MASK));
2124
2125 error:
2126 if (iova)
2127 __free_iova(&domain->iovad, iova);
2128 printk(KERN_ERR"Device %s request: %lx@%llx dir %d --- failed\n",
2129 pci_name(pdev), size, (unsigned long long)paddr, dir);
2130 return 0;
2131 }
2132
2133 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2134 unsigned long offset, size_t size,
2135 enum dma_data_direction dir,
2136 struct dma_attrs *attrs)
2137 {
2138 return __intel_map_single(dev, page_to_phys(page) + offset, size,
2139 dir, to_pci_dev(dev)->dma_mask);
2140 }
2141
2142 static void flush_unmaps(void)
2143 {
2144 int i, j;
2145
2146 timer_on = 0;
2147
2148 /* just flush them all */
2149 for (i = 0; i < g_num_of_iommus; i++) {
2150 struct intel_iommu *iommu = g_iommus[i];
2151 if (!iommu)
2152 continue;
2153
2154 if (deferred_flush[i].next) {
2155 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2156 DMA_TLB_GLOBAL_FLUSH, 0);
2157 for (j = 0; j < deferred_flush[i].next; j++) {
2158 __free_iova(&deferred_flush[i].domain[j]->iovad,
2159 deferred_flush[i].iova[j]);
2160 }
2161 deferred_flush[i].next = 0;
2162 }
2163 }
2164
2165 list_size = 0;
2166 }
2167
2168 static void flush_unmaps_timeout(unsigned long data)
2169 {
2170 unsigned long flags;
2171
2172 spin_lock_irqsave(&async_umap_flush_lock, flags);
2173 flush_unmaps();
2174 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2175 }
2176
2177 static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2178 {
2179 unsigned long flags;
2180 int next, iommu_id;
2181 struct intel_iommu *iommu;
2182
2183 spin_lock_irqsave(&async_umap_flush_lock, flags);
2184 if (list_size == HIGH_WATER_MARK)
2185 flush_unmaps();
2186
2187 iommu = domain_get_iommu(dom);
2188 iommu_id = iommu->seq_id;
2189
2190 next = deferred_flush[iommu_id].next;
2191 deferred_flush[iommu_id].domain[next] = dom;
2192 deferred_flush[iommu_id].iova[next] = iova;
2193 deferred_flush[iommu_id].next++;
2194
2195 if (!timer_on) {
2196 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2197 timer_on = 1;
2198 }
2199 list_size++;
2200 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2201 }
2202
2203 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2204 size_t size, enum dma_data_direction dir,
2205 struct dma_attrs *attrs)
2206 {
2207 struct pci_dev *pdev = to_pci_dev(dev);
2208 struct dmar_domain *domain;
2209 unsigned long start_addr;
2210 struct iova *iova;
2211 struct intel_iommu *iommu;
2212
2213 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2214 return;
2215 domain = find_domain(pdev);
2216 BUG_ON(!domain);
2217
2218 iommu = domain_get_iommu(domain);
2219
2220 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
2221 if (!iova)
2222 return;
2223
2224 start_addr = iova->pfn_lo << PAGE_SHIFT;
2225 size = aligned_size((u64)dev_addr, size);
2226
2227 pr_debug("Device %s unmapping: %lx@%llx\n",
2228 pci_name(pdev), size, (unsigned long long)start_addr);
2229
2230 /* clear the whole page */
2231 dma_pte_clear_range(domain, start_addr, start_addr + size);
2232 /* free page tables */
2233 dma_pte_free_pagetable(domain, start_addr, start_addr + size);
2234 if (intel_iommu_strict) {
2235 if (iommu_flush_iotlb_psi(iommu,
2236 domain->id, start_addr, size >> VTD_PAGE_SHIFT, 0))
2237 iommu_flush_write_buffer(iommu);
2238 /* free iova */
2239 __free_iova(&domain->iovad, iova);
2240 } else {
2241 add_unmap(domain, iova);
2242 /*
2243 * queue up the release of the unmap to save the 1/6th of the
2244 * cpu used up by the iotlb flush operation...
2245 */
2246 }
2247 }
2248
2249 static void intel_unmap_single(struct device *dev, dma_addr_t dev_addr, size_t size,
2250 int dir)
2251 {
2252 intel_unmap_page(dev, dev_addr, size, dir, NULL);
2253 }
2254
2255 static void *intel_alloc_coherent(struct device *hwdev, size_t size,
2256 dma_addr_t *dma_handle, gfp_t flags)
2257 {
2258 void *vaddr;
2259 int order;
2260
2261 size = PAGE_ALIGN(size);
2262 order = get_order(size);
2263 flags &= ~(GFP_DMA | GFP_DMA32);
2264
2265 vaddr = (void *)__get_free_pages(flags, order);
2266 if (!vaddr)
2267 return NULL;
2268 memset(vaddr, 0, size);
2269
2270 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2271 DMA_BIDIRECTIONAL,
2272 hwdev->coherent_dma_mask);
2273 if (*dma_handle)
2274 return vaddr;
2275 free_pages((unsigned long)vaddr, order);
2276 return NULL;
2277 }
2278
2279 static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2280 dma_addr_t dma_handle)
2281 {
2282 int order;
2283
2284 size = PAGE_ALIGN(size);
2285 order = get_order(size);
2286
2287 intel_unmap_single(hwdev, dma_handle, size, DMA_BIDIRECTIONAL);
2288 free_pages((unsigned long)vaddr, order);
2289 }
2290
2291 #define SG_ENT_VIRT_ADDRESS(sg) (sg_virt((sg)))
2292
2293 static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2294 int nelems, enum dma_data_direction dir,
2295 struct dma_attrs *attrs)
2296 {
2297 int i;
2298 struct pci_dev *pdev = to_pci_dev(hwdev);
2299 struct dmar_domain *domain;
2300 unsigned long start_addr;
2301 struct iova *iova;
2302 size_t size = 0;
2303 void *addr;
2304 struct scatterlist *sg;
2305 struct intel_iommu *iommu;
2306
2307 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2308 return;
2309
2310 domain = find_domain(pdev);
2311 BUG_ON(!domain);
2312
2313 iommu = domain_get_iommu(domain);
2314
2315 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
2316 if (!iova)
2317 return;
2318 for_each_sg(sglist, sg, nelems, i) {
2319 addr = SG_ENT_VIRT_ADDRESS(sg);
2320 size += aligned_size((u64)addr, sg->length);
2321 }
2322
2323 start_addr = iova->pfn_lo << PAGE_SHIFT;
2324
2325 /* clear the whole page */
2326 dma_pte_clear_range(domain, start_addr, start_addr + size);
2327 /* free page tables */
2328 dma_pte_free_pagetable(domain, start_addr, start_addr + size);
2329
2330 if (iommu_flush_iotlb_psi(iommu, domain->id, start_addr,
2331 size >> VTD_PAGE_SHIFT, 0))
2332 iommu_flush_write_buffer(iommu);
2333
2334 /* free iova */
2335 __free_iova(&domain->iovad, iova);
2336 }
2337
2338 static int intel_nontranslate_map_sg(struct device *hddev,
2339 struct scatterlist *sglist, int nelems, int dir)
2340 {
2341 int i;
2342 struct scatterlist *sg;
2343
2344 for_each_sg(sglist, sg, nelems, i) {
2345 BUG_ON(!sg_page(sg));
2346 sg->dma_address = virt_to_bus(SG_ENT_VIRT_ADDRESS(sg));
2347 sg->dma_length = sg->length;
2348 }
2349 return nelems;
2350 }
2351
2352 static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
2353 enum dma_data_direction dir, struct dma_attrs *attrs)
2354 {
2355 void *addr;
2356 int i;
2357 struct pci_dev *pdev = to_pci_dev(hwdev);
2358 struct dmar_domain *domain;
2359 size_t size = 0;
2360 int prot = 0;
2361 size_t offset = 0;
2362 struct iova *iova = NULL;
2363 int ret;
2364 struct scatterlist *sg;
2365 unsigned long start_addr;
2366 struct intel_iommu *iommu;
2367
2368 BUG_ON(dir == DMA_NONE);
2369 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2370 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
2371
2372 domain = get_valid_domain_for_dev(pdev);
2373 if (!domain)
2374 return 0;
2375
2376 iommu = domain_get_iommu(domain);
2377
2378 for_each_sg(sglist, sg, nelems, i) {
2379 addr = SG_ENT_VIRT_ADDRESS(sg);
2380 addr = (void *)virt_to_phys(addr);
2381 size += aligned_size((u64)addr, sg->length);
2382 }
2383
2384 iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
2385 if (!iova) {
2386 sglist->dma_length = 0;
2387 return 0;
2388 }
2389
2390 /*
2391 * Check if DMAR supports zero-length reads on write only
2392 * mappings..
2393 */
2394 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2395 !cap_zlr(iommu->cap))
2396 prot |= DMA_PTE_READ;
2397 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2398 prot |= DMA_PTE_WRITE;
2399
2400 start_addr = iova->pfn_lo << PAGE_SHIFT;
2401 offset = 0;
2402 for_each_sg(sglist, sg, nelems, i) {
2403 addr = SG_ENT_VIRT_ADDRESS(sg);
2404 addr = (void *)virt_to_phys(addr);
2405 size = aligned_size((u64)addr, sg->length);
2406 ret = domain_page_mapping(domain, start_addr + offset,
2407 ((u64)addr) & PAGE_MASK,
2408 size, prot);
2409 if (ret) {
2410 /* clear the page */
2411 dma_pte_clear_range(domain, start_addr,
2412 start_addr + offset);
2413 /* free page tables */
2414 dma_pte_free_pagetable(domain, start_addr,
2415 start_addr + offset);
2416 /* free iova */
2417 __free_iova(&domain->iovad, iova);
2418 return 0;
2419 }
2420 sg->dma_address = start_addr + offset +
2421 ((u64)addr & (~PAGE_MASK));
2422 sg->dma_length = sg->length;
2423 offset += size;
2424 }
2425
2426 /* it's a non-present to present mapping */
2427 if (iommu_flush_iotlb_psi(iommu, domain->id,
2428 start_addr, offset >> VTD_PAGE_SHIFT, 1))
2429 iommu_flush_write_buffer(iommu);
2430 return nelems;
2431 }
2432
2433 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
2434 {
2435 return !dma_addr;
2436 }
2437
2438 struct dma_map_ops intel_dma_ops = {
2439 .alloc_coherent = intel_alloc_coherent,
2440 .free_coherent = intel_free_coherent,
2441 .map_sg = intel_map_sg,
2442 .unmap_sg = intel_unmap_sg,
2443 .map_page = intel_map_page,
2444 .unmap_page = intel_unmap_page,
2445 .mapping_error = intel_mapping_error,
2446 };
2447
2448 static inline int iommu_domain_cache_init(void)
2449 {
2450 int ret = 0;
2451
2452 iommu_domain_cache = kmem_cache_create("iommu_domain",
2453 sizeof(struct dmar_domain),
2454 0,
2455 SLAB_HWCACHE_ALIGN,
2456
2457 NULL);
2458 if (!iommu_domain_cache) {
2459 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
2460 ret = -ENOMEM;
2461 }
2462
2463 return ret;
2464 }
2465
2466 static inline int iommu_devinfo_cache_init(void)
2467 {
2468 int ret = 0;
2469
2470 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
2471 sizeof(struct device_domain_info),
2472 0,
2473 SLAB_HWCACHE_ALIGN,
2474 NULL);
2475 if (!iommu_devinfo_cache) {
2476 printk(KERN_ERR "Couldn't create devinfo cache\n");
2477 ret = -ENOMEM;
2478 }
2479
2480 return ret;
2481 }
2482
2483 static inline int iommu_iova_cache_init(void)
2484 {
2485 int ret = 0;
2486
2487 iommu_iova_cache = kmem_cache_create("iommu_iova",
2488 sizeof(struct iova),
2489 0,
2490 SLAB_HWCACHE_ALIGN,
2491 NULL);
2492 if (!iommu_iova_cache) {
2493 printk(KERN_ERR "Couldn't create iova cache\n");
2494 ret = -ENOMEM;
2495 }
2496
2497 return ret;
2498 }
2499
2500 static int __init iommu_init_mempool(void)
2501 {
2502 int ret;
2503 ret = iommu_iova_cache_init();
2504 if (ret)
2505 return ret;
2506
2507 ret = iommu_domain_cache_init();
2508 if (ret)
2509 goto domain_error;
2510
2511 ret = iommu_devinfo_cache_init();
2512 if (!ret)
2513 return ret;
2514
2515 kmem_cache_destroy(iommu_domain_cache);
2516 domain_error:
2517 kmem_cache_destroy(iommu_iova_cache);
2518
2519 return -ENOMEM;
2520 }
2521
2522 static void __init iommu_exit_mempool(void)
2523 {
2524 kmem_cache_destroy(iommu_devinfo_cache);
2525 kmem_cache_destroy(iommu_domain_cache);
2526 kmem_cache_destroy(iommu_iova_cache);
2527
2528 }
2529
2530 static void __init init_no_remapping_devices(void)
2531 {
2532 struct dmar_drhd_unit *drhd;
2533
2534 for_each_drhd_unit(drhd) {
2535 if (!drhd->include_all) {
2536 int i;
2537 for (i = 0; i < drhd->devices_cnt; i++)
2538 if (drhd->devices[i] != NULL)
2539 break;
2540 /* ignore DMAR unit if no pci devices exist */
2541 if (i == drhd->devices_cnt)
2542 drhd->ignored = 1;
2543 }
2544 }
2545
2546 if (dmar_map_gfx)
2547 return;
2548
2549 for_each_drhd_unit(drhd) {
2550 int i;
2551 if (drhd->ignored || drhd->include_all)
2552 continue;
2553
2554 for (i = 0; i < drhd->devices_cnt; i++)
2555 if (drhd->devices[i] &&
2556 !IS_GFX_DEVICE(drhd->devices[i]))
2557 break;
2558
2559 if (i < drhd->devices_cnt)
2560 continue;
2561
2562 /* bypass IOMMU if it is just for gfx devices */
2563 drhd->ignored = 1;
2564 for (i = 0; i < drhd->devices_cnt; i++) {
2565 if (!drhd->devices[i])
2566 continue;
2567 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
2568 }
2569 }
2570 }
2571
2572 int __init intel_iommu_init(void)
2573 {
2574 int ret = 0;
2575
2576 if (dmar_table_init())
2577 return -ENODEV;
2578
2579 if (dmar_dev_scope_init())
2580 return -ENODEV;
2581
2582 /*
2583 * Check the need for DMA-remapping initialization now.
2584 * Above initialization will also be used by Interrupt-remapping.
2585 */
2586 if (no_iommu || swiotlb || dmar_disabled)
2587 return -ENODEV;
2588
2589 iommu_init_mempool();
2590 dmar_init_reserved_ranges();
2591
2592 init_no_remapping_devices();
2593
2594 ret = init_dmars();
2595 if (ret) {
2596 printk(KERN_ERR "IOMMU: dmar init failed\n");
2597 put_iova_domain(&reserved_iova_list);
2598 iommu_exit_mempool();
2599 return ret;
2600 }
2601 printk(KERN_INFO
2602 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
2603
2604 init_timer(&unmap_timer);
2605 force_iommu = 1;
2606 dma_ops = &intel_dma_ops;
2607
2608 register_iommu(&intel_iommu_ops);
2609
2610 return 0;
2611 }
2612
2613 static int vm_domain_add_dev_info(struct dmar_domain *domain,
2614 struct pci_dev *pdev)
2615 {
2616 struct device_domain_info *info;
2617 unsigned long flags;
2618
2619 info = alloc_devinfo_mem();
2620 if (!info)
2621 return -ENOMEM;
2622
2623 info->bus = pdev->bus->number;
2624 info->devfn = pdev->devfn;
2625 info->dev = pdev;
2626 info->domain = domain;
2627
2628 spin_lock_irqsave(&device_domain_lock, flags);
2629 list_add(&info->link, &domain->devices);
2630 list_add(&info->global, &device_domain_list);
2631 pdev->dev.archdata.iommu = info;
2632 spin_unlock_irqrestore(&device_domain_lock, flags);
2633
2634 return 0;
2635 }
2636
2637 static void vm_domain_remove_one_dev_info(struct dmar_domain *domain,
2638 struct pci_dev *pdev)
2639 {
2640 struct device_domain_info *info;
2641 struct intel_iommu *iommu;
2642 unsigned long flags;
2643 int found = 0;
2644 struct list_head *entry, *tmp;
2645
2646 iommu = device_to_iommu(pdev->bus->number, pdev->devfn);
2647 if (!iommu)
2648 return;
2649
2650 spin_lock_irqsave(&device_domain_lock, flags);
2651 list_for_each_safe(entry, tmp, &domain->devices) {
2652 info = list_entry(entry, struct device_domain_info, link);
2653 if (info->bus == pdev->bus->number &&
2654 info->devfn == pdev->devfn) {
2655 list_del(&info->link);
2656 list_del(&info->global);
2657 if (info->dev)
2658 info->dev->dev.archdata.iommu = NULL;
2659 spin_unlock_irqrestore(&device_domain_lock, flags);
2660
2661 iommu_detach_dev(iommu, info->bus, info->devfn);
2662 free_devinfo_mem(info);
2663
2664 spin_lock_irqsave(&device_domain_lock, flags);
2665
2666 if (found)
2667 break;
2668 else
2669 continue;
2670 }
2671
2672 /* if there is no other devices under the same iommu
2673 * owned by this domain, clear this iommu in iommu_bmp
2674 * update iommu count and coherency
2675 */
2676 if (device_to_iommu(info->bus, info->devfn) == iommu)
2677 found = 1;
2678 }
2679
2680 if (found == 0) {
2681 unsigned long tmp_flags;
2682 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
2683 clear_bit(iommu->seq_id, &domain->iommu_bmp);
2684 domain->iommu_count--;
2685 domain_update_iommu_coherency(domain);
2686 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
2687 }
2688
2689 spin_unlock_irqrestore(&device_domain_lock, flags);
2690 }
2691
2692 static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
2693 {
2694 struct device_domain_info *info;
2695 struct intel_iommu *iommu;
2696 unsigned long flags1, flags2;
2697
2698 spin_lock_irqsave(&device_domain_lock, flags1);
2699 while (!list_empty(&domain->devices)) {
2700 info = list_entry(domain->devices.next,
2701 struct device_domain_info, link);
2702 list_del(&info->link);
2703 list_del(&info->global);
2704 if (info->dev)
2705 info->dev->dev.archdata.iommu = NULL;
2706
2707 spin_unlock_irqrestore(&device_domain_lock, flags1);
2708
2709 iommu = device_to_iommu(info->bus, info->devfn);
2710 iommu_detach_dev(iommu, info->bus, info->devfn);
2711
2712 /* clear this iommu in iommu_bmp, update iommu count
2713 * and coherency
2714 */
2715 spin_lock_irqsave(&domain->iommu_lock, flags2);
2716 if (test_and_clear_bit(iommu->seq_id,
2717 &domain->iommu_bmp)) {
2718 domain->iommu_count--;
2719 domain_update_iommu_coherency(domain);
2720 }
2721 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
2722
2723 free_devinfo_mem(info);
2724 spin_lock_irqsave(&device_domain_lock, flags1);
2725 }
2726 spin_unlock_irqrestore(&device_domain_lock, flags1);
2727 }
2728
2729 /* domain id for virtual machine, it won't be set in context */
2730 static unsigned long vm_domid;
2731
2732 static int vm_domain_min_agaw(struct dmar_domain *domain)
2733 {
2734 int i;
2735 int min_agaw = domain->agaw;
2736
2737 i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
2738 for (; i < g_num_of_iommus; ) {
2739 if (min_agaw > g_iommus[i]->agaw)
2740 min_agaw = g_iommus[i]->agaw;
2741
2742 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
2743 }
2744
2745 return min_agaw;
2746 }
2747
2748 static struct dmar_domain *iommu_alloc_vm_domain(void)
2749 {
2750 struct dmar_domain *domain;
2751
2752 domain = alloc_domain_mem();
2753 if (!domain)
2754 return NULL;
2755
2756 domain->id = vm_domid++;
2757 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
2758 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
2759
2760 return domain;
2761 }
2762
2763 static int vm_domain_init(struct dmar_domain *domain, int guest_width)
2764 {
2765 int adjust_width;
2766
2767 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
2768 spin_lock_init(&domain->mapping_lock);
2769 spin_lock_init(&domain->iommu_lock);
2770
2771 domain_reserve_special_ranges(domain);
2772
2773 /* calculate AGAW */
2774 domain->gaw = guest_width;
2775 adjust_width = guestwidth_to_adjustwidth(guest_width);
2776 domain->agaw = width_to_agaw(adjust_width);
2777
2778 INIT_LIST_HEAD(&domain->devices);
2779
2780 domain->iommu_count = 0;
2781 domain->iommu_coherency = 0;
2782 domain->max_addr = 0;
2783
2784 /* always allocate the top pgd */
2785 domain->pgd = (struct dma_pte *)alloc_pgtable_page();
2786 if (!domain->pgd)
2787 return -ENOMEM;
2788 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
2789 return 0;
2790 }
2791
2792 static void iommu_free_vm_domain(struct dmar_domain *domain)
2793 {
2794 unsigned long flags;
2795 struct dmar_drhd_unit *drhd;
2796 struct intel_iommu *iommu;
2797 unsigned long i;
2798 unsigned long ndomains;
2799
2800 for_each_drhd_unit(drhd) {
2801 if (drhd->ignored)
2802 continue;
2803 iommu = drhd->iommu;
2804
2805 ndomains = cap_ndoms(iommu->cap);
2806 i = find_first_bit(iommu->domain_ids, ndomains);
2807 for (; i < ndomains; ) {
2808 if (iommu->domains[i] == domain) {
2809 spin_lock_irqsave(&iommu->lock, flags);
2810 clear_bit(i, iommu->domain_ids);
2811 iommu->domains[i] = NULL;
2812 spin_unlock_irqrestore(&iommu->lock, flags);
2813 break;
2814 }
2815 i = find_next_bit(iommu->domain_ids, ndomains, i+1);
2816 }
2817 }
2818 }
2819
2820 static void vm_domain_exit(struct dmar_domain *domain)
2821 {
2822 u64 end;
2823
2824 /* Domain 0 is reserved, so dont process it */
2825 if (!domain)
2826 return;
2827
2828 vm_domain_remove_all_dev_info(domain);
2829 /* destroy iovas */
2830 put_iova_domain(&domain->iovad);
2831 end = DOMAIN_MAX_ADDR(domain->gaw);
2832 end = end & (~VTD_PAGE_MASK);
2833
2834 /* clear ptes */
2835 dma_pte_clear_range(domain, 0, end);
2836
2837 /* free page tables */
2838 dma_pte_free_pagetable(domain, 0, end);
2839
2840 iommu_free_vm_domain(domain);
2841 free_domain_mem(domain);
2842 }
2843
2844 static int intel_iommu_domain_init(struct iommu_domain *domain)
2845 {
2846 struct dmar_domain *dmar_domain;
2847
2848 dmar_domain = iommu_alloc_vm_domain();
2849 if (!dmar_domain) {
2850 printk(KERN_ERR
2851 "intel_iommu_domain_init: dmar_domain == NULL\n");
2852 return -ENOMEM;
2853 }
2854 if (vm_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2855 printk(KERN_ERR
2856 "intel_iommu_domain_init() failed\n");
2857 vm_domain_exit(dmar_domain);
2858 return -ENOMEM;
2859 }
2860 domain->priv = dmar_domain;
2861
2862 return 0;
2863 }
2864
2865 static void intel_iommu_domain_destroy(struct iommu_domain *domain)
2866 {
2867 struct dmar_domain *dmar_domain = domain->priv;
2868
2869 domain->priv = NULL;
2870 vm_domain_exit(dmar_domain);
2871 }
2872
2873 static int intel_iommu_attach_device(struct iommu_domain *domain,
2874 struct device *dev)
2875 {
2876 struct dmar_domain *dmar_domain = domain->priv;
2877 struct pci_dev *pdev = to_pci_dev(dev);
2878 struct intel_iommu *iommu;
2879 int addr_width;
2880 u64 end;
2881 int ret;
2882
2883 /* normally pdev is not mapped */
2884 if (unlikely(domain_context_mapped(pdev))) {
2885 struct dmar_domain *old_domain;
2886
2887 old_domain = find_domain(pdev);
2888 if (old_domain) {
2889 if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
2890 vm_domain_remove_one_dev_info(old_domain, pdev);
2891 else
2892 domain_remove_dev_info(old_domain);
2893 }
2894 }
2895
2896 iommu = device_to_iommu(pdev->bus->number, pdev->devfn);
2897 if (!iommu)
2898 return -ENODEV;
2899
2900 /* check if this iommu agaw is sufficient for max mapped address */
2901 addr_width = agaw_to_width(iommu->agaw);
2902 end = DOMAIN_MAX_ADDR(addr_width);
2903 end = end & VTD_PAGE_MASK;
2904 if (end < dmar_domain->max_addr) {
2905 printk(KERN_ERR "%s: iommu agaw (%d) is not "
2906 "sufficient for the mapped address (%llx)\n",
2907 __func__, iommu->agaw, dmar_domain->max_addr);
2908 return -EFAULT;
2909 }
2910
2911 ret = domain_context_mapping(dmar_domain, pdev);
2912 if (ret)
2913 return ret;
2914
2915 ret = vm_domain_add_dev_info(dmar_domain, pdev);
2916 return ret;
2917 }
2918
2919 static void intel_iommu_detach_device(struct iommu_domain *domain,
2920 struct device *dev)
2921 {
2922 struct dmar_domain *dmar_domain = domain->priv;
2923 struct pci_dev *pdev = to_pci_dev(dev);
2924
2925 vm_domain_remove_one_dev_info(dmar_domain, pdev);
2926 }
2927
2928 static int intel_iommu_map_range(struct iommu_domain *domain,
2929 unsigned long iova, phys_addr_t hpa,
2930 size_t size, int iommu_prot)
2931 {
2932 struct dmar_domain *dmar_domain = domain->priv;
2933 u64 max_addr;
2934 int addr_width;
2935 int prot = 0;
2936 int ret;
2937
2938 if (iommu_prot & IOMMU_READ)
2939 prot |= DMA_PTE_READ;
2940 if (iommu_prot & IOMMU_WRITE)
2941 prot |= DMA_PTE_WRITE;
2942
2943 max_addr = (iova & VTD_PAGE_MASK) + VTD_PAGE_ALIGN(size);
2944 if (dmar_domain->max_addr < max_addr) {
2945 int min_agaw;
2946 u64 end;
2947
2948 /* check if minimum agaw is sufficient for mapped address */
2949 min_agaw = vm_domain_min_agaw(dmar_domain);
2950 addr_width = agaw_to_width(min_agaw);
2951 end = DOMAIN_MAX_ADDR(addr_width);
2952 end = end & VTD_PAGE_MASK;
2953 if (end < max_addr) {
2954 printk(KERN_ERR "%s: iommu agaw (%d) is not "
2955 "sufficient for the mapped address (%llx)\n",
2956 __func__, min_agaw, max_addr);
2957 return -EFAULT;
2958 }
2959 dmar_domain->max_addr = max_addr;
2960 }
2961
2962 ret = domain_page_mapping(dmar_domain, iova, hpa, size, prot);
2963 return ret;
2964 }
2965
2966 static void intel_iommu_unmap_range(struct iommu_domain *domain,
2967 unsigned long iova, size_t size)
2968 {
2969 struct dmar_domain *dmar_domain = domain->priv;
2970 dma_addr_t base;
2971
2972 /* The address might not be aligned */
2973 base = iova & VTD_PAGE_MASK;
2974 size = VTD_PAGE_ALIGN(size);
2975 dma_pte_clear_range(dmar_domain, base, base + size);
2976
2977 if (dmar_domain->max_addr == base + size)
2978 dmar_domain->max_addr = base;
2979 }
2980
2981 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
2982 unsigned long iova)
2983 {
2984 struct dmar_domain *dmar_domain = domain->priv;
2985 struct dma_pte *pte;
2986 u64 phys = 0;
2987
2988 pte = addr_to_dma_pte(dmar_domain, iova);
2989 if (pte)
2990 phys = dma_pte_addr(pte);
2991
2992 return phys;
2993 }
2994
2995 static struct iommu_ops intel_iommu_ops = {
2996 .domain_init = intel_iommu_domain_init,
2997 .domain_destroy = intel_iommu_domain_destroy,
2998 .attach_dev = intel_iommu_attach_device,
2999 .detach_dev = intel_iommu_detach_device,
3000 .map = intel_iommu_map_range,
3001 .unmap = intel_iommu_unmap_range,
3002 .iova_to_phys = intel_iommu_iova_to_phys,
3003 };
3004
3005 static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
3006 {
3007 /*
3008 * Mobile 4 Series Chipset neglects to set RWBF capability,
3009 * but needs it:
3010 */
3011 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
3012 rwbf_quirk = 1;
3013 }
3014
3015 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
(Inofficial) Linux kernel tree running on the Nao robot