?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
x86/amd-iommu: Add per IOMMU reference counting
[linux/fpc-iii.git] / arch / x86 / kernel / amd_iommu_init.c
blob8567d16980274879bda5352f9e0de4a4cd9236ac
1 /*
2 * Copyright (C) 2007-2009 Advanced Micro Devices, Inc.
3 * Author: Joerg Roedel <joerg.roedel@amd.com>
4 * Leo Duran <leo.duran@amd.com>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19
20 #include <linux/pci.h>
21 #include <linux/acpi.h>
22 #include <linux/gfp.h>
23 #include <linux/list.h>
24 #include <linux/sysdev.h>
25 #include <linux/interrupt.h>
26 #include <linux/msi.h>
27 #include <asm/pci-direct.h>
28 #include <asm/amd_iommu_proto.h>
29 #include <asm/amd_iommu_types.h>
30 #include <asm/amd_iommu.h>
31 #include <asm/iommu.h>
32 #include <asm/gart.h>
33 #include <asm/x86_init.h>
34
35 /*
36 * definitions for the ACPI scanning code
37 */
38 #define IVRS_HEADER_LENGTH 48
39
40 #define ACPI_IVHD_TYPE 0x10
41 #define ACPI_IVMD_TYPE_ALL 0x20
42 #define ACPI_IVMD_TYPE 0x21
43 #define ACPI_IVMD_TYPE_RANGE 0x22
44
45 #define IVHD_DEV_ALL 0x01
46 #define IVHD_DEV_SELECT 0x02
47 #define IVHD_DEV_SELECT_RANGE_START 0x03
48 #define IVHD_DEV_RANGE_END 0x04
49 #define IVHD_DEV_ALIAS 0x42
50 #define IVHD_DEV_ALIAS_RANGE 0x43
51 #define IVHD_DEV_EXT_SELECT 0x46
52 #define IVHD_DEV_EXT_SELECT_RANGE 0x47
53
54 #define IVHD_FLAG_HT_TUN_EN_MASK 0x01
55 #define IVHD_FLAG_PASSPW_EN_MASK 0x02
56 #define IVHD_FLAG_RESPASSPW_EN_MASK 0x04
57 #define IVHD_FLAG_ISOC_EN_MASK 0x08
58
59 #define IVMD_FLAG_EXCL_RANGE 0x08
60 #define IVMD_FLAG_UNITY_MAP 0x01
61
62 #define ACPI_DEVFLAG_INITPASS 0x01
63 #define ACPI_DEVFLAG_EXTINT 0x02
64 #define ACPI_DEVFLAG_NMI 0x04
65 #define ACPI_DEVFLAG_SYSMGT1 0x10
66 #define ACPI_DEVFLAG_SYSMGT2 0x20
67 #define ACPI_DEVFLAG_LINT0 0x40
68 #define ACPI_DEVFLAG_LINT1 0x80
69 #define ACPI_DEVFLAG_ATSDIS 0x10000000
70
71 /*
72 * ACPI table definitions
73 *
74 * These data structures are laid over the table to parse the important values
75 * out of it.
76 */
77
78 /*
79 * structure describing one IOMMU in the ACPI table. Typically followed by one
80 * or more ivhd_entrys.
81 */
82 struct ivhd_header {
83 u8 type;
84 u8 flags;
85 u16 length;
86 u16 devid;
87 u16 cap_ptr;
88 u64 mmio_phys;
89 u16 pci_seg;
90 u16 info;
91 u32 reserved;
92 } __attribute__((packed));
93
94 /*
95 * A device entry describing which devices a specific IOMMU translates and
96 * which requestor ids they use.
97 */
98 struct ivhd_entry {
99 u8 type;
100 u16 devid;
101 u8 flags;
102 u32 ext;
103 } __attribute__((packed));
104
105 /*
106 * An AMD IOMMU memory definition structure. It defines things like exclusion
107 * ranges for devices and regions that should be unity mapped.
108 */
109 struct ivmd_header {
110 u8 type;
111 u8 flags;
112 u16 length;
113 u16 devid;
114 u16 aux;
115 u64 resv;
116 u64 range_start;
117 u64 range_length;
118 } __attribute__((packed));
119
120 bool amd_iommu_dump;
121
122 static int __initdata amd_iommu_detected;
123
124 u16 amd_iommu_last_bdf; /* largest PCI device id we have
125 to handle */
126 LIST_HEAD(amd_iommu_unity_map); /* a list of required unity mappings
127 we find in ACPI */
128 #ifdef CONFIG_IOMMU_STRESS
129 bool amd_iommu_isolate = false;
130 #else
131 bool amd_iommu_isolate = true; /* if true, device isolation is
132 enabled */
133 #endif
134
135 bool amd_iommu_unmap_flush; /* if true, flush on every unmap */
136
137 LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the
138 system */
139
140 /* Array to assign indices to IOMMUs*/
141 struct amd_iommu *amd_iommus[MAX_IOMMUS];
142 int amd_iommus_present;
143
144 /*
145 * Pointer to the device table which is shared by all AMD IOMMUs
146 * it is indexed by the PCI device id or the HT unit id and contains
147 * information about the domain the device belongs to as well as the
148 * page table root pointer.
149 */
150 struct dev_table_entry *amd_iommu_dev_table;
151
152 /*
153 * The alias table is a driver specific data structure which contains the
154 * mappings of the PCI device ids to the actual requestor ids on the IOMMU.
155 * More than one device can share the same requestor id.
156 */
157 u16 *amd_iommu_alias_table;
158
159 /*
160 * The rlookup table is used to find the IOMMU which is responsible
161 * for a specific device. It is also indexed by the PCI device id.
162 */
163 struct amd_iommu **amd_iommu_rlookup_table;
164
165 /*
166 * The pd table (protection domain table) is used to find the protection domain
167 * data structure a device belongs to. Indexed with the PCI device id too.
168 */
169 struct protection_domain **amd_iommu_pd_table;
170
171 /*
172 * AMD IOMMU allows up to 2^16 differend protection domains. This is a bitmap
173 * to know which ones are already in use.
174 */
175 unsigned long *amd_iommu_pd_alloc_bitmap;
176
177 static u32 dev_table_size; /* size of the device table */
178 static u32 alias_table_size; /* size of the alias table */
179 static u32 rlookup_table_size; /* size if the rlookup table */
180
181 static inline void update_last_devid(u16 devid)
182 {
183 if (devid > amd_iommu_last_bdf)
184 amd_iommu_last_bdf = devid;
185 }
186
187 static inline unsigned long tbl_size(int entry_size)
188 {
189 unsigned shift = PAGE_SHIFT +
190 get_order(((int)amd_iommu_last_bdf + 1) * entry_size);
191
192 return 1UL << shift;
193 }
194
195 /****************************************************************************
196 *
197 * AMD IOMMU MMIO register space handling functions
198 *
199 * These functions are used to program the IOMMU device registers in
200 * MMIO space required for that driver.
201 *
202 ****************************************************************************/
203
204 /*
205 * This function set the exclusion range in the IOMMU. DMA accesses to the
206 * exclusion range are passed through untranslated
207 */
208 static void iommu_set_exclusion_range(struct amd_iommu *iommu)
209 {
210 u64 start = iommu->exclusion_start & PAGE_MASK;
211 u64 limit = (start + iommu->exclusion_length) & PAGE_MASK;
212 u64 entry;
213
214 if (!iommu->exclusion_start)
215 return;
216
217 entry = start | MMIO_EXCL_ENABLE_MASK;
218 memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,
219 &entry, sizeof(entry));
220
221 entry = limit;
222 memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,
223 &entry, sizeof(entry));
224 }
225
226 /* Programs the physical address of the device table into the IOMMU hardware */
227 static void __init iommu_set_device_table(struct amd_iommu *iommu)
228 {
229 u64 entry;
230
231 BUG_ON(iommu->mmio_base == NULL);
232
233 entry = virt_to_phys(amd_iommu_dev_table);
234 entry |= (dev_table_size >> 12) - 1;
235 memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET,
236 &entry, sizeof(entry));
237 }
238
239 /* Generic functions to enable/disable certain features of the IOMMU. */
240 static void iommu_feature_enable(struct amd_iommu *iommu, u8 bit)
241 {
242 u32 ctrl;
243
244 ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);
245 ctrl |= (1 << bit);
246 writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
247 }
248
249 static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit)
250 {
251 u32 ctrl;
252
253 ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);
254 ctrl &= ~(1 << bit);
255 writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
256 }
257
258 /* Function to enable the hardware */
259 static void iommu_enable(struct amd_iommu *iommu)
260 {
261 printk(KERN_INFO "AMD-Vi: Enabling IOMMU at %s cap 0x%hx\n",
262 dev_name(&iommu->dev->dev), iommu->cap_ptr);
263
264 iommu_feature_enable(iommu, CONTROL_IOMMU_EN);
265 }
266
267 static void iommu_disable(struct amd_iommu *iommu)
268 {
269 /* Disable command buffer */
270 iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
271
272 /* Disable event logging and event interrupts */
273 iommu_feature_disable(iommu, CONTROL_EVT_INT_EN);
274 iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);
275
276 /* Disable IOMMU hardware itself */
277 iommu_feature_disable(iommu, CONTROL_IOMMU_EN);
278 }
279
280 /*
281 * mapping and unmapping functions for the IOMMU MMIO space. Each AMD IOMMU in
282 * the system has one.
283 */
284 static u8 * __init iommu_map_mmio_space(u64 address)
285 {
286 u8 *ret;
287
288 if (!request_mem_region(address, MMIO_REGION_LENGTH, "amd_iommu"))
289 return NULL;
290
291 ret = ioremap_nocache(address, MMIO_REGION_LENGTH);
292 if (ret != NULL)
293 return ret;
294
295 release_mem_region(address, MMIO_REGION_LENGTH);
296
297 return NULL;
298 }
299
300 static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu)
301 {
302 if (iommu->mmio_base)
303 iounmap(iommu->mmio_base);
304 release_mem_region(iommu->mmio_phys, MMIO_REGION_LENGTH);
305 }
306
307 /****************************************************************************
308 *
309 * The functions below belong to the first pass of AMD IOMMU ACPI table
310 * parsing. In this pass we try to find out the highest device id this
311 * code has to handle. Upon this information the size of the shared data
312 * structures is determined later.
313 *
314 ****************************************************************************/
315
316 /*
317 * This function calculates the length of a given IVHD entry
318 */
319 static inline int ivhd_entry_length(u8 *ivhd)
320 {
321 return 0x04 << (*ivhd >> 6);
322 }
323
324 /*
325 * This function reads the last device id the IOMMU has to handle from the PCI
326 * capability header for this IOMMU
327 */
328 static int __init find_last_devid_on_pci(int bus, int dev, int fn, int cap_ptr)
329 {
330 u32 cap;
331
332 cap = read_pci_config(bus, dev, fn, cap_ptr+MMIO_RANGE_OFFSET);
333 update_last_devid(calc_devid(MMIO_GET_BUS(cap), MMIO_GET_LD(cap)));
334
335 return 0;
336 }
337
338 /*
339 * After reading the highest device id from the IOMMU PCI capability header
340 * this function looks if there is a higher device id defined in the ACPI table
341 */
342 static int __init find_last_devid_from_ivhd(struct ivhd_header *h)
343 {
344 u8 *p = (void *)h, *end = (void *)h;
345 struct ivhd_entry *dev;
346
347 p += sizeof(*h);
348 end += h->length;
349
350 find_last_devid_on_pci(PCI_BUS(h->devid),
351 PCI_SLOT(h->devid),
352 PCI_FUNC(h->devid),
353 h->cap_ptr);
354
355 while (p < end) {
356 dev = (struct ivhd_entry *)p;
357 switch (dev->type) {
358 case IVHD_DEV_SELECT:
359 case IVHD_DEV_RANGE_END:
360 case IVHD_DEV_ALIAS:
361 case IVHD_DEV_EXT_SELECT:
362 /* all the above subfield types refer to device ids */
363 update_last_devid(dev->devid);
364 break;
365 default:
366 break;
367 }
368 p += ivhd_entry_length(p);
369 }
370
371 WARN_ON(p != end);
372
373 return 0;
374 }
375
376 /*
377 * Iterate over all IVHD entries in the ACPI table and find the highest device
378 * id which we need to handle. This is the first of three functions which parse
379 * the ACPI table. So we check the checksum here.
380 */
381 static int __init find_last_devid_acpi(struct acpi_table_header *table)
382 {
383 int i;
384 u8 checksum = 0, *p = (u8 *)table, *end = (u8 *)table;
385 struct ivhd_header *h;
386
387 /*
388 * Validate checksum here so we don't need to do it when
389 * we actually parse the table
390 */
391 for (i = 0; i < table->length; ++i)
392 checksum += p[i];
393 if (checksum != 0)
394 /* ACPI table corrupt */
395 return -ENODEV;
396
397 p += IVRS_HEADER_LENGTH;
398
399 end += table->length;
400 while (p < end) {
401 h = (struct ivhd_header *)p;
402 switch (h->type) {
403 case ACPI_IVHD_TYPE:
404 find_last_devid_from_ivhd(h);
405 break;
406 default:
407 break;
408 }
409 p += h->length;
410 }
411 WARN_ON(p != end);
412
413 return 0;
414 }
415
416 /****************************************************************************
417 *
418 * The following functions belong the the code path which parses the ACPI table
419 * the second time. In this ACPI parsing iteration we allocate IOMMU specific
420 * data structures, initialize the device/alias/rlookup table and also
421 * basically initialize the hardware.
422 *
423 ****************************************************************************/
424
425 /*
426 * Allocates the command buffer. This buffer is per AMD IOMMU. We can
427 * write commands to that buffer later and the IOMMU will execute them
428 * asynchronously
429 */
430 static u8 * __init alloc_command_buffer(struct amd_iommu *iommu)
431 {
432 u8 *cmd_buf = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
433 get_order(CMD_BUFFER_SIZE));
434
435 if (cmd_buf == NULL)
436 return NULL;
437
438 iommu->cmd_buf_size = CMD_BUFFER_SIZE;
439
440 return cmd_buf;
441 }
442
443 /*
444 * This function resets the command buffer if the IOMMU stopped fetching
445 * commands from it.
446 */
447 void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu)
448 {
449 iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
450
451 writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
452 writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
453
454 iommu_feature_enable(iommu, CONTROL_CMDBUF_EN);
455 }
456
457 /*
458 * This function writes the command buffer address to the hardware and
459 * enables it.
460 */
461 static void iommu_enable_command_buffer(struct amd_iommu *iommu)
462 {
463 u64 entry;
464
465 BUG_ON(iommu->cmd_buf == NULL);
466
467 entry = (u64)virt_to_phys(iommu->cmd_buf);
468 entry |= MMIO_CMD_SIZE_512;
469
470 memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET,
471 &entry, sizeof(entry));
472
473 amd_iommu_reset_cmd_buffer(iommu);
474 }
475
476 static void __init free_command_buffer(struct amd_iommu *iommu)
477 {
478 free_pages((unsigned long)iommu->cmd_buf,
479 get_order(iommu->cmd_buf_size));
480 }
481
482 /* allocates the memory where the IOMMU will log its events to */
483 static u8 * __init alloc_event_buffer(struct amd_iommu *iommu)
484 {
485 iommu->evt_buf = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
486 get_order(EVT_BUFFER_SIZE));
487
488 if (iommu->evt_buf == NULL)
489 return NULL;
490
491 iommu->evt_buf_size = EVT_BUFFER_SIZE;
492
493 return iommu->evt_buf;
494 }
495
496 static void iommu_enable_event_buffer(struct amd_iommu *iommu)
497 {
498 u64 entry;
499
500 BUG_ON(iommu->evt_buf == NULL);
501
502 entry = (u64)virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK;
503
504 memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET,
505 &entry, sizeof(entry));
506
507 /* set head and tail to zero manually */
508 writel(0x00, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
509 writel(0x00, iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
510
511 iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);
512 }
513
514 static void __init free_event_buffer(struct amd_iommu *iommu)
515 {
516 free_pages((unsigned long)iommu->evt_buf, get_order(EVT_BUFFER_SIZE));
517 }
518
519 /* sets a specific bit in the device table entry. */
520 static void set_dev_entry_bit(u16 devid, u8 bit)
521 {
522 int i = (bit >> 5) & 0x07;
523 int _bit = bit & 0x1f;
524
525 amd_iommu_dev_table[devid].data[i] |= (1 << _bit);
526 }
527
528 static int get_dev_entry_bit(u16 devid, u8 bit)
529 {
530 int i = (bit >> 5) & 0x07;
531 int _bit = bit & 0x1f;
532
533 return (amd_iommu_dev_table[devid].data[i] & (1 << _bit)) >> _bit;
534 }
535
536
537 void amd_iommu_apply_erratum_63(u16 devid)
538 {
539 int sysmgt;
540
541 sysmgt = get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1) |
542 (get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2) << 1);
543
544 if (sysmgt == 0x01)
545 set_dev_entry_bit(devid, DEV_ENTRY_IW);
546 }
547
548 /* Writes the specific IOMMU for a device into the rlookup table */
549 static void __init set_iommu_for_device(struct amd_iommu *iommu, u16 devid)
550 {
551 amd_iommu_rlookup_table[devid] = iommu;
552 }
553
554 /*
555 * This function takes the device specific flags read from the ACPI
556 * table and sets up the device table entry with that information
557 */
558 static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu,
559 u16 devid, u32 flags, u32 ext_flags)
560 {
561 if (flags & ACPI_DEVFLAG_INITPASS)
562 set_dev_entry_bit(devid, DEV_ENTRY_INIT_PASS);
563 if (flags & ACPI_DEVFLAG_EXTINT)
564 set_dev_entry_bit(devid, DEV_ENTRY_EINT_PASS);
565 if (flags & ACPI_DEVFLAG_NMI)
566 set_dev_entry_bit(devid, DEV_ENTRY_NMI_PASS);
567 if (flags & ACPI_DEVFLAG_SYSMGT1)
568 set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1);
569 if (flags & ACPI_DEVFLAG_SYSMGT2)
570 set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2);
571 if (flags & ACPI_DEVFLAG_LINT0)
572 set_dev_entry_bit(devid, DEV_ENTRY_LINT0_PASS);
573 if (flags & ACPI_DEVFLAG_LINT1)
574 set_dev_entry_bit(devid, DEV_ENTRY_LINT1_PASS);
575
576 amd_iommu_apply_erratum_63(devid);
577
578 set_iommu_for_device(iommu, devid);
579 }
580
581 /*
582 * Reads the device exclusion range from ACPI and initialize IOMMU with
583 * it
584 */
585 static void __init set_device_exclusion_range(u16 devid, struct ivmd_header *m)
586 {
587 struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
588
589 if (!(m->flags & IVMD_FLAG_EXCL_RANGE))
590 return;
591
592 if (iommu) {
593 /*
594 * We only can configure exclusion ranges per IOMMU, not
595 * per device. But we can enable the exclusion range per
596 * device. This is done here
597 */
598 set_dev_entry_bit(m->devid, DEV_ENTRY_EX);
599 iommu->exclusion_start = m->range_start;
600 iommu->exclusion_length = m->range_length;
601 }
602 }
603
604 /*
605 * This function reads some important data from the IOMMU PCI space and
606 * initializes the driver data structure with it. It reads the hardware
607 * capabilities and the first/last device entries
608 */
609 static void __init init_iommu_from_pci(struct amd_iommu *iommu)
610 {
611 int cap_ptr = iommu->cap_ptr;
612 u32 range, misc;
613
614 pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET,
615 &iommu->cap);
616 pci_read_config_dword(iommu->dev, cap_ptr + MMIO_RANGE_OFFSET,
617 &range);
618 pci_read_config_dword(iommu->dev, cap_ptr + MMIO_MISC_OFFSET,
619 &misc);
620
621 iommu->first_device = calc_devid(MMIO_GET_BUS(range),
622 MMIO_GET_FD(range));
623 iommu->last_device = calc_devid(MMIO_GET_BUS(range),
624 MMIO_GET_LD(range));
625 iommu->evt_msi_num = MMIO_MSI_NUM(misc);
626 }
627
628 /*
629 * Takes a pointer to an AMD IOMMU entry in the ACPI table and
630 * initializes the hardware and our data structures with it.
631 */
632 static void __init init_iommu_from_acpi(struct amd_iommu *iommu,
633 struct ivhd_header *h)
634 {
635 u8 *p = (u8 *)h;
636 u8 *end = p, flags = 0;
637 u16 dev_i, devid = 0, devid_start = 0, devid_to = 0;
638 u32 ext_flags = 0;
639 bool alias = false;
640 struct ivhd_entry *e;
641
642 /*
643 * First set the recommended feature enable bits from ACPI
644 * into the IOMMU control registers
645 */
646 h->flags & IVHD_FLAG_HT_TUN_EN_MASK ?
647 iommu_feature_enable(iommu, CONTROL_HT_TUN_EN) :
648 iommu_feature_disable(iommu, CONTROL_HT_TUN_EN);
649
650 h->flags & IVHD_FLAG_PASSPW_EN_MASK ?
651 iommu_feature_enable(iommu, CONTROL_PASSPW_EN) :
652 iommu_feature_disable(iommu, CONTROL_PASSPW_EN);
653
654 h->flags & IVHD_FLAG_RESPASSPW_EN_MASK ?
655 iommu_feature_enable(iommu, CONTROL_RESPASSPW_EN) :
656 iommu_feature_disable(iommu, CONTROL_RESPASSPW_EN);
657
658 h->flags & IVHD_FLAG_ISOC_EN_MASK ?
659 iommu_feature_enable(iommu, CONTROL_ISOC_EN) :
660 iommu_feature_disable(iommu, CONTROL_ISOC_EN);
661
662 /*
663 * make IOMMU memory accesses cache coherent
664 */
665 iommu_feature_enable(iommu, CONTROL_COHERENT_EN);
666
667 /*
668 * Done. Now parse the device entries
669 */
670 p += sizeof(struct ivhd_header);
671 end += h->length;
672
673
674 while (p < end) {
675 e = (struct ivhd_entry *)p;
676 switch (e->type) {
677 case IVHD_DEV_ALL:
678
679 DUMP_printk(" DEV_ALL\t\t\t first devid: %02x:%02x.%x"
680 " last device %02x:%02x.%x flags: %02x\n",
681 PCI_BUS(iommu->first_device),
682 PCI_SLOT(iommu->first_device),
683 PCI_FUNC(iommu->first_device),
684 PCI_BUS(iommu->last_device),
685 PCI_SLOT(iommu->last_device),
686 PCI_FUNC(iommu->last_device),
687 e->flags);
688
689 for (dev_i = iommu->first_device;
690 dev_i <= iommu->last_device; ++dev_i)
691 set_dev_entry_from_acpi(iommu, dev_i,
692 e->flags, 0);
693 break;
694 case IVHD_DEV_SELECT:
695
696 DUMP_printk(" DEV_SELECT\t\t\t devid: %02x:%02x.%x "
697 "flags: %02x\n",
698 PCI_BUS(e->devid),
699 PCI_SLOT(e->devid),
700 PCI_FUNC(e->devid),
701 e->flags);
702
703 devid = e->devid;
704 set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
705 break;
706 case IVHD_DEV_SELECT_RANGE_START:
707
708 DUMP_printk(" DEV_SELECT_RANGE_START\t "
709 "devid: %02x:%02x.%x flags: %02x\n",
710 PCI_BUS(e->devid),
711 PCI_SLOT(e->devid),
712 PCI_FUNC(e->devid),
713 e->flags);
714
715 devid_start = e->devid;
716 flags = e->flags;
717 ext_flags = 0;
718 alias = false;
719 break;
720 case IVHD_DEV_ALIAS:
721
722 DUMP_printk(" DEV_ALIAS\t\t\t devid: %02x:%02x.%x "
723 "flags: %02x devid_to: %02x:%02x.%x\n",
724 PCI_BUS(e->devid),
725 PCI_SLOT(e->devid),
726 PCI_FUNC(e->devid),
727 e->flags,
728 PCI_BUS(e->ext >> 8),
729 PCI_SLOT(e->ext >> 8),
730 PCI_FUNC(e->ext >> 8));
731
732 devid = e->devid;
733 devid_to = e->ext >> 8;
734 set_dev_entry_from_acpi(iommu, devid , e->flags, 0);
735 set_dev_entry_from_acpi(iommu, devid_to, e->flags, 0);
736 amd_iommu_alias_table[devid] = devid_to;
737 break;
738 case IVHD_DEV_ALIAS_RANGE:
739
740 DUMP_printk(" DEV_ALIAS_RANGE\t\t "
741 "devid: %02x:%02x.%x flags: %02x "
742 "devid_to: %02x:%02x.%x\n",
743 PCI_BUS(e->devid),
744 PCI_SLOT(e->devid),
745 PCI_FUNC(e->devid),
746 e->flags,
747 PCI_BUS(e->ext >> 8),
748 PCI_SLOT(e->ext >> 8),
749 PCI_FUNC(e->ext >> 8));
750
751 devid_start = e->devid;
752 flags = e->flags;
753 devid_to = e->ext >> 8;
754 ext_flags = 0;
755 alias = true;
756 break;
757 case IVHD_DEV_EXT_SELECT:
758
759 DUMP_printk(" DEV_EXT_SELECT\t\t devid: %02x:%02x.%x "
760 "flags: %02x ext: %08x\n",
761 PCI_BUS(e->devid),
762 PCI_SLOT(e->devid),
763 PCI_FUNC(e->devid),
764 e->flags, e->ext);
765
766 devid = e->devid;
767 set_dev_entry_from_acpi(iommu, devid, e->flags,
768 e->ext);
769 break;
770 case IVHD_DEV_EXT_SELECT_RANGE:
771
772 DUMP_printk(" DEV_EXT_SELECT_RANGE\t devid: "
773 "%02x:%02x.%x flags: %02x ext: %08x\n",
774 PCI_BUS(e->devid),
775 PCI_SLOT(e->devid),
776 PCI_FUNC(e->devid),
777 e->flags, e->ext);
778
779 devid_start = e->devid;
780 flags = e->flags;
781 ext_flags = e->ext;
782 alias = false;
783 break;
784 case IVHD_DEV_RANGE_END:
785
786 DUMP_printk(" DEV_RANGE_END\t\t devid: %02x:%02x.%x\n",
787 PCI_BUS(e->devid),
788 PCI_SLOT(e->devid),
789 PCI_FUNC(e->devid));
790
791 devid = e->devid;
792 for (dev_i = devid_start; dev_i <= devid; ++dev_i) {
793 if (alias) {
794 amd_iommu_alias_table[dev_i] = devid_to;
795 set_dev_entry_from_acpi(iommu,
796 devid_to, flags, ext_flags);
797 }
798 set_dev_entry_from_acpi(iommu, dev_i,
799 flags, ext_flags);
800 }
801 break;
802 default:
803 break;
804 }
805
806 p += ivhd_entry_length(p);
807 }
808 }
809
810 /* Initializes the device->iommu mapping for the driver */
811 static int __init init_iommu_devices(struct amd_iommu *iommu)
812 {
813 u16 i;
814
815 for (i = iommu->first_device; i <= iommu->last_device; ++i)
816 set_iommu_for_device(iommu, i);
817
818 return 0;
819 }
820
821 static void __init free_iommu_one(struct amd_iommu *iommu)
822 {
823 free_command_buffer(iommu);
824 free_event_buffer(iommu);
825 iommu_unmap_mmio_space(iommu);
826 }
827
828 static void __init free_iommu_all(void)
829 {
830 struct amd_iommu *iommu, *next;
831
832 for_each_iommu_safe(iommu, next) {
833 list_del(&iommu->list);
834 free_iommu_one(iommu);
835 kfree(iommu);
836 }
837 }
838
839 /*
840 * This function clues the initialization function for one IOMMU
841 * together and also allocates the command buffer and programs the
842 * hardware. It does NOT enable the IOMMU. This is done afterwards.
843 */
844 static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h)
845 {
846 spin_lock_init(&iommu->lock);
847
848 /* Add IOMMU to internal data structures */
849 list_add_tail(&iommu->list, &amd_iommu_list);
850 iommu->index = amd_iommus_present++;
851
852 if (unlikely(iommu->index >= MAX_IOMMUS)) {
853 WARN(1, "AMD-Vi: System has more IOMMUs than supported by this driver\n");
854 return -ENOSYS;
855 }
856
857 /* Index is fine - add IOMMU to the array */
858 amd_iommus[iommu->index] = iommu;
859
860 /*
861 * Copy data from ACPI table entry to the iommu struct
862 */
863 iommu->dev = pci_get_bus_and_slot(PCI_BUS(h->devid), h->devid & 0xff);
864 if (!iommu->dev)
865 return 1;
866
867 iommu->cap_ptr = h->cap_ptr;
868 iommu->pci_seg = h->pci_seg;
869 iommu->mmio_phys = h->mmio_phys;
870 iommu->mmio_base = iommu_map_mmio_space(h->mmio_phys);
871 if (!iommu->mmio_base)
872 return -ENOMEM;
873
874 iommu->cmd_buf = alloc_command_buffer(iommu);
875 if (!iommu->cmd_buf)
876 return -ENOMEM;
877
878 iommu->evt_buf = alloc_event_buffer(iommu);
879 if (!iommu->evt_buf)
880 return -ENOMEM;
881
882 iommu->int_enabled = false;
883
884 init_iommu_from_pci(iommu);
885 init_iommu_from_acpi(iommu, h);
886 init_iommu_devices(iommu);
887
888 return pci_enable_device(iommu->dev);
889 }
890
891 /*
892 * Iterates over all IOMMU entries in the ACPI table, allocates the
893 * IOMMU structure and initializes it with init_iommu_one()
894 */
895 static int __init init_iommu_all(struct acpi_table_header *table)
896 {
897 u8 *p = (u8 *)table, *end = (u8 *)table;
898 struct ivhd_header *h;
899 struct amd_iommu *iommu;
900 int ret;
901
902 end += table->length;
903 p += IVRS_HEADER_LENGTH;
904
905 while (p < end) {
906 h = (struct ivhd_header *)p;
907 switch (*p) {
908 case ACPI_IVHD_TYPE:
909
910 DUMP_printk("device: %02x:%02x.%01x cap: %04x "
911 "seg: %d flags: %01x info %04x\n",
912 PCI_BUS(h->devid), PCI_SLOT(h->devid),
913 PCI_FUNC(h->devid), h->cap_ptr,
914 h->pci_seg, h->flags, h->info);
915 DUMP_printk(" mmio-addr: %016llx\n",
916 h->mmio_phys);
917
918 iommu = kzalloc(sizeof(struct amd_iommu), GFP_KERNEL);
919 if (iommu == NULL)
920 return -ENOMEM;
921 ret = init_iommu_one(iommu, h);
922 if (ret)
923 return ret;
924 break;
925 default:
926 break;
927 }
928 p += h->length;
929
930 }
931 WARN_ON(p != end);
932
933 return 0;
934 }
935
936 /****************************************************************************
937 *
938 * The following functions initialize the MSI interrupts for all IOMMUs
939 * in the system. Its a bit challenging because there could be multiple
940 * IOMMUs per PCI BDF but we can call pci_enable_msi(x) only once per
941 * pci_dev.
942 *
943 ****************************************************************************/
944
945 static int iommu_setup_msi(struct amd_iommu *iommu)
946 {
947 int r;
948
949 if (pci_enable_msi(iommu->dev))
950 return 1;
951
952 r = request_irq(iommu->dev->irq, amd_iommu_int_handler,
953 IRQF_SAMPLE_RANDOM,
954 "AMD-Vi",
955 NULL);
956
957 if (r) {
958 pci_disable_msi(iommu->dev);
959 return 1;
960 }
961
962 iommu->int_enabled = true;
963 iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);
964
965 return 0;
966 }
967
968 static int iommu_init_msi(struct amd_iommu *iommu)
969 {
970 if (iommu->int_enabled)
971 return 0;
972
973 if (pci_find_capability(iommu->dev, PCI_CAP_ID_MSI))
974 return iommu_setup_msi(iommu);
975
976 return 1;
977 }
978
979 /****************************************************************************
980 *
981 * The next functions belong to the third pass of parsing the ACPI
982 * table. In this last pass the memory mapping requirements are
983 * gathered (like exclusion and unity mapping reanges).
984 *
985 ****************************************************************************/
986
987 static void __init free_unity_maps(void)
988 {
989 struct unity_map_entry *entry, *next;
990
991 list_for_each_entry_safe(entry, next, &amd_iommu_unity_map, list) {
992 list_del(&entry->list);
993 kfree(entry);
994 }
995 }
996
997 /* called when we find an exclusion range definition in ACPI */
998 static int __init init_exclusion_range(struct ivmd_header *m)
999 {
1000 int i;
1001
1002 switch (m->type) {
1003 case ACPI_IVMD_TYPE:
1004 set_device_exclusion_range(m->devid, m);
1005 break;
1006 case ACPI_IVMD_TYPE_ALL:
1007 for (i = 0; i <= amd_iommu_last_bdf; ++i)
1008 set_device_exclusion_range(i, m);
1009 break;
1010 case ACPI_IVMD_TYPE_RANGE:
1011 for (i = m->devid; i <= m->aux; ++i)
1012 set_device_exclusion_range(i, m);
1013 break;
1014 default:
1015 break;
1016 }
1017
1018 return 0;
1019 }
1020
1021 /* called for unity map ACPI definition */
1022 static int __init init_unity_map_range(struct ivmd_header *m)
1023 {
1024 struct unity_map_entry *e = 0;
1025 char *s;
1026
1027 e = kzalloc(sizeof(*e), GFP_KERNEL);
1028 if (e == NULL)
1029 return -ENOMEM;
1030
1031 switch (m->type) {
1032 default:
1033 kfree(e);
1034 return 0;
1035 case ACPI_IVMD_TYPE:
1036 s = "IVMD_TYPEi\t\t\t";
1037 e->devid_start = e->devid_end = m->devid;
1038 break;
1039 case ACPI_IVMD_TYPE_ALL:
1040 s = "IVMD_TYPE_ALL\t\t";
1041 e->devid_start = 0;
1042 e->devid_end = amd_iommu_last_bdf;
1043 break;
1044 case ACPI_IVMD_TYPE_RANGE:
1045 s = "IVMD_TYPE_RANGE\t\t";
1046 e->devid_start = m->devid;
1047 e->devid_end = m->aux;
1048 break;
1049 }
1050 e->address_start = PAGE_ALIGN(m->range_start);
1051 e->address_end = e->address_start + PAGE_ALIGN(m->range_length);
1052 e->prot = m->flags >> 1;
1053
1054 DUMP_printk("%s devid_start: %02x:%02x.%x devid_end: %02x:%02x.%x"
1055 " range_start: %016llx range_end: %016llx flags: %x\n", s,
1056 PCI_BUS(e->devid_start), PCI_SLOT(e->devid_start),
1057 PCI_FUNC(e->devid_start), PCI_BUS(e->devid_end),
1058 PCI_SLOT(e->devid_end), PCI_FUNC(e->devid_end),
1059 e->address_start, e->address_end, m->flags);
1060
1061 list_add_tail(&e->list, &amd_iommu_unity_map);
1062
1063 return 0;
1064 }
1065
1066 /* iterates over all memory definitions we find in the ACPI table */
1067 static int __init init_memory_definitions(struct acpi_table_header *table)
1068 {
1069 u8 *p = (u8 *)table, *end = (u8 *)table;
1070 struct ivmd_header *m;
1071
1072 end += table->length;
1073 p += IVRS_HEADER_LENGTH;
1074
1075 while (p < end) {
1076 m = (struct ivmd_header *)p;
1077 if (m->flags & IVMD_FLAG_EXCL_RANGE)
1078 init_exclusion_range(m);
1079 else if (m->flags & IVMD_FLAG_UNITY_MAP)
1080 init_unity_map_range(m);
1081
1082 p += m->length;
1083 }
1084
1085 return 0;
1086 }
1087
1088 /*
1089 * Init the device table to not allow DMA access for devices and
1090 * suppress all page faults
1091 */
1092 static void init_device_table(void)
1093 {
1094 u16 devid;
1095
1096 for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) {
1097 set_dev_entry_bit(devid, DEV_ENTRY_VALID);
1098 set_dev_entry_bit(devid, DEV_ENTRY_TRANSLATION);
1099 }
1100 }
1101
1102 /*
1103 * This function finally enables all IOMMUs found in the system after
1104 * they have been initialized
1105 */
1106 static void enable_iommus(void)
1107 {
1108 struct amd_iommu *iommu;
1109
1110 for_each_iommu(iommu) {
1111 iommu_disable(iommu);
1112 iommu_set_device_table(iommu);
1113 iommu_enable_command_buffer(iommu);
1114 iommu_enable_event_buffer(iommu);
1115 iommu_set_exclusion_range(iommu);
1116 iommu_init_msi(iommu);
1117 iommu_enable(iommu);
1118 }
1119 }
1120
1121 static void disable_iommus(void)
1122 {
1123 struct amd_iommu *iommu;
1124
1125 for_each_iommu(iommu)
1126 iommu_disable(iommu);
1127 }
1128
1129 /*
1130 * Suspend/Resume support
1131 * disable suspend until real resume implemented
1132 */
1133
1134 static int amd_iommu_resume(struct sys_device *dev)
1135 {
1136 /* re-load the hardware */
1137 enable_iommus();
1138
1139 /*
1140 * we have to flush after the IOMMUs are enabled because a
1141 * disabled IOMMU will never execute the commands we send
1142 */
1143 amd_iommu_flush_all_devices();
1144 amd_iommu_flush_all_domains();
1145
1146 return 0;
1147 }
1148
1149 static int amd_iommu_suspend(struct sys_device *dev, pm_message_t state)
1150 {
1151 /* disable IOMMUs to go out of the way for BIOS */
1152 disable_iommus();
1153
1154 return 0;
1155 }
1156
1157 static struct sysdev_class amd_iommu_sysdev_class = {
1158 .name = "amd_iommu",
1159 .suspend = amd_iommu_suspend,
1160 .resume = amd_iommu_resume,
1161 };
1162
1163 static struct sys_device device_amd_iommu = {
1164 .id = 0,
1165 .cls = &amd_iommu_sysdev_class,
1166 };
1167
1168 /*
1169 * This is the core init function for AMD IOMMU hardware in the system.
1170 * This function is called from the generic x86 DMA layer initialization
1171 * code.
1172 *
1173 * This function basically parses the ACPI table for AMD IOMMU (IVRS)
1174 * three times:
1175 *
1176 * 1 pass) Find the highest PCI device id the driver has to handle.
1177 * Upon this information the size of the data structures is
1178 * determined that needs to be allocated.
1179 *
1180 * 2 pass) Initialize the data structures just allocated with the
1181 * information in the ACPI table about available AMD IOMMUs
1182 * in the system. It also maps the PCI devices in the
1183 * system to specific IOMMUs
1184 *
1185 * 3 pass) After the basic data structures are allocated and
1186 * initialized we update them with information about memory
1187 * remapping requirements parsed out of the ACPI table in
1188 * this last pass.
1189 *
1190 * After that the hardware is initialized and ready to go. In the last
1191 * step we do some Linux specific things like registering the driver in
1192 * the dma_ops interface and initializing the suspend/resume support
1193 * functions. Finally it prints some information about AMD IOMMUs and
1194 * the driver state and enables the hardware.
1195 */
1196 static int __init amd_iommu_init(void)
1197 {
1198 int i, ret = 0;
1199
1200 /*
1201 * First parse ACPI tables to find the largest Bus/Dev/Func
1202 * we need to handle. Upon this information the shared data
1203 * structures for the IOMMUs in the system will be allocated
1204 */
1205 if (acpi_table_parse("IVRS", find_last_devid_acpi) != 0)
1206 return -ENODEV;
1207
1208 dev_table_size = tbl_size(DEV_TABLE_ENTRY_SIZE);
1209 alias_table_size = tbl_size(ALIAS_TABLE_ENTRY_SIZE);
1210 rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE);
1211
1212 ret = -ENOMEM;
1213
1214 /* Device table - directly used by all IOMMUs */
1215 amd_iommu_dev_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
1216 get_order(dev_table_size));
1217 if (amd_iommu_dev_table == NULL)
1218 goto out;
1219
1220 /*
1221 * Alias table - map PCI Bus/Dev/Func to Bus/Dev/Func the
1222 * IOMMU see for that device
1223 */
1224 amd_iommu_alias_table = (void *)__get_free_pages(GFP_KERNEL,
1225 get_order(alias_table_size));
1226 if (amd_iommu_alias_table == NULL)
1227 goto free;
1228
1229 /* IOMMU rlookup table - find the IOMMU for a specific device */
1230 amd_iommu_rlookup_table = (void *)__get_free_pages(
1231 GFP_KERNEL | __GFP_ZERO,
1232 get_order(rlookup_table_size));
1233 if (amd_iommu_rlookup_table == NULL)
1234 goto free;
1235
1236 /*
1237 * Protection Domain table - maps devices to protection domains
1238 * This table has the same size as the rlookup_table
1239 */
1240 amd_iommu_pd_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
1241 get_order(rlookup_table_size));
1242 if (amd_iommu_pd_table == NULL)
1243 goto free;
1244
1245 amd_iommu_pd_alloc_bitmap = (void *)__get_free_pages(
1246 GFP_KERNEL | __GFP_ZERO,
1247 get_order(MAX_DOMAIN_ID/8));
1248 if (amd_iommu_pd_alloc_bitmap == NULL)
1249 goto free;
1250
1251 /* init the device table */
1252 init_device_table();
1253
1254 /*
1255 * let all alias entries point to itself
1256 */
1257 for (i = 0; i <= amd_iommu_last_bdf; ++i)
1258 amd_iommu_alias_table[i] = i;
1259
1260 /*
1261 * never allocate domain 0 because its used as the non-allocated and
1262 * error value placeholder
1263 */
1264 amd_iommu_pd_alloc_bitmap[0] = 1;
1265
1266 /*
1267 * now the data structures are allocated and basically initialized
1268 * start the real acpi table scan
1269 */
1270 ret = -ENODEV;
1271 if (acpi_table_parse("IVRS", init_iommu_all) != 0)
1272 goto free;
1273
1274 if (acpi_table_parse("IVRS", init_memory_definitions) != 0)
1275 goto free;
1276
1277 ret = sysdev_class_register(&amd_iommu_sysdev_class);
1278 if (ret)
1279 goto free;
1280
1281 ret = sysdev_register(&device_amd_iommu);
1282 if (ret)
1283 goto free;
1284
1285 if (iommu_pass_through)
1286 ret = amd_iommu_init_passthrough();
1287 else
1288 ret = amd_iommu_init_dma_ops();
1289 if (ret)
1290 goto free;
1291
1292 enable_iommus();
1293
1294 if (iommu_pass_through)
1295 goto out;
1296
1297 printk(KERN_INFO "AMD-Vi: device isolation ");
1298 if (amd_iommu_isolate)
1299 printk("enabled\n");
1300 else
1301 printk("disabled\n");
1302
1303 if (amd_iommu_unmap_flush)
1304 printk(KERN_INFO "AMD-Vi: IO/TLB flush on unmap enabled\n");
1305 else
1306 printk(KERN_INFO "AMD-Vi: Lazy IO/TLB flushing enabled\n");
1307
1308 x86_platform.iommu_shutdown = disable_iommus;
1309 out:
1310 return ret;
1311
1312 free:
1313 free_pages((unsigned long)amd_iommu_pd_alloc_bitmap,
1314 get_order(MAX_DOMAIN_ID/8));
1315
1316 free_pages((unsigned long)amd_iommu_pd_table,
1317 get_order(rlookup_table_size));
1318
1319 free_pages((unsigned long)amd_iommu_rlookup_table,
1320 get_order(rlookup_table_size));
1321
1322 free_pages((unsigned long)amd_iommu_alias_table,
1323 get_order(alias_table_size));
1324
1325 free_pages((unsigned long)amd_iommu_dev_table,
1326 get_order(dev_table_size));
1327
1328 free_iommu_all();
1329
1330 free_unity_maps();
1331
1332 goto out;
1333 }
1334
1335 /****************************************************************************
1336 *
1337 * Early detect code. This code runs at IOMMU detection time in the DMA
1338 * layer. It just looks if there is an IVRS ACPI table to detect AMD
1339 * IOMMUs
1340 *
1341 ****************************************************************************/
1342 static int __init early_amd_iommu_detect(struct acpi_table_header *table)
1343 {
1344 return 0;
1345 }
1346
1347 void __init amd_iommu_detect(void)
1348 {
1349 if (no_iommu || (iommu_detected && !gart_iommu_aperture))
1350 return;
1351
1352 if (acpi_table_parse("IVRS", early_amd_iommu_detect) == 0) {
1353 iommu_detected = 1;
1354 amd_iommu_detected = 1;
1355 x86_init.iommu.iommu_init = amd_iommu_init;
1356 }
1357 }
1358
1359 /****************************************************************************
1360 *
1361 * Parsing functions for the AMD IOMMU specific kernel command line
1362 * options.
1363 *
1364 ****************************************************************************/
1365
1366 static int __init parse_amd_iommu_dump(char *str)
1367 {
1368 amd_iommu_dump = true;
1369
1370 return 1;
1371 }
1372
1373 static int __init parse_amd_iommu_options(char *str)
1374 {
1375 for (; *str; ++str) {
1376 if (strncmp(str, "isolate", 7) == 0)
1377 amd_iommu_isolate = true;
1378 if (strncmp(str, "share", 5) == 0)
1379 amd_iommu_isolate = false;
1380 if (strncmp(str, "fullflush", 9) == 0)
1381 amd_iommu_unmap_flush = true;
1382 }
1383
1384 return 1;
1385 }
1386
1387 __setup("amd_iommu_dump", parse_amd_iommu_dump);
1388 __setup("amd_iommu=", parse_amd_iommu_options);
Linux with added FPC-III support