arch/x86/kernel/pci-dma.c

   1 #include <linux/dma-mapping.h>
   2 #include <linux/dma-debug.h>
   3 #include <linux/dmar.h>
   4 #include <linux/export.h>
   5 #include <linux/bootmem.h>
   6 #include <linux/gfp.h>
   7 #include <linux/pci.h>
   8 #include <linux/kmemleak.h>
   9
  10 #include <asm/proto.h>
  11 #include <asm/dma.h>
  12 #include <asm/iommu.h>
  13 #include <asm/gart.h>
  14 #include <asm/calgary.h>
  15 #include <asm/x86_init.h>
  16 #include <asm/iommu_table.h>
  17
  18 static int forbid_dac __read_mostly;
  19
  20 struct dma_map_ops *dma_ops = &nommu_dma_ops;
  21 EXPORT_SYMBOL(dma_ops);
  22
  23 static int iommu_sac_force __read_mostly;
  24
  25 #ifdef CONFIG_IOMMU_DEBUG
  26 int panic_on_overflow __read_mostly = 1;
  27 int force_iommu __read_mostly = 1;
  28 #else
  29 int panic_on_overflow __read_mostly = 0;
  30 int force_iommu __read_mostly = 0;
  31 #endif
  32
  33 int iommu_merge __read_mostly = 0;
  34
  35 int no_iommu __read_mostly;
  36 /* Set this to 1 if there is a HW IOMMU in the system */
  37 int iommu_detected __read_mostly = 0;
  38
  39 /*
  40  * This variable becomes 1 if iommu=pt is passed on the kernel command line.
  41  * If this variable is 1, IOMMU implementations do no DMA translation for
  42  * devices and allow every device to access to whole physical memory. This is
  43  * useful if a user wants to use an IOMMU only for KVM device assignment to
  44  * guests and not for driver dma translation.
  45  */
  46 int iommu_pass_through __read_mostly;
  47
  48 /*
  49  * Group multi-function PCI devices into a single device-group for the
  50  * iommu_device_group interface.  This tells the iommu driver to pretend
  51  * it cannot distinguish between functions of a device, exposing only one
  52  * group for the device.  Useful for disallowing use of individual PCI
  53  * functions from userspace drivers.
  54  */
  55 int iommu_group_mf __read_mostly;
  56
  57 extern struct iommu_table_entry __iommu_table[], __iommu_table_end[];
  58
  59 /* Dummy device used for NULL arguments (normally ISA). */
  60 struct device x86_dma_fallback_dev = {
  61         .init_name = "fallback device",
  62         .coherent_dma_mask = ISA_DMA_BIT_MASK,
  63         .dma_mask = &x86_dma_fallback_dev.coherent_dma_mask,
  64 };
  65 EXPORT_SYMBOL(x86_dma_fallback_dev);
  66
  67 /* Number of entries preallocated for DMA-API debugging */
  68 #define PREALLOC_DMA_DEBUG_ENTRIES       32768
  69
  70 int dma_set_mask(struct device *dev, u64 mask)
  71 {
  72         if (!dev->dma_mask || !dma_supported(dev, mask))
  73                 return -EIO;
  74
  75         *dev->dma_mask = mask;
  76
  77         return 0;
  78 }
  79 EXPORT_SYMBOL(dma_set_mask);
  80
  81 void __init pci_iommu_alloc(void)
  82 {
  83         struct iommu_table_entry *p;
  84
  85         sort_iommu_table(__iommu_table, __iommu_table_end);
  86         check_iommu_entries(__iommu_table, __iommu_table_end);
  87
  88         for (p = __iommu_table; p < __iommu_table_end; p++) {
  89                 if (p && p->detect && p->detect() > 0) {
  90                         p->flags |= IOMMU_DETECTED;
  91                         if (p->early_init)
  92                                 p->early_init();
  93                         if (p->flags & IOMMU_FINISH_IF_DETECTED)
  94                                 break;
  95                 }
  96         }
  97 }
  98 void *dma_generic_alloc_coherent(struct device *dev, size_t size,
  99                                  dma_addr_t *dma_addr, gfp_t flag)
 100 {
 101         unsigned long dma_mask;
 102         struct page *page;
 103         dma_addr_t addr;
 104
 105         dma_mask = dma_alloc_coherent_mask(dev, flag);
 106
 107         flag |= __GFP_ZERO;
 108 again:
 109         page = alloc_pages_node(dev_to_node(dev), flag, get_order(size));
 110         if (!page)
 111                 return NULL;
 112
 113         addr = page_to_phys(page);
 114         if (addr + size > dma_mask) {
 115                 __free_pages(page, get_order(size));
 116
 117                 if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) {
 118                         flag = (flag & ~GFP_DMA32) | GFP_DMA;
 119                         goto again;
 120                 }
 121
 122                 return NULL;
 123         }
 124
 125         *dma_addr = addr;
 126         return page_address(page);
 127 }
 128
 129 /*
 130  * See <Documentation/x86/x86_64/boot-options.txt> for the iommu kernel
 131  * parameter documentation.
 132  */
 133 static __init int iommu_setup(char *p)
 134 {
 135         iommu_merge = 1;
 136
 137         if (!p)
 138                 return -EINVAL;
 139
 140         while (*p) {
 141                 if (!strncmp(p, "off", 3))
 142                         no_iommu = 1;
 143                 /* gart_parse_options has more force support */
 144                 if (!strncmp(p, "force", 5))
 145                         force_iommu = 1;
 146                 if (!strncmp(p, "noforce", 7)) {
 147                         iommu_merge = 0;
 148                         force_iommu = 0;
 149                 }
 150
 151                 if (!strncmp(p, "biomerge", 8)) {
 152                         iommu_merge = 1;
 153                         force_iommu = 1;
 154                 }
 155                 if (!strncmp(p, "panic", 5))
 156                         panic_on_overflow = 1;
 157                 if (!strncmp(p, "nopanic", 7))
 158                         panic_on_overflow = 0;
 159                 if (!strncmp(p, "merge", 5)) {
 160                         iommu_merge = 1;
 161                         force_iommu = 1;
 162                 }
 163                 if (!strncmp(p, "nomerge", 7))
 164                         iommu_merge = 0;
 165                 if (!strncmp(p, "forcesac", 8))
 166                         iommu_sac_force = 1;
 167                 if (!strncmp(p, "allowdac", 8))
 168                         forbid_dac = 0;
 169                 if (!strncmp(p, "nodac", 5))
 170                         forbid_dac = 1;
 171                 if (!strncmp(p, "usedac", 6)) {
 172                         forbid_dac = -1;
 173                         return 1;
 174                 }
 175 #ifdef CONFIG_SWIOTLB
 176                 if (!strncmp(p, "soft", 4))
 177                         swiotlb = 1;
 178 #endif
 179                 if (!strncmp(p, "pt", 2))
 180                         iommu_pass_through = 1;
 181                 if (!strncmp(p, "group_mf", 8))
 182                         iommu_group_mf = 1;
 183
 184                 gart_parse_options(p);
 185
 186 #ifdef CONFIG_CALGARY_IOMMU
 187                 if (!strncmp(p, "calgary", 7))
 188                         use_calgary = 1;
 189 #endif /* CONFIG_CALGARY_IOMMU */
 190
 191                 p += strcspn(p, ",");
 192                 if (*p == ',')
 193                         ++p;
 194         }
 195         return 0;
 196 }
 197 early_param("iommu", iommu_setup);
 198
 199 int dma_supported(struct device *dev, u64 mask)
 200 {
 201         struct dma_map_ops *ops = get_dma_ops(dev);
 202
 203 #ifdef CONFIG_PCI
 204         if (mask > 0xffffffff && forbid_dac > 0) {
 205                 dev_info(dev, "PCI: Disallowing DAC for device\n");
 206                 return 0;
 207         }
 208 #endif
 209
 210         if (ops->dma_supported)
 211                 return ops->dma_supported(dev, mask);
 212
 213         /* Copied from i386. Doesn't make much sense, because it will
 214            only work for pci_alloc_coherent.
 215            The caller just has to use GFP_DMA in this case. */
 216         if (mask < DMA_BIT_MASK(24))
 217                 return 0;
 218
 219         /* Tell the device to use SAC when IOMMU force is on.  This
 220            allows the driver to use cheaper accesses in some cases.
 221
 222            Problem with this is that if we overflow the IOMMU area and
 223            return DAC as fallback address the device may not handle it
 224            correctly.
 225
 226            As a special case some controllers have a 39bit address
 227            mode that is as efficient as 32bit (aic79xx). Don't force
 228            SAC for these.  Assume all masks <= 40 bits are of this
 229            type. Normally this doesn't make any difference, but gives
 230            more gentle handling of IOMMU overflow. */
 231         if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) {
 232                 dev_info(dev, "Force SAC with mask %Lx\n", mask);
 233                 return 0;
 234         }
 235
 236         return 1;
 237 }
 238 EXPORT_SYMBOL(dma_supported);
 239
 240 static int __init pci_iommu_init(void)
 241 {
 242         struct iommu_table_entry *p;
 243         dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
 244
 245 #ifdef CONFIG_PCI
 246         dma_debug_add_bus(&pci_bus_type);
 247 #endif
 248         x86_init.iommu.iommu_init();
 249
 250         for (p = __iommu_table; p < __iommu_table_end; p++) {
 251                 if (p && (p->flags & IOMMU_DETECTED) && p->late_init)
 252                         p->late_init();
 253         }
 254
 255         return 0;
 256 }
 257 /* Must execute after PCI subsystem */
 258 rootfs_initcall(pci_iommu_init);
 259
 260 #ifdef CONFIG_PCI
 261 /* Many VIA bridges seem to corrupt data for DAC. Disable it here */
 262
 263 static __devinit void via_no_dac(struct pci_dev *dev)
 264 {
 265         if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
 266                 dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n");
 267                 forbid_dac = 1;
 268         }
 269 }
 270 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
 271 #endif