arch/x86/kernel/pci-dma.c

   1 #include <linux/dma-mapping.h>
   2 #include <linux/dma-debug.h>
   3 #include <linux/dmar.h>
   4 #include <linux/bootmem.h>
   5 #include <linux/pci.h>
   6 #include <linux/kmemleak.h>
   7
   8 #include <asm/proto.h>
   9 #include <asm/dma.h>
  10 #include <asm/iommu.h>
  11 #include <asm/gart.h>
  12 #include <asm/calgary.h>
  13 #include <asm/amd_iommu.h>
  14
  15 static int forbid_dac __read_mostly;
  16
  17 struct dma_map_ops *dma_ops;
  18 EXPORT_SYMBOL(dma_ops);
  19
  20 static int iommu_sac_force __read_mostly;
  21
  22 #ifdef CONFIG_IOMMU_DEBUG
  23 int panic_on_overflow __read_mostly = 1;
  24 int force_iommu __read_mostly = 1;
  25 #else
  26 int panic_on_overflow __read_mostly = 0;
  27 int force_iommu __read_mostly = 0;
  28 #endif
  29
  30 int iommu_merge __read_mostly = 0;
  31
  32 int no_iommu __read_mostly;
  33 /* Set this to 1 if there is a HW IOMMU in the system */
  34 int iommu_detected __read_mostly = 0;
  35
  36 /*
  37  * This variable becomes 1 if iommu=pt is passed on the kernel command line.
  38  * If this variable is 1, IOMMU implementations do no DMA ranslation for
  39  * devices and allow every device to access to whole physical memory. This is
  40  * useful if a user want to use an IOMMU only for KVM device assignment to
  41  * guests and not for driver dma translation.
  42  */
  43 int iommu_pass_through __read_mostly;
  44
  45 dma_addr_t bad_dma_address __read_mostly = 0;
  46 EXPORT_SYMBOL(bad_dma_address);
  47
  48 /* Dummy device used for NULL arguments (normally ISA). Better would
  49    be probably a smaller DMA mask, but this is bug-to-bug compatible
  50    to older i386. */
  51 struct device x86_dma_fallback_dev = {
  52         .init_name = "fallback device",
  53         .coherent_dma_mask = DMA_BIT_MASK(32),
  54         .dma_mask = &x86_dma_fallback_dev.coherent_dma_mask,
  55 };
  56 EXPORT_SYMBOL(x86_dma_fallback_dev);
  57
  58 /* Number of entries preallocated for DMA-API debugging */
  59 #define PREALLOC_DMA_DEBUG_ENTRIES       32768
  60
  61 int dma_set_mask(struct device *dev, u64 mask)
  62 {
  63         if (!dev->dma_mask || !dma_supported(dev, mask))
  64                 return -EIO;
  65
  66         *dev->dma_mask = mask;
  67
  68         return 0;
  69 }
  70 EXPORT_SYMBOL(dma_set_mask);
  71
  72 #ifdef CONFIG_X86_64
  73 static __initdata void *dma32_bootmem_ptr;
  74 static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);
  75
  76 static int __init parse_dma32_size_opt(char *p)
  77 {
  78         if (!p)
  79                 return -EINVAL;
  80         dma32_bootmem_size = memparse(p, &p);
  81         return 0;
  82 }
  83 early_param("dma32_size", parse_dma32_size_opt);
  84
  85 void __init dma32_reserve_bootmem(void)
  86 {
  87         unsigned long size, align;
  88         if (max_pfn <= MAX_DMA32_PFN)
  89                 return;
  90
  91         /*
  92          * check aperture_64.c allocate_aperture() for reason about
  93          * using 512M as goal
  94          */
  95         align = 64ULL<<20;
  96         size = roundup(dma32_bootmem_size, align);
  97         dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
  98                                  512ULL<<20);
  99         /*
 100          * Kmemleak should not scan this block as it may not be mapped via the
 101          * kernel direct mapping.
 102          */
 103         kmemleak_ignore(dma32_bootmem_ptr);
 104         if (dma32_bootmem_ptr)
 105                 dma32_bootmem_size = size;
 106         else
 107                 dma32_bootmem_size = 0;
 108 }
 109 static void __init dma32_free_bootmem(void)
 110 {
 111
 112         if (max_pfn <= MAX_DMA32_PFN)
 113                 return;
 114
 115         if (!dma32_bootmem_ptr)
 116                 return;
 117
 118         free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size);
 119
 120         dma32_bootmem_ptr = NULL;
 121         dma32_bootmem_size = 0;
 122 }
 123 #endif
 124
 125 void __init pci_iommu_alloc(void)
 126 {
 127 #ifdef CONFIG_X86_64
 128         /* free the range so iommu could get some range less than 4G */
 129         dma32_free_bootmem();
 130 #endif
 131
 132         /*
 133          * The order of these functions is important for
 134          * fall-back/fail-over reasons
 135          */
 136         gart_iommu_hole_init();
 137
 138         detect_calgary();
 139
 140         detect_intel_iommu();
 141
 142         amd_iommu_detect();
 143
 144         pci_swiotlb_init();
 145 }
 146
 147 void *dma_generic_alloc_coherent(struct device *dev, size_t size,
 148                                  dma_addr_t *dma_addr, gfp_t flag)
 149 {
 150         unsigned long dma_mask;
 151         struct page *page;
 152         dma_addr_t addr;
 153
 154         dma_mask = dma_alloc_coherent_mask(dev, flag);
 155
 156         flag |= __GFP_ZERO;
 157 again:
 158         page = alloc_pages_node(dev_to_node(dev), flag, get_order(size));
 159         if (!page)
 160                 return NULL;
 161
 162         addr = page_to_phys(page);
 163         if (addr + size > dma_mask) {
 164                 __free_pages(page, get_order(size));
 165
 166                 if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) {
 167                         flag = (flag & ~GFP_DMA32) | GFP_DMA;
 168                         goto again;
 169                 }
 170
 171                 return NULL;
 172         }
 173
 174         *dma_addr = addr;
 175         return page_address(page);
 176 }
 177
 178 /*
 179  * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
 180  * documentation.
 181  */
 182 static __init int iommu_setup(char *p)
 183 {
 184         iommu_merge = 1;
 185
 186         if (!p)
 187                 return -EINVAL;
 188
 189         while (*p) {
 190                 if (!strncmp(p, "off", 3))
 191                         no_iommu = 1;
 192                 /* gart_parse_options has more force support */
 193                 if (!strncmp(p, "force", 5))
 194                         force_iommu = 1;
 195                 if (!strncmp(p, "noforce", 7)) {
 196                         iommu_merge = 0;
 197                         force_iommu = 0;
 198                 }
 199
 200                 if (!strncmp(p, "biomerge", 8)) {
 201                         iommu_merge = 1;
 202                         force_iommu = 1;
 203                 }
 204                 if (!strncmp(p, "panic", 5))
 205                         panic_on_overflow = 1;
 206                 if (!strncmp(p, "nopanic", 7))
 207                         panic_on_overflow = 0;
 208                 if (!strncmp(p, "merge", 5)) {
 209                         iommu_merge = 1;
 210                         force_iommu = 1;
 211                 }
 212                 if (!strncmp(p, "nomerge", 7))
 213                         iommu_merge = 0;
 214                 if (!strncmp(p, "forcesac", 8))
 215                         iommu_sac_force = 1;
 216                 if (!strncmp(p, "allowdac", 8))
 217                         forbid_dac = 0;
 218                 if (!strncmp(p, "nodac", 5))
 219                         forbid_dac = -1;
 220                 if (!strncmp(p, "usedac", 6)) {
 221                         forbid_dac = -1;
 222                         return 1;
 223                 }
 224 #ifdef CONFIG_SWIOTLB
 225                 if (!strncmp(p, "soft", 4))
 226                         swiotlb = 1;
 227 #endif
 228                 if (!strncmp(p, "pt", 2)) {
 229                         iommu_pass_through = 1;
 230                         return 1;
 231                 }
 232
 233                 gart_parse_options(p);
 234
 235 #ifdef CONFIG_CALGARY_IOMMU
 236                 if (!strncmp(p, "calgary", 7))
 237                         use_calgary = 1;
 238 #endif /* CONFIG_CALGARY_IOMMU */
 239
 240                 p += strcspn(p, ",");
 241                 if (*p == ',')
 242                         ++p;
 243         }
 244         return 0;
 245 }
 246 early_param("iommu", iommu_setup);
 247
 248 int dma_supported(struct device *dev, u64 mask)
 249 {
 250         struct dma_map_ops *ops = get_dma_ops(dev);
 251
 252 #ifdef CONFIG_PCI
 253         if (mask > 0xffffffff && forbid_dac > 0) {
 254                 dev_info(dev, "PCI: Disallowing DAC for device\n");
 255                 return 0;
 256         }
 257 #endif
 258
 259         if (ops->dma_supported)
 260                 return ops->dma_supported(dev, mask);
 261
 262         /* Copied from i386. Doesn't make much sense, because it will
 263            only work for pci_alloc_coherent.
 264            The caller just has to use GFP_DMA in this case. */
 265         if (mask < DMA_BIT_MASK(24))
 266                 return 0;
 267
 268         /* Tell the device to use SAC when IOMMU force is on.  This
 269            allows the driver to use cheaper accesses in some cases.
 270
 271            Problem with this is that if we overflow the IOMMU area and
 272            return DAC as fallback address the device may not handle it
 273            correctly.
 274
 275            As a special case some controllers have a 39bit address
 276            mode that is as efficient as 32bit (aic79xx). Don't force
 277            SAC for these.  Assume all masks <= 40 bits are of this
 278            type. Normally this doesn't make any difference, but gives
 279            more gentle handling of IOMMU overflow. */
 280         if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) {
 281                 dev_info(dev, "Force SAC with mask %Lx\n", mask);
 282                 return 0;
 283         }
 284
 285         return 1;
 286 }
 287 EXPORT_SYMBOL(dma_supported);
 288
 289 static int __init pci_iommu_init(void)
 290 {
 291         dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
 292
 293 #ifdef CONFIG_PCI
 294         dma_debug_add_bus(&pci_bus_type);
 295 #endif
 296
 297         calgary_iommu_init();
 298
 299         intel_iommu_init();
 300
 301         amd_iommu_init();
 302
 303         gart_iommu_init();
 304
 305         no_iommu_init();
 306         return 0;
 307 }
 308
 309 void pci_iommu_shutdown(void)
 310 {
 311         gart_iommu_shutdown();
 312
 313         amd_iommu_shutdown();
 314 }
 315 /* Must execute after PCI subsystem */
 316 fs_initcall(pci_iommu_init);
 317
 318 #ifdef CONFIG_PCI
 319 /* Many VIA bridges seem to corrupt data for DAC. Disable it here */
 320
 321 static __devinit void via_no_dac(struct pci_dev *dev)
 322 {
 323         if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
 324                 dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n");
 325                 forbid_dac = 1;
 326         }
 327 }
 328 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
 329 #endif