arch/x86/kernel/pci-dma_64.c

   1 /*
   2  * Dynamic DMA mapping support.
   3  */
   4
   5 #include <linux/types.h>
   6 #include <linux/mm.h>
   7 #include <linux/string.h>
   8 #include <linux/pci.h>
   9 #include <linux/module.h>
  10 #include <linux/dmar.h>
  11 #include <linux/bootmem.h>
  12 #include <asm/proto.h>
  13 #include <asm/io.h>
  14 #include <asm/gart.h>
  15 #include <asm/calgary.h>
  16
  17 int iommu_merge __read_mostly = 0;
  18
  19 dma_addr_t bad_dma_address __read_mostly;
  20 EXPORT_SYMBOL(bad_dma_address);
  21
  22 /* This tells the BIO block layer to assume merging. Default to off
  23    because we cannot guarantee merging later. */
  24 int iommu_bio_merge __read_mostly = 0;
  25 EXPORT_SYMBOL(iommu_bio_merge);
  26
  27 static int iommu_sac_force __read_mostly = 0;
  28
  29 int no_iommu __read_mostly;
  30 #ifdef CONFIG_IOMMU_DEBUG
  31 int panic_on_overflow __read_mostly = 1;
  32 int force_iommu __read_mostly = 1;
  33 #else
  34 int panic_on_overflow __read_mostly = 0;
  35 int force_iommu __read_mostly= 0;
  36 #endif
  37
  38 /* Set this to 1 if there is a HW IOMMU in the system */
  39 int iommu_detected __read_mostly = 0;
  40
  41 /* Dummy device used for NULL arguments (normally ISA). Better would
  42    be probably a smaller DMA mask, but this is bug-to-bug compatible
  43    to i386. */
  44 struct device fallback_dev = {
  45         .bus_id = "fallback device",
  46         .coherent_dma_mask = DMA_32BIT_MASK,
  47         .dma_mask = &fallback_dev.coherent_dma_mask,
  48 };
  49
  50 /* Allocate DMA memory on node near device */
  51 noinline static void *
  52 dma_alloc_pages(struct device *dev, gfp_t gfp, unsigned order)
  53 {
  54         struct page *page;
  55         int node;
  56
  57         node = dev_to_node(dev);
  58
  59         page = alloc_pages_node(node, gfp, order);
  60         return page ? page_address(page) : NULL;
  61 }
  62
  63 /*
  64  * Allocate memory for a coherent mapping.
  65  */
  66 void *
  67 dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
  68                    gfp_t gfp)
  69 {
  70         void *memory;
  71         unsigned long dma_mask = 0;
  72         u64 bus;
  73
  74         if (!dev)
  75                 dev = &fallback_dev;
  76         dma_mask = dev->coherent_dma_mask;
  77         if (dma_mask == 0)
  78                 dma_mask = DMA_32BIT_MASK;
  79
  80         /* Device not DMA able */
  81         if (dev->dma_mask == NULL)
  82                 return NULL;
  83
  84         /* Don't invoke OOM killer */
  85         gfp |= __GFP_NORETRY;
  86
  87         /* Kludge to make it bug-to-bug compatible with i386. i386
  88            uses the normal dma_mask for alloc_coherent. */
  89         dma_mask &= *dev->dma_mask;
  90
  91         /* Why <=? Even when the mask is smaller than 4GB it is often
  92            larger than 16MB and in this case we have a chance of
  93            finding fitting memory in the next higher zone first. If
  94            not retry with true GFP_DMA. -AK */
  95         if (dma_mask <= DMA_32BIT_MASK)
  96                 gfp |= GFP_DMA32;
  97
  98  again:
  99         memory = dma_alloc_pages(dev, gfp, get_order(size));
 100         if (memory == NULL)
 101                 return NULL;
 102
 103         {
 104                 int high, mmu;
 105                 bus = virt_to_bus(memory);
 106                 high = (bus + size) >= dma_mask;
 107                 mmu = high;
 108                 if (force_iommu && !(gfp & GFP_DMA))
 109                         mmu = 1;
 110                 else if (high) {
 111                         free_pages((unsigned long)memory,
 112                                    get_order(size));
 113
 114                         /* Don't use the 16MB ZONE_DMA unless absolutely
 115                            needed. It's better to use remapping first. */
 116                         if (dma_mask < DMA_32BIT_MASK && !(gfp & GFP_DMA)) {
 117                                 gfp = (gfp & ~GFP_DMA32) | GFP_DMA;
 118                                 goto again;
 119                         }
 120
 121                         /* Let low level make its own zone decisions */
 122                         gfp &= ~(GFP_DMA32|GFP_DMA);
 123
 124                         if (dma_ops->alloc_coherent)
 125                                 return dma_ops->alloc_coherent(dev, size,
 126                                                            dma_handle, gfp);
 127                         return NULL;
 128                 }
 129
 130                 memset(memory, 0, size);
 131                 if (!mmu) {
 132                         *dma_handle = virt_to_bus(memory);
 133                         return memory;
 134                 }
 135         }
 136
 137         if (dma_ops->alloc_coherent) {
 138                 free_pages((unsigned long)memory, get_order(size));
 139                 gfp &= ~(GFP_DMA|GFP_DMA32);
 140                 return dma_ops->alloc_coherent(dev, size, dma_handle, gfp);
 141         }
 142
 143         if (dma_ops->map_simple) {
 144                 *dma_handle = dma_ops->map_simple(dev, virt_to_phys(memory),
 145                                               size,
 146                                               PCI_DMA_BIDIRECTIONAL);
 147                 if (*dma_handle != bad_dma_address)
 148                         return memory;
 149         }
 150
 151         if (panic_on_overflow)
 152                 panic("dma_alloc_coherent: IOMMU overflow by %lu bytes\n",size);
 153         free_pages((unsigned long)memory, get_order(size));
 154         return NULL;
 155 }
 156 EXPORT_SYMBOL(dma_alloc_coherent);
 157
 158 /*
 159  * Unmap coherent memory.
 160  * The caller must ensure that the device has finished accessing the mapping.
 161  */
 162 void dma_free_coherent(struct device *dev, size_t size,
 163                          void *vaddr, dma_addr_t bus)
 164 {
 165         WARN_ON(irqs_disabled());       /* for portability */
 166         if (dma_ops->unmap_single)
 167                 dma_ops->unmap_single(dev, bus, size, 0);
 168         free_pages((unsigned long)vaddr, get_order(size));
 169 }
 170 EXPORT_SYMBOL(dma_free_coherent);
 171
 172 static int forbid_dac __read_mostly;
 173
 174 int dma_supported(struct device *dev, u64 mask)
 175 {
 176 #ifdef CONFIG_PCI
 177         if (mask > 0xffffffff && forbid_dac > 0) {
 178
 179
 180
 181                 printk(KERN_INFO "PCI: Disallowing DAC for device %s\n", dev->bus_id);
 182                 return 0;
 183         }
 184 #endif
 185
 186         if (dma_ops->dma_supported)
 187                 return dma_ops->dma_supported(dev, mask);
 188
 189         /* Copied from i386. Doesn't make much sense, because it will
 190            only work for pci_alloc_coherent.
 191            The caller just has to use GFP_DMA in this case. */
 192         if (mask < DMA_24BIT_MASK)
 193                 return 0;
 194
 195         /* Tell the device to use SAC when IOMMU force is on.  This
 196            allows the driver to use cheaper accesses in some cases.
 197
 198            Problem with this is that if we overflow the IOMMU area and
 199            return DAC as fallback address the device may not handle it
 200            correctly.
 201
 202            As a special case some controllers have a 39bit address
 203            mode that is as efficient as 32bit (aic79xx). Don't force
 204            SAC for these.  Assume all masks <= 40 bits are of this
 205            type. Normally this doesn't make any difference, but gives
 206            more gentle handling of IOMMU overflow. */
 207         if (iommu_sac_force && (mask >= DMA_40BIT_MASK)) {
 208                 printk(KERN_INFO "%s: Force SAC with mask %Lx\n", dev->bus_id,mask);
 209                 return 0;
 210         }
 211
 212         return 1;
 213 }
 214 EXPORT_SYMBOL(dma_supported);
 215
 216 /*
 217  * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
 218  * documentation.
 219  */
 220 static __init int iommu_setup(char *p)
 221 {
 222         iommu_merge = 1;
 223
 224         if (!p)
 225                 return -EINVAL;
 226
 227         while (*p) {
 228                 if (!strncmp(p, "off", 3))
 229                         no_iommu = 1;
 230                 /* gart_parse_options has more force support */
 231                 if (!strncmp(p, "force", 5))
 232                         force_iommu = 1;
 233                 if (!strncmp(p, "noforce", 7)) {
 234                         iommu_merge = 0;
 235                         force_iommu = 0;
 236                 }
 237
 238                 if (!strncmp(p, "biomerge", 8)) {
 239                         iommu_bio_merge = 4096;
 240                         iommu_merge = 1;
 241                         force_iommu = 1;
 242                 }
 243                 if (!strncmp(p, "panic", 5))
 244                         panic_on_overflow = 1;
 245                 if (!strncmp(p, "nopanic", 7))
 246                         panic_on_overflow = 0;
 247                 if (!strncmp(p, "merge", 5)) {
 248                         iommu_merge = 1;
 249                         force_iommu = 1;
 250                 }
 251                 if (!strncmp(p, "nomerge", 7))
 252                         iommu_merge = 0;
 253                 if (!strncmp(p, "forcesac", 8))
 254                         iommu_sac_force = 1;
 255                 if (!strncmp(p, "allowdac", 8))
 256                         forbid_dac = 0;
 257                 if (!strncmp(p, "nodac", 5))
 258                         forbid_dac = -1;
 259
 260 #ifdef CONFIG_SWIOTLB
 261                 if (!strncmp(p, "soft", 4))
 262                         swiotlb = 1;
 263 #endif
 264
 265 #ifdef CONFIG_GART_IOMMU
 266                 gart_parse_options(p);
 267 #endif
 268
 269 #ifdef CONFIG_CALGARY_IOMMU
 270                 if (!strncmp(p, "calgary", 7))
 271                         use_calgary = 1;
 272 #endif /* CONFIG_CALGARY_IOMMU */
 273
 274                 p += strcspn(p, ",");
 275                 if (*p == ',')
 276                         ++p;
 277         }
 278         return 0;
 279 }
 280 early_param("iommu", iommu_setup);
 281
 282 static __initdata void *dma32_bootmem_ptr;
 283 static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);
 284
 285 static int __init parse_dma32_size_opt(char *p)
 286 {
 287         if (!p)
 288                 return -EINVAL;
 289         dma32_bootmem_size = memparse(p, &p);
 290         return 0;
 291 }
 292 early_param("dma32_size", parse_dma32_size_opt);
 293
 294 void __init dma32_reserve_bootmem(void)
 295 {
 296         unsigned long size, align;
 297         if (end_pfn <= MAX_DMA32_PFN)
 298                 return;
 299
 300         align = 64ULL<<20;
 301         size = round_up(dma32_bootmem_size, align);
 302         dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
 303                                  __pa(MAX_DMA_ADDRESS));
 304         if (dma32_bootmem_ptr)
 305                 dma32_bootmem_size = size;
 306         else
 307                 dma32_bootmem_size = 0;
 308 }
 309 static void __init dma32_free_bootmem(void)
 310 {
 311         int node;
 312
 313         if (end_pfn <= MAX_DMA32_PFN)
 314                 return;
 315
 316         if (!dma32_bootmem_ptr)
 317                 return;
 318
 319         for_each_online_node(node)
 320                 free_bootmem_node(NODE_DATA(node), __pa(dma32_bootmem_ptr),
 321                                   dma32_bootmem_size);
 322
 323         dma32_bootmem_ptr = NULL;
 324         dma32_bootmem_size = 0;
 325 }
 326
 327 void __init pci_iommu_alloc(void)
 328 {
 329         /* free the range so iommu could get some range less than 4G */
 330         dma32_free_bootmem();
 331         /*
 332          * The order of these functions is important for
 333          * fall-back/fail-over reasons
 334          */
 335 #ifdef CONFIG_GART_IOMMU
 336         gart_iommu_hole_init();
 337 #endif
 338
 339 #ifdef CONFIG_CALGARY_IOMMU
 340         detect_calgary();
 341 #endif
 342
 343         detect_intel_iommu();
 344
 345 #ifdef CONFIG_SWIOTLB
 346         pci_swiotlb_init();
 347 #endif
 348 }
 349
 350 static int __init pci_iommu_init(void)
 351 {
 352 #ifdef CONFIG_CALGARY_IOMMU
 353         calgary_iommu_init();
 354 #endif
 355
 356         intel_iommu_init();
 357
 358 #ifdef CONFIG_GART_IOMMU
 359         gart_iommu_init();
 360 #endif
 361
 362         no_iommu_init();
 363         return 0;
 364 }
 365
 366 void pci_iommu_shutdown(void)
 367 {
 368         gart_iommu_shutdown();
 369 }
 370
 371 #ifdef CONFIG_PCI
 372 /* Many VIA bridges seem to corrupt data for DAC. Disable it here */
 373
 374 static __devinit void via_no_dac(struct pci_dev *dev)
 375 {
 376         if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
 377                 printk(KERN_INFO "PCI: VIA PCI bridge detected. Disabling DAC.\n");
 378                 forbid_dac = 1;
 379         }
 380 }
 381 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
 382 #endif
 383 /* Must execute after PCI subsystem */
 384 fs_initcall(pci_iommu_init);