drivers/base/dma-mapping.c

   1 /*
   2  * drivers/base/dma-mapping.c - arch-independent dma-mapping routines
   3  *
   4  * Copyright (c) 2006  SUSE Linux Products GmbH
   5  * Copyright (c) 2006  Tejun Heo <teheo@suse.de>
   6  *
   7  * This file is released under the GPLv2.
   8  */
   9
  10 #include <linux/dma-mapping.h>
  11 #include <linux/export.h>
  12 #include <linux/gfp.h>
  13 #include <linux/slab.h>
  14 #include <linux/vmalloc.h>
  15
  16 /*
  17  * Managed DMA API
  18  */
  19 struct dma_devres {
  20         size_t          size;
  21         void            *vaddr;
  22         dma_addr_t      dma_handle;
  23 };
  24
  25 static void dmam_coherent_release(struct device *dev, void *res)
  26 {
  27         struct dma_devres *this = res;
  28
  29         dma_free_coherent(dev, this->size, this->vaddr, this->dma_handle);
  30 }
  31
  32 static void dmam_noncoherent_release(struct device *dev, void *res)
  33 {
  34         struct dma_devres *this = res;
  35
  36         dma_free_noncoherent(dev, this->size, this->vaddr, this->dma_handle);
  37 }
  38
  39 static int dmam_match(struct device *dev, void *res, void *match_data)
  40 {
  41         struct dma_devres *this = res, *match = match_data;
  42
  43         if (this->vaddr == match->vaddr) {
  44                 WARN_ON(this->size != match->size ||
  45                         this->dma_handle != match->dma_handle);
  46                 return 1;
  47         }
  48         return 0;
  49 }
  50
  51 /**
  52  * dmam_alloc_coherent - Managed dma_alloc_coherent()
  53  * @dev: Device to allocate coherent memory for
  54  * @size: Size of allocation
  55  * @dma_handle: Out argument for allocated DMA handle
  56  * @gfp: Allocation flags
  57  *
  58  * Managed dma_alloc_coherent().  Memory allocated using this function
  59  * will be automatically released on driver detach.
  60  *
  61  * RETURNS:
  62  * Pointer to allocated memory on success, NULL on failure.
  63  */
  64 void *dmam_alloc_coherent(struct device *dev, size_t size,
  65                            dma_addr_t *dma_handle, gfp_t gfp)
  66 {
  67         struct dma_devres *dr;
  68         void *vaddr;
  69
  70         dr = devres_alloc(dmam_coherent_release, sizeof(*dr), gfp);
  71         if (!dr)
  72                 return NULL;
  73
  74         vaddr = dma_alloc_coherent(dev, size, dma_handle, gfp);
  75         if (!vaddr) {
  76                 devres_free(dr);
  77                 return NULL;
  78         }
  79
  80         dr->vaddr = vaddr;
  81         dr->dma_handle = *dma_handle;
  82         dr->size = size;
  83
  84         devres_add(dev, dr);
  85
  86         return vaddr;
  87 }
  88 EXPORT_SYMBOL(dmam_alloc_coherent);
  89
  90 /**
  91  * dmam_free_coherent - Managed dma_free_coherent()
  92  * @dev: Device to free coherent memory for
  93  * @size: Size of allocation
  94  * @vaddr: Virtual address of the memory to free
  95  * @dma_handle: DMA handle of the memory to free
  96  *
  97  * Managed dma_free_coherent().
  98  */
  99 void dmam_free_coherent(struct device *dev, size_t size, void *vaddr,
 100                         dma_addr_t dma_handle)
 101 {
 102         struct dma_devres match_data = { size, vaddr, dma_handle };
 103
 104         dma_free_coherent(dev, size, vaddr, dma_handle);
 105         WARN_ON(devres_destroy(dev, dmam_coherent_release, dmam_match,
 106                                &match_data));
 107 }
 108 EXPORT_SYMBOL(dmam_free_coherent);
 109
 110 /**
 111  * dmam_alloc_non_coherent - Managed dma_alloc_non_coherent()
 112  * @dev: Device to allocate non_coherent memory for
 113  * @size: Size of allocation
 114  * @dma_handle: Out argument for allocated DMA handle
 115  * @gfp: Allocation flags
 116  *
 117  * Managed dma_alloc_non_coherent().  Memory allocated using this
 118  * function will be automatically released on driver detach.
 119  *
 120  * RETURNS:
 121  * Pointer to allocated memory on success, NULL on failure.
 122  */
 123 void *dmam_alloc_noncoherent(struct device *dev, size_t size,
 124                              dma_addr_t *dma_handle, gfp_t gfp)
 125 {
 126         struct dma_devres *dr;
 127         void *vaddr;
 128
 129         dr = devres_alloc(dmam_noncoherent_release, sizeof(*dr), gfp);
 130         if (!dr)
 131                 return NULL;
 132
 133         vaddr = dma_alloc_noncoherent(dev, size, dma_handle, gfp);
 134         if (!vaddr) {
 135                 devres_free(dr);
 136                 return NULL;
 137         }
 138
 139         dr->vaddr = vaddr;
 140         dr->dma_handle = *dma_handle;
 141         dr->size = size;
 142
 143         devres_add(dev, dr);
 144
 145         return vaddr;
 146 }
 147 EXPORT_SYMBOL(dmam_alloc_noncoherent);
 148
 149 /**
 150  * dmam_free_coherent - Managed dma_free_noncoherent()
 151  * @dev: Device to free noncoherent memory for
 152  * @size: Size of allocation
 153  * @vaddr: Virtual address of the memory to free
 154  * @dma_handle: DMA handle of the memory to free
 155  *
 156  * Managed dma_free_noncoherent().
 157  */
 158 void dmam_free_noncoherent(struct device *dev, size_t size, void *vaddr,
 159                            dma_addr_t dma_handle)
 160 {
 161         struct dma_devres match_data = { size, vaddr, dma_handle };
 162
 163         dma_free_noncoherent(dev, size, vaddr, dma_handle);
 164         WARN_ON(!devres_destroy(dev, dmam_noncoherent_release, dmam_match,
 165                                 &match_data));
 166 }
 167 EXPORT_SYMBOL(dmam_free_noncoherent);
 168
 169 #ifdef CONFIG_HAVE_GENERIC_DMA_COHERENT
 170
 171 static void dmam_coherent_decl_release(struct device *dev, void *res)
 172 {
 173         dma_release_declared_memory(dev);
 174 }
 175
 176 /**
 177  * dmam_declare_coherent_memory - Managed dma_declare_coherent_memory()
 178  * @dev: Device to declare coherent memory for
 179  * @phys_addr: Physical address of coherent memory to be declared
 180  * @device_addr: Device address of coherent memory to be declared
 181  * @size: Size of coherent memory to be declared
 182  * @flags: Flags
 183  *
 184  * Managed dma_declare_coherent_memory().
 185  *
 186  * RETURNS:
 187  * 0 on success, -errno on failure.
 188  */
 189 int dmam_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
 190                                  dma_addr_t device_addr, size_t size, int flags)
 191 {
 192         void *res;
 193         int rc;
 194
 195         res = devres_alloc(dmam_coherent_decl_release, 0, GFP_KERNEL);
 196         if (!res)
 197                 return -ENOMEM;
 198
 199         rc = dma_declare_coherent_memory(dev, phys_addr, device_addr, size,
 200                                          flags);
 201         if (rc == 0)
 202                 devres_add(dev, res);
 203         else
 204                 devres_free(res);
 205
 206         return rc;
 207 }
 208 EXPORT_SYMBOL(dmam_declare_coherent_memory);
 209
 210 /**
 211  * dmam_release_declared_memory - Managed dma_release_declared_memory().
 212  * @dev: Device to release declared coherent memory for
 213  *
 214  * Managed dmam_release_declared_memory().
 215  */
 216 void dmam_release_declared_memory(struct device *dev)
 217 {
 218         WARN_ON(devres_destroy(dev, dmam_coherent_decl_release, NULL, NULL));
 219 }
 220 EXPORT_SYMBOL(dmam_release_declared_memory);
 221
 222 #endif
 223
 224 /*
 225  * Create scatter-list for the already allocated DMA buffer.
 226  */
 227 int dma_common_get_sgtable(struct device *dev, struct sg_table *sgt,
 228                  void *cpu_addr, dma_addr_t handle, size_t size)
 229 {
 230         struct page *page = virt_to_page(cpu_addr);
 231         int ret;
 232
 233         ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
 234         if (unlikely(ret))
 235                 return ret;
 236
 237         sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
 238         return 0;
 239 }
 240 EXPORT_SYMBOL(dma_common_get_sgtable);
 241
 242 /*
 243  * Create userspace mapping for the DMA-coherent memory.
 244  */
 245 int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
 246                     void *cpu_addr, dma_addr_t dma_addr, size_t size)
 247 {
 248         int ret = -ENXIO;
 249 #if defined(CONFIG_MMU) && !defined(CONFIG_ARCH_NO_COHERENT_DMA_MMAP)
 250         unsigned long user_count = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
 251         unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
 252         unsigned long pfn = page_to_pfn(virt_to_page(cpu_addr));
 253         unsigned long off = vma->vm_pgoff;
 254
 255         vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 256
 257         if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
 258                 return ret;
 259
 260         if (off < count && user_count <= (count - off)) {
 261                 ret = remap_pfn_range(vma, vma->vm_start,
 262                                       pfn + off,
 263                                       user_count << PAGE_SHIFT,
 264                                       vma->vm_page_prot);
 265         }
 266 #endif  /* CONFIG_MMU && !CONFIG_ARCH_NO_COHERENT_DMA_MMAP */
 267
 268         return ret;
 269 }
 270 EXPORT_SYMBOL(dma_common_mmap);
 271
 272 #ifdef CONFIG_MMU
 273 /*
 274  * remaps an array of PAGE_SIZE pages into another vm_area
 275  * Cannot be used in non-sleeping contexts
 276  */
 277 void *dma_common_pages_remap(struct page **pages, size_t size,
 278                         unsigned long vm_flags, pgprot_t prot,
 279                         const void *caller)
 280 {
 281         struct vm_struct *area;
 282
 283         area = get_vm_area_caller(size, vm_flags, caller);
 284         if (!area)
 285                 return NULL;
 286
 287         area->pages = pages;
 288
 289         if (map_vm_area(area, prot, pages)) {
 290                 vunmap(area->addr);
 291                 return NULL;
 292         }
 293
 294         return area->addr;
 295 }
 296
 297 /*
 298  * remaps an allocated contiguous region into another vm_area.
 299  * Cannot be used in non-sleeping contexts
 300  */
 301
 302 void *dma_common_contiguous_remap(struct page *page, size_t size,
 303                         unsigned long vm_flags,
 304                         pgprot_t prot, const void *caller)
 305 {
 306         int i;
 307         struct page **pages;
 308         void *ptr;
 309         unsigned long pfn;
 310
 311         pages = kmalloc(sizeof(struct page *) << get_order(size), GFP_KERNEL);
 312         if (!pages)
 313                 return NULL;
 314
 315         for (i = 0, pfn = page_to_pfn(page); i < (size >> PAGE_SHIFT); i++)
 316                 pages[i] = pfn_to_page(pfn + i);
 317
 318         ptr = dma_common_pages_remap(pages, size, vm_flags, prot, caller);
 319
 320         kfree(pages);
 321
 322         return ptr;
 323 }
 324
 325 /*
 326  * unmaps a range previously mapped by dma_common_*_remap
 327  */
 328 void dma_common_free_remap(void *cpu_addr, size_t size, unsigned long vm_flags)
 329 {
 330         struct vm_struct *area = find_vm_area(cpu_addr);
 331
 332         if (!area || (area->flags & vm_flags) != vm_flags) {
 333                 WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
 334                 return;
 335         }
 336
 337         unmap_kernel_range((unsigned long)cpu_addr, size);
 338         vunmap(cpu_addr);
 339 }
 340 #endif