arch/xtensa/kernel/pci-dma.c

   1 /*
   2  * DMA coherent memory allocation.
   3  *
   4  * This program is free software; you can redistribute  it and/or modify it
   5  * under  the terms of  the GNU General  Public License as published by the
   6  * Free Software Foundation;  either version 2 of the  License, or (at your
   7  * option) any later version.
   8  *
   9  * Copyright (C) 2002 - 2005 Tensilica Inc.
  10  * Copyright (C) 2015 Cadence Design Systems Inc.
  11  *
  12  * Based on version for i386.
  13  *
  14  * Chris Zankel <chris@zankel.net>
  15  * Joe Taylor <joe@tensilica.com, joetylr@yahoo.com>
  16  */
  17
  18 #include <linux/dma-contiguous.h>
  19 #include <linux/gfp.h>
  20 #include <linux/highmem.h>
  21 #include <linux/mm.h>
  22 #include <linux/module.h>
  23 #include <linux/pci.h>
  24 #include <linux/string.h>
  25 #include <linux/types.h>
  26 #include <asm/cacheflush.h>
  27 #include <asm/io.h>
  28
  29 static void do_cache_op(dma_addr_t dma_handle, size_t size,
  30                         void (*fn)(unsigned long, unsigned long))
  31 {
  32         unsigned long off = dma_handle & (PAGE_SIZE - 1);
  33         unsigned long pfn = PFN_DOWN(dma_handle);
  34         struct page *page = pfn_to_page(pfn);
  35
  36         if (!PageHighMem(page))
  37                 fn((unsigned long)bus_to_virt(dma_handle), size);
  38         else
  39                 while (size > 0) {
  40                         size_t sz = min_t(size_t, size, PAGE_SIZE - off);
  41                         void *vaddr = kmap_atomic(page);
  42
  43                         fn((unsigned long)vaddr + off, sz);
  44                         kunmap_atomic(vaddr);
  45                         off = 0;
  46                         ++page;
  47                         size -= sz;
  48                 }
  49 }
  50
  51 static void xtensa_sync_single_for_cpu(struct device *dev,
  52                                        dma_addr_t dma_handle, size_t size,
  53                                        enum dma_data_direction dir)
  54 {
  55         switch (dir) {
  56         case DMA_BIDIRECTIONAL:
  57         case DMA_FROM_DEVICE:
  58                 do_cache_op(dma_handle, size, __invalidate_dcache_range);
  59                 break;
  60
  61         case DMA_NONE:
  62                 BUG();
  63                 break;
  64
  65         default:
  66                 break;
  67         }
  68 }
  69
  70 static void xtensa_sync_single_for_device(struct device *dev,
  71                                           dma_addr_t dma_handle, size_t size,
  72                                           enum dma_data_direction dir)
  73 {
  74         switch (dir) {
  75         case DMA_BIDIRECTIONAL:
  76         case DMA_TO_DEVICE:
  77                 if (XCHAL_DCACHE_IS_WRITEBACK)
  78                         do_cache_op(dma_handle, size, __flush_dcache_range);
  79                 break;
  80
  81         case DMA_NONE:
  82                 BUG();
  83                 break;
  84
  85         default:
  86                 break;
  87         }
  88 }
  89
  90 static void xtensa_sync_sg_for_cpu(struct device *dev,
  91                                    struct scatterlist *sg, int nents,
  92                                    enum dma_data_direction dir)
  93 {
  94         struct scatterlist *s;
  95         int i;
  96
  97         for_each_sg(sg, s, nents, i) {
  98                 xtensa_sync_single_for_cpu(dev, sg_dma_address(s),
  99                                            sg_dma_len(s), dir);
 100         }
 101 }
 102
 103 static void xtensa_sync_sg_for_device(struct device *dev,
 104                                       struct scatterlist *sg, int nents,
 105                                       enum dma_data_direction dir)
 106 {
 107         struct scatterlist *s;
 108         int i;
 109
 110         for_each_sg(sg, s, nents, i) {
 111                 xtensa_sync_single_for_device(dev, sg_dma_address(s),
 112                                               sg_dma_len(s), dir);
 113         }
 114 }
 115
 116 /*
 117  * Note: We assume that the full memory space is always mapped to 'kseg'
 118  *       Otherwise we have to use page attributes (not implemented).
 119  */
 120
 121 static void *xtensa_dma_alloc(struct device *dev, size_t size,
 122                               dma_addr_t *handle, gfp_t flag,
 123                               unsigned long attrs)
 124 {
 125         unsigned long ret;
 126         unsigned long uncached = 0;
 127         unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
 128         struct page *page = NULL;
 129
 130         /* ignore region speicifiers */
 131
 132         flag &= ~(__GFP_DMA | __GFP_HIGHMEM);
 133
 134         if (dev == NULL || (dev->coherent_dma_mask < 0xffffffff))
 135                 flag |= GFP_DMA;
 136
 137         if (gfpflags_allow_blocking(flag))
 138                 page = dma_alloc_from_contiguous(dev, count, get_order(size),
 139                                                  flag);
 140
 141         if (!page)
 142                 page = alloc_pages(flag, get_order(size));
 143
 144         if (!page)
 145                 return NULL;
 146
 147         ret = (unsigned long)page_address(page);
 148
 149         /* We currently don't support coherent memory outside KSEG */
 150
 151         BUG_ON(ret < XCHAL_KSEG_CACHED_VADDR ||
 152                ret > XCHAL_KSEG_CACHED_VADDR + XCHAL_KSEG_SIZE - 1);
 153
 154         uncached = ret + XCHAL_KSEG_BYPASS_VADDR - XCHAL_KSEG_CACHED_VADDR;
 155         *handle = virt_to_bus((void *)ret);
 156         __invalidate_dcache_range(ret, size);
 157
 158         return (void *)uncached;
 159 }
 160
 161 static void xtensa_dma_free(struct device *dev, size_t size, void *vaddr,
 162                             dma_addr_t dma_handle, unsigned long attrs)
 163 {
 164         unsigned long addr = (unsigned long)vaddr +
 165                 XCHAL_KSEG_CACHED_VADDR - XCHAL_KSEG_BYPASS_VADDR;
 166         struct page *page = virt_to_page(addr);
 167         unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
 168
 169         BUG_ON(addr < XCHAL_KSEG_CACHED_VADDR ||
 170                addr > XCHAL_KSEG_CACHED_VADDR + XCHAL_KSEG_SIZE - 1);
 171
 172         if (!dma_release_from_contiguous(dev, page, count))
 173                 __free_pages(page, get_order(size));
 174 }
 175
 176 static dma_addr_t xtensa_map_page(struct device *dev, struct page *page,
 177                                   unsigned long offset, size_t size,
 178                                   enum dma_data_direction dir,
 179                                   unsigned long attrs)
 180 {
 181         dma_addr_t dma_handle = page_to_phys(page) + offset;
 182
 183         if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
 184                 xtensa_sync_single_for_device(dev, dma_handle, size, dir);
 185
 186         return dma_handle;
 187 }
 188
 189 static void xtensa_unmap_page(struct device *dev, dma_addr_t dma_handle,
 190                               size_t size, enum dma_data_direction dir,
 191                               unsigned long attrs)
 192 {
 193         if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
 194                 xtensa_sync_single_for_cpu(dev, dma_handle, size, dir);
 195 }
 196
 197 static int xtensa_map_sg(struct device *dev, struct scatterlist *sg,
 198                          int nents, enum dma_data_direction dir,
 199                          unsigned long attrs)
 200 {
 201         struct scatterlist *s;
 202         int i;
 203
 204         for_each_sg(sg, s, nents, i) {
 205                 s->dma_address = xtensa_map_page(dev, sg_page(s), s->offset,
 206                                                  s->length, dir, attrs);
 207         }
 208         return nents;
 209 }
 210
 211 static void xtensa_unmap_sg(struct device *dev,
 212                             struct scatterlist *sg, int nents,
 213                             enum dma_data_direction dir,
 214                             unsigned long attrs)
 215 {
 216         struct scatterlist *s;
 217         int i;
 218
 219         for_each_sg(sg, s, nents, i) {
 220                 xtensa_unmap_page(dev, sg_dma_address(s),
 221                                   sg_dma_len(s), dir, attrs);
 222         }
 223 }
 224
 225 int xtensa_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
 226 {
 227         return 0;
 228 }
 229
 230 const struct dma_map_ops xtensa_dma_map_ops = {
 231         .alloc = xtensa_dma_alloc,
 232         .free = xtensa_dma_free,
 233         .map_page = xtensa_map_page,
 234         .unmap_page = xtensa_unmap_page,
 235         .map_sg = xtensa_map_sg,
 236         .unmap_sg = xtensa_unmap_sg,
 237         .sync_single_for_cpu = xtensa_sync_single_for_cpu,
 238         .sync_single_for_device = xtensa_sync_single_for_device,
 239         .sync_sg_for_cpu = xtensa_sync_sg_for_cpu,
 240         .sync_sg_for_device = xtensa_sync_sg_for_device,
 241         .mapping_error = xtensa_dma_mapping_error,
 242 };
 243 EXPORT_SYMBOL(xtensa_dma_map_ops);
 244
 245 #define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
 246
 247 static int __init xtensa_dma_init(void)
 248 {
 249         dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
 250         return 0;
 251 }
 252 fs_initcall(xtensa_dma_init);