drivers/misc/bcm-vk/bcm_vk_sg.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright 2018-2020 Broadcom.
   4  */
   5 #include <linux/dma-mapping.h>
   6 #include <linux/mm.h>
   7 #include <linux/pagemap.h>
   8 #include <linux/pgtable.h>
   9 #include <linux/vmalloc.h>
  10
  11 #include <asm/page.h>
  12 #include <linux/unaligned.h>
  13
  14 #include <uapi/linux/misc/bcm_vk.h>
  15
  16 #include "bcm_vk.h"
  17 #include "bcm_vk_msg.h"
  18 #include "bcm_vk_sg.h"
  19
  20 /*
  21  * Valkyrie has a hardware limitation of 16M transfer size.
  22  * So limit the SGL chunks to 16M.
  23  */
  24 #define BCM_VK_MAX_SGL_CHUNK SZ_16M
  25
  26 static int bcm_vk_dma_alloc(struct device *dev,
  27                             struct bcm_vk_dma *dma,
  28                             int dir,
  29                             struct _vk_data *vkdata);
  30 static int bcm_vk_dma_free(struct device *dev, struct bcm_vk_dma *dma);
  31
  32 /* Uncomment to dump SGLIST */
  33 /* #define BCM_VK_DUMP_SGLIST */
  34
  35 static int bcm_vk_dma_alloc(struct device *dev,
  36                             struct bcm_vk_dma *dma,
  37                             int direction,
  38                             struct _vk_data *vkdata)
  39 {
  40         dma_addr_t addr, sg_addr;
  41         int err;
  42         int i;
  43         int offset;
  44         u32 size;
  45         u32 remaining_size;
  46         u32 transfer_size;
  47         u64 data;
  48         unsigned long first, last;
  49         struct _vk_data *sgdata;
  50
  51         /* Get 64-bit user address */
  52         data = get_unaligned(&vkdata->address);
  53
  54         /* offset into first page */
  55         offset = offset_in_page(data);
  56
  57         /* Calculate number of pages */
  58         first = (data & PAGE_MASK) >> PAGE_SHIFT;
  59         last  = ((data + vkdata->size - 1) & PAGE_MASK) >> PAGE_SHIFT;
  60         dma->nr_pages = last - first + 1;
  61
  62         /* Allocate DMA pages */
  63         dma->pages = kmalloc_array(dma->nr_pages,
  64                                    sizeof(struct page *),
  65                                    GFP_KERNEL);
  66         if (!dma->pages)
  67                 return -ENOMEM;
  68
  69         dev_dbg(dev, "Alloc DMA Pages [0x%llx+0x%x => %d pages]\n",
  70                 data, vkdata->size, dma->nr_pages);
  71
  72         dma->direction = direction;
  73
  74         /* Get user pages into memory */
  75         err = get_user_pages_fast(data & PAGE_MASK,
  76                                   dma->nr_pages,
  77                                   direction == DMA_FROM_DEVICE,
  78                                   dma->pages);
  79         if (err != dma->nr_pages) {
  80                 dma->nr_pages = (err >= 0) ? err : 0;
  81                 dev_err(dev, "get_user_pages_fast, err=%d [%d]\n",
  82                         err, dma->nr_pages);
  83                 return err < 0 ? err : -EINVAL;
  84         }
  85
  86         /* Max size of sg list is 1 per mapped page + fields at start */
  87         dma->sglen = (dma->nr_pages * sizeof(*sgdata)) +
  88                      (sizeof(u32) * SGLIST_VKDATA_START);
  89
  90         /* Allocate sglist */
  91         dma->sglist = dma_alloc_coherent(dev,
  92                                          dma->sglen,
  93                                          &dma->handle,
  94                                          GFP_KERNEL);
  95         if (!dma->sglist)
  96                 return -ENOMEM;
  97
  98         dma->sglist[SGLIST_NUM_SG] = 0;
  99         dma->sglist[SGLIST_TOTALSIZE] = vkdata->size;
 100         remaining_size = vkdata->size;
 101         sgdata = (struct _vk_data *)&dma->sglist[SGLIST_VKDATA_START];
 102
 103         /* Map all pages into DMA */
 104         size = min_t(size_t, PAGE_SIZE - offset, remaining_size);
 105         remaining_size -= size;
 106         sg_addr = dma_map_page(dev,
 107                                dma->pages[0],
 108                                offset,
 109                                size,
 110                                dma->direction);
 111         transfer_size = size;
 112         if (unlikely(dma_mapping_error(dev, sg_addr))) {
 113                 __free_page(dma->pages[0]);
 114                 return -EIO;
 115         }
 116
 117         for (i = 1; i < dma->nr_pages; i++) {
 118                 size = min_t(size_t, PAGE_SIZE, remaining_size);
 119                 remaining_size -= size;
 120                 addr = dma_map_page(dev,
 121                                     dma->pages[i],
 122                                     0,
 123                                     size,
 124                                     dma->direction);
 125                 if (unlikely(dma_mapping_error(dev, addr))) {
 126                         __free_page(dma->pages[i]);
 127                         return -EIO;
 128                 }
 129
 130                 /*
 131                  * Compress SG list entry when pages are contiguous
 132                  * and transfer size less or equal to BCM_VK_MAX_SGL_CHUNK
 133                  */
 134                 if ((addr == (sg_addr + transfer_size)) &&
 135                     ((transfer_size + size) <= BCM_VK_MAX_SGL_CHUNK)) {
 136                         /* pages are contiguous, add to same sg entry */
 137                         transfer_size += size;
 138                 } else {
 139                         /* pages are not contiguous, write sg entry */
 140                         sgdata->size = transfer_size;
 141                         put_unaligned(sg_addr, (u64 *)&sgdata->address);
 142                         dma->sglist[SGLIST_NUM_SG]++;
 143
 144                         /* start new sg entry */
 145                         sgdata++;
 146                         sg_addr = addr;
 147                         transfer_size = size;
 148                 }
 149         }
 150         /* Write last sg list entry */
 151         sgdata->size = transfer_size;
 152         put_unaligned(sg_addr, (u64 *)&sgdata->address);
 153         dma->sglist[SGLIST_NUM_SG]++;
 154
 155         /* Update pointers and size field to point to sglist */
 156         put_unaligned((u64)dma->handle, &vkdata->address);
 157         vkdata->size = (dma->sglist[SGLIST_NUM_SG] * sizeof(*sgdata)) +
 158                        (sizeof(u32) * SGLIST_VKDATA_START);
 159
 160 #ifdef BCM_VK_DUMP_SGLIST
 161         dev_dbg(dev,
 162                 "sgl 0x%llx handle 0x%llx, sglen: 0x%x sgsize: 0x%x\n",
 163                 (u64)dma->sglist,
 164                 dma->handle,
 165                 dma->sglen,
 166                 vkdata->size);
 167         for (i = 0; i < vkdata->size / sizeof(u32); i++)
 168                 dev_dbg(dev, "i:0x%x 0x%x\n", i, dma->sglist[i]);
 169 #endif
 170
 171         return 0;
 172 }
 173
 174 int bcm_vk_sg_alloc(struct device *dev,
 175                     struct bcm_vk_dma *dma,
 176                     int dir,
 177                     struct _vk_data *vkdata,
 178                     int num)
 179 {
 180         int i;
 181         int rc = -EINVAL;
 182
 183         /* Convert user addresses to DMA SG List */
 184         for (i = 0; i < num; i++) {
 185                 if (vkdata[i].size && vkdata[i].address) {
 186                         /*
 187                          * If both size and address are non-zero
 188                          * then DMA alloc.
 189                          */
 190                         rc = bcm_vk_dma_alloc(dev,
 191                                               &dma[i],
 192                                               dir,
 193                                               &vkdata[i]);
 194                 } else if (vkdata[i].size ||
 195                            vkdata[i].address) {
 196                         /*
 197                          * If one of size and address are zero
 198                          * there is a problem.
 199                          */
 200                         dev_err(dev,
 201                                 "Invalid vkdata %x 0x%x 0x%llx\n",
 202                                 i, vkdata[i].size, vkdata[i].address);
 203                         rc = -EINVAL;
 204                 } else {
 205                         /*
 206                          * If size and address are both zero
 207                          * don't convert, but return success.
 208                          */
 209                         rc = 0;
 210                 }
 211
 212                 if (rc)
 213                         goto fail_alloc;
 214         }
 215         return rc;
 216
 217 fail_alloc:
 218         while (i > 0) {
 219                 i--;
 220                 if (dma[i].sglist)
 221                         bcm_vk_dma_free(dev, &dma[i]);
 222         }
 223         return rc;
 224 }
 225
 226 static int bcm_vk_dma_free(struct device *dev, struct bcm_vk_dma *dma)
 227 {
 228         dma_addr_t addr;
 229         int i;
 230         int num_sg;
 231         u32 size;
 232         struct _vk_data *vkdata;
 233
 234         dev_dbg(dev, "free sglist=%p sglen=0x%x\n", dma->sglist, dma->sglen);
 235
 236         /* Unmap all pages in the sglist */
 237         num_sg = dma->sglist[SGLIST_NUM_SG];
 238         vkdata = (struct _vk_data *)&dma->sglist[SGLIST_VKDATA_START];
 239         for (i = 0; i < num_sg; i++) {
 240                 size = vkdata[i].size;
 241                 addr = get_unaligned(&vkdata[i].address);
 242
 243                 dma_unmap_page(dev, addr, size, dma->direction);
 244         }
 245
 246         /* Free allocated sglist */
 247         dma_free_coherent(dev, dma->sglen, dma->sglist, dma->handle);
 248
 249         /* Release lock on all pages */
 250         for (i = 0; i < dma->nr_pages; i++)
 251                 put_page(dma->pages[i]);
 252
 253         /* Free allocated dma pages */
 254         kfree(dma->pages);
 255         dma->sglist = NULL;
 256
 257         return 0;
 258 }
 259
 260 int bcm_vk_sg_free(struct device *dev, struct bcm_vk_dma *dma, int num,
 261                    int *proc_cnt)
 262 {
 263         int i;
 264
 265         *proc_cnt = 0;
 266         /* Unmap and free all pages and sglists */
 267         for (i = 0; i < num; i++) {
 268                 if (dma[i].sglist) {
 269                         bcm_vk_dma_free(dev, &dma[i]);
 270                         *proc_cnt += 1;
 271                 }
 272         }
 273
 274         return 0;
 275 }