drivers/media/pci/intel/ipu6/ipu6-dma.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2013--2024 Intel Corporation
   4  */
   5
   6 #include <linux/cacheflush.h>
   7 #include <linux/dma-mapping.h>
   8 #include <linux/iova.h>
   9 #include <linux/list.h>
  10 #include <linux/mm.h>
  11 #include <linux/vmalloc.h>
  12 #include <linux/scatterlist.h>
  13 #include <linux/slab.h>
  14 #include <linux/types.h>
  15
  16 #include "ipu6.h"
  17 #include "ipu6-bus.h"
  18 #include "ipu6-dma.h"
  19 #include "ipu6-mmu.h"
  20
  21 struct vm_info {
  22         struct list_head list;
  23         struct page **pages;
  24         dma_addr_t ipu6_iova;
  25         void *vaddr;
  26         unsigned long size;
  27 };
  28
  29 static struct vm_info *get_vm_info(struct ipu6_mmu *mmu, dma_addr_t iova)
  30 {
  31         struct vm_info *info, *save;
  32
  33         list_for_each_entry_safe(info, save, &mmu->vma_list, list) {
  34                 if (iova >= info->ipu6_iova &&
  35                     iova < (info->ipu6_iova + info->size))
  36                         return info;
  37         }
  38
  39         return NULL;
  40 }
  41
  42 static void __clear_buffer(struct page *page, size_t size, unsigned long attrs)
  43 {
  44         void *ptr;
  45
  46         if (!page)
  47                 return;
  48         /*
  49          * Ensure that the allocated pages are zeroed, and that any data
  50          * lurking in the kernel direct-mapped region is invalidated.
  51          */
  52         ptr = page_address(page);
  53         memset(ptr, 0, size);
  54         if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0)
  55                 clflush_cache_range(ptr, size);
  56 }
  57
  58 static struct page **__alloc_buffer(size_t size, gfp_t gfp, unsigned long attrs)
  59 {
  60         int count = PHYS_PFN(size);
  61         int array_size = count * sizeof(struct page *);
  62         struct page **pages;
  63         int i = 0;
  64
  65         pages = kvzalloc(array_size, GFP_KERNEL);
  66         if (!pages)
  67                 return NULL;
  68
  69         gfp |= __GFP_NOWARN;
  70
  71         while (count) {
  72                 int j, order = __fls(count);
  73
  74                 pages[i] = alloc_pages(gfp, order);
  75                 while (!pages[i] && order)
  76                         pages[i] = alloc_pages(gfp, --order);
  77                 if (!pages[i])
  78                         goto error;
  79
  80                 if (order) {
  81                         split_page(pages[i], order);
  82                         j = 1 << order;
  83                         while (j--)
  84                                 pages[i + j] = pages[i] + j;
  85                 }
  86
  87                 __clear_buffer(pages[i], PAGE_SIZE << order, attrs);
  88                 i += 1 << order;
  89                 count -= 1 << order;
  90         }
  91
  92         return pages;
  93 error:
  94         while (i--)
  95                 if (pages[i])
  96                         __free_pages(pages[i], 0);
  97         kvfree(pages);
  98         return NULL;
  99 }
 100
 101 static void __free_buffer(struct page **pages, size_t size, unsigned long attrs)
 102 {
 103         int count = PHYS_PFN(size);
 104         unsigned int i;
 105
 106         for (i = 0; i < count && pages[i]; i++) {
 107                 __clear_buffer(pages[i], PAGE_SIZE, attrs);
 108                 __free_pages(pages[i], 0);
 109         }
 110
 111         kvfree(pages);
 112 }
 113
 114 void ipu6_dma_sync_single(struct ipu6_bus_device *sys, dma_addr_t dma_handle,
 115                           size_t size)
 116 {
 117         void *vaddr;
 118         u32 offset;
 119         struct vm_info *info;
 120         struct ipu6_mmu *mmu = sys->mmu;
 121
 122         info = get_vm_info(mmu, dma_handle);
 123         if (WARN_ON(!info))
 124                 return;
 125
 126         offset = dma_handle - info->ipu6_iova;
 127         if (WARN_ON(size > (info->size - offset)))
 128                 return;
 129
 130         vaddr = info->vaddr + offset;
 131         clflush_cache_range(vaddr, size);
 132 }
 133 EXPORT_SYMBOL_NS_GPL(ipu6_dma_sync_single, INTEL_IPU6);
 134
 135 void ipu6_dma_sync_sg(struct ipu6_bus_device *sys, struct scatterlist *sglist,
 136                       int nents)
 137 {
 138         struct scatterlist *sg;
 139         int i;
 140
 141         for_each_sg(sglist, sg, nents, i)
 142                 clflush_cache_range(sg_virt(sg), sg->length);
 143 }
 144 EXPORT_SYMBOL_NS_GPL(ipu6_dma_sync_sg, INTEL_IPU6);
 145
 146 void ipu6_dma_sync_sgtable(struct ipu6_bus_device *sys, struct sg_table *sgt)
 147 {
 148         ipu6_dma_sync_sg(sys, sgt->sgl, sgt->orig_nents);
 149 }
 150 EXPORT_SYMBOL_NS_GPL(ipu6_dma_sync_sgtable, INTEL_IPU6);
 151
 152 void *ipu6_dma_alloc(struct ipu6_bus_device *sys, size_t size,
 153                      dma_addr_t *dma_handle, gfp_t gfp,
 154                      unsigned long attrs)
 155 {
 156         struct device *dev = &sys->auxdev.dev;
 157         struct pci_dev *pdev = sys->isp->pdev;
 158         dma_addr_t pci_dma_addr, ipu6_iova;
 159         struct ipu6_mmu *mmu = sys->mmu;
 160         struct vm_info *info;
 161         unsigned long count;
 162         struct page **pages;
 163         struct iova *iova;
 164         unsigned int i;
 165         int ret;
 166
 167         info = kzalloc(sizeof(*info), GFP_KERNEL);
 168         if (!info)
 169                 return NULL;
 170
 171         size = PAGE_ALIGN(size);
 172         count = PHYS_PFN(size);
 173
 174         iova = alloc_iova(&mmu->dmap->iovad, count,
 175                           PHYS_PFN(dma_get_mask(dev)), 0);
 176         if (!iova)
 177                 goto out_kfree;
 178
 179         pages = __alloc_buffer(size, gfp, attrs);
 180         if (!pages)
 181                 goto out_free_iova;
 182
 183         dev_dbg(dev, "dma_alloc: size %zu iova low pfn %lu, high pfn %lu\n",
 184                 size, iova->pfn_lo, iova->pfn_hi);
 185         for (i = 0; iova->pfn_lo + i <= iova->pfn_hi; i++) {
 186                 pci_dma_addr = dma_map_page_attrs(&pdev->dev, pages[i], 0,
 187                                                   PAGE_SIZE, DMA_BIDIRECTIONAL,
 188                                                   attrs);
 189                 dev_dbg(dev, "dma_alloc: mapped pci_dma_addr %pad\n",
 190                         &pci_dma_addr);
 191                 if (dma_mapping_error(&pdev->dev, pci_dma_addr)) {
 192                         dev_err(dev, "pci_dma_mapping for page[%d] failed", i);
 193                         goto out_unmap;
 194                 }
 195
 196                 ret = ipu6_mmu_map(mmu->dmap->mmu_info,
 197                                    PFN_PHYS(iova->pfn_lo + i), pci_dma_addr,
 198                                    PAGE_SIZE);
 199                 if (ret) {
 200                         dev_err(dev, "ipu6_mmu_map for pci_dma[%d] %pad failed",
 201                                 i, &pci_dma_addr);
 202                         dma_unmap_page_attrs(&pdev->dev, pci_dma_addr,
 203                                              PAGE_SIZE, DMA_BIDIRECTIONAL,
 204                                              attrs);
 205                         goto out_unmap;
 206                 }
 207         }
 208
 209         info->vaddr = vmap(pages, count, VM_USERMAP, PAGE_KERNEL);
 210         if (!info->vaddr)
 211                 goto out_unmap;
 212
 213         *dma_handle = PFN_PHYS(iova->pfn_lo);
 214
 215         info->pages = pages;
 216         info->ipu6_iova = *dma_handle;
 217         info->size = size;
 218         list_add(&info->list, &mmu->vma_list);
 219
 220         return info->vaddr;
 221
 222 out_unmap:
 223         while (i--) {
 224                 ipu6_iova = PFN_PHYS(iova->pfn_lo + i);
 225                 pci_dma_addr = ipu6_mmu_iova_to_phys(mmu->dmap->mmu_info,
 226                                                      ipu6_iova);
 227                 dma_unmap_page_attrs(&pdev->dev, pci_dma_addr, PAGE_SIZE,
 228                                      DMA_BIDIRECTIONAL, attrs);
 229
 230                 ipu6_mmu_unmap(mmu->dmap->mmu_info, ipu6_iova, PAGE_SIZE);
 231         }
 232
 233         __free_buffer(pages, size, attrs);
 234
 235 out_free_iova:
 236         __free_iova(&mmu->dmap->iovad, iova);
 237 out_kfree:
 238         kfree(info);
 239
 240         return NULL;
 241 }
 242 EXPORT_SYMBOL_NS_GPL(ipu6_dma_alloc, INTEL_IPU6);
 243
 244 void ipu6_dma_free(struct ipu6_bus_device *sys, size_t size, void *vaddr,
 245                    dma_addr_t dma_handle, unsigned long attrs)
 246 {
 247         struct ipu6_mmu *mmu = sys->mmu;
 248         struct pci_dev *pdev = sys->isp->pdev;
 249         struct iova *iova = find_iova(&mmu->dmap->iovad, PHYS_PFN(dma_handle));
 250         dma_addr_t pci_dma_addr, ipu6_iova;
 251         struct vm_info *info;
 252         struct page **pages;
 253         unsigned int i;
 254
 255         if (WARN_ON(!iova))
 256                 return;
 257
 258         info = get_vm_info(mmu, dma_handle);
 259         if (WARN_ON(!info))
 260                 return;
 261
 262         if (WARN_ON(!info->vaddr))
 263                 return;
 264
 265         if (WARN_ON(!info->pages))
 266                 return;
 267
 268         list_del(&info->list);
 269
 270         size = PAGE_ALIGN(size);
 271
 272         pages = info->pages;
 273
 274         vunmap(vaddr);
 275
 276         for (i = 0; i < PHYS_PFN(size); i++) {
 277                 ipu6_iova = PFN_PHYS(iova->pfn_lo + i);
 278                 pci_dma_addr = ipu6_mmu_iova_to_phys(mmu->dmap->mmu_info,
 279                                                      ipu6_iova);
 280                 dma_unmap_page_attrs(&pdev->dev, pci_dma_addr, PAGE_SIZE,
 281                                      DMA_BIDIRECTIONAL, attrs);
 282         }
 283
 284         ipu6_mmu_unmap(mmu->dmap->mmu_info, PFN_PHYS(iova->pfn_lo),
 285                        PFN_PHYS(iova_size(iova)));
 286
 287         __free_buffer(pages, size, attrs);
 288
 289         mmu->tlb_invalidate(mmu);
 290
 291         __free_iova(&mmu->dmap->iovad, iova);
 292
 293         kfree(info);
 294 }
 295 EXPORT_SYMBOL_NS_GPL(ipu6_dma_free, INTEL_IPU6);
 296
 297 int ipu6_dma_mmap(struct ipu6_bus_device *sys, struct vm_area_struct *vma,
 298                   void *addr, dma_addr_t iova, size_t size,
 299                   unsigned long attrs)
 300 {
 301         struct ipu6_mmu *mmu = sys->mmu;
 302         size_t count = PFN_UP(size);
 303         struct vm_info *info;
 304         size_t i;
 305         int ret;
 306
 307         info = get_vm_info(mmu, iova);
 308         if (!info)
 309                 return -EFAULT;
 310
 311         if (!info->vaddr)
 312                 return -EFAULT;
 313
 314         if (vma->vm_start & ~PAGE_MASK)
 315                 return -EINVAL;
 316
 317         if (size > info->size)
 318                 return -EFAULT;
 319
 320         for (i = 0; i < count; i++) {
 321                 ret = vm_insert_page(vma, vma->vm_start + PFN_PHYS(i),
 322                                      info->pages[i]);
 323                 if (ret < 0)
 324                         return ret;
 325         }
 326
 327         return 0;
 328 }
 329
 330 void ipu6_dma_unmap_sg(struct ipu6_bus_device *sys, struct scatterlist *sglist,
 331                        int nents, enum dma_data_direction dir,
 332                        unsigned long attrs)
 333 {
 334         struct device *dev = &sys->auxdev.dev;
 335         struct ipu6_mmu *mmu = sys->mmu;
 336         struct iova *iova = find_iova(&mmu->dmap->iovad,
 337                                       PHYS_PFN(sg_dma_address(sglist)));
 338         struct scatterlist *sg;
 339         dma_addr_t pci_dma_addr;
 340         unsigned int i;
 341
 342         if (!nents)
 343                 return;
 344
 345         if (WARN_ON(!iova))
 346                 return;
 347
 348         /*
 349          * Before IPU6 mmu unmap, return the pci dma address back to sg
 350          * assume the nents is less than orig_nents as the least granule
 351          * is 1 SZ_4K page
 352          */
 353         dev_dbg(dev, "trying to unmap concatenated %u ents\n", nents);
 354         for_each_sg(sglist, sg, nents, i) {
 355                 dev_dbg(dev, "unmap sg[%d] %pad size %u\n", i,
 356                         &sg_dma_address(sg), sg_dma_len(sg));
 357                 pci_dma_addr = ipu6_mmu_iova_to_phys(mmu->dmap->mmu_info,
 358                                                      sg_dma_address(sg));
 359                 dev_dbg(dev, "return pci_dma_addr %pad back to sg[%d]\n",
 360                         &pci_dma_addr, i);
 361                 sg_dma_address(sg) = pci_dma_addr;
 362         }
 363
 364         dev_dbg(dev, "ipu6_mmu_unmap low pfn %lu high pfn %lu\n",
 365                 iova->pfn_lo, iova->pfn_hi);
 366         ipu6_mmu_unmap(mmu->dmap->mmu_info, PFN_PHYS(iova->pfn_lo),
 367                        PFN_PHYS(iova_size(iova)));
 368
 369         mmu->tlb_invalidate(mmu);
 370         __free_iova(&mmu->dmap->iovad, iova);
 371 }
 372 EXPORT_SYMBOL_NS_GPL(ipu6_dma_unmap_sg, INTEL_IPU6);
 373
 374 int ipu6_dma_map_sg(struct ipu6_bus_device *sys, struct scatterlist *sglist,
 375                     int nents, enum dma_data_direction dir,
 376                     unsigned long attrs)
 377 {
 378         struct device *dev = &sys->auxdev.dev;
 379         struct ipu6_mmu *mmu = sys->mmu;
 380         struct scatterlist *sg;
 381         struct iova *iova;
 382         size_t npages = 0;
 383         unsigned long iova_addr;
 384         int i;
 385
 386         for_each_sg(sglist, sg, nents, i) {
 387                 if (sg->offset) {
 388                         dev_err(dev, "Unsupported non-zero sg[%d].offset %x\n",
 389                                 i, sg->offset);
 390                         return -EFAULT;
 391                 }
 392         }
 393
 394         for_each_sg(sglist, sg, nents, i)
 395                 npages += PFN_UP(sg_dma_len(sg));
 396
 397         dev_dbg(dev, "dmamap trying to map %d ents %zu pages\n",
 398                 nents, npages);
 399
 400         iova = alloc_iova(&mmu->dmap->iovad, npages,
 401                           PHYS_PFN(dma_get_mask(dev)), 0);
 402         if (!iova)
 403                 return 0;
 404
 405         dev_dbg(dev, "dmamap: iova low pfn %lu, high pfn %lu\n", iova->pfn_lo,
 406                 iova->pfn_hi);
 407
 408         iova_addr = iova->pfn_lo;
 409         for_each_sg(sglist, sg, nents, i) {
 410                 phys_addr_t iova_pa;
 411                 int ret;
 412
 413                 iova_pa = PFN_PHYS(iova_addr);
 414                 dev_dbg(dev, "mapping entry %d: iova %pap phy %pap size %d\n",
 415                         i, &iova_pa, &sg_dma_address(sg), sg_dma_len(sg));
 416
 417                 ret = ipu6_mmu_map(mmu->dmap->mmu_info, PFN_PHYS(iova_addr),
 418                                    sg_dma_address(sg),
 419                                    PAGE_ALIGN(sg_dma_len(sg)));
 420                 if (ret)
 421                         goto out_fail;
 422
 423                 sg_dma_address(sg) = PFN_PHYS(iova_addr);
 424
 425                 iova_addr += PFN_UP(sg_dma_len(sg));
 426         }
 427
 428         dev_dbg(dev, "dmamap %d ents %zu pages mapped\n", nents, npages);
 429
 430         return nents;
 431
 432 out_fail:
 433         ipu6_dma_unmap_sg(sys, sglist, i, dir, attrs);
 434
 435         return 0;
 436 }
 437 EXPORT_SYMBOL_NS_GPL(ipu6_dma_map_sg, INTEL_IPU6);
 438
 439 int ipu6_dma_map_sgtable(struct ipu6_bus_device *sys, struct sg_table *sgt,
 440                          enum dma_data_direction dir, unsigned long attrs)
 441 {
 442         int nents;
 443
 444         nents = ipu6_dma_map_sg(sys, sgt->sgl, sgt->nents, dir, attrs);
 445         if (nents < 0)
 446                 return nents;
 447
 448         sgt->nents = nents;
 449
 450         return 0;
 451 }
 452 EXPORT_SYMBOL_NS_GPL(ipu6_dma_map_sgtable, INTEL_IPU6);
 453
 454 void ipu6_dma_unmap_sgtable(struct ipu6_bus_device *sys, struct sg_table *sgt,
 455                             enum dma_data_direction dir, unsigned long attrs)
 456 {
 457         ipu6_dma_unmap_sg(sys, sgt->sgl, sgt->nents, dir, attrs);
 458 }
 459 EXPORT_SYMBOL_NS_GPL(ipu6_dma_unmap_sgtable, INTEL_IPU6);
 460
 461 /*
 462  * Create scatter-list for the already allocated DMA buffer
 463  */
 464 int ipu6_dma_get_sgtable(struct ipu6_bus_device *sys, struct sg_table *sgt,
 465                          void *cpu_addr, dma_addr_t handle, size_t size,
 466                          unsigned long attrs)
 467 {
 468         struct device *dev = &sys->auxdev.dev;
 469         struct ipu6_mmu *mmu = sys->mmu;
 470         struct vm_info *info;
 471         int n_pages;
 472         int ret = 0;
 473
 474         info = get_vm_info(mmu, handle);
 475         if (!info)
 476                 return -EFAULT;
 477
 478         if (!info->vaddr)
 479                 return -EFAULT;
 480
 481         if (WARN_ON(!info->pages))
 482                 return -ENOMEM;
 483
 484         n_pages = PFN_UP(size);
 485
 486         ret = sg_alloc_table_from_pages(sgt, info->pages, n_pages, 0, size,
 487                                         GFP_KERNEL);
 488         if (ret)
 489                 dev_warn(dev, "get sgt table failed\n");
 490
 491         return ret;
 492 }