arch/riscv/mm/dma-noncoherent.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * RISC-V specific functions to support DMA for non-coherent devices
   4  *
   5  * Copyright (c) 2021 Western Digital Corporation or its affiliates.
   6  */
   7
   8 #include <linux/dma-direct.h>
   9 #include <linux/dma-map-ops.h>
  10 #include <linux/mm.h>
  11 #include <asm/cacheflush.h>
  12 #include <asm/dma-noncoherent.h>
  13
  14 static bool noncoherent_supported __ro_after_init;
  15 int dma_cache_alignment __ro_after_init = ARCH_DMA_MINALIGN;
  16 EXPORT_SYMBOL_GPL(dma_cache_alignment);
  17
  18 static inline void arch_dma_cache_wback(phys_addr_t paddr, size_t size)
  19 {
  20         void *vaddr = phys_to_virt(paddr);
  21
  22 #ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS
  23         if (unlikely(noncoherent_cache_ops.wback)) {
  24                 noncoherent_cache_ops.wback(paddr, size);
  25                 return;
  26         }
  27 #endif
  28         ALT_CMO_OP(CLEAN, vaddr, size, riscv_cbom_block_size);
  29 }
  30
  31 static inline void arch_dma_cache_inv(phys_addr_t paddr, size_t size)
  32 {
  33         void *vaddr = phys_to_virt(paddr);
  34
  35 #ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS
  36         if (unlikely(noncoherent_cache_ops.inv)) {
  37                 noncoherent_cache_ops.inv(paddr, size);
  38                 return;
  39         }
  40 #endif
  41
  42         ALT_CMO_OP(INVAL, vaddr, size, riscv_cbom_block_size);
  43 }
  44
  45 static inline void arch_dma_cache_wback_inv(phys_addr_t paddr, size_t size)
  46 {
  47         void *vaddr = phys_to_virt(paddr);
  48
  49 #ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS
  50         if (unlikely(noncoherent_cache_ops.wback_inv)) {
  51                 noncoherent_cache_ops.wback_inv(paddr, size);
  52                 return;
  53         }
  54 #endif
  55
  56         ALT_CMO_OP(FLUSH, vaddr, size, riscv_cbom_block_size);
  57 }
  58
  59 static inline bool arch_sync_dma_clean_before_fromdevice(void)
  60 {
  61         return true;
  62 }
  63
  64 static inline bool arch_sync_dma_cpu_needs_post_dma_flush(void)
  65 {
  66         return true;
  67 }
  68
  69 void arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
  70                               enum dma_data_direction dir)
  71 {
  72         switch (dir) {
  73         case DMA_TO_DEVICE:
  74                 arch_dma_cache_wback(paddr, size);
  75                 break;
  76
  77         case DMA_FROM_DEVICE:
  78                 if (!arch_sync_dma_clean_before_fromdevice()) {
  79                         arch_dma_cache_inv(paddr, size);
  80                         break;
  81                 }
  82                 fallthrough;
  83
  84         case DMA_BIDIRECTIONAL:
  85                 /* Skip the invalidate here if it's done later */
  86                 if (IS_ENABLED(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) &&
  87                     arch_sync_dma_cpu_needs_post_dma_flush())
  88                         arch_dma_cache_wback(paddr, size);
  89                 else
  90                         arch_dma_cache_wback_inv(paddr, size);
  91                 break;
  92
  93         default:
  94                 break;
  95         }
  96 }
  97
  98 void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
  99                            enum dma_data_direction dir)
 100 {
 101         switch (dir) {
 102         case DMA_TO_DEVICE:
 103                 break;
 104
 105         case DMA_FROM_DEVICE:
 106         case DMA_BIDIRECTIONAL:
 107                 /* FROM_DEVICE invalidate needed if speculative CPU prefetch only */
 108                 if (arch_sync_dma_cpu_needs_post_dma_flush())
 109                         arch_dma_cache_inv(paddr, size);
 110                 break;
 111
 112         default:
 113                 break;
 114         }
 115 }
 116
 117 void arch_dma_prep_coherent(struct page *page, size_t size)
 118 {
 119         void *flush_addr = page_address(page);
 120
 121 #ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS
 122         if (unlikely(noncoherent_cache_ops.wback_inv)) {
 123                 noncoherent_cache_ops.wback_inv(page_to_phys(page), size);
 124                 return;
 125         }
 126 #endif
 127
 128         ALT_CMO_OP(FLUSH, flush_addr, size, riscv_cbom_block_size);
 129 }
 130
 131 void arch_setup_dma_ops(struct device *dev, bool coherent)
 132 {
 133         WARN_TAINT(!coherent && riscv_cbom_block_size > ARCH_DMA_MINALIGN,
 134                    TAINT_CPU_OUT_OF_SPEC,
 135                    "%s %s: ARCH_DMA_MINALIGN smaller than riscv,cbom-block-size (%d < %d)",
 136                    dev_driver_string(dev), dev_name(dev),
 137                    ARCH_DMA_MINALIGN, riscv_cbom_block_size);
 138
 139         WARN_TAINT(!coherent && !noncoherent_supported, TAINT_CPU_OUT_OF_SPEC,
 140                    "%s %s: device non-coherent but no non-coherent operations supported",
 141                    dev_driver_string(dev), dev_name(dev));
 142
 143         dev->dma_coherent = coherent;
 144 }
 145
 146 void riscv_noncoherent_supported(void)
 147 {
 148         WARN(!riscv_cbom_block_size,
 149              "Non-coherent DMA support enabled without a block size\n");
 150         noncoherent_supported = true;
 151 }
 152
 153 void __init riscv_set_dma_cache_alignment(void)
 154 {
 155         if (!noncoherent_supported)
 156                 dma_cache_alignment = 1;
 157 }