drivers/net/ethernet/qlogic/qed/qed_chain.c

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
   2 /* Copyright (c) 2020 Marvell International Ltd. */
   3
   4 #include <linux/dma-mapping.h>
   5 #include <linux/qed/qed_chain.h>
   6 #include <linux/vmalloc.h>
   7
   8 #include "qed_dev_api.h"
   9
  10 static void qed_chain_init(struct qed_chain *chain,
  11                            const struct qed_chain_init_params *params,
  12                            u32 page_cnt)
  13 {
  14         memset(chain, 0, sizeof(*chain));
  15
  16         chain->elem_size = params->elem_size;
  17         chain->intended_use = params->intended_use;
  18         chain->mode = params->mode;
  19         chain->cnt_type = params->cnt_type;
  20
  21         chain->elem_per_page = ELEMS_PER_PAGE(params->elem_size,
  22                                               params->page_size);
  23         chain->usable_per_page = USABLE_ELEMS_PER_PAGE(params->elem_size,
  24                                                        params->page_size,
  25                                                        params->mode);
  26         chain->elem_unusable = UNUSABLE_ELEMS_PER_PAGE(params->elem_size,
  27                                                        params->mode);
  28
  29         chain->elem_per_page_mask = chain->elem_per_page - 1;
  30         chain->next_page_mask = chain->usable_per_page &
  31                                 chain->elem_per_page_mask;
  32
  33         chain->page_size = params->page_size;
  34         chain->page_cnt = page_cnt;
  35         chain->capacity = chain->usable_per_page * page_cnt;
  36         chain->size = chain->elem_per_page * page_cnt;
  37
  38         if (params->ext_pbl_virt) {
  39                 chain->pbl_sp.table_virt = params->ext_pbl_virt;
  40                 chain->pbl_sp.table_phys = params->ext_pbl_phys;
  41
  42                 chain->b_external_pbl = true;
  43         }
  44 }
  45
  46 static void qed_chain_init_next_ptr_elem(const struct qed_chain *chain,
  47                                          void *virt_curr, void *virt_next,
  48                                          dma_addr_t phys_next)
  49 {
  50         struct qed_chain_next *next;
  51         u32 size;
  52
  53         size = chain->elem_size * chain->usable_per_page;
  54         next = virt_curr + size;
  55
  56         DMA_REGPAIR_LE(next->next_phys, phys_next);
  57         next->next_virt = virt_next;
  58 }
  59
  60 static void qed_chain_init_mem(struct qed_chain *chain, void *virt_addr,
  61                                dma_addr_t phys_addr)
  62 {
  63         chain->p_virt_addr = virt_addr;
  64         chain->p_phys_addr = phys_addr;
  65 }
  66
  67 static void qed_chain_free_next_ptr(struct qed_dev *cdev,
  68                                     struct qed_chain *chain)
  69 {
  70         struct device *dev = &cdev->pdev->dev;
  71         struct qed_chain_next *next;
  72         dma_addr_t phys, phys_next;
  73         void *virt, *virt_next;
  74         u32 size, i;
  75
  76         size = chain->elem_size * chain->usable_per_page;
  77         virt = chain->p_virt_addr;
  78         phys = chain->p_phys_addr;
  79
  80         for (i = 0; i < chain->page_cnt; i++) {
  81                 if (!virt)
  82                         break;
  83
  84                 next = virt + size;
  85                 virt_next = next->next_virt;
  86                 phys_next = HILO_DMA_REGPAIR(next->next_phys);
  87
  88                 dma_free_coherent(dev, chain->page_size, virt, phys);
  89
  90                 virt = virt_next;
  91                 phys = phys_next;
  92         }
  93 }
  94
  95 static void qed_chain_free_single(struct qed_dev *cdev,
  96                                   struct qed_chain *chain)
  97 {
  98         if (!chain->p_virt_addr)
  99                 return;
 100
 101         dma_free_coherent(&cdev->pdev->dev, chain->page_size,
 102                           chain->p_virt_addr, chain->p_phys_addr);
 103 }
 104
 105 static void qed_chain_free_pbl(struct qed_dev *cdev, struct qed_chain *chain)
 106 {
 107         struct device *dev = &cdev->pdev->dev;
 108         struct addr_tbl_entry *entry;
 109         u32 i;
 110
 111         if (!chain->pbl.pp_addr_tbl)
 112                 return;
 113
 114         for (i = 0; i < chain->page_cnt; i++) {
 115                 entry = chain->pbl.pp_addr_tbl + i;
 116                 if (!entry->virt_addr)
 117                         break;
 118
 119                 dma_free_coherent(dev, chain->page_size, entry->virt_addr,
 120                                   entry->dma_map);
 121         }
 122
 123         if (!chain->b_external_pbl)
 124                 dma_free_coherent(dev, chain->pbl_sp.table_size,
 125                                   chain->pbl_sp.table_virt,
 126                                   chain->pbl_sp.table_phys);
 127
 128         vfree(chain->pbl.pp_addr_tbl);
 129         chain->pbl.pp_addr_tbl = NULL;
 130 }
 131
 132 /**
 133  * qed_chain_free() - Free chain DMA memory.
 134  *
 135  * @cdev: Main device structure.
 136  * @chain: Chain to free.
 137  */
 138 void qed_chain_free(struct qed_dev *cdev, struct qed_chain *chain)
 139 {
 140         switch (chain->mode) {
 141         case QED_CHAIN_MODE_NEXT_PTR:
 142                 qed_chain_free_next_ptr(cdev, chain);
 143                 break;
 144         case QED_CHAIN_MODE_SINGLE:
 145                 qed_chain_free_single(cdev, chain);
 146                 break;
 147         case QED_CHAIN_MODE_PBL:
 148                 qed_chain_free_pbl(cdev, chain);
 149                 break;
 150         default:
 151                 return;
 152         }
 153
 154         qed_chain_init_mem(chain, NULL, 0);
 155 }
 156
 157 static int
 158 qed_chain_alloc_sanity_check(struct qed_dev *cdev,
 159                              const struct qed_chain_init_params *params,
 160                              u32 page_cnt)
 161 {
 162         u64 chain_size;
 163
 164         chain_size = ELEMS_PER_PAGE(params->elem_size, params->page_size);
 165         chain_size *= page_cnt;
 166
 167         if (!chain_size)
 168                 return -EINVAL;
 169
 170         /* The actual chain size can be larger than the maximal possible value
 171          * after rounding up the requested elements number to pages, and after
 172          * taking into account the unusuable elements (next-ptr elements).
 173          * The size of a "u16" chain can be (U16_MAX + 1) since the chain
 174          * size/capacity fields are of u32 type.
 175          */
 176         switch (params->cnt_type) {
 177         case QED_CHAIN_CNT_TYPE_U16:
 178                 if (chain_size > U16_MAX + 1)
 179                         break;
 180
 181                 return 0;
 182         case QED_CHAIN_CNT_TYPE_U32:
 183                 if (chain_size > U32_MAX)
 184                         break;
 185
 186                 return 0;
 187         default:
 188                 return -EINVAL;
 189         }
 190
 191         DP_NOTICE(cdev,
 192                   "The actual chain size (0x%llx) is larger than the maximal possible value\n",
 193                   chain_size);
 194
 195         return -EINVAL;
 196 }
 197
 198 static int qed_chain_alloc_next_ptr(struct qed_dev *cdev,
 199                                     struct qed_chain *chain)
 200 {
 201         struct device *dev = &cdev->pdev->dev;
 202         void *virt, *virt_prev = NULL;
 203         dma_addr_t phys;
 204         u32 i;
 205
 206         for (i = 0; i < chain->page_cnt; i++) {
 207                 virt = dma_alloc_coherent(dev, chain->page_size, &phys,
 208                                           GFP_KERNEL);
 209                 if (!virt)
 210                         return -ENOMEM;
 211
 212                 if (i == 0) {
 213                         qed_chain_init_mem(chain, virt, phys);
 214                         qed_chain_reset(chain);
 215                 } else {
 216                         qed_chain_init_next_ptr_elem(chain, virt_prev, virt,
 217                                                      phys);
 218                 }
 219
 220                 virt_prev = virt;
 221         }
 222
 223         /* Last page's next element should point to the beginning of the
 224          * chain.
 225          */
 226         qed_chain_init_next_ptr_elem(chain, virt_prev, chain->p_virt_addr,
 227                                      chain->p_phys_addr);
 228
 229         return 0;
 230 }
 231
 232 static int qed_chain_alloc_single(struct qed_dev *cdev,
 233                                   struct qed_chain *chain)
 234 {
 235         dma_addr_t phys;
 236         void *virt;
 237
 238         virt = dma_alloc_coherent(&cdev->pdev->dev, chain->page_size,
 239                                   &phys, GFP_KERNEL);
 240         if (!virt)
 241                 return -ENOMEM;
 242
 243         qed_chain_init_mem(chain, virt, phys);
 244         qed_chain_reset(chain);
 245
 246         return 0;
 247 }
 248
 249 static int qed_chain_alloc_pbl(struct qed_dev *cdev, struct qed_chain *chain)
 250 {
 251         struct device *dev = &cdev->pdev->dev;
 252         struct addr_tbl_entry *addr_tbl;
 253         dma_addr_t phys, pbl_phys;
 254         __le64 *pbl_virt;
 255         u32 page_cnt, i;
 256         size_t size;
 257         void *virt;
 258
 259         page_cnt = chain->page_cnt;
 260
 261         size = array_size(page_cnt, sizeof(*addr_tbl));
 262         if (unlikely(size == SIZE_MAX))
 263                 return -EOVERFLOW;
 264
 265         addr_tbl = vzalloc(size);
 266         if (!addr_tbl)
 267                 return -ENOMEM;
 268
 269         chain->pbl.pp_addr_tbl = addr_tbl;
 270
 271         if (chain->b_external_pbl) {
 272                 pbl_virt = chain->pbl_sp.table_virt;
 273                 goto alloc_pages;
 274         }
 275
 276         size = array_size(page_cnt, sizeof(*pbl_virt));
 277         if (unlikely(size == SIZE_MAX))
 278                 return -EOVERFLOW;
 279
 280         pbl_virt = dma_alloc_coherent(dev, size, &pbl_phys, GFP_KERNEL);
 281         if (!pbl_virt)
 282                 return -ENOMEM;
 283
 284         chain->pbl_sp.table_virt = pbl_virt;
 285         chain->pbl_sp.table_phys = pbl_phys;
 286         chain->pbl_sp.table_size = size;
 287
 288 alloc_pages:
 289         for (i = 0; i < page_cnt; i++) {
 290                 virt = dma_alloc_coherent(dev, chain->page_size, &phys,
 291                                           GFP_KERNEL);
 292                 if (!virt)
 293                         return -ENOMEM;
 294
 295                 if (i == 0) {
 296                         qed_chain_init_mem(chain, virt, phys);
 297                         qed_chain_reset(chain);
 298                 }
 299
 300                 /* Fill the PBL table with the physical address of the page */
 301                 pbl_virt[i] = cpu_to_le64(phys);
 302
 303                 /* Keep the virtual address of the page */
 304                 addr_tbl[i].virt_addr = virt;
 305                 addr_tbl[i].dma_map = phys;
 306         }
 307
 308         return 0;
 309 }
 310
 311 /**
 312  * qed_chain_alloc() - Allocate and initialize a chain.
 313  *
 314  * @cdev: Main device structure.
 315  * @chain: Chain to be processed.
 316  * @params: Chain initialization parameters.
 317  *
 318  * Return: 0 on success, negative errno otherwise.
 319  */
 320 int qed_chain_alloc(struct qed_dev *cdev, struct qed_chain *chain,
 321                     struct qed_chain_init_params *params)
 322 {
 323         u32 page_cnt;
 324         int rc;
 325
 326         if (!params->page_size)
 327                 params->page_size = QED_CHAIN_PAGE_SIZE;
 328
 329         if (params->mode == QED_CHAIN_MODE_SINGLE)
 330                 page_cnt = 1;
 331         else
 332                 page_cnt = QED_CHAIN_PAGE_CNT(params->num_elems,
 333                                               params->elem_size,
 334                                               params->page_size,
 335                                               params->mode);
 336
 337         rc = qed_chain_alloc_sanity_check(cdev, params, page_cnt);
 338         if (rc) {
 339                 DP_NOTICE(cdev,
 340                           "Cannot allocate a chain with the given arguments:\n");
 341                 DP_NOTICE(cdev,
 342                           "[use_mode %d, mode %d, cnt_type %d, num_elems %d, elem_size %zu, page_size %u]\n",
 343                           params->intended_use, params->mode, params->cnt_type,
 344                           params->num_elems, params->elem_size,
 345                           params->page_size);
 346                 return rc;
 347         }
 348
 349         qed_chain_init(chain, params, page_cnt);
 350
 351         switch (params->mode) {
 352         case QED_CHAIN_MODE_NEXT_PTR:
 353                 rc = qed_chain_alloc_next_ptr(cdev, chain);
 354                 break;
 355         case QED_CHAIN_MODE_SINGLE:
 356                 rc = qed_chain_alloc_single(cdev, chain);
 357                 break;
 358         case QED_CHAIN_MODE_PBL:
 359                 rc = qed_chain_alloc_pbl(cdev, chain);
 360                 break;
 361         default:
 362                 return -EINVAL;
 363         }
 364
 365         if (!rc)
 366                 return 0;
 367
 368         qed_chain_free(cdev, chain);
 369
 370         return rc;
 371 }