drivers/infiniband/hw/irdma/pble.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /* Copyright (c) 2015 - 2021 Intel Corporation */
   3 #include "osdep.h"
   4 #include "hmc.h"
   5 #include "defs.h"
   6 #include "type.h"
   7 #include "protos.h"
   8 #include "pble.h"
   9
  10 static int add_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc);
  11
  12 /**
  13  * irdma_destroy_pble_prm - destroy prm during module unload
  14  * @pble_rsrc: pble resources
  15  */
  16 void irdma_destroy_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc)
  17 {
  18         struct irdma_chunk *chunk;
  19         struct irdma_pble_prm *pinfo = &pble_rsrc->pinfo;
  20
  21         while (!list_empty(&pinfo->clist)) {
  22                 chunk = (struct irdma_chunk *) pinfo->clist.next;
  23                 list_del(&chunk->list);
  24                 if (chunk->type == PBLE_SD_PAGED)
  25                         irdma_pble_free_paged_mem(chunk);
  26                 bitmap_free(chunk->bitmapbuf);
  27                 kfree(chunk->chunkmem.va);
  28         }
  29 }
  30
  31 /**
  32  * irdma_hmc_init_pble - Initialize pble resources during module load
  33  * @dev: irdma_sc_dev struct
  34  * @pble_rsrc: pble resources
  35  */
  36 int irdma_hmc_init_pble(struct irdma_sc_dev *dev,
  37                         struct irdma_hmc_pble_rsrc *pble_rsrc)
  38 {
  39         struct irdma_hmc_info *hmc_info;
  40         u32 fpm_idx = 0;
  41         int status = 0;
  42
  43         hmc_info = dev->hmc_info;
  44         pble_rsrc->dev = dev;
  45         pble_rsrc->fpm_base_addr = hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].base;
  46         /* Start pble' on 4k boundary */
  47         if (pble_rsrc->fpm_base_addr & 0xfff)
  48                 fpm_idx = (4096 - (pble_rsrc->fpm_base_addr & 0xfff)) >> 3;
  49         pble_rsrc->unallocated_pble =
  50                 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt - fpm_idx;
  51         pble_rsrc->next_fpm_addr = pble_rsrc->fpm_base_addr + (fpm_idx << 3);
  52         pble_rsrc->pinfo.pble_shift = PBLE_SHIFT;
  53
  54         mutex_init(&pble_rsrc->pble_mutex_lock);
  55
  56         spin_lock_init(&pble_rsrc->pinfo.prm_lock);
  57         INIT_LIST_HEAD(&pble_rsrc->pinfo.clist);
  58         if (add_pble_prm(pble_rsrc)) {
  59                 irdma_destroy_pble_prm(pble_rsrc);
  60                 status = -ENOMEM;
  61         }
  62
  63         return status;
  64 }
  65
  66 /**
  67  * get_sd_pd_idx -  Returns sd index, pd index and rel_pd_idx from fpm address
  68  * @pble_rsrc: structure containing fpm address
  69  * @idx: where to return indexes
  70  */
  71 static void get_sd_pd_idx(struct irdma_hmc_pble_rsrc *pble_rsrc,
  72                           struct sd_pd_idx *idx)
  73 {
  74         idx->sd_idx = (u32)pble_rsrc->next_fpm_addr / IRDMA_HMC_DIRECT_BP_SIZE;
  75         idx->pd_idx = (u32)(pble_rsrc->next_fpm_addr / IRDMA_HMC_PAGED_BP_SIZE);
  76         idx->rel_pd_idx = (idx->pd_idx % IRDMA_HMC_PD_CNT_IN_SD);
  77 }
  78
  79 /**
  80  * add_sd_direct - add sd direct for pble
  81  * @pble_rsrc: pble resource ptr
  82  * @info: page info for sd
  83  */
  84 static int add_sd_direct(struct irdma_hmc_pble_rsrc *pble_rsrc,
  85                          struct irdma_add_page_info *info)
  86 {
  87         struct irdma_sc_dev *dev = pble_rsrc->dev;
  88         int ret_code = 0;
  89         struct sd_pd_idx *idx = &info->idx;
  90         struct irdma_chunk *chunk = info->chunk;
  91         struct irdma_hmc_info *hmc_info = info->hmc_info;
  92         struct irdma_hmc_sd_entry *sd_entry = info->sd_entry;
  93         u32 offset = 0;
  94
  95         if (!sd_entry->valid) {
  96                 ret_code = irdma_add_sd_table_entry(dev->hw, hmc_info,
  97                                                     info->idx.sd_idx,
  98                                                     IRDMA_SD_TYPE_DIRECT,
  99                                                     IRDMA_HMC_DIRECT_BP_SIZE);
 100                 if (ret_code)
 101                         return ret_code;
 102
 103                 chunk->type = PBLE_SD_CONTIGOUS;
 104         }
 105
 106         offset = idx->rel_pd_idx << HMC_PAGED_BP_SHIFT;
 107         chunk->size = info->pages << HMC_PAGED_BP_SHIFT;
 108         chunk->vaddr = sd_entry->u.bp.addr.va + offset;
 109         chunk->fpm_addr = pble_rsrc->next_fpm_addr;
 110         ibdev_dbg(to_ibdev(dev),
 111                   "PBLE: chunk_size[%lld] = 0x%llx vaddr=0x%pK fpm_addr = %llx\n",
 112                   chunk->size, chunk->size, chunk->vaddr, chunk->fpm_addr);
 113
 114         return 0;
 115 }
 116
 117 /**
 118  * fpm_to_idx - given fpm address, get pble index
 119  * @pble_rsrc: pble resource management
 120  * @addr: fpm address for index
 121  */
 122 static u32 fpm_to_idx(struct irdma_hmc_pble_rsrc *pble_rsrc, u64 addr)
 123 {
 124         u64 idx;
 125
 126         idx = (addr - (pble_rsrc->fpm_base_addr)) >> 3;
 127
 128         return (u32)idx;
 129 }
 130
 131 /**
 132  * add_bp_pages - add backing pages for sd
 133  * @pble_rsrc: pble resource management
 134  * @info: page info for sd
 135  */
 136 static int add_bp_pages(struct irdma_hmc_pble_rsrc *pble_rsrc,
 137                         struct irdma_add_page_info *info)
 138 {
 139         struct irdma_sc_dev *dev = pble_rsrc->dev;
 140         u8 *addr;
 141         struct irdma_dma_mem mem;
 142         struct irdma_hmc_pd_entry *pd_entry;
 143         struct irdma_hmc_sd_entry *sd_entry = info->sd_entry;
 144         struct irdma_hmc_info *hmc_info = info->hmc_info;
 145         struct irdma_chunk *chunk = info->chunk;
 146         int status = 0;
 147         u32 rel_pd_idx = info->idx.rel_pd_idx;
 148         u32 pd_idx = info->idx.pd_idx;
 149         u32 i;
 150
 151         if (irdma_pble_get_paged_mem(chunk, info->pages))
 152                 return -ENOMEM;
 153
 154         status = irdma_add_sd_table_entry(dev->hw, hmc_info, info->idx.sd_idx,
 155                                           IRDMA_SD_TYPE_PAGED,
 156                                           IRDMA_HMC_DIRECT_BP_SIZE);
 157         if (status)
 158                 goto error;
 159
 160         addr = chunk->vaddr;
 161         for (i = 0; i < info->pages; i++) {
 162                 mem.pa = (u64)chunk->dmainfo.dmaaddrs[i];
 163                 mem.size = 4096;
 164                 mem.va = addr;
 165                 pd_entry = &sd_entry->u.pd_table.pd_entry[rel_pd_idx++];
 166                 if (!pd_entry->valid) {
 167                         status = irdma_add_pd_table_entry(dev, hmc_info,
 168                                                           pd_idx++, &mem);
 169                         if (status)
 170                                 goto error;
 171
 172                         addr += 4096;
 173                 }
 174         }
 175
 176         chunk->fpm_addr = pble_rsrc->next_fpm_addr;
 177         return 0;
 178
 179 error:
 180         irdma_pble_free_paged_mem(chunk);
 181
 182         return status;
 183 }
 184
 185 /**
 186  * irdma_get_type - add a sd entry type for sd
 187  * @dev: irdma_sc_dev struct
 188  * @idx: index of sd
 189  * @pages: pages in the sd
 190  */
 191 static enum irdma_sd_entry_type irdma_get_type(struct irdma_sc_dev *dev,
 192                                                struct sd_pd_idx *idx, u32 pages)
 193 {
 194         enum irdma_sd_entry_type sd_entry_type;
 195
 196         sd_entry_type = !idx->rel_pd_idx && pages == IRDMA_HMC_PD_CNT_IN_SD ?
 197                         IRDMA_SD_TYPE_DIRECT : IRDMA_SD_TYPE_PAGED;
 198         return sd_entry_type;
 199 }
 200
 201 /**
 202  * add_pble_prm - add a sd entry for pble resoure
 203  * @pble_rsrc: pble resource management
 204  */
 205 static int add_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc)
 206 {
 207         struct irdma_sc_dev *dev = pble_rsrc->dev;
 208         struct irdma_hmc_sd_entry *sd_entry;
 209         struct irdma_hmc_info *hmc_info;
 210         struct irdma_chunk *chunk;
 211         struct irdma_add_page_info info;
 212         struct sd_pd_idx *idx = &info.idx;
 213         int ret_code = 0;
 214         enum irdma_sd_entry_type sd_entry_type;
 215         u64 sd_reg_val = 0;
 216         struct irdma_virt_mem chunkmem;
 217         u32 pages;
 218
 219         if (pble_rsrc->unallocated_pble < PBLE_PER_PAGE)
 220                 return -ENOMEM;
 221
 222         if (pble_rsrc->next_fpm_addr & 0xfff)
 223                 return -EINVAL;
 224
 225         chunkmem.size = sizeof(*chunk);
 226         chunkmem.va = kzalloc(chunkmem.size, GFP_KERNEL);
 227         if (!chunkmem.va)
 228                 return -ENOMEM;
 229
 230         chunk = chunkmem.va;
 231         chunk->chunkmem = chunkmem;
 232         hmc_info = dev->hmc_info;
 233         chunk->dev = dev;
 234         chunk->fpm_addr = pble_rsrc->next_fpm_addr;
 235         get_sd_pd_idx(pble_rsrc, idx);
 236         sd_entry = &hmc_info->sd_table.sd_entry[idx->sd_idx];
 237         pages = (idx->rel_pd_idx) ? (IRDMA_HMC_PD_CNT_IN_SD - idx->rel_pd_idx) :
 238                                     IRDMA_HMC_PD_CNT_IN_SD;
 239         pages = min(pages, pble_rsrc->unallocated_pble >> PBLE_512_SHIFT);
 240         info.chunk = chunk;
 241         info.hmc_info = hmc_info;
 242         info.pages = pages;
 243         info.sd_entry = sd_entry;
 244         if (!sd_entry->valid)
 245                 sd_entry_type = irdma_get_type(dev, idx, pages);
 246         else
 247                 sd_entry_type = sd_entry->entry_type;
 248
 249         ibdev_dbg(to_ibdev(dev),
 250                   "PBLE: pages = %d, unallocated_pble[%d] current_fpm_addr = %llx\n",
 251                   pages, pble_rsrc->unallocated_pble,
 252                   pble_rsrc->next_fpm_addr);
 253         ibdev_dbg(to_ibdev(dev), "PBLE: sd_entry_type = %d\n", sd_entry_type);
 254         if (sd_entry_type == IRDMA_SD_TYPE_DIRECT)
 255                 ret_code = add_sd_direct(pble_rsrc, &info);
 256
 257         if (ret_code)
 258                 sd_entry_type = IRDMA_SD_TYPE_PAGED;
 259         else
 260                 pble_rsrc->stats_direct_sds++;
 261
 262         if (sd_entry_type == IRDMA_SD_TYPE_PAGED) {
 263                 ret_code = add_bp_pages(pble_rsrc, &info);
 264                 if (ret_code)
 265                         goto error;
 266                 else
 267                         pble_rsrc->stats_paged_sds++;
 268         }
 269
 270         ret_code = irdma_prm_add_pble_mem(&pble_rsrc->pinfo, chunk);
 271         if (ret_code)
 272                 goto error;
 273
 274         pble_rsrc->next_fpm_addr += chunk->size;
 275         ibdev_dbg(to_ibdev(dev),
 276                   "PBLE: next_fpm_addr = %llx chunk_size[%llu] = 0x%llx\n",
 277                   pble_rsrc->next_fpm_addr, chunk->size, chunk->size);
 278         pble_rsrc->unallocated_pble -= (u32)(chunk->size >> 3);
 279         sd_reg_val = (sd_entry_type == IRDMA_SD_TYPE_PAGED) ?
 280                              sd_entry->u.pd_table.pd_page_addr.pa :
 281                              sd_entry->u.bp.addr.pa;
 282
 283         if (!sd_entry->valid) {
 284                 ret_code = irdma_hmc_sd_one(dev, hmc_info->hmc_fn_id, sd_reg_val,
 285                                             idx->sd_idx, sd_entry->entry_type, true);
 286                 if (ret_code)
 287                         goto error;
 288         }
 289
 290         list_add(&chunk->list, &pble_rsrc->pinfo.clist);
 291         sd_entry->valid = true;
 292         return 0;
 293
 294 error:
 295         bitmap_free(chunk->bitmapbuf);
 296         kfree(chunk->chunkmem.va);
 297
 298         return ret_code;
 299 }
 300
 301 /**
 302  * free_lvl2 - fee level 2 pble
 303  * @pble_rsrc: pble resource management
 304  * @palloc: level 2 pble allocation
 305  */
 306 static void free_lvl2(struct irdma_hmc_pble_rsrc *pble_rsrc,
 307                       struct irdma_pble_alloc *palloc)
 308 {
 309         u32 i;
 310         struct irdma_pble_level2 *lvl2 = &palloc->level2;
 311         struct irdma_pble_info *root = &lvl2->root;
 312         struct irdma_pble_info *leaf = lvl2->leaf;
 313
 314         for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) {
 315                 if (leaf->addr)
 316                         irdma_prm_return_pbles(&pble_rsrc->pinfo,
 317                                                &leaf->chunkinfo);
 318                 else
 319                         break;
 320         }
 321
 322         if (root->addr)
 323                 irdma_prm_return_pbles(&pble_rsrc->pinfo, &root->chunkinfo);
 324
 325         kfree(lvl2->leafmem.va);
 326         lvl2->leaf = NULL;
 327 }
 328
 329 /**
 330  * get_lvl2_pble - get level 2 pble resource
 331  * @pble_rsrc: pble resource management
 332  * @palloc: level 2 pble allocation
 333  */
 334 static int get_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
 335                          struct irdma_pble_alloc *palloc)
 336 {
 337         u32 lf4k, lflast, total, i;
 338         u32 pblcnt = PBLE_PER_PAGE;
 339         u64 *addr;
 340         struct irdma_pble_level2 *lvl2 = &palloc->level2;
 341         struct irdma_pble_info *root = &lvl2->root;
 342         struct irdma_pble_info *leaf;
 343         int ret_code;
 344         u64 fpm_addr;
 345
 346         /* number of full 512 (4K) leafs) */
 347         lf4k = palloc->total_cnt >> 9;
 348         lflast = palloc->total_cnt % PBLE_PER_PAGE;
 349         total = (lflast == 0) ? lf4k : lf4k + 1;
 350         lvl2->leaf_cnt = total;
 351
 352         lvl2->leafmem.size = (sizeof(*leaf) * total);
 353         lvl2->leafmem.va = kzalloc(lvl2->leafmem.size, GFP_KERNEL);
 354         if (!lvl2->leafmem.va)
 355                 return -ENOMEM;
 356
 357         lvl2->leaf = lvl2->leafmem.va;
 358         leaf = lvl2->leaf;
 359         ret_code = irdma_prm_get_pbles(&pble_rsrc->pinfo, &root->chunkinfo,
 360                                        total << 3, &root->addr, &fpm_addr);
 361         if (ret_code) {
 362                 kfree(lvl2->leafmem.va);
 363                 lvl2->leaf = NULL;
 364                 return -ENOMEM;
 365         }
 366
 367         root->idx = fpm_to_idx(pble_rsrc, fpm_addr);
 368         root->cnt = total;
 369         addr = root->addr;
 370         for (i = 0; i < total; i++, leaf++) {
 371                 pblcnt = (lflast && ((i + 1) == total)) ?
 372                                 lflast : PBLE_PER_PAGE;
 373                 ret_code = irdma_prm_get_pbles(&pble_rsrc->pinfo,
 374                                                &leaf->chunkinfo, pblcnt << 3,
 375                                                &leaf->addr, &fpm_addr);
 376                 if (ret_code)
 377                         goto error;
 378
 379                 leaf->idx = fpm_to_idx(pble_rsrc, fpm_addr);
 380
 381                 leaf->cnt = pblcnt;
 382                 *addr = (u64)leaf->idx;
 383                 addr++;
 384         }
 385
 386         palloc->level = PBLE_LEVEL_2;
 387         pble_rsrc->stats_lvl2++;
 388         return 0;
 389
 390 error:
 391         free_lvl2(pble_rsrc, palloc);
 392
 393         return -ENOMEM;
 394 }
 395
 396 /**
 397  * get_lvl1_pble - get level 1 pble resource
 398  * @pble_rsrc: pble resource management
 399  * @palloc: level 1 pble allocation
 400  */
 401 static int get_lvl1_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
 402                          struct irdma_pble_alloc *palloc)
 403 {
 404         int ret_code;
 405         u64 fpm_addr;
 406         struct irdma_pble_info *lvl1 = &palloc->level1;
 407
 408         ret_code = irdma_prm_get_pbles(&pble_rsrc->pinfo, &lvl1->chunkinfo,
 409                                        palloc->total_cnt << 3, &lvl1->addr,
 410                                        &fpm_addr);
 411         if (ret_code)
 412                 return -ENOMEM;
 413
 414         palloc->level = PBLE_LEVEL_1;
 415         lvl1->idx = fpm_to_idx(pble_rsrc, fpm_addr);
 416         lvl1->cnt = palloc->total_cnt;
 417         pble_rsrc->stats_lvl1++;
 418
 419         return 0;
 420 }
 421
 422 /**
 423  * get_lvl1_lvl2_pble - calls get_lvl1 and get_lvl2 pble routine
 424  * @pble_rsrc: pble resources
 425  * @palloc: contains all inforamtion regarding pble (idx + pble addr)
 426  * @lvl: Bitmask for requested pble level
 427  */
 428 static int get_lvl1_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
 429                               struct irdma_pble_alloc *palloc, u8 lvl)
 430 {
 431         int status = 0;
 432
 433         status = get_lvl1_pble(pble_rsrc, palloc);
 434         if (!status || lvl == PBLE_LEVEL_1 || palloc->total_cnt <= PBLE_PER_PAGE)
 435                 return status;
 436
 437         status = get_lvl2_pble(pble_rsrc, palloc);
 438
 439         return status;
 440 }
 441
 442 /**
 443  * irdma_get_pble - allocate pbles from the prm
 444  * @pble_rsrc: pble resources
 445  * @palloc: contains all inforamtion regarding pble (idx + pble addr)
 446  * @pble_cnt: #of pbles requested
 447  * @lvl: requested pble level mask
 448  */
 449 int irdma_get_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
 450                    struct irdma_pble_alloc *palloc, u32 pble_cnt,
 451                    u8 lvl)
 452 {
 453         int status = 0;
 454         int max_sds = 0;
 455         int i;
 456
 457         palloc->total_cnt = pble_cnt;
 458         palloc->level = PBLE_LEVEL_0;
 459
 460         mutex_lock(&pble_rsrc->pble_mutex_lock);
 461
 462         /*check first to see if we can get pble's without acquiring
 463          * additional sd's
 464          */
 465         status = get_lvl1_lvl2_pble(pble_rsrc, palloc, lvl);
 466         if (!status)
 467                 goto exit;
 468
 469         max_sds = (palloc->total_cnt >> 18) + 1;
 470         for (i = 0; i < max_sds; i++) {
 471                 status = add_pble_prm(pble_rsrc);
 472                 if (status)
 473                         break;
 474
 475                 status = get_lvl1_lvl2_pble(pble_rsrc, palloc, lvl);
 476                 /* if level1_only, only go through it once */
 477                 if (!status || lvl)
 478                         break;
 479         }
 480
 481 exit:
 482         if (!status) {
 483                 pble_rsrc->allocdpbles += pble_cnt;
 484                 pble_rsrc->stats_alloc_ok++;
 485         } else {
 486                 pble_rsrc->stats_alloc_fail++;
 487         }
 488         mutex_unlock(&pble_rsrc->pble_mutex_lock);
 489
 490         return status;
 491 }
 492
 493 /**
 494  * irdma_free_pble - put pbles back into prm
 495  * @pble_rsrc: pble resources
 496  * @palloc: contains all information regarding pble resource being freed
 497  */
 498 void irdma_free_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
 499                      struct irdma_pble_alloc *palloc)
 500 {
 501         pble_rsrc->freedpbles += palloc->total_cnt;
 502
 503         if (palloc->level == PBLE_LEVEL_2)
 504                 free_lvl2(pble_rsrc, palloc);
 505         else
 506                 irdma_prm_return_pbles(&pble_rsrc->pinfo,
 507                                        &palloc->level1.chunkinfo);
 508         pble_rsrc->stats_alloc_freed++;
 509 }