drivers/vdpa/mlx5/core/mr.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /* Copyright (c) 2020 Mellanox Technologies Ltd. */
   3
   4 #include <linux/vhost_types.h>
   5 #include <linux/vdpa.h>
   6 #include <linux/gcd.h>
   7 #include <linux/string.h>
   8 #include <linux/mlx5/qp.h>
   9 #include "mlx5_vdpa.h"
  10
  11 /* DIV_ROUND_UP where the divider is a power of 2 give by its log base 2 value */
  12 #define MLX5_DIV_ROUND_UP_POW2(_n, _s) \
  13 ({ \
  14         u64 __s = _s; \
  15         u64 _res; \
  16         _res = (((_n) + (1 << (__s)) - 1) >> (__s)); \
  17         _res; \
  18 })
  19
  20 static int get_octo_len(u64 len, int page_shift)
  21 {
  22         u64 page_size = 1ULL << page_shift;
  23         int npages;
  24
  25         npages = ALIGN(len, page_size) >> page_shift;
  26         return (npages + 1) / 2;
  27 }
  28
  29 static void mlx5_set_access_mode(void *mkc, int mode)
  30 {
  31         MLX5_SET(mkc, mkc, access_mode_1_0, mode & 0x3);
  32         MLX5_SET(mkc, mkc, access_mode_4_2, mode >> 2);
  33 }
  34
  35 static void populate_mtts(struct mlx5_vdpa_direct_mr *mr, __be64 *mtt)
  36 {
  37         struct scatterlist *sg;
  38         int nsg = mr->nsg;
  39         u64 dma_addr;
  40         u64 dma_len;
  41         int j = 0;
  42         int i;
  43
  44         for_each_sg(mr->sg_head.sgl, sg, mr->nent, i) {
  45                 for (dma_addr = sg_dma_address(sg), dma_len = sg_dma_len(sg);
  46                      nsg && dma_len;
  47                      nsg--, dma_addr += BIT(mr->log_size), dma_len -= BIT(mr->log_size))
  48                         mtt[j++] = cpu_to_be64(dma_addr);
  49         }
  50 }
  51
  52 struct mlx5_create_mkey_mem {
  53         u8 out[MLX5_ST_SZ_BYTES(create_mkey_out)];
  54         u8 in[MLX5_ST_SZ_BYTES(create_mkey_in)];
  55         __be64 mtt[];
  56 };
  57
  58 struct mlx5_destroy_mkey_mem {
  59         u8 out[MLX5_ST_SZ_BYTES(destroy_mkey_out)];
  60         u8 in[MLX5_ST_SZ_BYTES(destroy_mkey_in)];
  61 };
  62
  63 static void fill_create_direct_mr(struct mlx5_vdpa_dev *mvdev,
  64                                   struct mlx5_vdpa_direct_mr *mr,
  65                                   struct mlx5_create_mkey_mem *mem)
  66 {
  67         void *in = &mem->in;
  68         void *mkc;
  69
  70         MLX5_SET(create_mkey_in, in, uid, mvdev->res.uid);
  71         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
  72         MLX5_SET(mkc, mkc, lw, !!(mr->perm & VHOST_MAP_WO));
  73         MLX5_SET(mkc, mkc, lr, !!(mr->perm & VHOST_MAP_RO));
  74         mlx5_set_access_mode(mkc, MLX5_MKC_ACCESS_MODE_MTT);
  75         MLX5_SET(mkc, mkc, qpn, 0xffffff);
  76         MLX5_SET(mkc, mkc, pd, mvdev->res.pdn);
  77         MLX5_SET64(mkc, mkc, start_addr, mr->offset);
  78         MLX5_SET64(mkc, mkc, len, mr->end - mr->start);
  79         MLX5_SET(mkc, mkc, log_page_size, mr->log_size);
  80         MLX5_SET(mkc, mkc, translations_octword_size,
  81                  get_octo_len(mr->end - mr->start, mr->log_size));
  82         MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
  83                  get_octo_len(mr->end - mr->start, mr->log_size));
  84         populate_mtts(mr, MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt));
  85
  86         MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY);
  87         MLX5_SET(create_mkey_in, in, uid, mvdev->res.uid);
  88 }
  89
  90 static void create_direct_mr_end(struct mlx5_vdpa_dev *mvdev,
  91                                  struct mlx5_vdpa_direct_mr *mr,
  92                                  struct mlx5_create_mkey_mem *mem)
  93 {
  94         u32 mkey_index = MLX5_GET(create_mkey_out, mem->out, mkey_index);
  95
  96         mr->mr = mlx5_idx_to_mkey(mkey_index);
  97 }
  98
  99 static void fill_destroy_direct_mr(struct mlx5_vdpa_dev *mvdev,
 100                                    struct mlx5_vdpa_direct_mr *mr,
 101                                    struct mlx5_destroy_mkey_mem *mem)
 102 {
 103         void *in = &mem->in;
 104
 105         MLX5_SET(destroy_mkey_in, in, uid, mvdev->res.uid);
 106         MLX5_SET(destroy_mkey_in, in, opcode, MLX5_CMD_OP_DESTROY_MKEY);
 107         MLX5_SET(destroy_mkey_in, in, mkey_index, mlx5_mkey_to_idx(mr->mr));
 108 }
 109
 110 static void destroy_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr)
 111 {
 112         if (!mr->mr)
 113                 return;
 114
 115         mlx5_vdpa_destroy_mkey(mvdev, mr->mr);
 116 }
 117
 118 static u64 map_start(struct vhost_iotlb_map *map, struct mlx5_vdpa_direct_mr *mr)
 119 {
 120         return max_t(u64, map->start, mr->start);
 121 }
 122
 123 static u64 map_end(struct vhost_iotlb_map *map, struct mlx5_vdpa_direct_mr *mr)
 124 {
 125         return min_t(u64, map->last + 1, mr->end);
 126 }
 127
 128 static u64 maplen(struct vhost_iotlb_map *map, struct mlx5_vdpa_direct_mr *mr)
 129 {
 130         return map_end(map, mr) - map_start(map, mr);
 131 }
 132
 133 #define MLX5_VDPA_INVALID_START_ADDR ((u64)-1)
 134 #define MLX5_VDPA_INVALID_LEN ((u64)-1)
 135
 136 static u64 indir_start_addr(struct mlx5_vdpa_mr *mkey)
 137 {
 138         struct mlx5_vdpa_direct_mr *s;
 139
 140         s = list_first_entry_or_null(&mkey->head, struct mlx5_vdpa_direct_mr, list);
 141         if (!s)
 142                 return MLX5_VDPA_INVALID_START_ADDR;
 143
 144         return s->start;
 145 }
 146
 147 static u64 indir_len(struct mlx5_vdpa_mr *mkey)
 148 {
 149         struct mlx5_vdpa_direct_mr *s;
 150         struct mlx5_vdpa_direct_mr *e;
 151
 152         s = list_first_entry_or_null(&mkey->head, struct mlx5_vdpa_direct_mr, list);
 153         if (!s)
 154                 return MLX5_VDPA_INVALID_LEN;
 155
 156         e = list_last_entry(&mkey->head, struct mlx5_vdpa_direct_mr, list);
 157
 158         return e->end - s->start;
 159 }
 160
 161 #define LOG_MAX_KLM_SIZE 30
 162 #define MAX_KLM_SIZE BIT(LOG_MAX_KLM_SIZE)
 163
 164 static u32 klm_bcount(u64 size)
 165 {
 166         return (u32)size;
 167 }
 168
 169 static void fill_indir(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mkey, void *in)
 170 {
 171         struct mlx5_vdpa_direct_mr *dmr;
 172         struct mlx5_klm *klmarr;
 173         struct mlx5_klm *klm;
 174         bool first = true;
 175         u64 preve;
 176         int i;
 177
 178         klmarr = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
 179         i = 0;
 180         list_for_each_entry(dmr, &mkey->head, list) {
 181 again:
 182                 klm = &klmarr[i++];
 183                 if (first) {
 184                         preve = dmr->start;
 185                         first = false;
 186                 }
 187
 188                 if (preve == dmr->start) {
 189                         klm->key = cpu_to_be32(dmr->mr);
 190                         klm->bcount = cpu_to_be32(klm_bcount(dmr->end - dmr->start));
 191                         preve = dmr->end;
 192                 } else {
 193                         klm->key = cpu_to_be32(mvdev->res.null_mkey);
 194                         klm->bcount = cpu_to_be32(klm_bcount(dmr->start - preve));
 195                         preve = dmr->start;
 196                         goto again;
 197                 }
 198         }
 199 }
 200
 201 static int klm_byte_size(int nklms)
 202 {
 203         return 16 * ALIGN(nklms, 4);
 204 }
 205
 206 #define MLX5_VDPA_MTT_ALIGN 16
 207
 208 static int create_direct_keys(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)
 209 {
 210         struct mlx5_vdpa_async_cmd *cmds;
 211         struct mlx5_vdpa_direct_mr *dmr;
 212         int err = 0;
 213         int i = 0;
 214
 215         cmds = kvcalloc(mr->num_directs, sizeof(*cmds), GFP_KERNEL);
 216         if (!cmds)
 217                 return -ENOMEM;
 218
 219         list_for_each_entry(dmr, &mr->head, list) {
 220                 struct mlx5_create_mkey_mem *cmd_mem;
 221                 int mttlen, mttcount;
 222
 223                 mttlen = roundup(MLX5_ST_SZ_BYTES(mtt) * dmr->nsg, MLX5_VDPA_MTT_ALIGN);
 224                 mttcount = mttlen / sizeof(cmd_mem->mtt[0]);
 225                 cmd_mem = kvcalloc(1, struct_size(cmd_mem, mtt, mttcount), GFP_KERNEL);
 226                 if (!cmd_mem) {
 227                         err = -ENOMEM;
 228                         goto done;
 229                 }
 230
 231                 cmds[i].out = cmd_mem->out;
 232                 cmds[i].outlen = sizeof(cmd_mem->out);
 233                 cmds[i].in = cmd_mem->in;
 234                 cmds[i].inlen = struct_size(cmd_mem, mtt, mttcount);
 235
 236                 fill_create_direct_mr(mvdev, dmr, cmd_mem);
 237
 238                 i++;
 239         }
 240
 241         err = mlx5_vdpa_exec_async_cmds(mvdev, cmds, mr->num_directs);
 242         if (err) {
 243
 244                 mlx5_vdpa_err(mvdev, "error issuing MTT mkey creation for direct mrs: %d\n", err);
 245                 goto done;
 246         }
 247
 248         i = 0;
 249         list_for_each_entry(dmr, &mr->head, list) {
 250                 struct mlx5_vdpa_async_cmd *cmd = &cmds[i++];
 251                 struct mlx5_create_mkey_mem *cmd_mem;
 252
 253                 cmd_mem = container_of(cmd->out, struct mlx5_create_mkey_mem, out);
 254
 255                 if (!cmd->err) {
 256                         create_direct_mr_end(mvdev, dmr, cmd_mem);
 257                 } else {
 258                         err = err ? err : cmd->err;
 259                         mlx5_vdpa_err(mvdev, "error creating MTT mkey [0x%llx, 0x%llx]: %d\n",
 260                                 dmr->start, dmr->end, cmd->err);
 261                 }
 262         }
 263
 264 done:
 265         for (i = i-1; i >= 0; i--) {
 266                 struct mlx5_create_mkey_mem *cmd_mem;
 267
 268                 cmd_mem = container_of(cmds[i].out, struct mlx5_create_mkey_mem, out);
 269                 kvfree(cmd_mem);
 270         }
 271
 272         kvfree(cmds);
 273         return err;
 274 }
 275
 276 DEFINE_FREE(free_cmds, struct mlx5_vdpa_async_cmd *, kvfree(_T))
 277 DEFINE_FREE(free_cmd_mem, struct mlx5_destroy_mkey_mem *, kvfree(_T))
 278
 279 static int destroy_direct_keys(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)
 280 {
 281         struct mlx5_destroy_mkey_mem *cmd_mem __free(free_cmd_mem) = NULL;
 282         struct mlx5_vdpa_async_cmd *cmds __free(free_cmds) = NULL;
 283         struct mlx5_vdpa_direct_mr *dmr;
 284         int err = 0;
 285         int i = 0;
 286
 287         cmds = kvcalloc(mr->num_directs, sizeof(*cmds), GFP_KERNEL);
 288         cmd_mem = kvcalloc(mr->num_directs, sizeof(*cmd_mem), GFP_KERNEL);
 289         if (!cmds || !cmd_mem)
 290                 return -ENOMEM;
 291
 292         list_for_each_entry(dmr, &mr->head, list) {
 293                 cmds[i].out = cmd_mem[i].out;
 294                 cmds[i].outlen = sizeof(cmd_mem[i].out);
 295                 cmds[i].in = cmd_mem[i].in;
 296                 cmds[i].inlen = sizeof(cmd_mem[i].in);
 297                 fill_destroy_direct_mr(mvdev, dmr, &cmd_mem[i]);
 298                 i++;
 299         }
 300
 301         err = mlx5_vdpa_exec_async_cmds(mvdev, cmds, mr->num_directs);
 302         if (err) {
 303
 304                 mlx5_vdpa_err(mvdev, "error issuing MTT mkey deletion for direct mrs: %d\n", err);
 305                 return err;
 306         }
 307
 308         i = 0;
 309         list_for_each_entry(dmr, &mr->head, list) {
 310                 struct mlx5_vdpa_async_cmd *cmd = &cmds[i++];
 311
 312                 dmr->mr = 0;
 313                 if (cmd->err) {
 314                         err = err ? err : cmd->err;
 315                         mlx5_vdpa_err(mvdev, "error deleting MTT mkey [0x%llx, 0x%llx]: %d\n",
 316                                 dmr->start, dmr->end, cmd->err);
 317                 }
 318         }
 319
 320         return err;
 321 }
 322
 323 static int create_indirect_key(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)
 324 {
 325         int inlen;
 326         void *mkc;
 327         void *in;
 328         int err;
 329         u64 start;
 330         u64 len;
 331
 332         start = indir_start_addr(mr);
 333         len = indir_len(mr);
 334         if (start == MLX5_VDPA_INVALID_START_ADDR || len == MLX5_VDPA_INVALID_LEN)
 335                 return -EINVAL;
 336
 337         inlen = MLX5_ST_SZ_BYTES(create_mkey_in) + klm_byte_size(mr->num_klms);
 338         in = kzalloc(inlen, GFP_KERNEL);
 339         if (!in)
 340                 return -ENOMEM;
 341
 342         MLX5_SET(create_mkey_in, in, uid, mvdev->res.uid);
 343         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
 344         MLX5_SET(mkc, mkc, lw, 1);
 345         MLX5_SET(mkc, mkc, lr, 1);
 346         mlx5_set_access_mode(mkc, MLX5_MKC_ACCESS_MODE_KLMS);
 347         MLX5_SET(mkc, mkc, qpn, 0xffffff);
 348         MLX5_SET(mkc, mkc, pd, mvdev->res.pdn);
 349         MLX5_SET64(mkc, mkc, start_addr, start);
 350         MLX5_SET64(mkc, mkc, len, len);
 351         MLX5_SET(mkc, mkc, translations_octword_size, klm_byte_size(mr->num_klms) / 16);
 352         MLX5_SET(create_mkey_in, in, translations_octword_actual_size, mr->num_klms);
 353         fill_indir(mvdev, mr, in);
 354         err = mlx5_vdpa_create_mkey(mvdev, &mr->mkey, in, inlen);
 355         kfree(in);
 356         return err;
 357 }
 358
 359 static void destroy_indirect_key(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mkey)
 360 {
 361         mlx5_vdpa_destroy_mkey(mvdev, mkey->mkey);
 362 }
 363
 364 static int map_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr,
 365                          struct vhost_iotlb *iotlb)
 366 {
 367         struct vhost_iotlb_map *map;
 368         unsigned long lgcd = 0;
 369         int log_entity_size;
 370         unsigned long size;
 371         int err;
 372         struct page *pg;
 373         unsigned int nsg;
 374         int sglen;
 375         u64 pa, offset;
 376         u64 paend;
 377         struct scatterlist *sg;
 378         struct device *dma = mvdev->vdev.dma_dev;
 379
 380         for (map = vhost_iotlb_itree_first(iotlb, mr->start, mr->end - 1);
 381              map; map = vhost_iotlb_itree_next(map, mr->start, mr->end - 1)) {
 382                 size = maplen(map, mr);
 383                 lgcd = gcd(lgcd, size);
 384         }
 385         log_entity_size = ilog2(lgcd);
 386
 387         sglen = 1 << log_entity_size;
 388         nsg = MLX5_DIV_ROUND_UP_POW2(mr->end - mr->start, log_entity_size);
 389
 390         err = sg_alloc_table(&mr->sg_head, nsg, GFP_KERNEL);
 391         if (err)
 392                 return err;
 393
 394         sg = mr->sg_head.sgl;
 395         for (map = vhost_iotlb_itree_first(iotlb, mr->start, mr->end - 1);
 396              map; map = vhost_iotlb_itree_next(map, mr->start, mr->end - 1)) {
 397                 offset = mr->start > map->start ? mr->start - map->start : 0;
 398                 pa = map->addr + offset;
 399                 paend = map->addr + offset + maplen(map, mr);
 400                 for (; pa < paend; pa += sglen) {
 401                         pg = pfn_to_page(__phys_to_pfn(pa));
 402                         if (!sg) {
 403                                 mlx5_vdpa_warn(mvdev, "sg null. start 0x%llx, end 0x%llx\n",
 404                                                map->start, map->last + 1);
 405                                 err = -ENOMEM;
 406                                 goto err_map;
 407                         }
 408                         sg_set_page(sg, pg, sglen, 0);
 409                         sg = sg_next(sg);
 410                         if (!sg)
 411                                 goto done;
 412                 }
 413         }
 414 done:
 415         mr->log_size = log_entity_size;
 416         mr->nsg = nsg;
 417         mr->nent = dma_map_sg_attrs(dma, mr->sg_head.sgl, mr->nsg, DMA_BIDIRECTIONAL, 0);
 418         if (!mr->nent) {
 419                 err = -ENOMEM;
 420                 goto err_map;
 421         }
 422
 423         return 0;
 424
 425 err_map:
 426         sg_free_table(&mr->sg_head);
 427         return err;
 428 }
 429
 430 static void unmap_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr)
 431 {
 432         struct device *dma = mvdev->vdev.dma_dev;
 433
 434         destroy_direct_mr(mvdev, mr);
 435         dma_unmap_sg_attrs(dma, mr->sg_head.sgl, mr->nsg, DMA_BIDIRECTIONAL, 0);
 436         sg_free_table(&mr->sg_head);
 437 }
 438
 439 static int add_direct_chain(struct mlx5_vdpa_dev *mvdev,
 440                             struct mlx5_vdpa_mr *mr,
 441                             u64 start,
 442                             u64 size,
 443                             u8 perm,
 444                             struct vhost_iotlb *iotlb)
 445 {
 446         struct mlx5_vdpa_direct_mr *dmr;
 447         struct mlx5_vdpa_direct_mr *n;
 448         LIST_HEAD(tmp);
 449         u64 st;
 450         u64 sz;
 451         int err;
 452
 453         st = start;
 454         while (size) {
 455                 sz = (u32)min_t(u64, MAX_KLM_SIZE, size);
 456                 dmr = kzalloc(sizeof(*dmr), GFP_KERNEL);
 457                 if (!dmr) {
 458                         err = -ENOMEM;
 459                         goto err_alloc;
 460                 }
 461
 462                 dmr->start = st;
 463                 dmr->end = st + sz;
 464                 dmr->perm = perm;
 465                 err = map_direct_mr(mvdev, dmr, iotlb);
 466                 if (err) {
 467                         kfree(dmr);
 468                         goto err_alloc;
 469                 }
 470
 471                 list_add_tail(&dmr->list, &tmp);
 472                 size -= sz;
 473                 mr->num_directs++;
 474                 mr->num_klms++;
 475                 st += sz;
 476         }
 477         list_splice_tail(&tmp, &mr->head);
 478         return 0;
 479
 480 err_alloc:
 481         list_for_each_entry_safe(dmr, n, &mr->head, list) {
 482                 list_del_init(&dmr->list);
 483                 unmap_direct_mr(mvdev, dmr);
 484                 kfree(dmr);
 485         }
 486         return err;
 487 }
 488
 489 /* The iotlb pointer contains a list of maps. Go over the maps, possibly
 490  * merging mergeable maps, and create direct memory keys that provide the
 491  * device access to memory. The direct mkeys are then referred to by the
 492  * indirect memory key that provides access to the enitre address space given
 493  * by iotlb.
 494  */
 495 static int create_user_mr(struct mlx5_vdpa_dev *mvdev,
 496                           struct mlx5_vdpa_mr *mr,
 497                           struct vhost_iotlb *iotlb)
 498 {
 499         struct mlx5_vdpa_direct_mr *dmr;
 500         struct mlx5_vdpa_direct_mr *n;
 501         struct vhost_iotlb_map *map;
 502         u32 pperm = U16_MAX;
 503         u64 last = U64_MAX;
 504         u64 ps = U64_MAX;
 505         u64 pe = U64_MAX;
 506         u64 start = 0;
 507         int err = 0;
 508         int nnuls;
 509
 510         INIT_LIST_HEAD(&mr->head);
 511         for (map = vhost_iotlb_itree_first(iotlb, start, last); map;
 512              map = vhost_iotlb_itree_next(map, start, last)) {
 513                 start = map->start;
 514                 if (pe == map->start && pperm == map->perm) {
 515                         pe = map->last + 1;
 516                 } else {
 517                         if (ps != U64_MAX) {
 518                                 if (pe < map->start) {
 519                                         /* We have a hole in the map. Check how
 520                                          * many null keys are required to fill it.
 521                                          */
 522                                         nnuls = MLX5_DIV_ROUND_UP_POW2(map->start - pe,
 523                                                                        LOG_MAX_KLM_SIZE);
 524                                         mr->num_klms += nnuls;
 525                                 }
 526                                 err = add_direct_chain(mvdev, mr, ps, pe - ps, pperm, iotlb);
 527                                 if (err)
 528                                         goto err_chain;
 529                         }
 530                         ps = map->start;
 531                         pe = map->last + 1;
 532                         pperm = map->perm;
 533                 }
 534         }
 535         err = add_direct_chain(mvdev, mr, ps, pe - ps, pperm, iotlb);
 536         if (err)
 537                 goto err_chain;
 538
 539         err = create_direct_keys(mvdev, mr);
 540         if (err)
 541                 goto err_chain;
 542
 543         /* Create the memory key that defines the guests's address space. This
 544          * memory key refers to the direct keys that contain the MTT
 545          * translations
 546          */
 547         err = create_indirect_key(mvdev, mr);
 548         if (err)
 549                 goto err_chain;
 550
 551         mr->user_mr = true;
 552         return 0;
 553
 554 err_chain:
 555         list_for_each_entry_safe_reverse(dmr, n, &mr->head, list) {
 556                 list_del_init(&dmr->list);
 557                 unmap_direct_mr(mvdev, dmr);
 558                 kfree(dmr);
 559         }
 560         return err;
 561 }
 562
 563 static int create_dma_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)
 564 {
 565         int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
 566         void *mkc;
 567         u32 *in;
 568         int err;
 569
 570         in = kzalloc(inlen, GFP_KERNEL);
 571         if (!in)
 572                 return -ENOMEM;
 573
 574         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
 575
 576         MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA);
 577         MLX5_SET(mkc, mkc, length64, 1);
 578         MLX5_SET(mkc, mkc, lw, 1);
 579         MLX5_SET(mkc, mkc, lr, 1);
 580         MLX5_SET(mkc, mkc, pd, mvdev->res.pdn);
 581         MLX5_SET(mkc, mkc, qpn, 0xffffff);
 582
 583         err = mlx5_vdpa_create_mkey(mvdev, &mr->mkey, in, inlen);
 584         if (!err)
 585                 mr->user_mr = false;
 586
 587         kfree(in);
 588         return err;
 589 }
 590
 591 static void destroy_dma_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)
 592 {
 593         mlx5_vdpa_destroy_mkey(mvdev, mr->mkey);
 594 }
 595
 596 static int dup_iotlb(struct vhost_iotlb *dst, struct vhost_iotlb *src)
 597 {
 598         struct vhost_iotlb_map *map;
 599         u64 start = 0, last = ULLONG_MAX;
 600         int err;
 601
 602         if (dst == src)
 603                 return -EINVAL;
 604
 605         if (!src) {
 606                 err = vhost_iotlb_add_range(dst, start, last, start, VHOST_ACCESS_RW);
 607                 return err;
 608         }
 609
 610         for (map = vhost_iotlb_itree_first(src, start, last); map;
 611                 map = vhost_iotlb_itree_next(map, start, last)) {
 612                 err = vhost_iotlb_add_range(dst, map->start, map->last,
 613                                             map->addr, map->perm);
 614                 if (err)
 615                         return err;
 616         }
 617         return 0;
 618 }
 619
 620 static void prune_iotlb(struct vhost_iotlb *iotlb)
 621 {
 622         vhost_iotlb_del_range(iotlb, 0, ULLONG_MAX);
 623 }
 624
 625 static void destroy_user_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)
 626 {
 627         struct mlx5_vdpa_direct_mr *dmr;
 628         struct mlx5_vdpa_direct_mr *n;
 629
 630         destroy_indirect_key(mvdev, mr);
 631         destroy_direct_keys(mvdev, mr);
 632         list_for_each_entry_safe_reverse(dmr, n, &mr->head, list) {
 633                 list_del_init(&dmr->list);
 634                 unmap_direct_mr(mvdev, dmr);
 635                 kfree(dmr);
 636         }
 637 }
 638
 639 static void _mlx5_vdpa_destroy_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)
 640 {
 641         if (WARN_ON(!mr))
 642                 return;
 643
 644         if (mr->user_mr)
 645                 destroy_user_mr(mvdev, mr);
 646         else
 647                 destroy_dma_mr(mvdev, mr);
 648
 649         vhost_iotlb_free(mr->iotlb);
 650
 651         list_del(&mr->mr_list);
 652
 653         kfree(mr);
 654 }
 655
 656 /* There can be multiple .set_map() operations in quick succession.
 657  * This large delay is a simple way to prevent the MR cleanup from blocking
 658  * .set_map() MR creation in this scenario.
 659  */
 660 #define MLX5_VDPA_MR_GC_TRIGGER_MS 2000
 661
 662 static void mlx5_vdpa_mr_gc_handler(struct work_struct *work)
 663 {
 664         struct mlx5_vdpa_mr_resources *mres;
 665         struct mlx5_vdpa_mr *mr, *tmp;
 666         struct mlx5_vdpa_dev *mvdev;
 667
 668         mres = container_of(work, struct mlx5_vdpa_mr_resources, gc_dwork_ent.work);
 669
 670         if (atomic_read(&mres->shutdown)) {
 671                 mutex_lock(&mres->lock);
 672         } else if (!mutex_trylock(&mres->lock)) {
 673                 queue_delayed_work(mres->wq_gc, &mres->gc_dwork_ent,
 674                                    msecs_to_jiffies(MLX5_VDPA_MR_GC_TRIGGER_MS));
 675                 return;
 676         }
 677
 678         mvdev = container_of(mres, struct mlx5_vdpa_dev, mres);
 679
 680         list_for_each_entry_safe(mr, tmp, &mres->mr_gc_list_head, mr_list) {
 681                 _mlx5_vdpa_destroy_mr(mvdev, mr);
 682         }
 683
 684         mutex_unlock(&mres->lock);
 685 }
 686
 687 static void _mlx5_vdpa_put_mr(struct mlx5_vdpa_dev *mvdev,
 688                               struct mlx5_vdpa_mr *mr)
 689 {
 690         struct mlx5_vdpa_mr_resources *mres = &mvdev->mres;
 691
 692         if (!mr)
 693                 return;
 694
 695         if (refcount_dec_and_test(&mr->refcount)) {
 696                 list_move_tail(&mr->mr_list, &mres->mr_gc_list_head);
 697                 queue_delayed_work(mres->wq_gc, &mres->gc_dwork_ent,
 698                                    msecs_to_jiffies(MLX5_VDPA_MR_GC_TRIGGER_MS));
 699         }
 700 }
 701
 702 void mlx5_vdpa_put_mr(struct mlx5_vdpa_dev *mvdev,
 703                       struct mlx5_vdpa_mr *mr)
 704 {
 705         mutex_lock(&mvdev->mres.lock);
 706         _mlx5_vdpa_put_mr(mvdev, mr);
 707         mutex_unlock(&mvdev->mres.lock);
 708 }
 709
 710 static void _mlx5_vdpa_get_mr(struct mlx5_vdpa_dev *mvdev,
 711                               struct mlx5_vdpa_mr *mr)
 712 {
 713         if (!mr)
 714                 return;
 715
 716         refcount_inc(&mr->refcount);
 717 }
 718
 719 void mlx5_vdpa_get_mr(struct mlx5_vdpa_dev *mvdev,
 720                       struct mlx5_vdpa_mr *mr)
 721 {
 722         mutex_lock(&mvdev->mres.lock);
 723         _mlx5_vdpa_get_mr(mvdev, mr);
 724         mutex_unlock(&mvdev->mres.lock);
 725 }
 726
 727 void mlx5_vdpa_update_mr(struct mlx5_vdpa_dev *mvdev,
 728                          struct mlx5_vdpa_mr *new_mr,
 729                          unsigned int asid)
 730 {
 731         struct mlx5_vdpa_mr *old_mr = mvdev->mres.mr[asid];
 732
 733         mutex_lock(&mvdev->mres.lock);
 734
 735         _mlx5_vdpa_put_mr(mvdev, old_mr);
 736         mvdev->mres.mr[asid] = new_mr;
 737
 738         mutex_unlock(&mvdev->mres.lock);
 739 }
 740
 741 static void mlx5_vdpa_show_mr_leaks(struct mlx5_vdpa_dev *mvdev)
 742 {
 743         struct mlx5_vdpa_mr *mr;
 744
 745         mutex_lock(&mvdev->mres.lock);
 746
 747         list_for_each_entry(mr, &mvdev->mres.mr_list_head, mr_list) {
 748
 749                 mlx5_vdpa_warn(mvdev, "mkey still alive after resource delete: "
 750                                       "mr: %p, mkey: 0x%x, refcount: %u\n",
 751                                        mr, mr->mkey, refcount_read(&mr->refcount));
 752         }
 753
 754         mutex_unlock(&mvdev->mres.lock);
 755
 756 }
 757
 758 void mlx5_vdpa_clean_mrs(struct mlx5_vdpa_dev *mvdev)
 759 {
 760         if (!mvdev->res.valid)
 761                 return;
 762
 763         for (int i = 0; i < MLX5_VDPA_NUM_AS; i++)
 764                 mlx5_vdpa_update_mr(mvdev, NULL, i);
 765
 766         prune_iotlb(mvdev->cvq.iotlb);
 767
 768         mlx5_vdpa_show_mr_leaks(mvdev);
 769 }
 770
 771 static int _mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev,
 772                                 struct mlx5_vdpa_mr *mr,
 773                                 struct vhost_iotlb *iotlb)
 774 {
 775         int err;
 776
 777         if (iotlb)
 778                 err = create_user_mr(mvdev, mr, iotlb);
 779         else
 780                 err = create_dma_mr(mvdev, mr);
 781
 782         if (err)
 783                 return err;
 784
 785         mr->iotlb = vhost_iotlb_alloc(0, 0);
 786         if (!mr->iotlb) {
 787                 err = -ENOMEM;
 788                 goto err_mr;
 789         }
 790
 791         err = dup_iotlb(mr->iotlb, iotlb);
 792         if (err)
 793                 goto err_iotlb;
 794
 795         list_add_tail(&mr->mr_list, &mvdev->mres.mr_list_head);
 796
 797         return 0;
 798
 799 err_iotlb:
 800         vhost_iotlb_free(mr->iotlb);
 801
 802 err_mr:
 803         if (iotlb)
 804                 destroy_user_mr(mvdev, mr);
 805         else
 806                 destroy_dma_mr(mvdev, mr);
 807
 808         return err;
 809 }
 810
 811 struct mlx5_vdpa_mr *mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev,
 812                                          struct vhost_iotlb *iotlb)
 813 {
 814         struct mlx5_vdpa_mr *mr;
 815         int err;
 816
 817         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
 818         if (!mr)
 819                 return ERR_PTR(-ENOMEM);
 820
 821         mutex_lock(&mvdev->mres.lock);
 822         err = _mlx5_vdpa_create_mr(mvdev, mr, iotlb);
 823         mutex_unlock(&mvdev->mres.lock);
 824
 825         if (err)
 826                 goto out_err;
 827
 828         refcount_set(&mr->refcount, 1);
 829
 830         return mr;
 831
 832 out_err:
 833         kfree(mr);
 834         return ERR_PTR(err);
 835 }
 836
 837 int mlx5_vdpa_update_cvq_iotlb(struct mlx5_vdpa_dev *mvdev,
 838                                 struct vhost_iotlb *iotlb,
 839                                 unsigned int asid)
 840 {
 841         int err;
 842
 843         if (mvdev->mres.group2asid[MLX5_VDPA_CVQ_GROUP] != asid)
 844                 return 0;
 845
 846         spin_lock(&mvdev->cvq.iommu_lock);
 847
 848         prune_iotlb(mvdev->cvq.iotlb);
 849         err = dup_iotlb(mvdev->cvq.iotlb, iotlb);
 850
 851         spin_unlock(&mvdev->cvq.iommu_lock);
 852
 853         return err;
 854 }
 855
 856 int mlx5_vdpa_create_dma_mr(struct mlx5_vdpa_dev *mvdev)
 857 {
 858         struct mlx5_vdpa_mr *mr;
 859
 860         mr = mlx5_vdpa_create_mr(mvdev, NULL);
 861         if (IS_ERR(mr))
 862                 return PTR_ERR(mr);
 863
 864         mlx5_vdpa_update_mr(mvdev, mr, 0);
 865
 866         return mlx5_vdpa_update_cvq_iotlb(mvdev, NULL, 0);
 867 }
 868
 869 int mlx5_vdpa_reset_mr(struct mlx5_vdpa_dev *mvdev, unsigned int asid)
 870 {
 871         if (asid >= MLX5_VDPA_NUM_AS)
 872                 return -EINVAL;
 873
 874         mlx5_vdpa_update_mr(mvdev, NULL, asid);
 875
 876         if (asid == 0 && MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) {
 877                 if (mlx5_vdpa_create_dma_mr(mvdev))
 878                         mlx5_vdpa_warn(mvdev, "create DMA MR failed\n");
 879         } else {
 880                 mlx5_vdpa_update_cvq_iotlb(mvdev, NULL, asid);
 881         }
 882
 883         return 0;
 884 }
 885
 886 int mlx5_vdpa_init_mr_resources(struct mlx5_vdpa_dev *mvdev)
 887 {
 888         struct mlx5_vdpa_mr_resources *mres = &mvdev->mres;
 889
 890         mres->wq_gc = create_singlethread_workqueue("mlx5_vdpa_mr_gc");
 891         if (!mres->wq_gc)
 892                 return -ENOMEM;
 893
 894         INIT_DELAYED_WORK(&mres->gc_dwork_ent, mlx5_vdpa_mr_gc_handler);
 895
 896         mutex_init(&mres->lock);
 897
 898         INIT_LIST_HEAD(&mres->mr_list_head);
 899         INIT_LIST_HEAD(&mres->mr_gc_list_head);
 900
 901         return 0;
 902 }
 903
 904 void mlx5_vdpa_destroy_mr_resources(struct mlx5_vdpa_dev *mvdev)
 905 {
 906         struct mlx5_vdpa_mr_resources *mres = &mvdev->mres;
 907
 908         atomic_set(&mres->shutdown, 1);
 909
 910         flush_delayed_work(&mres->gc_dwork_ent);
 911         destroy_workqueue(mres->wq_gc);
 912         mres->wq_gc = NULL;
 913         mutex_destroy(&mres->lock);
 914 }