drivers/infiniband/sw/rxe/rxe_pool.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
   4  * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
   5  */
   6
   7 #include "rxe.h"
   8 #include "rxe_loc.h"
   9
  10 /* info about object pools
  11  * note that mr and mw share a single index space
  12  * so that one can map an lkey to the correct type of object
  13  */
  14 struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
  15         [RXE_TYPE_UC] = {
  16                 .name           = "rxe-uc",
  17                 .size           = sizeof(struct rxe_ucontext),
  18                 .flags          = RXE_POOL_NO_ALLOC,
  19         },
  20         [RXE_TYPE_PD] = {
  21                 .name           = "rxe-pd",
  22                 .size           = sizeof(struct rxe_pd),
  23                 .flags          = RXE_POOL_NO_ALLOC,
  24         },
  25         [RXE_TYPE_AH] = {
  26                 .name           = "rxe-ah",
  27                 .size           = sizeof(struct rxe_ah),
  28                 .flags          = RXE_POOL_ATOMIC | RXE_POOL_NO_ALLOC,
  29         },
  30         [RXE_TYPE_SRQ] = {
  31                 .name           = "rxe-srq",
  32                 .size           = sizeof(struct rxe_srq),
  33                 .flags          = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC,
  34                 .min_index      = RXE_MIN_SRQ_INDEX,
  35                 .max_index      = RXE_MAX_SRQ_INDEX,
  36         },
  37         [RXE_TYPE_QP] = {
  38                 .name           = "rxe-qp",
  39                 .size           = sizeof(struct rxe_qp),
  40                 .cleanup        = rxe_qp_cleanup,
  41                 .flags          = RXE_POOL_INDEX,
  42                 .min_index      = RXE_MIN_QP_INDEX,
  43                 .max_index      = RXE_MAX_QP_INDEX,
  44         },
  45         [RXE_TYPE_CQ] = {
  46                 .name           = "rxe-cq",
  47                 .size           = sizeof(struct rxe_cq),
  48                 .flags          = RXE_POOL_NO_ALLOC,
  49                 .cleanup        = rxe_cq_cleanup,
  50         },
  51         [RXE_TYPE_MR] = {
  52                 .name           = "rxe-mr",
  53                 .size           = sizeof(struct rxe_mem),
  54                 .cleanup        = rxe_mem_cleanup,
  55                 .flags          = RXE_POOL_INDEX,
  56                 .max_index      = RXE_MAX_MR_INDEX,
  57                 .min_index      = RXE_MIN_MR_INDEX,
  58         },
  59         [RXE_TYPE_MW] = {
  60                 .name           = "rxe-mw",
  61                 .size           = sizeof(struct rxe_mem),
  62                 .flags          = RXE_POOL_INDEX,
  63                 .max_index      = RXE_MAX_MW_INDEX,
  64                 .min_index      = RXE_MIN_MW_INDEX,
  65         },
  66         [RXE_TYPE_MC_GRP] = {
  67                 .name           = "rxe-mc_grp",
  68                 .size           = sizeof(struct rxe_mc_grp),
  69                 .cleanup        = rxe_mc_cleanup,
  70                 .flags          = RXE_POOL_KEY,
  71                 .key_offset     = offsetof(struct rxe_mc_grp, mgid),
  72                 .key_size       = sizeof(union ib_gid),
  73         },
  74         [RXE_TYPE_MC_ELEM] = {
  75                 .name           = "rxe-mc_elem",
  76                 .size           = sizeof(struct rxe_mc_elem),
  77                 .flags          = RXE_POOL_ATOMIC,
  78         },
  79 };
  80
  81 static inline const char *pool_name(struct rxe_pool *pool)
  82 {
  83         return rxe_type_info[pool->type].name;
  84 }
  85
  86 static int rxe_pool_init_index(struct rxe_pool *pool, u32 max, u32 min)
  87 {
  88         int err = 0;
  89         size_t size;
  90
  91         if ((max - min + 1) < pool->max_elem) {
  92                 pr_warn("not enough indices for max_elem\n");
  93                 err = -EINVAL;
  94                 goto out;
  95         }
  96
  97         pool->max_index = max;
  98         pool->min_index = min;
  99
 100         size = BITS_TO_LONGS(max - min + 1) * sizeof(long);
 101         pool->table = kmalloc(size, GFP_KERNEL);
 102         if (!pool->table) {
 103                 err = -ENOMEM;
 104                 goto out;
 105         }
 106
 107         pool->table_size = size;
 108         bitmap_zero(pool->table, max - min + 1);
 109
 110 out:
 111         return err;
 112 }
 113
 114 int rxe_pool_init(
 115         struct rxe_dev          *rxe,
 116         struct rxe_pool         *pool,
 117         enum rxe_elem_type      type,
 118         unsigned int            max_elem)
 119 {
 120         int                     err = 0;
 121         size_t                  size = rxe_type_info[type].size;
 122
 123         memset(pool, 0, sizeof(*pool));
 124
 125         pool->rxe               = rxe;
 126         pool->type              = type;
 127         pool->max_elem          = max_elem;
 128         pool->elem_size         = ALIGN(size, RXE_POOL_ALIGN);
 129         pool->flags             = rxe_type_info[type].flags;
 130         pool->tree              = RB_ROOT;
 131         pool->cleanup           = rxe_type_info[type].cleanup;
 132
 133         atomic_set(&pool->num_elem, 0);
 134
 135         kref_init(&pool->ref_cnt);
 136
 137         rwlock_init(&pool->pool_lock);
 138
 139         if (rxe_type_info[type].flags & RXE_POOL_INDEX) {
 140                 err = rxe_pool_init_index(pool,
 141                                           rxe_type_info[type].max_index,
 142                                           rxe_type_info[type].min_index);
 143                 if (err)
 144                         goto out;
 145         }
 146
 147         if (rxe_type_info[type].flags & RXE_POOL_KEY) {
 148                 pool->key_offset = rxe_type_info[type].key_offset;
 149                 pool->key_size = rxe_type_info[type].key_size;
 150         }
 151
 152         pool->state = RXE_POOL_STATE_VALID;
 153
 154 out:
 155         return err;
 156 }
 157
 158 static void rxe_pool_release(struct kref *kref)
 159 {
 160         struct rxe_pool *pool = container_of(kref, struct rxe_pool, ref_cnt);
 161
 162         pool->state = RXE_POOL_STATE_INVALID;
 163         kfree(pool->table);
 164 }
 165
 166 static void rxe_pool_put(struct rxe_pool *pool)
 167 {
 168         kref_put(&pool->ref_cnt, rxe_pool_release);
 169 }
 170
 171 void rxe_pool_cleanup(struct rxe_pool *pool)
 172 {
 173         unsigned long flags;
 174
 175         write_lock_irqsave(&pool->pool_lock, flags);
 176         pool->state = RXE_POOL_STATE_INVALID;
 177         if (atomic_read(&pool->num_elem) > 0)
 178                 pr_warn("%s pool destroyed with unfree'd elem\n",
 179                         pool_name(pool));
 180         write_unlock_irqrestore(&pool->pool_lock, flags);
 181
 182         rxe_pool_put(pool);
 183 }
 184
 185 static u32 alloc_index(struct rxe_pool *pool)
 186 {
 187         u32 index;
 188         u32 range = pool->max_index - pool->min_index + 1;
 189
 190         index = find_next_zero_bit(pool->table, range, pool->last);
 191         if (index >= range)
 192                 index = find_first_zero_bit(pool->table, range);
 193
 194         WARN_ON_ONCE(index >= range);
 195         set_bit(index, pool->table);
 196         pool->last = index;
 197         return index + pool->min_index;
 198 }
 199
 200 static void insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new)
 201 {
 202         struct rb_node **link = &pool->tree.rb_node;
 203         struct rb_node *parent = NULL;
 204         struct rxe_pool_entry *elem;
 205
 206         while (*link) {
 207                 parent = *link;
 208                 elem = rb_entry(parent, struct rxe_pool_entry, node);
 209
 210                 if (elem->index == new->index) {
 211                         pr_warn("element already exists!\n");
 212                         goto out;
 213                 }
 214
 215                 if (elem->index > new->index)
 216                         link = &(*link)->rb_left;
 217                 else
 218                         link = &(*link)->rb_right;
 219         }
 220
 221         rb_link_node(&new->node, parent, link);
 222         rb_insert_color(&new->node, &pool->tree);
 223 out:
 224         return;
 225 }
 226
 227 static void insert_key(struct rxe_pool *pool, struct rxe_pool_entry *new)
 228 {
 229         struct rb_node **link = &pool->tree.rb_node;
 230         struct rb_node *parent = NULL;
 231         struct rxe_pool_entry *elem;
 232         int cmp;
 233
 234         while (*link) {
 235                 parent = *link;
 236                 elem = rb_entry(parent, struct rxe_pool_entry, node);
 237
 238                 cmp = memcmp((u8 *)elem + pool->key_offset,
 239                              (u8 *)new + pool->key_offset, pool->key_size);
 240
 241                 if (cmp == 0) {
 242                         pr_warn("key already exists!\n");
 243                         goto out;
 244                 }
 245
 246                 if (cmp > 0)
 247                         link = &(*link)->rb_left;
 248                 else
 249                         link = &(*link)->rb_right;
 250         }
 251
 252         rb_link_node(&new->node, parent, link);
 253         rb_insert_color(&new->node, &pool->tree);
 254 out:
 255         return;
 256 }
 257
 258 void rxe_add_key(void *arg, void *key)
 259 {
 260         struct rxe_pool_entry *elem = arg;
 261         struct rxe_pool *pool = elem->pool;
 262         unsigned long flags;
 263
 264         write_lock_irqsave(&pool->pool_lock, flags);
 265         memcpy((u8 *)elem + pool->key_offset, key, pool->key_size);
 266         insert_key(pool, elem);
 267         write_unlock_irqrestore(&pool->pool_lock, flags);
 268 }
 269
 270 void rxe_drop_key(void *arg)
 271 {
 272         struct rxe_pool_entry *elem = arg;
 273         struct rxe_pool *pool = elem->pool;
 274         unsigned long flags;
 275
 276         write_lock_irqsave(&pool->pool_lock, flags);
 277         rb_erase(&elem->node, &pool->tree);
 278         write_unlock_irqrestore(&pool->pool_lock, flags);
 279 }
 280
 281 void rxe_add_index(void *arg)
 282 {
 283         struct rxe_pool_entry *elem = arg;
 284         struct rxe_pool *pool = elem->pool;
 285         unsigned long flags;
 286
 287         write_lock_irqsave(&pool->pool_lock, flags);
 288         elem->index = alloc_index(pool);
 289         insert_index(pool, elem);
 290         write_unlock_irqrestore(&pool->pool_lock, flags);
 291 }
 292
 293 void rxe_drop_index(void *arg)
 294 {
 295         struct rxe_pool_entry *elem = arg;
 296         struct rxe_pool *pool = elem->pool;
 297         unsigned long flags;
 298
 299         write_lock_irqsave(&pool->pool_lock, flags);
 300         clear_bit(elem->index - pool->min_index, pool->table);
 301         rb_erase(&elem->node, &pool->tree);
 302         write_unlock_irqrestore(&pool->pool_lock, flags);
 303 }
 304
 305 void *rxe_alloc(struct rxe_pool *pool)
 306 {
 307         struct rxe_pool_entry *elem;
 308         unsigned long flags;
 309
 310         might_sleep_if(!(pool->flags & RXE_POOL_ATOMIC));
 311
 312         read_lock_irqsave(&pool->pool_lock, flags);
 313         if (pool->state != RXE_POOL_STATE_VALID) {
 314                 read_unlock_irqrestore(&pool->pool_lock, flags);
 315                 return NULL;
 316         }
 317         kref_get(&pool->ref_cnt);
 318         read_unlock_irqrestore(&pool->pool_lock, flags);
 319
 320         if (!ib_device_try_get(&pool->rxe->ib_dev))
 321                 goto out_put_pool;
 322
 323         if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
 324                 goto out_cnt;
 325
 326         elem = kzalloc(rxe_type_info[pool->type].size,
 327                                  (pool->flags & RXE_POOL_ATOMIC) ?
 328                                  GFP_ATOMIC : GFP_KERNEL);
 329         if (!elem)
 330                 goto out_cnt;
 331
 332         elem->pool = pool;
 333         kref_init(&elem->ref_cnt);
 334
 335         return elem;
 336
 337 out_cnt:
 338         atomic_dec(&pool->num_elem);
 339         ib_device_put(&pool->rxe->ib_dev);
 340 out_put_pool:
 341         rxe_pool_put(pool);
 342         return NULL;
 343 }
 344
 345 int rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_entry *elem)
 346 {
 347         unsigned long flags;
 348
 349         might_sleep_if(!(pool->flags & RXE_POOL_ATOMIC));
 350
 351         read_lock_irqsave(&pool->pool_lock, flags);
 352         if (pool->state != RXE_POOL_STATE_VALID) {
 353                 read_unlock_irqrestore(&pool->pool_lock, flags);
 354                 return -EINVAL;
 355         }
 356         kref_get(&pool->ref_cnt);
 357         read_unlock_irqrestore(&pool->pool_lock, flags);
 358
 359         if (!ib_device_try_get(&pool->rxe->ib_dev))
 360                 goto out_put_pool;
 361
 362         if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
 363                 goto out_cnt;
 364
 365         elem->pool = pool;
 366         kref_init(&elem->ref_cnt);
 367
 368         return 0;
 369
 370 out_cnt:
 371         atomic_dec(&pool->num_elem);
 372         ib_device_put(&pool->rxe->ib_dev);
 373 out_put_pool:
 374         rxe_pool_put(pool);
 375         return -EINVAL;
 376 }
 377
 378 void rxe_elem_release(struct kref *kref)
 379 {
 380         struct rxe_pool_entry *elem =
 381                 container_of(kref, struct rxe_pool_entry, ref_cnt);
 382         struct rxe_pool *pool = elem->pool;
 383
 384         if (pool->cleanup)
 385                 pool->cleanup(elem);
 386
 387         if (!(pool->flags & RXE_POOL_NO_ALLOC))
 388                 kfree(elem);
 389         atomic_dec(&pool->num_elem);
 390         ib_device_put(&pool->rxe->ib_dev);
 391         rxe_pool_put(pool);
 392 }
 393
 394 void *rxe_pool_get_index(struct rxe_pool *pool, u32 index)
 395 {
 396         struct rb_node *node = NULL;
 397         struct rxe_pool_entry *elem = NULL;
 398         unsigned long flags;
 399
 400         read_lock_irqsave(&pool->pool_lock, flags);
 401
 402         if (pool->state != RXE_POOL_STATE_VALID)
 403                 goto out;
 404
 405         node = pool->tree.rb_node;
 406
 407         while (node) {
 408                 elem = rb_entry(node, struct rxe_pool_entry, node);
 409
 410                 if (elem->index > index)
 411                         node = node->rb_left;
 412                 else if (elem->index < index)
 413                         node = node->rb_right;
 414                 else {
 415                         kref_get(&elem->ref_cnt);
 416                         break;
 417                 }
 418         }
 419
 420 out:
 421         read_unlock_irqrestore(&pool->pool_lock, flags);
 422         return node ? elem : NULL;
 423 }
 424
 425 void *rxe_pool_get_key(struct rxe_pool *pool, void *key)
 426 {
 427         struct rb_node *node = NULL;
 428         struct rxe_pool_entry *elem = NULL;
 429         int cmp;
 430         unsigned long flags;
 431
 432         read_lock_irqsave(&pool->pool_lock, flags);
 433
 434         if (pool->state != RXE_POOL_STATE_VALID)
 435                 goto out;
 436
 437         node = pool->tree.rb_node;
 438
 439         while (node) {
 440                 elem = rb_entry(node, struct rxe_pool_entry, node);
 441
 442                 cmp = memcmp((u8 *)elem + pool->key_offset,
 443                              key, pool->key_size);
 444
 445                 if (cmp > 0)
 446                         node = node->rb_left;
 447                 else if (cmp < 0)
 448                         node = node->rb_right;
 449                 else
 450                         break;
 451         }
 452
 453         if (node)
 454                 kref_get(&elem->ref_cnt);
 455
 456 out:
 457         read_unlock_irqrestore(&pool->pool_lock, flags);
 458         return node ? elem : NULL;
 459 }