?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 4.19.133
[linux/fpc-iii.git] / drivers / infiniband / sw / rxe / rxe_pool.c
blob0e2425f2823351778c1b63bd135864c59b32d4af
1 /*
2 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
3 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34 #include "rxe.h"
35 #include "rxe_loc.h"
36
37 /* info about object pools
38 * note that mr and mw share a single index space
39 * so that one can map an lkey to the correct type of object
40 */
41 struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
42 [RXE_TYPE_UC] = {
43 .name = "rxe-uc",
44 .size = sizeof(struct rxe_ucontext),
45 },
46 [RXE_TYPE_PD] = {
47 .name = "rxe-pd",
48 .size = sizeof(struct rxe_pd),
49 },
50 [RXE_TYPE_AH] = {
51 .name = "rxe-ah",
52 .size = sizeof(struct rxe_ah),
53 .flags = RXE_POOL_ATOMIC,
54 },
55 [RXE_TYPE_SRQ] = {
56 .name = "rxe-srq",
57 .size = sizeof(struct rxe_srq),
58 .flags = RXE_POOL_INDEX,
59 .min_index = RXE_MIN_SRQ_INDEX,
60 .max_index = RXE_MAX_SRQ_INDEX,
61 },
62 [RXE_TYPE_QP] = {
63 .name = "rxe-qp",
64 .size = sizeof(struct rxe_qp),
65 .cleanup = rxe_qp_cleanup,
66 .flags = RXE_POOL_INDEX,
67 .min_index = RXE_MIN_QP_INDEX,
68 .max_index = RXE_MAX_QP_INDEX,
69 },
70 [RXE_TYPE_CQ] = {
71 .name = "rxe-cq",
72 .size = sizeof(struct rxe_cq),
73 .cleanup = rxe_cq_cleanup,
74 },
75 [RXE_TYPE_MR] = {
76 .name = "rxe-mr",
77 .size = sizeof(struct rxe_mem),
78 .cleanup = rxe_mem_cleanup,
79 .flags = RXE_POOL_INDEX,
80 .max_index = RXE_MAX_MR_INDEX,
81 .min_index = RXE_MIN_MR_INDEX,
82 },
83 [RXE_TYPE_MW] = {
84 .name = "rxe-mw",
85 .size = sizeof(struct rxe_mem),
86 .flags = RXE_POOL_INDEX,
87 .max_index = RXE_MAX_MW_INDEX,
88 .min_index = RXE_MIN_MW_INDEX,
89 },
90 [RXE_TYPE_MC_GRP] = {
91 .name = "rxe-mc_grp",
92 .size = sizeof(struct rxe_mc_grp),
93 .cleanup = rxe_mc_cleanup,
94 .flags = RXE_POOL_KEY,
95 .key_offset = offsetof(struct rxe_mc_grp, mgid),
96 .key_size = sizeof(union ib_gid),
97 },
98 [RXE_TYPE_MC_ELEM] = {
99 .name = "rxe-mc_elem",
100 .size = sizeof(struct rxe_mc_elem),
101 .flags = RXE_POOL_ATOMIC,
102 },
103 };
104
105 static inline const char *pool_name(struct rxe_pool *pool)
106 {
107 return rxe_type_info[pool->type].name;
108 }
109
110 static inline struct kmem_cache *pool_cache(struct rxe_pool *pool)
111 {
112 return rxe_type_info[pool->type].cache;
113 }
114
115 static void rxe_cache_clean(size_t cnt)
116 {
117 int i;
118 struct rxe_type_info *type;
119
120 for (i = 0; i < cnt; i++) {
121 type = &rxe_type_info[i];
122 kmem_cache_destroy(type->cache);
123 type->cache = NULL;
124 }
125 }
126
127 int rxe_cache_init(void)
128 {
129 int err;
130 int i;
131 size_t size;
132 struct rxe_type_info *type;
133
134 for (i = 0; i < RXE_NUM_TYPES; i++) {
135 type = &rxe_type_info[i];
136 size = ALIGN(type->size, RXE_POOL_ALIGN);
137 type->cache = kmem_cache_create(type->name, size,
138 RXE_POOL_ALIGN,
139 RXE_POOL_CACHE_FLAGS, NULL);
140 if (!type->cache) {
141 pr_err("Unable to init kmem cache for %s\n",
142 type->name);
143 err = -ENOMEM;
144 goto err1;
145 }
146 }
147
148 return 0;
149
150 err1:
151 rxe_cache_clean(i);
152
153 return err;
154 }
155
156 void rxe_cache_exit(void)
157 {
158 rxe_cache_clean(RXE_NUM_TYPES);
159 }
160
161 static int rxe_pool_init_index(struct rxe_pool *pool, u32 max, u32 min)
162 {
163 int err = 0;
164 size_t size;
165
166 if ((max - min + 1) < pool->max_elem) {
167 pr_warn("not enough indices for max_elem\n");
168 err = -EINVAL;
169 goto out;
170 }
171
172 pool->max_index = max;
173 pool->min_index = min;
174
175 size = BITS_TO_LONGS(max - min + 1) * sizeof(long);
176 pool->table = kmalloc(size, GFP_KERNEL);
177 if (!pool->table) {
178 err = -ENOMEM;
179 goto out;
180 }
181
182 pool->table_size = size;
183 bitmap_zero(pool->table, max - min + 1);
184
185 out:
186 return err;
187 }
188
189 int rxe_pool_init(
190 struct rxe_dev *rxe,
191 struct rxe_pool *pool,
192 enum rxe_elem_type type,
193 unsigned int max_elem)
194 {
195 int err = 0;
196 size_t size = rxe_type_info[type].size;
197
198 memset(pool, 0, sizeof(*pool));
199
200 pool->rxe = rxe;
201 pool->type = type;
202 pool->max_elem = max_elem;
203 pool->elem_size = ALIGN(size, RXE_POOL_ALIGN);
204 pool->flags = rxe_type_info[type].flags;
205 pool->tree = RB_ROOT;
206 pool->cleanup = rxe_type_info[type].cleanup;
207
208 atomic_set(&pool->num_elem, 0);
209
210 kref_init(&pool->ref_cnt);
211
212 spin_lock_init(&pool->pool_lock);
213
214 if (rxe_type_info[type].flags & RXE_POOL_INDEX) {
215 err = rxe_pool_init_index(pool,
216 rxe_type_info[type].max_index,
217 rxe_type_info[type].min_index);
218 if (err)
219 goto out;
220 }
221
222 if (rxe_type_info[type].flags & RXE_POOL_KEY) {
223 pool->key_offset = rxe_type_info[type].key_offset;
224 pool->key_size = rxe_type_info[type].key_size;
225 }
226
227 pool->state = rxe_pool_valid;
228
229 out:
230 return err;
231 }
232
233 static void rxe_pool_release(struct kref *kref)
234 {
235 struct rxe_pool *pool = container_of(kref, struct rxe_pool, ref_cnt);
236
237 pool->state = rxe_pool_invalid;
238 kfree(pool->table);
239 }
240
241 static void rxe_pool_put(struct rxe_pool *pool)
242 {
243 kref_put(&pool->ref_cnt, rxe_pool_release);
244 }
245
246 int rxe_pool_cleanup(struct rxe_pool *pool)
247 {
248 unsigned long flags;
249
250 spin_lock_irqsave(&pool->pool_lock, flags);
251 pool->state = rxe_pool_invalid;
252 if (atomic_read(&pool->num_elem) > 0)
253 pr_warn("%s pool destroyed with unfree'd elem\n",
254 pool_name(pool));
255 spin_unlock_irqrestore(&pool->pool_lock, flags);
256
257 rxe_pool_put(pool);
258
259 return 0;
260 }
261
262 static u32 alloc_index(struct rxe_pool *pool)
263 {
264 u32 index;
265 u32 range = pool->max_index - pool->min_index + 1;
266
267 index = find_next_zero_bit(pool->table, range, pool->last);
268 if (index >= range)
269 index = find_first_zero_bit(pool->table, range);
270
271 WARN_ON_ONCE(index >= range);
272 set_bit(index, pool->table);
273 pool->last = index;
274 return index + pool->min_index;
275 }
276
277 static void insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new)
278 {
279 struct rb_node **link = &pool->tree.rb_node;
280 struct rb_node *parent = NULL;
281 struct rxe_pool_entry *elem;
282
283 while (*link) {
284 parent = *link;
285 elem = rb_entry(parent, struct rxe_pool_entry, node);
286
287 if (elem->index == new->index) {
288 pr_warn("element already exists!\n");
289 goto out;
290 }
291
292 if (elem->index > new->index)
293 link = &(*link)->rb_left;
294 else
295 link = &(*link)->rb_right;
296 }
297
298 rb_link_node(&new->node, parent, link);
299 rb_insert_color(&new->node, &pool->tree);
300 out:
301 return;
302 }
303
304 static void insert_key(struct rxe_pool *pool, struct rxe_pool_entry *new)
305 {
306 struct rb_node **link = &pool->tree.rb_node;
307 struct rb_node *parent = NULL;
308 struct rxe_pool_entry *elem;
309 int cmp;
310
311 while (*link) {
312 parent = *link;
313 elem = rb_entry(parent, struct rxe_pool_entry, node);
314
315 cmp = memcmp((u8 *)elem + pool->key_offset,
316 (u8 *)new + pool->key_offset, pool->key_size);
317
318 if (cmp == 0) {
319 pr_warn("key already exists!\n");
320 goto out;
321 }
322
323 if (cmp > 0)
324 link = &(*link)->rb_left;
325 else
326 link = &(*link)->rb_right;
327 }
328
329 rb_link_node(&new->node, parent, link);
330 rb_insert_color(&new->node, &pool->tree);
331 out:
332 return;
333 }
334
335 void rxe_add_key(void *arg, void *key)
336 {
337 struct rxe_pool_entry *elem = arg;
338 struct rxe_pool *pool = elem->pool;
339 unsigned long flags;
340
341 spin_lock_irqsave(&pool->pool_lock, flags);
342 memcpy((u8 *)elem + pool->key_offset, key, pool->key_size);
343 insert_key(pool, elem);
344 spin_unlock_irqrestore(&pool->pool_lock, flags);
345 }
346
347 void rxe_drop_key(void *arg)
348 {
349 struct rxe_pool_entry *elem = arg;
350 struct rxe_pool *pool = elem->pool;
351 unsigned long flags;
352
353 spin_lock_irqsave(&pool->pool_lock, flags);
354 rb_erase(&elem->node, &pool->tree);
355 spin_unlock_irqrestore(&pool->pool_lock, flags);
356 }
357
358 void rxe_add_index(void *arg)
359 {
360 struct rxe_pool_entry *elem = arg;
361 struct rxe_pool *pool = elem->pool;
362 unsigned long flags;
363
364 spin_lock_irqsave(&pool->pool_lock, flags);
365 elem->index = alloc_index(pool);
366 insert_index(pool, elem);
367 spin_unlock_irqrestore(&pool->pool_lock, flags);
368 }
369
370 void rxe_drop_index(void *arg)
371 {
372 struct rxe_pool_entry *elem = arg;
373 struct rxe_pool *pool = elem->pool;
374 unsigned long flags;
375
376 spin_lock_irqsave(&pool->pool_lock, flags);
377 clear_bit(elem->index - pool->min_index, pool->table);
378 rb_erase(&elem->node, &pool->tree);
379 spin_unlock_irqrestore(&pool->pool_lock, flags);
380 }
381
382 void *rxe_alloc(struct rxe_pool *pool)
383 {
384 struct rxe_pool_entry *elem;
385 unsigned long flags;
386
387 might_sleep_if(!(pool->flags & RXE_POOL_ATOMIC));
388
389 spin_lock_irqsave(&pool->pool_lock, flags);
390 if (pool->state != rxe_pool_valid) {
391 spin_unlock_irqrestore(&pool->pool_lock, flags);
392 return NULL;
393 }
394 kref_get(&pool->ref_cnt);
395 spin_unlock_irqrestore(&pool->pool_lock, flags);
396
397 kref_get(&pool->rxe->ref_cnt);
398
399 if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
400 goto out_put_pool;
401
402 elem = kmem_cache_zalloc(pool_cache(pool),
403 (pool->flags & RXE_POOL_ATOMIC) ?
404 GFP_ATOMIC : GFP_KERNEL);
405 if (!elem)
406 goto out_put_pool;
407
408 elem->pool = pool;
409 kref_init(&elem->ref_cnt);
410
411 return elem;
412
413 out_put_pool:
414 atomic_dec(&pool->num_elem);
415 rxe_dev_put(pool->rxe);
416 rxe_pool_put(pool);
417 return NULL;
418 }
419
420 void rxe_elem_release(struct kref *kref)
421 {
422 struct rxe_pool_entry *elem =
423 container_of(kref, struct rxe_pool_entry, ref_cnt);
424 struct rxe_pool *pool = elem->pool;
425
426 if (pool->cleanup)
427 pool->cleanup(elem);
428
429 kmem_cache_free(pool_cache(pool), elem);
430 atomic_dec(&pool->num_elem);
431 rxe_dev_put(pool->rxe);
432 rxe_pool_put(pool);
433 }
434
435 void *rxe_pool_get_index(struct rxe_pool *pool, u32 index)
436 {
437 struct rb_node *node = NULL;
438 struct rxe_pool_entry *elem = NULL;
439 unsigned long flags;
440
441 spin_lock_irqsave(&pool->pool_lock, flags);
442
443 if (pool->state != rxe_pool_valid)
444 goto out;
445
446 node = pool->tree.rb_node;
447
448 while (node) {
449 elem = rb_entry(node, struct rxe_pool_entry, node);
450
451 if (elem->index > index)
452 node = node->rb_left;
453 else if (elem->index < index)
454 node = node->rb_right;
455 else
456 break;
457 }
458
459 if (node)
460 kref_get(&elem->ref_cnt);
461
462 out:
463 spin_unlock_irqrestore(&pool->pool_lock, flags);
464 return node ? elem : NULL;
465 }
466
467 void *rxe_pool_get_key(struct rxe_pool *pool, void *key)
468 {
469 struct rb_node *node = NULL;
470 struct rxe_pool_entry *elem = NULL;
471 int cmp;
472 unsigned long flags;
473
474 spin_lock_irqsave(&pool->pool_lock, flags);
475
476 if (pool->state != rxe_pool_valid)
477 goto out;
478
479 node = pool->tree.rb_node;
480
481 while (node) {
482 elem = rb_entry(node, struct rxe_pool_entry, node);
483
484 cmp = memcmp((u8 *)elem + pool->key_offset,
485 key, pool->key_size);
486
487 if (cmp > 0)
488 node = node->rb_left;
489 else if (cmp < 0)
490 node = node->rb_right;
491 else
492 break;
493 }
494
495 if (node)
496 kref_get(&elem->ref_cnt);
497
498 out:
499 spin_unlock_irqrestore(&pool->pool_lock, flags);
500 return node ? elem : NULL;
501 }
Linux with added FPC-III support