?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
dm writecache: add cond_resched to loop in persistent_memory_claim()
[linux/fpc-iii.git] / drivers / infiniband / hw / mlx5 / mr.c
blob44683073be0c4a7988d6f3eb917936b444470de1
1 /*
2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33
34 #include <linux/kref.h>
35 #include <linux/random.h>
36 #include <linux/debugfs.h>
37 #include <linux/export.h>
38 #include <linux/delay.h>
39 #include <rdma/ib_umem.h>
40 #include <rdma/ib_umem_odp.h>
41 #include <rdma/ib_verbs.h>
42 #include "mlx5_ib.h"
43
44 enum {
45 MAX_PENDING_REG_MR = 8,
46 };
47
48 #define MLX5_UMR_ALIGN 2048
49
50 static void
51 create_mkey_callback(int status, struct mlx5_async_work *context);
52
53 static void
54 assign_mkey_variant(struct mlx5_ib_dev *dev, struct mlx5_core_mkey *mkey,
55 u32 *in)
56 {
57 u8 key = atomic_inc_return(&dev->mkey_var);
58 void *mkc;
59
60 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
61 MLX5_SET(mkc, mkc, mkey_7_0, key);
62 mkey->key = key;
63 }
64
65 static int
66 mlx5_ib_create_mkey(struct mlx5_ib_dev *dev, struct mlx5_core_mkey *mkey,
67 u32 *in, int inlen)
68 {
69 assign_mkey_variant(dev, mkey, in);
70 return mlx5_core_create_mkey(dev->mdev, mkey, in, inlen);
71 }
72
73 static int
74 mlx5_ib_create_mkey_cb(struct mlx5_ib_dev *dev,
75 struct mlx5_core_mkey *mkey,
76 struct mlx5_async_ctx *async_ctx,
77 u32 *in, int inlen, u32 *out, int outlen,
78 struct mlx5_async_work *context)
79 {
80 MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY);
81 assign_mkey_variant(dev, mkey, in);
82 return mlx5_cmd_exec_cb(async_ctx, in, inlen, out, outlen,
83 create_mkey_callback, context);
84 }
85
86 static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
87 static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
88 static int mr_cache_max_order(struct mlx5_ib_dev *dev);
89 static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent);
90
91 static bool umr_can_use_indirect_mkey(struct mlx5_ib_dev *dev)
92 {
93 return !MLX5_CAP_GEN(dev->mdev, umr_indirect_mkey_disabled);
94 }
95
96 static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
97 {
98 WARN_ON(xa_load(&dev->odp_mkeys, mlx5_base_mkey(mr->mmkey.key)));
99
100 return mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey);
101 }
102
103 static bool use_umr_mtt_update(struct mlx5_ib_mr *mr, u64 start, u64 length)
104 {
105 return ((u64)1 << mr->order) * MLX5_ADAPTER_PAGE_SIZE >=
106 length + (start & (MLX5_ADAPTER_PAGE_SIZE - 1));
107 }
108
109 static void create_mkey_callback(int status, struct mlx5_async_work *context)
110 {
111 struct mlx5_ib_mr *mr =
112 container_of(context, struct mlx5_ib_mr, cb_work);
113 struct mlx5_ib_dev *dev = mr->dev;
114 struct mlx5_cache_ent *ent = mr->cache_ent;
115 unsigned long flags;
116
117 if (status) {
118 mlx5_ib_warn(dev, "async reg mr failed. status %d\n", status);
119 kfree(mr);
120 spin_lock_irqsave(&ent->lock, flags);
121 ent->pending--;
122 WRITE_ONCE(dev->fill_delay, 1);
123 spin_unlock_irqrestore(&ent->lock, flags);
124 mod_timer(&dev->delay_timer, jiffies + HZ);
125 return;
126 }
127
128 mr->mmkey.type = MLX5_MKEY_MR;
129 mr->mmkey.key |= mlx5_idx_to_mkey(
130 MLX5_GET(create_mkey_out, mr->out, mkey_index));
131
132 WRITE_ONCE(dev->cache.last_add, jiffies);
133
134 spin_lock_irqsave(&ent->lock, flags);
135 list_add_tail(&mr->list, &ent->head);
136 ent->available_mrs++;
137 ent->total_mrs++;
138 /* If we are doing fill_to_high_water then keep going. */
139 queue_adjust_cache_locked(ent);
140 ent->pending--;
141 spin_unlock_irqrestore(&ent->lock, flags);
142 }
143
144 static struct mlx5_ib_mr *alloc_cache_mr(struct mlx5_cache_ent *ent, void *mkc)
145 {
146 struct mlx5_ib_mr *mr;
147
148 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
149 if (!mr)
150 return NULL;
151 mr->order = ent->order;
152 mr->cache_ent = ent;
153 mr->dev = ent->dev;
154
155 MLX5_SET(mkc, mkc, free, 1);
156 MLX5_SET(mkc, mkc, umr_en, 1);
157 MLX5_SET(mkc, mkc, access_mode_1_0, ent->access_mode & 0x3);
158 MLX5_SET(mkc, mkc, access_mode_4_2, (ent->access_mode >> 2) & 0x7);
159
160 MLX5_SET(mkc, mkc, qpn, 0xffffff);
161 MLX5_SET(mkc, mkc, translations_octword_size, ent->xlt);
162 MLX5_SET(mkc, mkc, log_page_size, ent->page);
163 return mr;
164 }
165
166 /* Asynchronously schedule new MRs to be populated in the cache. */
167 static int add_keys(struct mlx5_cache_ent *ent, unsigned int num)
168 {
169 size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
170 struct mlx5_ib_mr *mr;
171 void *mkc;
172 u32 *in;
173 int err = 0;
174 int i;
175
176 in = kzalloc(inlen, GFP_KERNEL);
177 if (!in)
178 return -ENOMEM;
179
180 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
181 for (i = 0; i < num; i++) {
182 mr = alloc_cache_mr(ent, mkc);
183 if (!mr) {
184 err = -ENOMEM;
185 break;
186 }
187 spin_lock_irq(&ent->lock);
188 if (ent->pending >= MAX_PENDING_REG_MR) {
189 err = -EAGAIN;
190 spin_unlock_irq(&ent->lock);
191 kfree(mr);
192 break;
193 }
194 ent->pending++;
195 spin_unlock_irq(&ent->lock);
196 err = mlx5_ib_create_mkey_cb(ent->dev, &mr->mmkey,
197 &ent->dev->async_ctx, in, inlen,
198 mr->out, sizeof(mr->out),
199 &mr->cb_work);
200 if (err) {
201 spin_lock_irq(&ent->lock);
202 ent->pending--;
203 spin_unlock_irq(&ent->lock);
204 mlx5_ib_warn(ent->dev, "create mkey failed %d\n", err);
205 kfree(mr);
206 break;
207 }
208 }
209
210 kfree(in);
211 return err;
212 }
213
214 /* Synchronously create a MR in the cache */
215 static struct mlx5_ib_mr *create_cache_mr(struct mlx5_cache_ent *ent)
216 {
217 size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
218 struct mlx5_ib_mr *mr;
219 void *mkc;
220 u32 *in;
221 int err;
222
223 in = kzalloc(inlen, GFP_KERNEL);
224 if (!in)
225 return ERR_PTR(-ENOMEM);
226 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
227
228 mr = alloc_cache_mr(ent, mkc);
229 if (!mr) {
230 err = -ENOMEM;
231 goto free_in;
232 }
233
234 err = mlx5_core_create_mkey(ent->dev->mdev, &mr->mmkey, in, inlen);
235 if (err)
236 goto free_mr;
237
238 mr->mmkey.type = MLX5_MKEY_MR;
239 WRITE_ONCE(ent->dev->cache.last_add, jiffies);
240 spin_lock_irq(&ent->lock);
241 ent->total_mrs++;
242 spin_unlock_irq(&ent->lock);
243 kfree(in);
244 return mr;
245 free_mr:
246 kfree(mr);
247 free_in:
248 kfree(in);
249 return ERR_PTR(err);
250 }
251
252 static void remove_cache_mr_locked(struct mlx5_cache_ent *ent)
253 {
254 struct mlx5_ib_mr *mr;
255
256 lockdep_assert_held(&ent->lock);
257 if (list_empty(&ent->head))
258 return;
259 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
260 list_del(&mr->list);
261 ent->available_mrs--;
262 ent->total_mrs--;
263 spin_unlock_irq(&ent->lock);
264 mlx5_core_destroy_mkey(ent->dev->mdev, &mr->mmkey);
265 kfree(mr);
266 spin_lock_irq(&ent->lock);
267 }
268
269 static int resize_available_mrs(struct mlx5_cache_ent *ent, unsigned int target,
270 bool limit_fill)
271 {
272 int err;
273
274 lockdep_assert_held(&ent->lock);
275
276 while (true) {
277 if (limit_fill)
278 target = ent->limit * 2;
279 if (target == ent->available_mrs + ent->pending)
280 return 0;
281 if (target > ent->available_mrs + ent->pending) {
282 u32 todo = target - (ent->available_mrs + ent->pending);
283
284 spin_unlock_irq(&ent->lock);
285 err = add_keys(ent, todo);
286 if (err == -EAGAIN)
287 usleep_range(3000, 5000);
288 spin_lock_irq(&ent->lock);
289 if (err) {
290 if (err != -EAGAIN)
291 return err;
292 } else
293 return 0;
294 } else {
295 remove_cache_mr_locked(ent);
296 }
297 }
298 }
299
300 static ssize_t size_write(struct file *filp, const char __user *buf,
301 size_t count, loff_t *pos)
302 {
303 struct mlx5_cache_ent *ent = filp->private_data;
304 u32 target;
305 int err;
306
307 err = kstrtou32_from_user(buf, count, 0, &target);
308 if (err)
309 return err;
310
311 /*
312 * Target is the new value of total_mrs the user requests, however we
313 * cannot free MRs that are in use. Compute the target value for
314 * available_mrs.
315 */
316 spin_lock_irq(&ent->lock);
317 if (target < ent->total_mrs - ent->available_mrs) {
318 err = -EINVAL;
319 goto err_unlock;
320 }
321 target = target - (ent->total_mrs - ent->available_mrs);
322 if (target < ent->limit || target > ent->limit*2) {
323 err = -EINVAL;
324 goto err_unlock;
325 }
326 err = resize_available_mrs(ent, target, false);
327 if (err)
328 goto err_unlock;
329 spin_unlock_irq(&ent->lock);
330
331 return count;
332
333 err_unlock:
334 spin_unlock_irq(&ent->lock);
335 return err;
336 }
337
338 static ssize_t size_read(struct file *filp, char __user *buf, size_t count,
339 loff_t *pos)
340 {
341 struct mlx5_cache_ent *ent = filp->private_data;
342 char lbuf[20];
343 int err;
344
345 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->total_mrs);
346 if (err < 0)
347 return err;
348
349 return simple_read_from_buffer(buf, count, pos, lbuf, err);
350 }
351
352 static const struct file_operations size_fops = {
353 .owner = THIS_MODULE,
354 .open = simple_open,
355 .write = size_write,
356 .read = size_read,
357 };
358
359 static ssize_t limit_write(struct file *filp, const char __user *buf,
360 size_t count, loff_t *pos)
361 {
362 struct mlx5_cache_ent *ent = filp->private_data;
363 u32 var;
364 int err;
365
366 err = kstrtou32_from_user(buf, count, 0, &var);
367 if (err)
368 return err;
369
370 /*
371 * Upon set we immediately fill the cache to high water mark implied by
372 * the limit.
373 */
374 spin_lock_irq(&ent->lock);
375 ent->limit = var;
376 err = resize_available_mrs(ent, 0, true);
377 spin_unlock_irq(&ent->lock);
378 if (err)
379 return err;
380 return count;
381 }
382
383 static ssize_t limit_read(struct file *filp, char __user *buf, size_t count,
384 loff_t *pos)
385 {
386 struct mlx5_cache_ent *ent = filp->private_data;
387 char lbuf[20];
388 int err;
389
390 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->limit);
391 if (err < 0)
392 return err;
393
394 return simple_read_from_buffer(buf, count, pos, lbuf, err);
395 }
396
397 static const struct file_operations limit_fops = {
398 .owner = THIS_MODULE,
399 .open = simple_open,
400 .write = limit_write,
401 .read = limit_read,
402 };
403
404 static bool someone_adding(struct mlx5_mr_cache *cache)
405 {
406 unsigned int i;
407
408 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
409 struct mlx5_cache_ent *ent = &cache->ent[i];
410 bool ret;
411
412 spin_lock_irq(&ent->lock);
413 ret = ent->available_mrs < ent->limit;
414 spin_unlock_irq(&ent->lock);
415 if (ret)
416 return true;
417 }
418 return false;
419 }
420
421 /*
422 * Check if the bucket is outside the high/low water mark and schedule an async
423 * update. The cache refill has hysteresis, once the low water mark is hit it is
424 * refilled up to the high mark.
425 */
426 static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent)
427 {
428 lockdep_assert_held(&ent->lock);
429
430 if (ent->disabled || READ_ONCE(ent->dev->fill_delay))
431 return;
432 if (ent->available_mrs < ent->limit) {
433 ent->fill_to_high_water = true;
434 queue_work(ent->dev->cache.wq, &ent->work);
435 } else if (ent->fill_to_high_water &&
436 ent->available_mrs + ent->pending < 2 * ent->limit) {
437 /*
438 * Once we start populating due to hitting a low water mark
439 * continue until we pass the high water mark.
440 */
441 queue_work(ent->dev->cache.wq, &ent->work);
442 } else if (ent->available_mrs == 2 * ent->limit) {
443 ent->fill_to_high_water = false;
444 } else if (ent->available_mrs > 2 * ent->limit) {
445 /* Queue deletion of excess entries */
446 ent->fill_to_high_water = false;
447 if (ent->pending)
448 queue_delayed_work(ent->dev->cache.wq, &ent->dwork,
449 msecs_to_jiffies(1000));
450 else
451 queue_work(ent->dev->cache.wq, &ent->work);
452 }
453 }
454
455 static void __cache_work_func(struct mlx5_cache_ent *ent)
456 {
457 struct mlx5_ib_dev *dev = ent->dev;
458 struct mlx5_mr_cache *cache = &dev->cache;
459 int err;
460
461 spin_lock_irq(&ent->lock);
462 if (ent->disabled)
463 goto out;
464
465 if (ent->fill_to_high_water &&
466 ent->available_mrs + ent->pending < 2 * ent->limit &&
467 !READ_ONCE(dev->fill_delay)) {
468 spin_unlock_irq(&ent->lock);
469 err = add_keys(ent, 1);
470 spin_lock_irq(&ent->lock);
471 if (ent->disabled)
472 goto out;
473 if (err) {
474 /*
475 * EAGAIN only happens if pending is positive, so we
476 * will be rescheduled from reg_mr_callback(). The only
477 * failure path here is ENOMEM.
478 */
479 if (err != -EAGAIN) {
480 mlx5_ib_warn(
481 dev,
482 "command failed order %d, err %d\n",
483 ent->order, err);
484 queue_delayed_work(cache->wq, &ent->dwork,
485 msecs_to_jiffies(1000));
486 }
487 }
488 } else if (ent->available_mrs > 2 * ent->limit) {
489 bool need_delay;
490
491 /*
492 * The remove_cache_mr() logic is performed as garbage
493 * collection task. Such task is intended to be run when no
494 * other active processes are running.
495 *
496 * The need_resched() will return TRUE if there are user tasks
497 * to be activated in near future.
498 *
499 * In such case, we don't execute remove_cache_mr() and postpone
500 * the garbage collection work to try to run in next cycle, in
501 * order to free CPU resources to other tasks.
502 */
503 spin_unlock_irq(&ent->lock);
504 need_delay = need_resched() || someone_adding(cache) ||
505 time_after(jiffies,
506 READ_ONCE(cache->last_add) + 300 * HZ);
507 spin_lock_irq(&ent->lock);
508 if (ent->disabled)
509 goto out;
510 if (need_delay)
511 queue_delayed_work(cache->wq, &ent->dwork, 300 * HZ);
512 remove_cache_mr_locked(ent);
513 queue_adjust_cache_locked(ent);
514 }
515 out:
516 spin_unlock_irq(&ent->lock);
517 }
518
519 static void delayed_cache_work_func(struct work_struct *work)
520 {
521 struct mlx5_cache_ent *ent;
522
523 ent = container_of(work, struct mlx5_cache_ent, dwork.work);
524 __cache_work_func(ent);
525 }
526
527 static void cache_work_func(struct work_struct *work)
528 {
529 struct mlx5_cache_ent *ent;
530
531 ent = container_of(work, struct mlx5_cache_ent, work);
532 __cache_work_func(ent);
533 }
534
535 /* Allocate a special entry from the cache */
536 struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,
537 unsigned int entry)
538 {
539 struct mlx5_mr_cache *cache = &dev->cache;
540 struct mlx5_cache_ent *ent;
541 struct mlx5_ib_mr *mr;
542
543 if (WARN_ON(entry <= MR_CACHE_LAST_STD_ENTRY ||
544 entry >= ARRAY_SIZE(cache->ent)))
545 return ERR_PTR(-EINVAL);
546
547 ent = &cache->ent[entry];
548 spin_lock_irq(&ent->lock);
549 if (list_empty(&ent->head)) {
550 spin_unlock_irq(&ent->lock);
551 mr = create_cache_mr(ent);
552 if (IS_ERR(mr))
553 return mr;
554 } else {
555 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
556 list_del(&mr->list);
557 ent->available_mrs--;
558 queue_adjust_cache_locked(ent);
559 spin_unlock_irq(&ent->lock);
560 }
561 return mr;
562 }
563
564 /* Return a MR already available in the cache */
565 static struct mlx5_ib_mr *get_cache_mr(struct mlx5_cache_ent *req_ent)
566 {
567 struct mlx5_ib_dev *dev = req_ent->dev;
568 struct mlx5_ib_mr *mr = NULL;
569 struct mlx5_cache_ent *ent = req_ent;
570
571 /* Try larger MR pools from the cache to satisfy the allocation */
572 for (; ent != &dev->cache.ent[MR_CACHE_LAST_STD_ENTRY + 1]; ent++) {
573 mlx5_ib_dbg(dev, "order %u, cache index %zu\n", ent->order,
574 ent - dev->cache.ent);
575
576 spin_lock_irq(&ent->lock);
577 if (!list_empty(&ent->head)) {
578 mr = list_first_entry(&ent->head, struct mlx5_ib_mr,
579 list);
580 list_del(&mr->list);
581 ent->available_mrs--;
582 queue_adjust_cache_locked(ent);
583 spin_unlock_irq(&ent->lock);
584 break;
585 }
586 queue_adjust_cache_locked(ent);
587 spin_unlock_irq(&ent->lock);
588 }
589
590 if (!mr)
591 req_ent->miss++;
592
593 return mr;
594 }
595
596 static void detach_mr_from_cache(struct mlx5_ib_mr *mr)
597 {
598 struct mlx5_cache_ent *ent = mr->cache_ent;
599
600 mr->cache_ent = NULL;
601 spin_lock_irq(&ent->lock);
602 ent->total_mrs--;
603 spin_unlock_irq(&ent->lock);
604 }
605
606 void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
607 {
608 struct mlx5_cache_ent *ent = mr->cache_ent;
609
610 if (!ent)
611 return;
612
613 if (mlx5_mr_cache_invalidate(mr)) {
614 detach_mr_from_cache(mr);
615 destroy_mkey(dev, mr);
616 return;
617 }
618
619 spin_lock_irq(&ent->lock);
620 list_add_tail(&mr->list, &ent->head);
621 ent->available_mrs++;
622 queue_adjust_cache_locked(ent);
623 spin_unlock_irq(&ent->lock);
624 }
625
626 static void clean_keys(struct mlx5_ib_dev *dev, int c)
627 {
628 struct mlx5_mr_cache *cache = &dev->cache;
629 struct mlx5_cache_ent *ent = &cache->ent[c];
630 struct mlx5_ib_mr *tmp_mr;
631 struct mlx5_ib_mr *mr;
632 LIST_HEAD(del_list);
633
634 cancel_delayed_work(&ent->dwork);
635 while (1) {
636 spin_lock_irq(&ent->lock);
637 if (list_empty(&ent->head)) {
638 spin_unlock_irq(&ent->lock);
639 break;
640 }
641 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list);
642 list_move(&mr->list, &del_list);
643 ent->available_mrs--;
644 ent->total_mrs--;
645 spin_unlock_irq(&ent->lock);
646 mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey);
647 }
648
649 list_for_each_entry_safe(mr, tmp_mr, &del_list, list) {
650 list_del(&mr->list);
651 kfree(mr);
652 }
653 }
654
655 static void mlx5_mr_cache_debugfs_cleanup(struct mlx5_ib_dev *dev)
656 {
657 if (!mlx5_debugfs_root || dev->is_rep)
658 return;
659
660 debugfs_remove_recursive(dev->cache.root);
661 dev->cache.root = NULL;
662 }
663
664 static void mlx5_mr_cache_debugfs_init(struct mlx5_ib_dev *dev)
665 {
666 struct mlx5_mr_cache *cache = &dev->cache;
667 struct mlx5_cache_ent *ent;
668 struct dentry *dir;
669 int i;
670
671 if (!mlx5_debugfs_root || dev->is_rep)
672 return;
673
674 cache->root = debugfs_create_dir("mr_cache", dev->mdev->priv.dbg_root);
675
676 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
677 ent = &cache->ent[i];
678 sprintf(ent->name, "%d", ent->order);
679 dir = debugfs_create_dir(ent->name, cache->root);
680 debugfs_create_file("size", 0600, dir, ent, &size_fops);
681 debugfs_create_file("limit", 0600, dir, ent, &limit_fops);
682 debugfs_create_u32("cur", 0400, dir, &ent->available_mrs);
683 debugfs_create_u32("miss", 0600, dir, &ent->miss);
684 }
685 }
686
687 static void delay_time_func(struct timer_list *t)
688 {
689 struct mlx5_ib_dev *dev = from_timer(dev, t, delay_timer);
690
691 WRITE_ONCE(dev->fill_delay, 0);
692 }
693
694 int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)
695 {
696 struct mlx5_mr_cache *cache = &dev->cache;
697 struct mlx5_cache_ent *ent;
698 int i;
699
700 mutex_init(&dev->slow_path_mutex);
701 cache->wq = alloc_ordered_workqueue("mkey_cache", WQ_MEM_RECLAIM);
702 if (!cache->wq) {
703 mlx5_ib_warn(dev, "failed to create work queue\n");
704 return -ENOMEM;
705 }
706
707 mlx5_cmd_init_async_ctx(dev->mdev, &dev->async_ctx);
708 timer_setup(&dev->delay_timer, delay_time_func, 0);
709 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
710 ent = &cache->ent[i];
711 INIT_LIST_HEAD(&ent->head);
712 spin_lock_init(&ent->lock);
713 ent->order = i + 2;
714 ent->dev = dev;
715 ent->limit = 0;
716
717 INIT_WORK(&ent->work, cache_work_func);
718 INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func);
719
720 if (i > MR_CACHE_LAST_STD_ENTRY) {
721 mlx5_odp_init_mr_cache_entry(ent);
722 continue;
723 }
724
725 if (ent->order > mr_cache_max_order(dev))
726 continue;
727
728 ent->page = PAGE_SHIFT;
729 ent->xlt = (1 << ent->order) * sizeof(struct mlx5_mtt) /
730 MLX5_IB_UMR_OCTOWORD;
731 ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
732 if ((dev->mdev->profile->mask & MLX5_PROF_MASK_MR_CACHE) &&
733 !dev->is_rep &&
734 mlx5_core_is_pf(dev->mdev))
735 ent->limit = dev->mdev->profile->mr_cache[i].limit;
736 else
737 ent->limit = 0;
738 spin_lock_irq(&ent->lock);
739 queue_adjust_cache_locked(ent);
740 spin_unlock_irq(&ent->lock);
741 }
742
743 mlx5_mr_cache_debugfs_init(dev);
744
745 return 0;
746 }
747
748 int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev)
749 {
750 unsigned int i;
751
752 if (!dev->cache.wq)
753 return 0;
754
755 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) {
756 struct mlx5_cache_ent *ent = &dev->cache.ent[i];
757
758 spin_lock_irq(&ent->lock);
759 ent->disabled = true;
760 spin_unlock_irq(&ent->lock);
761 cancel_work_sync(&ent->work);
762 cancel_delayed_work_sync(&ent->dwork);
763 }
764
765 mlx5_mr_cache_debugfs_cleanup(dev);
766 mlx5_cmd_cleanup_async_ctx(&dev->async_ctx);
767
768 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++)
769 clean_keys(dev, i);
770
771 destroy_workqueue(dev->cache.wq);
772 del_timer_sync(&dev->delay_timer);
773
774 return 0;
775 }
776
777 static void set_mkc_access_pd_addr_fields(void *mkc, int acc, u64 start_addr,
778 struct ib_pd *pd)
779 {
780 struct mlx5_ib_dev *dev = to_mdev(pd->device);
781
782 MLX5_SET(mkc, mkc, a, !!(acc & IB_ACCESS_REMOTE_ATOMIC));
783 MLX5_SET(mkc, mkc, rw, !!(acc & IB_ACCESS_REMOTE_WRITE));
784 MLX5_SET(mkc, mkc, rr, !!(acc & IB_ACCESS_REMOTE_READ));
785 MLX5_SET(mkc, mkc, lw, !!(acc & IB_ACCESS_LOCAL_WRITE));
786 MLX5_SET(mkc, mkc, lr, 1);
787
788 if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write))
789 MLX5_SET(mkc, mkc, relaxed_ordering_write,
790 !!(acc & IB_ACCESS_RELAXED_ORDERING));
791 if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read))
792 MLX5_SET(mkc, mkc, relaxed_ordering_read,
793 !!(acc & IB_ACCESS_RELAXED_ORDERING));
794
795 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
796 MLX5_SET(mkc, mkc, qpn, 0xffffff);
797 MLX5_SET64(mkc, mkc, start_addr, start_addr);
798 }
799
800 struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc)
801 {
802 struct mlx5_ib_dev *dev = to_mdev(pd->device);
803 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
804 struct mlx5_ib_mr *mr;
805 void *mkc;
806 u32 *in;
807 int err;
808
809 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
810 if (!mr)
811 return ERR_PTR(-ENOMEM);
812
813 in = kzalloc(inlen, GFP_KERNEL);
814 if (!in) {
815 err = -ENOMEM;
816 goto err_free;
817 }
818
819 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
820
821 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA);
822 MLX5_SET(mkc, mkc, length64, 1);
823 set_mkc_access_pd_addr_fields(mkc, acc, 0, pd);
824
825 err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen);
826 if (err)
827 goto err_in;
828
829 kfree(in);
830 mr->mmkey.type = MLX5_MKEY_MR;
831 mr->ibmr.lkey = mr->mmkey.key;
832 mr->ibmr.rkey = mr->mmkey.key;
833 mr->umem = NULL;
834
835 return &mr->ibmr;
836
837 err_in:
838 kfree(in);
839
840 err_free:
841 kfree(mr);
842
843 return ERR_PTR(err);
844 }
845
846 static int get_octo_len(u64 addr, u64 len, int page_shift)
847 {
848 u64 page_size = 1ULL << page_shift;
849 u64 offset;
850 int npages;
851
852 offset = addr & (page_size - 1);
853 npages = ALIGN(len + offset, page_size) >> page_shift;
854 return (npages + 1) / 2;
855 }
856
857 static int mr_cache_max_order(struct mlx5_ib_dev *dev)
858 {
859 if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset))
860 return MR_CACHE_LAST_STD_ENTRY + 2;
861 return MLX5_MAX_UMR_SHIFT;
862 }
863
864 static int mr_umem_get(struct mlx5_ib_dev *dev, u64 start, u64 length,
865 int access_flags, struct ib_umem **umem, int *npages,
866 int *page_shift, int *ncont, int *order)
867 {
868 struct ib_umem *u;
869
870 *umem = NULL;
871
872 if (access_flags & IB_ACCESS_ON_DEMAND) {
873 struct ib_umem_odp *odp;
874
875 odp = ib_umem_odp_get(&dev->ib_dev, start, length, access_flags,
876 &mlx5_mn_ops);
877 if (IS_ERR(odp)) {
878 mlx5_ib_dbg(dev, "umem get failed (%ld)\n",
879 PTR_ERR(odp));
880 return PTR_ERR(odp);
881 }
882
883 u = &odp->umem;
884
885 *page_shift = odp->page_shift;
886 *ncont = ib_umem_odp_num_pages(odp);
887 *npages = *ncont << (*page_shift - PAGE_SHIFT);
888 if (order)
889 *order = ilog2(roundup_pow_of_two(*ncont));
890 } else {
891 u = ib_umem_get(&dev->ib_dev, start, length, access_flags);
892 if (IS_ERR(u)) {
893 mlx5_ib_dbg(dev, "umem get failed (%ld)\n", PTR_ERR(u));
894 return PTR_ERR(u);
895 }
896
897 mlx5_ib_cont_pages(u, start, MLX5_MKEY_PAGE_SHIFT_MASK, npages,
898 page_shift, ncont, order);
899 }
900
901 if (!*npages) {
902 mlx5_ib_warn(dev, "avoid zero region\n");
903 ib_umem_release(u);
904 return -EINVAL;
905 }
906
907 *umem = u;
908
909 mlx5_ib_dbg(dev, "npages %d, ncont %d, order %d, page_shift %d\n",
910 *npages, *ncont, *order, *page_shift);
911
912 return 0;
913 }
914
915 static void mlx5_ib_umr_done(struct ib_cq *cq, struct ib_wc *wc)
916 {
917 struct mlx5_ib_umr_context *context =
918 container_of(wc->wr_cqe, struct mlx5_ib_umr_context, cqe);
919
920 context->status = wc->status;
921 complete(&context->done);
922 }
923
924 static inline void mlx5_ib_init_umr_context(struct mlx5_ib_umr_context *context)
925 {
926 context->cqe.done = mlx5_ib_umr_done;
927 context->status = -1;
928 init_completion(&context->done);
929 }
930
931 static int mlx5_ib_post_send_wait(struct mlx5_ib_dev *dev,
932 struct mlx5_umr_wr *umrwr)
933 {
934 struct umr_common *umrc = &dev->umrc;
935 const struct ib_send_wr *bad;
936 int err;
937 struct mlx5_ib_umr_context umr_context;
938
939 mlx5_ib_init_umr_context(&umr_context);
940 umrwr->wr.wr_cqe = &umr_context.cqe;
941
942 down(&umrc->sem);
943 err = ib_post_send(umrc->qp, &umrwr->wr, &bad);
944 if (err) {
945 mlx5_ib_warn(dev, "UMR post send failed, err %d\n", err);
946 } else {
947 wait_for_completion(&umr_context.done);
948 if (umr_context.status != IB_WC_SUCCESS) {
949 mlx5_ib_warn(dev, "reg umr failed (%u)\n",
950 umr_context.status);
951 err = -EFAULT;
952 }
953 }
954 up(&umrc->sem);
955 return err;
956 }
957
958 static struct mlx5_cache_ent *mr_cache_ent_from_order(struct mlx5_ib_dev *dev,
959 unsigned int order)
960 {
961 struct mlx5_mr_cache *cache = &dev->cache;
962
963 if (order < cache->ent[0].order)
964 return &cache->ent[0];
965 order = order - cache->ent[0].order;
966 if (order > MR_CACHE_LAST_STD_ENTRY)
967 return NULL;
968 return &cache->ent[order];
969 }
970
971 static struct mlx5_ib_mr *
972 alloc_mr_from_cache(struct ib_pd *pd, struct ib_umem *umem, u64 virt_addr,
973 u64 len, int npages, int page_shift, unsigned int order,
974 int access_flags)
975 {
976 struct mlx5_ib_dev *dev = to_mdev(pd->device);
977 struct mlx5_cache_ent *ent = mr_cache_ent_from_order(dev, order);
978 struct mlx5_ib_mr *mr;
979
980 if (!ent)
981 return ERR_PTR(-E2BIG);
982 mr = get_cache_mr(ent);
983 if (!mr) {
984 mr = create_cache_mr(ent);
985 if (IS_ERR(mr))
986 return mr;
987 }
988
989 mr->ibmr.pd = pd;
990 mr->umem = umem;
991 mr->access_flags = access_flags;
992 mr->desc_size = sizeof(struct mlx5_mtt);
993 mr->mmkey.iova = virt_addr;
994 mr->mmkey.size = len;
995 mr->mmkey.pd = to_mpd(pd)->pdn;
996
997 return mr;
998 }
999
1000 #define MLX5_MAX_UMR_CHUNK ((1 << (MLX5_MAX_UMR_SHIFT + 4)) - \
1001 MLX5_UMR_MTT_ALIGNMENT)
1002 #define MLX5_SPARE_UMR_CHUNK 0x10000
1003
1004 int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
1005 int page_shift, int flags)
1006 {
1007 struct mlx5_ib_dev *dev = mr->dev;
1008 struct device *ddev = dev->ib_dev.dev.parent;
1009 int size;
1010 void *xlt;
1011 dma_addr_t dma;
1012 struct mlx5_umr_wr wr;
1013 struct ib_sge sg;
1014 int err = 0;
1015 int desc_size = (flags & MLX5_IB_UPD_XLT_INDIRECT)
1016 ? sizeof(struct mlx5_klm)
1017 : sizeof(struct mlx5_mtt);
1018 const int page_align = MLX5_UMR_MTT_ALIGNMENT / desc_size;
1019 const int page_mask = page_align - 1;
1020 size_t pages_mapped = 0;
1021 size_t pages_to_map = 0;
1022 size_t pages_iter = 0;
1023 size_t size_to_map = 0;
1024 gfp_t gfp;
1025 bool use_emergency_page = false;
1026
1027 if ((flags & MLX5_IB_UPD_XLT_INDIRECT) &&
1028 !umr_can_use_indirect_mkey(dev))
1029 return -EPERM;
1030
1031 /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes,
1032 * so we need to align the offset and length accordingly
1033 */
1034 if (idx & page_mask) {
1035 npages += idx & page_mask;
1036 idx &= ~page_mask;
1037 }
1038
1039 gfp = flags & MLX5_IB_UPD_XLT_ATOMIC ? GFP_ATOMIC : GFP_KERNEL;
1040 gfp |= __GFP_ZERO | __GFP_NOWARN;
1041
1042 pages_to_map = ALIGN(npages, page_align);
1043 size = desc_size * pages_to_map;
1044 size = min_t(int, size, MLX5_MAX_UMR_CHUNK);
1045
1046 xlt = (void *)__get_free_pages(gfp, get_order(size));
1047 if (!xlt && size > MLX5_SPARE_UMR_CHUNK) {
1048 mlx5_ib_dbg(dev, "Failed to allocate %d bytes of order %d. fallback to spare UMR allocation od %d bytes\n",
1049 size, get_order(size), MLX5_SPARE_UMR_CHUNK);
1050
1051 size = MLX5_SPARE_UMR_CHUNK;
1052 xlt = (void *)__get_free_pages(gfp, get_order(size));
1053 }
1054
1055 if (!xlt) {
1056 mlx5_ib_warn(dev, "Using XLT emergency buffer\n");
1057 xlt = (void *)mlx5_ib_get_xlt_emergency_page();
1058 size = PAGE_SIZE;
1059 memset(xlt, 0, size);
1060 use_emergency_page = true;
1061 }
1062 pages_iter = size / desc_size;
1063 dma = dma_map_single(ddev, xlt, size, DMA_TO_DEVICE);
1064 if (dma_mapping_error(ddev, dma)) {
1065 mlx5_ib_err(dev, "unable to map DMA during XLT update.\n");
1066 err = -ENOMEM;
1067 goto free_xlt;
1068 }
1069
1070 if (mr->umem->is_odp) {
1071 if (!(flags & MLX5_IB_UPD_XLT_INDIRECT)) {
1072 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem);
1073 size_t max_pages = ib_umem_odp_num_pages(odp) - idx;
1074
1075 pages_to_map = min_t(size_t, pages_to_map, max_pages);
1076 }
1077 }
1078
1079 sg.addr = dma;
1080 sg.lkey = dev->umrc.pd->local_dma_lkey;
1081
1082 memset(&wr, 0, sizeof(wr));
1083 wr.wr.send_flags = MLX5_IB_SEND_UMR_UPDATE_XLT;
1084 if (!(flags & MLX5_IB_UPD_XLT_ENABLE))
1085 wr.wr.send_flags |= MLX5_IB_SEND_UMR_FAIL_IF_FREE;
1086 wr.wr.sg_list = &sg;
1087 wr.wr.num_sge = 1;
1088 wr.wr.opcode = MLX5_IB_WR_UMR;
1089
1090 wr.pd = mr->ibmr.pd;
1091 wr.mkey = mr->mmkey.key;
1092 wr.length = mr->mmkey.size;
1093 wr.virt_addr = mr->mmkey.iova;
1094 wr.access_flags = mr->access_flags;
1095 wr.page_shift = page_shift;
1096
1097 for (pages_mapped = 0;
1098 pages_mapped < pages_to_map && !err;
1099 pages_mapped += pages_iter, idx += pages_iter) {
1100 npages = min_t(int, pages_iter, pages_to_map - pages_mapped);
1101 size_to_map = npages * desc_size;
1102 dma_sync_single_for_cpu(ddev, dma, size, DMA_TO_DEVICE);
1103 if (mr->umem->is_odp) {
1104 mlx5_odp_populate_xlt(xlt, idx, npages, mr, flags);
1105 } else {
1106 __mlx5_ib_populate_pas(dev, mr->umem, page_shift, idx,
1107 npages, xlt,
1108 MLX5_IB_MTT_PRESENT);
1109 /* Clear padding after the pages
1110 * brought from the umem.
1111 */
1112 memset(xlt + size_to_map, 0, size - size_to_map);
1113 }
1114 dma_sync_single_for_device(ddev, dma, size, DMA_TO_DEVICE);
1115
1116 sg.length = ALIGN(size_to_map, MLX5_UMR_MTT_ALIGNMENT);
1117
1118 if (pages_mapped + pages_iter >= pages_to_map) {
1119 if (flags & MLX5_IB_UPD_XLT_ENABLE)
1120 wr.wr.send_flags |=
1121 MLX5_IB_SEND_UMR_ENABLE_MR |
1122 MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS |
1123 MLX5_IB_SEND_UMR_UPDATE_TRANSLATION;
1124 if (flags & MLX5_IB_UPD_XLT_PD ||
1125 flags & MLX5_IB_UPD_XLT_ACCESS)
1126 wr.wr.send_flags |=
1127 MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;
1128 if (flags & MLX5_IB_UPD_XLT_ADDR)
1129 wr.wr.send_flags |=
1130 MLX5_IB_SEND_UMR_UPDATE_TRANSLATION;
1131 }
1132
1133 wr.offset = idx * desc_size;
1134 wr.xlt_size = sg.length;
1135
1136 err = mlx5_ib_post_send_wait(dev, &wr);
1137 }
1138 dma_unmap_single(ddev, dma, size, DMA_TO_DEVICE);
1139
1140 free_xlt:
1141 if (use_emergency_page)
1142 mlx5_ib_put_xlt_emergency_page();
1143 else
1144 free_pages((unsigned long)xlt, get_order(size));
1145
1146 return err;
1147 }
1148
1149 /*
1150 * If ibmr is NULL it will be allocated by reg_create.
1151 * Else, the given ibmr will be used.
1152 */
1153 static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,
1154 u64 virt_addr, u64 length,
1155 struct ib_umem *umem, int npages,
1156 int page_shift, int access_flags,
1157 bool populate)
1158 {
1159 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1160 struct mlx5_ib_mr *mr;
1161 __be64 *pas;
1162 void *mkc;
1163 int inlen;
1164 u32 *in;
1165 int err;
1166 bool pg_cap = !!(MLX5_CAP_GEN(dev->mdev, pg));
1167
1168 mr = ibmr ? to_mmr(ibmr) : kzalloc(sizeof(*mr), GFP_KERNEL);
1169 if (!mr)
1170 return ERR_PTR(-ENOMEM);
1171
1172 mr->ibmr.pd = pd;
1173 mr->access_flags = access_flags;
1174
1175 inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
1176 if (populate)
1177 inlen += sizeof(*pas) * roundup(npages, 2);
1178 in = kvzalloc(inlen, GFP_KERNEL);
1179 if (!in) {
1180 err = -ENOMEM;
1181 goto err_1;
1182 }
1183 pas = (__be64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
1184 if (populate && !(access_flags & IB_ACCESS_ON_DEMAND))
1185 mlx5_ib_populate_pas(dev, umem, page_shift, pas,
1186 pg_cap ? MLX5_IB_MTT_PRESENT : 0);
1187
1188 /* The pg_access bit allows setting the access flags
1189 * in the page list submitted with the command. */
1190 MLX5_SET(create_mkey_in, in, pg_access, !!(pg_cap));
1191
1192 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1193 MLX5_SET(mkc, mkc, free, !populate);
1194 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
1195 if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write))
1196 MLX5_SET(mkc, mkc, relaxed_ordering_write,
1197 !!(access_flags & IB_ACCESS_RELAXED_ORDERING));
1198 if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read))
1199 MLX5_SET(mkc, mkc, relaxed_ordering_read,
1200 !!(access_flags & IB_ACCESS_RELAXED_ORDERING));
1201 MLX5_SET(mkc, mkc, a, !!(access_flags & IB_ACCESS_REMOTE_ATOMIC));
1202 MLX5_SET(mkc, mkc, rw, !!(access_flags & IB_ACCESS_REMOTE_WRITE));
1203 MLX5_SET(mkc, mkc, rr, !!(access_flags & IB_ACCESS_REMOTE_READ));
1204 MLX5_SET(mkc, mkc, lw, !!(access_flags & IB_ACCESS_LOCAL_WRITE));
1205 MLX5_SET(mkc, mkc, lr, 1);
1206 MLX5_SET(mkc, mkc, umr_en, 1);
1207
1208 MLX5_SET64(mkc, mkc, start_addr, virt_addr);
1209 MLX5_SET64(mkc, mkc, len, length);
1210 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
1211 MLX5_SET(mkc, mkc, bsf_octword_size, 0);
1212 MLX5_SET(mkc, mkc, translations_octword_size,
1213 get_octo_len(virt_addr, length, page_shift));
1214 MLX5_SET(mkc, mkc, log_page_size, page_shift);
1215 MLX5_SET(mkc, mkc, qpn, 0xffffff);
1216 if (populate) {
1217 MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
1218 get_octo_len(virt_addr, length, page_shift));
1219 }
1220
1221 err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen);
1222 if (err) {
1223 mlx5_ib_warn(dev, "create mkey failed\n");
1224 goto err_2;
1225 }
1226 mr->mmkey.type = MLX5_MKEY_MR;
1227 mr->desc_size = sizeof(struct mlx5_mtt);
1228 mr->dev = dev;
1229 kvfree(in);
1230
1231 mlx5_ib_dbg(dev, "mkey = 0x%x\n", mr->mmkey.key);
1232
1233 return mr;
1234
1235 err_2:
1236 kvfree(in);
1237
1238 err_1:
1239 if (!ibmr)
1240 kfree(mr);
1241
1242 return ERR_PTR(err);
1243 }
1244
1245 static void set_mr_fields(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr,
1246 int npages, u64 length, int access_flags)
1247 {
1248 mr->npages = npages;
1249 atomic_add(npages, &dev->mdev->priv.reg_pages);
1250 mr->ibmr.lkey = mr->mmkey.key;
1251 mr->ibmr.rkey = mr->mmkey.key;
1252 mr->ibmr.length = length;
1253 mr->access_flags = access_flags;
1254 }
1255
1256 static struct ib_mr *mlx5_ib_get_dm_mr(struct ib_pd *pd, u64 start_addr,
1257 u64 length, int acc, int mode)
1258 {
1259 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1260 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
1261 struct mlx5_ib_mr *mr;
1262 void *mkc;
1263 u32 *in;
1264 int err;
1265
1266 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1267 if (!mr)
1268 return ERR_PTR(-ENOMEM);
1269
1270 in = kzalloc(inlen, GFP_KERNEL);
1271 if (!in) {
1272 err = -ENOMEM;
1273 goto err_free;
1274 }
1275
1276 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1277
1278 MLX5_SET(mkc, mkc, access_mode_1_0, mode & 0x3);
1279 MLX5_SET(mkc, mkc, access_mode_4_2, (mode >> 2) & 0x7);
1280 MLX5_SET64(mkc, mkc, len, length);
1281 set_mkc_access_pd_addr_fields(mkc, acc, start_addr, pd);
1282
1283 err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen);
1284 if (err)
1285 goto err_in;
1286
1287 kfree(in);
1288
1289 mr->umem = NULL;
1290 set_mr_fields(dev, mr, 0, length, acc);
1291
1292 return &mr->ibmr;
1293
1294 err_in:
1295 kfree(in);
1296
1297 err_free:
1298 kfree(mr);
1299
1300 return ERR_PTR(err);
1301 }
1302
1303 int mlx5_ib_advise_mr(struct ib_pd *pd,
1304 enum ib_uverbs_advise_mr_advice advice,
1305 u32 flags,
1306 struct ib_sge *sg_list,
1307 u32 num_sge,
1308 struct uverbs_attr_bundle *attrs)
1309 {
1310 if (advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH &&
1311 advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE)
1312 return -EOPNOTSUPP;
1313
1314 return mlx5_ib_advise_mr_prefetch(pd, advice, flags,
1315 sg_list, num_sge);
1316 }
1317
1318 struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm,
1319 struct ib_dm_mr_attr *attr,
1320 struct uverbs_attr_bundle *attrs)
1321 {
1322 struct mlx5_ib_dm *mdm = to_mdm(dm);
1323 struct mlx5_core_dev *dev = to_mdev(dm->device)->mdev;
1324 u64 start_addr = mdm->dev_addr + attr->offset;
1325 int mode;
1326
1327 switch (mdm->type) {
1328 case MLX5_IB_UAPI_DM_TYPE_MEMIC:
1329 if (attr->access_flags & ~MLX5_IB_DM_MEMIC_ALLOWED_ACCESS)
1330 return ERR_PTR(-EINVAL);
1331
1332 mode = MLX5_MKC_ACCESS_MODE_MEMIC;
1333 start_addr -= pci_resource_start(dev->pdev, 0);
1334 break;
1335 case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
1336 case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:
1337 if (attr->access_flags & ~MLX5_IB_DM_SW_ICM_ALLOWED_ACCESS)
1338 return ERR_PTR(-EINVAL);
1339
1340 mode = MLX5_MKC_ACCESS_MODE_SW_ICM;
1341 break;
1342 default:
1343 return ERR_PTR(-EINVAL);
1344 }
1345
1346 return mlx5_ib_get_dm_mr(pd, start_addr, attr->length,
1347 attr->access_flags, mode);
1348 }
1349
1350 struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
1351 u64 virt_addr, int access_flags,
1352 struct ib_udata *udata)
1353 {
1354 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1355 struct mlx5_ib_mr *mr = NULL;
1356 bool use_umr;
1357 struct ib_umem *umem;
1358 int page_shift;
1359 int npages;
1360 int ncont;
1361 int order;
1362 int err;
1363
1364 if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM))
1365 return ERR_PTR(-EOPNOTSUPP);
1366
1367 mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n",
1368 start, virt_addr, length, access_flags);
1369
1370 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) && !start &&
1371 length == U64_MAX) {
1372 if (virt_addr != start)
1373 return ERR_PTR(-EINVAL);
1374 if (!(access_flags & IB_ACCESS_ON_DEMAND) ||
1375 !(dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT))
1376 return ERR_PTR(-EINVAL);
1377
1378 mr = mlx5_ib_alloc_implicit_mr(to_mpd(pd), udata, access_flags);
1379 if (IS_ERR(mr))
1380 return ERR_CAST(mr);
1381 return &mr->ibmr;
1382 }
1383
1384 err = mr_umem_get(dev, start, length, access_flags, &umem,
1385 &npages, &page_shift, &ncont, &order);
1386
1387 if (err < 0)
1388 return ERR_PTR(err);
1389
1390 use_umr = mlx5_ib_can_use_umr(dev, true, access_flags);
1391
1392 if (order <= mr_cache_max_order(dev) && use_umr) {
1393 mr = alloc_mr_from_cache(pd, umem, virt_addr, length, ncont,
1394 page_shift, order, access_flags);
1395 if (PTR_ERR(mr) == -EAGAIN) {
1396 mlx5_ib_dbg(dev, "cache empty for order %d\n", order);
1397 mr = NULL;
1398 }
1399 } else if (!MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) {
1400 if (access_flags & IB_ACCESS_ON_DEMAND) {
1401 err = -EINVAL;
1402 pr_err("Got MR registration for ODP MR > 512MB, not supported for Connect-IB\n");
1403 goto error;
1404 }
1405 use_umr = false;
1406 }
1407
1408 if (!mr) {
1409 mutex_lock(&dev->slow_path_mutex);
1410 mr = reg_create(NULL, pd, virt_addr, length, umem, ncont,
1411 page_shift, access_flags, !use_umr);
1412 mutex_unlock(&dev->slow_path_mutex);
1413 }
1414
1415 if (IS_ERR(mr)) {
1416 err = PTR_ERR(mr);
1417 goto error;
1418 }
1419
1420 mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmkey.key);
1421
1422 mr->umem = umem;
1423 set_mr_fields(dev, mr, npages, length, access_flags);
1424
1425 if (use_umr) {
1426 int update_xlt_flags = MLX5_IB_UPD_XLT_ENABLE;
1427
1428 if (access_flags & IB_ACCESS_ON_DEMAND)
1429 update_xlt_flags |= MLX5_IB_UPD_XLT_ZAP;
1430
1431 err = mlx5_ib_update_xlt(mr, 0, ncont, page_shift,
1432 update_xlt_flags);
1433
1434 if (err) {
1435 dereg_mr(dev, mr);
1436 return ERR_PTR(err);
1437 }
1438 }
1439
1440 if (is_odp_mr(mr)) {
1441 to_ib_umem_odp(mr->umem)->private = mr;
1442 init_waitqueue_head(&mr->q_deferred_work);
1443 atomic_set(&mr->num_deferred_work, 0);
1444 err = xa_err(xa_store(&dev->odp_mkeys,
1445 mlx5_base_mkey(mr->mmkey.key), &mr->mmkey,
1446 GFP_KERNEL));
1447 if (err) {
1448 dereg_mr(dev, mr);
1449 return ERR_PTR(err);
1450 }
1451 }
1452
1453 return &mr->ibmr;
1454 error:
1455 ib_umem_release(umem);
1456 return ERR_PTR(err);
1457 }
1458
1459 /**
1460 * mlx5_mr_cache_invalidate - Fence all DMA on the MR
1461 * @mr: The MR to fence
1462 *
1463 * Upon return the NIC will not be doing any DMA to the pages under the MR,
1464 * and any DMA inprogress will be completed. Failure of this function
1465 * indicates the HW has failed catastrophically.
1466 */
1467 int mlx5_mr_cache_invalidate(struct mlx5_ib_mr *mr)
1468 {
1469 struct mlx5_umr_wr umrwr = {};
1470
1471 if (mr->dev->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
1472 return 0;
1473
1474 umrwr.wr.send_flags = MLX5_IB_SEND_UMR_DISABLE_MR |
1475 MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;
1476 umrwr.wr.opcode = MLX5_IB_WR_UMR;
1477 umrwr.pd = mr->dev->umrc.pd;
1478 umrwr.mkey = mr->mmkey.key;
1479 umrwr.ignore_free_state = 1;
1480
1481 return mlx5_ib_post_send_wait(mr->dev, &umrwr);
1482 }
1483
1484 static int rereg_umr(struct ib_pd *pd, struct mlx5_ib_mr *mr,
1485 int access_flags, int flags)
1486 {
1487 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1488 struct mlx5_umr_wr umrwr = {};
1489 int err;
1490
1491 umrwr.wr.send_flags = MLX5_IB_SEND_UMR_FAIL_IF_FREE;
1492
1493 umrwr.wr.opcode = MLX5_IB_WR_UMR;
1494 umrwr.mkey = mr->mmkey.key;
1495
1496 if (flags & IB_MR_REREG_PD || flags & IB_MR_REREG_ACCESS) {
1497 umrwr.pd = pd;
1498 umrwr.access_flags = access_flags;
1499 umrwr.wr.send_flags |= MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;
1500 }
1501
1502 err = mlx5_ib_post_send_wait(dev, &umrwr);
1503
1504 return err;
1505 }
1506
1507 int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
1508 u64 length, u64 virt_addr, int new_access_flags,
1509 struct ib_pd *new_pd, struct ib_udata *udata)
1510 {
1511 struct mlx5_ib_dev *dev = to_mdev(ib_mr->device);
1512 struct mlx5_ib_mr *mr = to_mmr(ib_mr);
1513 struct ib_pd *pd = (flags & IB_MR_REREG_PD) ? new_pd : ib_mr->pd;
1514 int access_flags = flags & IB_MR_REREG_ACCESS ?
1515 new_access_flags :
1516 mr->access_flags;
1517 int page_shift = 0;
1518 int upd_flags = 0;
1519 int npages = 0;
1520 int ncont = 0;
1521 int order = 0;
1522 u64 addr, len;
1523 int err;
1524
1525 mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n",
1526 start, virt_addr, length, access_flags);
1527
1528 atomic_sub(mr->npages, &dev->mdev->priv.reg_pages);
1529
1530 if (!mr->umem)
1531 return -EINVAL;
1532
1533 if (is_odp_mr(mr))
1534 return -EOPNOTSUPP;
1535
1536 if (flags & IB_MR_REREG_TRANS) {
1537 addr = virt_addr;
1538 len = length;
1539 } else {
1540 addr = mr->umem->address;
1541 len = mr->umem->length;
1542 }
1543
1544 if (flags != IB_MR_REREG_PD) {
1545 /*
1546 * Replace umem. This needs to be done whether or not UMR is
1547 * used.
1548 */
1549 flags |= IB_MR_REREG_TRANS;
1550 ib_umem_release(mr->umem);
1551 mr->umem = NULL;
1552 err = mr_umem_get(dev, addr, len, access_flags, &mr->umem,
1553 &npages, &page_shift, &ncont, &order);
1554 if (err)
1555 goto err;
1556 }
1557
1558 if (!mlx5_ib_can_use_umr(dev, true, access_flags) ||
1559 (flags & IB_MR_REREG_TRANS && !use_umr_mtt_update(mr, addr, len))) {
1560 /*
1561 * UMR can't be used - MKey needs to be replaced.
1562 */
1563 if (mr->cache_ent)
1564 detach_mr_from_cache(mr);
1565 err = destroy_mkey(dev, mr);
1566 if (err)
1567 goto err;
1568
1569 mr = reg_create(ib_mr, pd, addr, len, mr->umem, ncont,
1570 page_shift, access_flags, true);
1571
1572 if (IS_ERR(mr)) {
1573 err = PTR_ERR(mr);
1574 mr = to_mmr(ib_mr);
1575 goto err;
1576 }
1577 } else {
1578 /*
1579 * Send a UMR WQE
1580 */
1581 mr->ibmr.pd = pd;
1582 mr->access_flags = access_flags;
1583 mr->mmkey.iova = addr;
1584 mr->mmkey.size = len;
1585 mr->mmkey.pd = to_mpd(pd)->pdn;
1586
1587 if (flags & IB_MR_REREG_TRANS) {
1588 upd_flags = MLX5_IB_UPD_XLT_ADDR;
1589 if (flags & IB_MR_REREG_PD)
1590 upd_flags |= MLX5_IB_UPD_XLT_PD;
1591 if (flags & IB_MR_REREG_ACCESS)
1592 upd_flags |= MLX5_IB_UPD_XLT_ACCESS;
1593 err = mlx5_ib_update_xlt(mr, 0, npages, page_shift,
1594 upd_flags);
1595 } else {
1596 err = rereg_umr(pd, mr, access_flags, flags);
1597 }
1598
1599 if (err)
1600 goto err;
1601 }
1602
1603 set_mr_fields(dev, mr, npages, len, access_flags);
1604
1605 return 0;
1606
1607 err:
1608 ib_umem_release(mr->umem);
1609 mr->umem = NULL;
1610
1611 clean_mr(dev, mr);
1612 return err;
1613 }
1614
1615 static int
1616 mlx5_alloc_priv_descs(struct ib_device *device,
1617 struct mlx5_ib_mr *mr,
1618 int ndescs,
1619 int desc_size)
1620 {
1621 int size = ndescs * desc_size;
1622 int add_size;
1623 int ret;
1624
1625 add_size = max_t(int, MLX5_UMR_ALIGN - ARCH_KMALLOC_MINALIGN, 0);
1626
1627 mr->descs_alloc = kzalloc(size + add_size, GFP_KERNEL);
1628 if (!mr->descs_alloc)
1629 return -ENOMEM;
1630
1631 mr->descs = PTR_ALIGN(mr->descs_alloc, MLX5_UMR_ALIGN);
1632
1633 mr->desc_map = dma_map_single(device->dev.parent, mr->descs,
1634 size, DMA_TO_DEVICE);
1635 if (dma_mapping_error(device->dev.parent, mr->desc_map)) {
1636 ret = -ENOMEM;
1637 goto err;
1638 }
1639
1640 return 0;
1641 err:
1642 kfree(mr->descs_alloc);
1643
1644 return ret;
1645 }
1646
1647 static void
1648 mlx5_free_priv_descs(struct mlx5_ib_mr *mr)
1649 {
1650 if (mr->descs) {
1651 struct ib_device *device = mr->ibmr.device;
1652 int size = mr->max_descs * mr->desc_size;
1653
1654 dma_unmap_single(device->dev.parent, mr->desc_map,
1655 size, DMA_TO_DEVICE);
1656 kfree(mr->descs_alloc);
1657 mr->descs = NULL;
1658 }
1659 }
1660
1661 static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
1662 {
1663 if (mr->sig) {
1664 if (mlx5_core_destroy_psv(dev->mdev,
1665 mr->sig->psv_memory.psv_idx))
1666 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n",
1667 mr->sig->psv_memory.psv_idx);
1668 if (mlx5_core_destroy_psv(dev->mdev,
1669 mr->sig->psv_wire.psv_idx))
1670 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",
1671 mr->sig->psv_wire.psv_idx);
1672 xa_erase(&dev->sig_mrs, mlx5_base_mkey(mr->mmkey.key));
1673 kfree(mr->sig);
1674 mr->sig = NULL;
1675 }
1676
1677 if (!mr->cache_ent) {
1678 destroy_mkey(dev, mr);
1679 mlx5_free_priv_descs(mr);
1680 }
1681 }
1682
1683 static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
1684 {
1685 int npages = mr->npages;
1686 struct ib_umem *umem = mr->umem;
1687
1688 /* Stop all DMA */
1689 if (is_odp_mr(mr))
1690 mlx5_ib_fence_odp_mr(mr);
1691 else
1692 clean_mr(dev, mr);
1693
1694 if (mr->cache_ent)
1695 mlx5_mr_cache_free(dev, mr);
1696 else
1697 kfree(mr);
1698
1699 ib_umem_release(umem);
1700 atomic_sub(npages, &dev->mdev->priv.reg_pages);
1701
1702 }
1703
1704 int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
1705 {
1706 struct mlx5_ib_mr *mmr = to_mmr(ibmr);
1707
1708 if (ibmr->type == IB_MR_TYPE_INTEGRITY) {
1709 dereg_mr(to_mdev(mmr->mtt_mr->ibmr.device), mmr->mtt_mr);
1710 dereg_mr(to_mdev(mmr->klm_mr->ibmr.device), mmr->klm_mr);
1711 }
1712
1713 if (is_odp_mr(mmr) && to_ib_umem_odp(mmr->umem)->is_implicit_odp) {
1714 mlx5_ib_free_implicit_mr(mmr);
1715 return 0;
1716 }
1717
1718 dereg_mr(to_mdev(ibmr->device), mmr);
1719
1720 return 0;
1721 }
1722
1723 static void mlx5_set_umr_free_mkey(struct ib_pd *pd, u32 *in, int ndescs,
1724 int access_mode, int page_shift)
1725 {
1726 void *mkc;
1727
1728 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1729
1730 MLX5_SET(mkc, mkc, free, 1);
1731 MLX5_SET(mkc, mkc, qpn, 0xffffff);
1732 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
1733 MLX5_SET(mkc, mkc, translations_octword_size, ndescs);
1734 MLX5_SET(mkc, mkc, access_mode_1_0, access_mode & 0x3);
1735 MLX5_SET(mkc, mkc, access_mode_4_2, (access_mode >> 2) & 0x7);
1736 MLX5_SET(mkc, mkc, umr_en, 1);
1737 MLX5_SET(mkc, mkc, log_page_size, page_shift);
1738 }
1739
1740 static int _mlx5_alloc_mkey_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,
1741 int ndescs, int desc_size, int page_shift,
1742 int access_mode, u32 *in, int inlen)
1743 {
1744 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1745 int err;
1746
1747 mr->access_mode = access_mode;
1748 mr->desc_size = desc_size;
1749 mr->max_descs = ndescs;
1750
1751 err = mlx5_alloc_priv_descs(pd->device, mr, ndescs, desc_size);
1752 if (err)
1753 return err;
1754
1755 mlx5_set_umr_free_mkey(pd, in, ndescs, access_mode, page_shift);
1756
1757 err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen);
1758 if (err)
1759 goto err_free_descs;
1760
1761 mr->mmkey.type = MLX5_MKEY_MR;
1762 mr->ibmr.lkey = mr->mmkey.key;
1763 mr->ibmr.rkey = mr->mmkey.key;
1764
1765 return 0;
1766
1767 err_free_descs:
1768 mlx5_free_priv_descs(mr);
1769 return err;
1770 }
1771
1772 static struct mlx5_ib_mr *mlx5_ib_alloc_pi_mr(struct ib_pd *pd,
1773 u32 max_num_sg, u32 max_num_meta_sg,
1774 int desc_size, int access_mode)
1775 {
1776 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
1777 int ndescs = ALIGN(max_num_sg + max_num_meta_sg, 4);
1778 int page_shift = 0;
1779 struct mlx5_ib_mr *mr;
1780 u32 *in;
1781 int err;
1782
1783 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1784 if (!mr)
1785 return ERR_PTR(-ENOMEM);
1786
1787 mr->ibmr.pd = pd;
1788 mr->ibmr.device = pd->device;
1789
1790 in = kzalloc(inlen, GFP_KERNEL);
1791 if (!in) {
1792 err = -ENOMEM;
1793 goto err_free;
1794 }
1795
1796 if (access_mode == MLX5_MKC_ACCESS_MODE_MTT)
1797 page_shift = PAGE_SHIFT;
1798
1799 err = _mlx5_alloc_mkey_descs(pd, mr, ndescs, desc_size, page_shift,
1800 access_mode, in, inlen);
1801 if (err)
1802 goto err_free_in;
1803
1804 mr->umem = NULL;
1805 kfree(in);
1806
1807 return mr;
1808
1809 err_free_in:
1810 kfree(in);
1811 err_free:
1812 kfree(mr);
1813 return ERR_PTR(err);
1814 }
1815
1816 static int mlx5_alloc_mem_reg_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,
1817 int ndescs, u32 *in, int inlen)
1818 {
1819 return _mlx5_alloc_mkey_descs(pd, mr, ndescs, sizeof(struct mlx5_mtt),
1820 PAGE_SHIFT, MLX5_MKC_ACCESS_MODE_MTT, in,
1821 inlen);
1822 }
1823
1824 static int mlx5_alloc_sg_gaps_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,
1825 int ndescs, u32 *in, int inlen)
1826 {
1827 return _mlx5_alloc_mkey_descs(pd, mr, ndescs, sizeof(struct mlx5_klm),
1828 0, MLX5_MKC_ACCESS_MODE_KLMS, in, inlen);
1829 }
1830
1831 static int mlx5_alloc_integrity_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr,
1832 int max_num_sg, int max_num_meta_sg,
1833 u32 *in, int inlen)
1834 {
1835 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1836 u32 psv_index[2];
1837 void *mkc;
1838 int err;
1839
1840 mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL);
1841 if (!mr->sig)
1842 return -ENOMEM;
1843
1844 /* create mem & wire PSVs */
1845 err = mlx5_core_create_psv(dev->mdev, to_mpd(pd)->pdn, 2, psv_index);
1846 if (err)
1847 goto err_free_sig;
1848
1849 mr->sig->psv_memory.psv_idx = psv_index[0];
1850 mr->sig->psv_wire.psv_idx = psv_index[1];
1851
1852 mr->sig->sig_status_checked = true;
1853 mr->sig->sig_err_exists = false;
1854 /* Next UMR, Arm SIGERR */
1855 ++mr->sig->sigerr_count;
1856 mr->klm_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, max_num_meta_sg,
1857 sizeof(struct mlx5_klm),
1858 MLX5_MKC_ACCESS_MODE_KLMS);
1859 if (IS_ERR(mr->klm_mr)) {
1860 err = PTR_ERR(mr->klm_mr);
1861 goto err_destroy_psv;
1862 }
1863 mr->mtt_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, max_num_meta_sg,
1864 sizeof(struct mlx5_mtt),
1865 MLX5_MKC_ACCESS_MODE_MTT);
1866 if (IS_ERR(mr->mtt_mr)) {
1867 err = PTR_ERR(mr->mtt_mr);
1868 goto err_free_klm_mr;
1869 }
1870
1871 /* Set bsf descriptors for mkey */
1872 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1873 MLX5_SET(mkc, mkc, bsf_en, 1);
1874 MLX5_SET(mkc, mkc, bsf_octword_size, MLX5_MKEY_BSF_OCTO_SIZE);
1875
1876 err = _mlx5_alloc_mkey_descs(pd, mr, 4, sizeof(struct mlx5_klm), 0,
1877 MLX5_MKC_ACCESS_MODE_KLMS, in, inlen);
1878 if (err)
1879 goto err_free_mtt_mr;
1880
1881 err = xa_err(xa_store(&dev->sig_mrs, mlx5_base_mkey(mr->mmkey.key),
1882 mr->sig, GFP_KERNEL));
1883 if (err)
1884 goto err_free_descs;
1885 return 0;
1886
1887 err_free_descs:
1888 destroy_mkey(dev, mr);
1889 mlx5_free_priv_descs(mr);
1890 err_free_mtt_mr:
1891 dereg_mr(to_mdev(mr->mtt_mr->ibmr.device), mr->mtt_mr);
1892 mr->mtt_mr = NULL;
1893 err_free_klm_mr:
1894 dereg_mr(to_mdev(mr->klm_mr->ibmr.device), mr->klm_mr);
1895 mr->klm_mr = NULL;
1896 err_destroy_psv:
1897 if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_memory.psv_idx))
1898 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n",
1899 mr->sig->psv_memory.psv_idx);
1900 if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_wire.psv_idx))
1901 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",
1902 mr->sig->psv_wire.psv_idx);
1903 err_free_sig:
1904 kfree(mr->sig);
1905
1906 return err;
1907 }
1908
1909 static struct ib_mr *__mlx5_ib_alloc_mr(struct ib_pd *pd,
1910 enum ib_mr_type mr_type, u32 max_num_sg,
1911 u32 max_num_meta_sg)
1912 {
1913 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1914 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
1915 int ndescs = ALIGN(max_num_sg, 4);
1916 struct mlx5_ib_mr *mr;
1917 u32 *in;
1918 int err;
1919
1920 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1921 if (!mr)
1922 return ERR_PTR(-ENOMEM);
1923
1924 in = kzalloc(inlen, GFP_KERNEL);
1925 if (!in) {
1926 err = -ENOMEM;
1927 goto err_free;
1928 }
1929
1930 mr->ibmr.device = pd->device;
1931 mr->umem = NULL;
1932
1933 switch (mr_type) {
1934 case IB_MR_TYPE_MEM_REG:
1935 err = mlx5_alloc_mem_reg_descs(pd, mr, ndescs, in, inlen);
1936 break;
1937 case IB_MR_TYPE_SG_GAPS:
1938 err = mlx5_alloc_sg_gaps_descs(pd, mr, ndescs, in, inlen);
1939 break;
1940 case IB_MR_TYPE_INTEGRITY:
1941 err = mlx5_alloc_integrity_descs(pd, mr, max_num_sg,
1942 max_num_meta_sg, in, inlen);
1943 break;
1944 default:
1945 mlx5_ib_warn(dev, "Invalid mr type %d\n", mr_type);
1946 err = -EINVAL;
1947 }
1948
1949 if (err)
1950 goto err_free_in;
1951
1952 kfree(in);
1953
1954 return &mr->ibmr;
1955
1956 err_free_in:
1957 kfree(in);
1958 err_free:
1959 kfree(mr);
1960 return ERR_PTR(err);
1961 }
1962
1963 struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
1964 u32 max_num_sg, struct ib_udata *udata)
1965 {
1966 return __mlx5_ib_alloc_mr(pd, mr_type, max_num_sg, 0);
1967 }
1968
1969 struct ib_mr *mlx5_ib_alloc_mr_integrity(struct ib_pd *pd,
1970 u32 max_num_sg, u32 max_num_meta_sg)
1971 {
1972 return __mlx5_ib_alloc_mr(pd, IB_MR_TYPE_INTEGRITY, max_num_sg,
1973 max_num_meta_sg);
1974 }
1975
1976 struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
1977 struct ib_udata *udata)
1978 {
1979 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1980 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
1981 struct mlx5_ib_mw *mw = NULL;
1982 u32 *in = NULL;
1983 void *mkc;
1984 int ndescs;
1985 int err;
1986 struct mlx5_ib_alloc_mw req = {};
1987 struct {
1988 __u32 comp_mask;
1989 __u32 response_length;
1990 } resp = {};
1991
1992 err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req)));
1993 if (err)
1994 return ERR_PTR(err);
1995
1996 if (req.comp_mask || req.reserved1 || req.reserved2)
1997 return ERR_PTR(-EOPNOTSUPP);
1998
1999 if (udata->inlen > sizeof(req) &&
2000 !ib_is_udata_cleared(udata, sizeof(req),
2001 udata->inlen - sizeof(req)))
2002 return ERR_PTR(-EOPNOTSUPP);
2003
2004 ndescs = req.num_klms ? roundup(req.num_klms, 4) : roundup(1, 4);
2005
2006 mw = kzalloc(sizeof(*mw), GFP_KERNEL);
2007 in = kzalloc(inlen, GFP_KERNEL);
2008 if (!mw || !in) {
2009 err = -ENOMEM;
2010 goto free;
2011 }
2012
2013 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
2014
2015 MLX5_SET(mkc, mkc, free, 1);
2016 MLX5_SET(mkc, mkc, translations_octword_size, ndescs);
2017 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
2018 MLX5_SET(mkc, mkc, umr_en, 1);
2019 MLX5_SET(mkc, mkc, lr, 1);
2020 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KLMS);
2021 MLX5_SET(mkc, mkc, en_rinval, !!((type == IB_MW_TYPE_2)));
2022 MLX5_SET(mkc, mkc, qpn, 0xffffff);
2023
2024 err = mlx5_ib_create_mkey(dev, &mw->mmkey, in, inlen);
2025 if (err)
2026 goto free;
2027
2028 mw->mmkey.type = MLX5_MKEY_MW;
2029 mw->ibmw.rkey = mw->mmkey.key;
2030 mw->ndescs = ndescs;
2031
2032 resp.response_length = min(offsetof(typeof(resp), response_length) +
2033 sizeof(resp.response_length), udata->outlen);
2034 if (resp.response_length) {
2035 err = ib_copy_to_udata(udata, &resp, resp.response_length);
2036 if (err) {
2037 mlx5_core_destroy_mkey(dev->mdev, &mw->mmkey);
2038 goto free;
2039 }
2040 }
2041
2042 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {
2043 err = xa_err(xa_store(&dev->odp_mkeys,
2044 mlx5_base_mkey(mw->mmkey.key), &mw->mmkey,
2045 GFP_KERNEL));
2046 if (err)
2047 goto free_mkey;
2048 }
2049
2050 kfree(in);
2051 return &mw->ibmw;
2052
2053 free_mkey:
2054 mlx5_core_destroy_mkey(dev->mdev, &mw->mmkey);
2055 free:
2056 kfree(mw);
2057 kfree(in);
2058 return ERR_PTR(err);
2059 }
2060
2061 int mlx5_ib_dealloc_mw(struct ib_mw *mw)
2062 {
2063 struct mlx5_ib_dev *dev = to_mdev(mw->device);
2064 struct mlx5_ib_mw *mmw = to_mmw(mw);
2065 int err;
2066
2067 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {
2068 xa_erase(&dev->odp_mkeys, mlx5_base_mkey(mmw->mmkey.key));
2069 /*
2070 * pagefault_single_data_segment() may be accessing mmw under
2071 * SRCU if the user bound an ODP MR to this MW.
2072 */
2073 synchronize_srcu(&dev->odp_srcu);
2074 }
2075
2076 err = mlx5_core_destroy_mkey(dev->mdev, &mmw->mmkey);
2077 if (err)
2078 return err;
2079 kfree(mmw);
2080 return 0;
2081 }
2082
2083 int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
2084 struct ib_mr_status *mr_status)
2085 {
2086 struct mlx5_ib_mr *mmr = to_mmr(ibmr);
2087 int ret = 0;
2088
2089 if (check_mask & ~IB_MR_CHECK_SIG_STATUS) {
2090 pr_err("Invalid status check mask\n");
2091 ret = -EINVAL;
2092 goto done;
2093 }
2094
2095 mr_status->fail_status = 0;
2096 if (check_mask & IB_MR_CHECK_SIG_STATUS) {
2097 if (!mmr->sig) {
2098 ret = -EINVAL;
2099 pr_err("signature status check requested on a non-signature enabled MR\n");
2100 goto done;
2101 }
2102
2103 mmr->sig->sig_status_checked = true;
2104 if (!mmr->sig->sig_err_exists)
2105 goto done;
2106
2107 if (ibmr->lkey == mmr->sig->err_item.key)
2108 memcpy(&mr_status->sig_err, &mmr->sig->err_item,
2109 sizeof(mr_status->sig_err));
2110 else {
2111 mr_status->sig_err.err_type = IB_SIG_BAD_GUARD;
2112 mr_status->sig_err.sig_err_offset = 0;
2113 mr_status->sig_err.key = mmr->sig->err_item.key;
2114 }
2115
2116 mmr->sig->sig_err_exists = false;
2117 mr_status->fail_status |= IB_MR_CHECK_SIG_STATUS;
2118 }
2119
2120 done:
2121 return ret;
2122 }
2123
2124 static int
2125 mlx5_ib_map_pa_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
2126 int data_sg_nents, unsigned int *data_sg_offset,
2127 struct scatterlist *meta_sg, int meta_sg_nents,
2128 unsigned int *meta_sg_offset)
2129 {
2130 struct mlx5_ib_mr *mr = to_mmr(ibmr);
2131 unsigned int sg_offset = 0;
2132 int n = 0;
2133
2134 mr->meta_length = 0;
2135 if (data_sg_nents == 1) {
2136 n++;
2137 mr->ndescs = 1;
2138 if (data_sg_offset)
2139 sg_offset = *data_sg_offset;
2140 mr->data_length = sg_dma_len(data_sg) - sg_offset;
2141 mr->data_iova = sg_dma_address(data_sg) + sg_offset;
2142 if (meta_sg_nents == 1) {
2143 n++;
2144 mr->meta_ndescs = 1;
2145 if (meta_sg_offset)
2146 sg_offset = *meta_sg_offset;
2147 else
2148 sg_offset = 0;
2149 mr->meta_length = sg_dma_len(meta_sg) - sg_offset;
2150 mr->pi_iova = sg_dma_address(meta_sg) + sg_offset;
2151 }
2152 ibmr->length = mr->data_length + mr->meta_length;
2153 }
2154
2155 return n;
2156 }
2157
2158 static int
2159 mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr,
2160 struct scatterlist *sgl,
2161 unsigned short sg_nents,
2162 unsigned int *sg_offset_p,
2163 struct scatterlist *meta_sgl,
2164 unsigned short meta_sg_nents,
2165 unsigned int *meta_sg_offset_p)
2166 {
2167 struct scatterlist *sg = sgl;
2168 struct mlx5_klm *klms = mr->descs;
2169 unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0;
2170 u32 lkey = mr->ibmr.pd->local_dma_lkey;
2171 int i, j = 0;
2172
2173 mr->ibmr.iova = sg_dma_address(sg) + sg_offset;
2174 mr->ibmr.length = 0;
2175
2176 for_each_sg(sgl, sg, sg_nents, i) {
2177 if (unlikely(i >= mr->max_descs))
2178 break;
2179 klms[i].va = cpu_to_be64(sg_dma_address(sg) + sg_offset);
2180 klms[i].bcount = cpu_to_be32(sg_dma_len(sg) - sg_offset);
2181 klms[i].key = cpu_to_be32(lkey);
2182 mr->ibmr.length += sg_dma_len(sg) - sg_offset;
2183
2184 sg_offset = 0;
2185 }
2186
2187 if (sg_offset_p)
2188 *sg_offset_p = sg_offset;
2189
2190 mr->ndescs = i;
2191 mr->data_length = mr->ibmr.length;
2192
2193 if (meta_sg_nents) {
2194 sg = meta_sgl;
2195 sg_offset = meta_sg_offset_p ? *meta_sg_offset_p : 0;
2196 for_each_sg(meta_sgl, sg, meta_sg_nents, j) {
2197 if (unlikely(i + j >= mr->max_descs))
2198 break;
2199 klms[i + j].va = cpu_to_be64(sg_dma_address(sg) +
2200 sg_offset);
2201 klms[i + j].bcount = cpu_to_be32(sg_dma_len(sg) -
2202 sg_offset);
2203 klms[i + j].key = cpu_to_be32(lkey);
2204 mr->ibmr.length += sg_dma_len(sg) - sg_offset;
2205
2206 sg_offset = 0;
2207 }
2208 if (meta_sg_offset_p)
2209 *meta_sg_offset_p = sg_offset;
2210
2211 mr->meta_ndescs = j;
2212 mr->meta_length = mr->ibmr.length - mr->data_length;
2213 }
2214
2215 return i + j;
2216 }
2217
2218 static int mlx5_set_page(struct ib_mr *ibmr, u64 addr)
2219 {
2220 struct mlx5_ib_mr *mr = to_mmr(ibmr);
2221 __be64 *descs;
2222
2223 if (unlikely(mr->ndescs == mr->max_descs))
2224 return -ENOMEM;
2225
2226 descs = mr->descs;
2227 descs[mr->ndescs++] = cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR);
2228
2229 return 0;
2230 }
2231
2232 static int mlx5_set_page_pi(struct ib_mr *ibmr, u64 addr)
2233 {
2234 struct mlx5_ib_mr *mr = to_mmr(ibmr);
2235 __be64 *descs;
2236
2237 if (unlikely(mr->ndescs + mr->meta_ndescs == mr->max_descs))
2238 return -ENOMEM;
2239
2240 descs = mr->descs;
2241 descs[mr->ndescs + mr->meta_ndescs++] =
2242 cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR);
2243
2244 return 0;
2245 }
2246
2247 static int
2248 mlx5_ib_map_mtt_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
2249 int data_sg_nents, unsigned int *data_sg_offset,
2250 struct scatterlist *meta_sg, int meta_sg_nents,
2251 unsigned int *meta_sg_offset)
2252 {
2253 struct mlx5_ib_mr *mr = to_mmr(ibmr);
2254 struct mlx5_ib_mr *pi_mr = mr->mtt_mr;
2255 int n;
2256
2257 pi_mr->ndescs = 0;
2258 pi_mr->meta_ndescs = 0;
2259 pi_mr->meta_length = 0;
2260
2261 ib_dma_sync_single_for_cpu(ibmr->device, pi_mr->desc_map,
2262 pi_mr->desc_size * pi_mr->max_descs,
2263 DMA_TO_DEVICE);
2264
2265 pi_mr->ibmr.page_size = ibmr->page_size;
2266 n = ib_sg_to_pages(&pi_mr->ibmr, data_sg, data_sg_nents, data_sg_offset,
2267 mlx5_set_page);
2268 if (n != data_sg_nents)
2269 return n;
2270
2271 pi_mr->data_iova = pi_mr->ibmr.iova;
2272 pi_mr->data_length = pi_mr->ibmr.length;
2273 pi_mr->ibmr.length = pi_mr->data_length;
2274 ibmr->length = pi_mr->data_length;
2275
2276 if (meta_sg_nents) {
2277 u64 page_mask = ~((u64)ibmr->page_size - 1);
2278 u64 iova = pi_mr->data_iova;
2279
2280 n += ib_sg_to_pages(&pi_mr->ibmr, meta_sg, meta_sg_nents,
2281 meta_sg_offset, mlx5_set_page_pi);
2282
2283 pi_mr->meta_length = pi_mr->ibmr.length;
2284 /*
2285 * PI address for the HW is the offset of the metadata address
2286 * relative to the first data page address.
2287 * It equals to first data page address + size of data pages +
2288 * metadata offset at the first metadata page
2289 */
2290 pi_mr->pi_iova = (iova & page_mask) +
2291 pi_mr->ndescs * ibmr->page_size +
2292 (pi_mr->ibmr.iova & ~page_mask);
2293 /*
2294 * In order to use one MTT MR for data and metadata, we register
2295 * also the gaps between the end of the data and the start of
2296 * the metadata (the sig MR will verify that the HW will access
2297 * to right addresses). This mapping is safe because we use
2298 * internal mkey for the registration.
2299 */
2300 pi_mr->ibmr.length = pi_mr->pi_iova + pi_mr->meta_length - iova;
2301 pi_mr->ibmr.iova = iova;
2302 ibmr->length += pi_mr->meta_length;
2303 }
2304
2305 ib_dma_sync_single_for_device(ibmr->device, pi_mr->desc_map,
2306 pi_mr->desc_size * pi_mr->max_descs,
2307 DMA_TO_DEVICE);
2308
2309 return n;
2310 }
2311
2312 static int
2313 mlx5_ib_map_klm_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
2314 int data_sg_nents, unsigned int *data_sg_offset,
2315 struct scatterlist *meta_sg, int meta_sg_nents,
2316 unsigned int *meta_sg_offset)
2317 {
2318 struct mlx5_ib_mr *mr = to_mmr(ibmr);
2319 struct mlx5_ib_mr *pi_mr = mr->klm_mr;
2320 int n;
2321
2322 pi_mr->ndescs = 0;
2323 pi_mr->meta_ndescs = 0;
2324 pi_mr->meta_length = 0;
2325
2326 ib_dma_sync_single_for_cpu(ibmr->device, pi_mr->desc_map,
2327 pi_mr->desc_size * pi_mr->max_descs,
2328 DMA_TO_DEVICE);
2329
2330 n = mlx5_ib_sg_to_klms(pi_mr, data_sg, data_sg_nents, data_sg_offset,
2331 meta_sg, meta_sg_nents, meta_sg_offset);
2332
2333 ib_dma_sync_single_for_device(ibmr->device, pi_mr->desc_map,
2334 pi_mr->desc_size * pi_mr->max_descs,
2335 DMA_TO_DEVICE);
2336
2337 /* This is zero-based memory region */
2338 pi_mr->data_iova = 0;
2339 pi_mr->ibmr.iova = 0;
2340 pi_mr->pi_iova = pi_mr->data_length;
2341 ibmr->length = pi_mr->ibmr.length;
2342
2343 return n;
2344 }
2345
2346 int mlx5_ib_map_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,
2347 int data_sg_nents, unsigned int *data_sg_offset,
2348 struct scatterlist *meta_sg, int meta_sg_nents,
2349 unsigned int *meta_sg_offset)
2350 {
2351 struct mlx5_ib_mr *mr = to_mmr(ibmr);
2352 struct mlx5_ib_mr *pi_mr = NULL;
2353 int n;
2354
2355 WARN_ON(ibmr->type != IB_MR_TYPE_INTEGRITY);
2356
2357 mr->ndescs = 0;
2358 mr->data_length = 0;
2359 mr->data_iova = 0;
2360 mr->meta_ndescs = 0;
2361 mr->pi_iova = 0;
2362 /*
2363 * As a performance optimization, if possible, there is no need to
2364 * perform UMR operation to register the data/metadata buffers.
2365 * First try to map the sg lists to PA descriptors with local_dma_lkey.
2366 * Fallback to UMR only in case of a failure.
2367 */
2368 n = mlx5_ib_map_pa_mr_sg_pi(ibmr, data_sg, data_sg_nents,
2369 data_sg_offset, meta_sg, meta_sg_nents,
2370 meta_sg_offset);
2371 if (n == data_sg_nents + meta_sg_nents)
2372 goto out;
2373 /*
2374 * As a performance optimization, if possible, there is no need to map
2375 * the sg lists to KLM descriptors. First try to map the sg lists to MTT
2376 * descriptors and fallback to KLM only in case of a failure.
2377 * It's more efficient for the HW to work with MTT descriptors
2378 * (especially in high load).
2379 * Use KLM (indirect access) only if it's mandatory.
2380 */
2381 pi_mr = mr->mtt_mr;
2382 n = mlx5_ib_map_mtt_mr_sg_pi(ibmr, data_sg, data_sg_nents,
2383 data_sg_offset, meta_sg, meta_sg_nents,
2384 meta_sg_offset);
2385 if (n == data_sg_nents + meta_sg_nents)
2386 goto out;
2387
2388 pi_mr = mr->klm_mr;
2389 n = mlx5_ib_map_klm_mr_sg_pi(ibmr, data_sg, data_sg_nents,
2390 data_sg_offset, meta_sg, meta_sg_nents,
2391 meta_sg_offset);
2392 if (unlikely(n != data_sg_nents + meta_sg_nents))
2393 return -ENOMEM;
2394
2395 out:
2396 /* This is zero-based memory region */
2397 ibmr->iova = 0;
2398 mr->pi_mr = pi_mr;
2399 if (pi_mr)
2400 ibmr->sig_attrs->meta_length = pi_mr->meta_length;
2401 else
2402 ibmr->sig_attrs->meta_length = mr->meta_length;
2403
2404 return 0;
2405 }
2406
2407 int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
2408 unsigned int *sg_offset)
2409 {
2410 struct mlx5_ib_mr *mr = to_mmr(ibmr);
2411 int n;
2412
2413 mr->ndescs = 0;
2414
2415 ib_dma_sync_single_for_cpu(ibmr->device, mr->desc_map,
2416 mr->desc_size * mr->max_descs,
2417 DMA_TO_DEVICE);
2418
2419 if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS)
2420 n = mlx5_ib_sg_to_klms(mr, sg, sg_nents, sg_offset, NULL, 0,
2421 NULL);
2422 else
2423 n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset,
2424 mlx5_set_page);
2425
2426 ib_dma_sync_single_for_device(ibmr->device, mr->desc_map,
2427 mr->desc_size * mr->max_descs,
2428 DMA_TO_DEVICE);
2429
2430 return n;
2431 }
Linux with added FPC-III support