?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[TG3]: Set minimal hw interrupt mitigation.
[linux-2.6/verdex.git] / drivers / infiniband / hw / mthca / mthca_mr.c
blob8960fc2306beb1c0052950e87288e5b7f6e469fa
1 /*
2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 * $Id: mthca_mr.c 1349 2004-12-16 21:09:43Z roland $
33 */
34
35 #include <linux/slab.h>
36 #include <linux/init.h>
37 #include <linux/errno.h>
38
39 #include "mthca_dev.h"
40 #include "mthca_cmd.h"
41 #include "mthca_memfree.h"
42
43 /*
44 * Must be packed because mtt_seg is 64 bits but only aligned to 32 bits.
45 */
46 struct mthca_mpt_entry {
47 u32 flags;
48 u32 page_size;
49 u32 key;
50 u32 pd;
51 u64 start;
52 u64 length;
53 u32 lkey;
54 u32 window_count;
55 u32 window_count_limit;
56 u64 mtt_seg;
57 u32 mtt_sz; /* Arbel only */
58 u32 reserved[2];
59 } __attribute__((packed));
60
61 #define MTHCA_MPT_FLAG_SW_OWNS (0xfUL << 28)
62 #define MTHCA_MPT_FLAG_MIO (1 << 17)
63 #define MTHCA_MPT_FLAG_BIND_ENABLE (1 << 15)
64 #define MTHCA_MPT_FLAG_PHYSICAL (1 << 9)
65 #define MTHCA_MPT_FLAG_REGION (1 << 8)
66
67 #define MTHCA_MTT_FLAG_PRESENT 1
68
69 #define MTHCA_MPT_STATUS_SW 0xF0
70 #define MTHCA_MPT_STATUS_HW 0x00
71
72 /*
73 * Buddy allocator for MTT segments (currently not very efficient
74 * since it doesn't keep a free list and just searches linearly
75 * through the bitmaps)
76 */
77
78 static u32 mthca_buddy_alloc(struct mthca_buddy *buddy, int order)
79 {
80 int o;
81 int m;
82 u32 seg;
83
84 spin_lock(&buddy->lock);
85
86 for (o = order; o <= buddy->max_order; ++o) {
87 m = 1 << (buddy->max_order - o);
88 seg = find_first_bit(buddy->bits[o], m);
89 if (seg < m)
90 goto found;
91 }
92
93 spin_unlock(&buddy->lock);
94 return -1;
95
96 found:
97 clear_bit(seg, buddy->bits[o]);
98
99 while (o > order) {
100 --o;
101 seg <<= 1;
102 set_bit(seg ^ 1, buddy->bits[o]);
103 }
104
105 spin_unlock(&buddy->lock);
106
107 seg <<= order;
108
109 return seg;
110 }
111
112 static void mthca_buddy_free(struct mthca_buddy *buddy, u32 seg, int order)
113 {
114 seg >>= order;
115
116 spin_lock(&buddy->lock);
117
118 while (test_bit(seg ^ 1, buddy->bits[order])) {
119 clear_bit(seg ^ 1, buddy->bits[order]);
120 seg >>= 1;
121 ++order;
122 }
123
124 set_bit(seg, buddy->bits[order]);
125
126 spin_unlock(&buddy->lock);
127 }
128
129 static int __devinit mthca_buddy_init(struct mthca_buddy *buddy, int max_order)
130 {
131 int i, s;
132
133 buddy->max_order = max_order;
134 spin_lock_init(&buddy->lock);
135
136 buddy->bits = kmalloc((buddy->max_order + 1) * sizeof (long *),
137 GFP_KERNEL);
138 if (!buddy->bits)
139 goto err_out;
140
141 memset(buddy->bits, 0, (buddy->max_order + 1) * sizeof (long *));
142
143 for (i = 0; i <= buddy->max_order; ++i) {
144 s = BITS_TO_LONGS(1 << (buddy->max_order - i));
145 buddy->bits[i] = kmalloc(s * sizeof (long), GFP_KERNEL);
146 if (!buddy->bits[i])
147 goto err_out_free;
148 bitmap_zero(buddy->bits[i],
149 1 << (buddy->max_order - i));
150 }
151
152 set_bit(0, buddy->bits[buddy->max_order]);
153
154 return 0;
155
156 err_out_free:
157 for (i = 0; i <= buddy->max_order; ++i)
158 kfree(buddy->bits[i]);
159
160 kfree(buddy->bits);
161
162 err_out:
163 return -ENOMEM;
164 }
165
166 static void __devexit mthca_buddy_cleanup(struct mthca_buddy *buddy)
167 {
168 int i;
169
170 for (i = 0; i <= buddy->max_order; ++i)
171 kfree(buddy->bits[i]);
172
173 kfree(buddy->bits);
174 }
175
176 static u32 mthca_alloc_mtt(struct mthca_dev *dev, int order,
177 struct mthca_buddy *buddy)
178 {
179 u32 seg = mthca_buddy_alloc(buddy, order);
180
181 if (seg == -1)
182 return -1;
183
184 if (mthca_is_memfree(dev))
185 if (mthca_table_get_range(dev, dev->mr_table.mtt_table, seg,
186 seg + (1 << order) - 1)) {
187 mthca_buddy_free(buddy, seg, order);
188 seg = -1;
189 }
190
191 return seg;
192 }
193
194 static void mthca_free_mtt(struct mthca_dev *dev, u32 seg, int order,
195 struct mthca_buddy* buddy)
196 {
197 mthca_buddy_free(buddy, seg, order);
198
199 if (mthca_is_memfree(dev))
200 mthca_table_put_range(dev, dev->mr_table.mtt_table, seg,
201 seg + (1 << order) - 1);
202 }
203
204 static inline u32 tavor_hw_index_to_key(u32 ind)
205 {
206 return ind;
207 }
208
209 static inline u32 tavor_key_to_hw_index(u32 key)
210 {
211 return key;
212 }
213
214 static inline u32 arbel_hw_index_to_key(u32 ind)
215 {
216 return (ind >> 24) | (ind << 8);
217 }
218
219 static inline u32 arbel_key_to_hw_index(u32 key)
220 {
221 return (key << 24) | (key >> 8);
222 }
223
224 static inline u32 hw_index_to_key(struct mthca_dev *dev, u32 ind)
225 {
226 if (mthca_is_memfree(dev))
227 return arbel_hw_index_to_key(ind);
228 else
229 return tavor_hw_index_to_key(ind);
230 }
231
232 static inline u32 key_to_hw_index(struct mthca_dev *dev, u32 key)
233 {
234 if (mthca_is_memfree(dev))
235 return arbel_key_to_hw_index(key);
236 else
237 return tavor_key_to_hw_index(key);
238 }
239
240 int mthca_mr_alloc_notrans(struct mthca_dev *dev, u32 pd,
241 u32 access, struct mthca_mr *mr)
242 {
243 void *mailbox = NULL;
244 struct mthca_mpt_entry *mpt_entry;
245 u32 key;
246 int err;
247 u8 status;
248
249 might_sleep();
250
251 mr->order = -1;
252 key = mthca_alloc(&dev->mr_table.mpt_alloc);
253 if (key == -1)
254 return -ENOMEM;
255 mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key);
256
257 if (mthca_is_memfree(dev)) {
258 err = mthca_table_get(dev, dev->mr_table.mpt_table, key);
259 if (err)
260 goto err_out_mpt_free;
261 }
262
263 mailbox = kmalloc(sizeof *mpt_entry + MTHCA_CMD_MAILBOX_EXTRA,
264 GFP_KERNEL);
265 if (!mailbox) {
266 err = -ENOMEM;
267 goto err_out_table;
268 }
269 mpt_entry = MAILBOX_ALIGN(mailbox);
270
271 mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS |
272 MTHCA_MPT_FLAG_MIO |
273 MTHCA_MPT_FLAG_PHYSICAL |
274 MTHCA_MPT_FLAG_REGION |
275 access);
276 mpt_entry->page_size = 0;
277 mpt_entry->key = cpu_to_be32(key);
278 mpt_entry->pd = cpu_to_be32(pd);
279 mpt_entry->start = 0;
280 mpt_entry->length = ~0ULL;
281
282 memset(&mpt_entry->lkey, 0,
283 sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, lkey));
284
285 err = mthca_SW2HW_MPT(dev, mpt_entry,
286 key & (dev->limits.num_mpts - 1),
287 &status);
288 if (err) {
289 mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err);
290 goto err_out_table;
291 } else if (status) {
292 mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n",
293 status);
294 err = -EINVAL;
295 goto err_out_table;
296 }
297
298 kfree(mailbox);
299 return err;
300
301 err_out_table:
302 if (mthca_is_memfree(dev))
303 mthca_table_put(dev, dev->mr_table.mpt_table, key);
304
305 err_out_mpt_free:
306 mthca_free(&dev->mr_table.mpt_alloc, key);
307 kfree(mailbox);
308 return err;
309 }
310
311 int mthca_mr_alloc_phys(struct mthca_dev *dev, u32 pd,
312 u64 *buffer_list, int buffer_size_shift,
313 int list_len, u64 iova, u64 total_size,
314 u32 access, struct mthca_mr *mr)
315 {
316 void *mailbox;
317 u64 *mtt_entry;
318 struct mthca_mpt_entry *mpt_entry;
319 u32 key;
320 int err = -ENOMEM;
321 u8 status;
322 int i;
323
324 might_sleep();
325 WARN_ON(buffer_size_shift >= 32);
326
327 key = mthca_alloc(&dev->mr_table.mpt_alloc);
328 if (key == -1)
329 return -ENOMEM;
330 mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key);
331
332 if (mthca_is_memfree(dev)) {
333 err = mthca_table_get(dev, dev->mr_table.mpt_table, key);
334 if (err)
335 goto err_out_mpt_free;
336 }
337
338 for (i = MTHCA_MTT_SEG_SIZE / 8, mr->order = 0;
339 i < list_len;
340 i <<= 1, ++mr->order)
341 ; /* nothing */
342
343 mr->first_seg = mthca_alloc_mtt(dev, mr->order,
344 &dev->mr_table.mtt_buddy);
345 if (mr->first_seg == -1)
346 goto err_out_table;
347
348 /*
349 * If list_len is odd, we add one more dummy entry for
350 * firmware efficiency.
351 */
352 mailbox = kmalloc(max(sizeof *mpt_entry,
353 (size_t) 8 * (list_len + (list_len & 1) + 2)) +
354 MTHCA_CMD_MAILBOX_EXTRA,
355 GFP_KERNEL);
356 if (!mailbox)
357 goto err_out_free_mtt;
358
359 mtt_entry = MAILBOX_ALIGN(mailbox);
360
361 mtt_entry[0] = cpu_to_be64(dev->mr_table.mtt_base +
362 mr->first_seg * MTHCA_MTT_SEG_SIZE);
363 mtt_entry[1] = 0;
364 for (i = 0; i < list_len; ++i)
365 mtt_entry[i + 2] = cpu_to_be64(buffer_list[i] |
366 MTHCA_MTT_FLAG_PRESENT);
367 if (list_len & 1) {
368 mtt_entry[i + 2] = 0;
369 ++list_len;
370 }
371
372 if (0) {
373 mthca_dbg(dev, "Dumping MPT entry\n");
374 for (i = 0; i < list_len + 2; ++i)
375 printk(KERN_ERR "[%2d] %016llx\n",
376 i, (unsigned long long) be64_to_cpu(mtt_entry[i]));
377 }
378
379 err = mthca_WRITE_MTT(dev, mtt_entry, list_len, &status);
380 if (err) {
381 mthca_warn(dev, "WRITE_MTT failed (%d)\n", err);
382 goto err_out_mailbox_free;
383 }
384 if (status) {
385 mthca_warn(dev, "WRITE_MTT returned status 0x%02x\n",
386 status);
387 err = -EINVAL;
388 goto err_out_mailbox_free;
389 }
390
391 mpt_entry = MAILBOX_ALIGN(mailbox);
392
393 mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS |
394 MTHCA_MPT_FLAG_MIO |
395 MTHCA_MPT_FLAG_REGION |
396 access);
397
398 mpt_entry->page_size = cpu_to_be32(buffer_size_shift - 12);
399 mpt_entry->key = cpu_to_be32(key);
400 mpt_entry->pd = cpu_to_be32(pd);
401 mpt_entry->start = cpu_to_be64(iova);
402 mpt_entry->length = cpu_to_be64(total_size);
403 memset(&mpt_entry->lkey, 0,
404 sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, lkey));
405 mpt_entry->mtt_seg = cpu_to_be64(dev->mr_table.mtt_base +
406 mr->first_seg * MTHCA_MTT_SEG_SIZE);
407
408 if (0) {
409 mthca_dbg(dev, "Dumping MPT entry %08x:\n", mr->ibmr.lkey);
410 for (i = 0; i < sizeof (struct mthca_mpt_entry) / 4; ++i) {
411 if (i % 4 == 0)
412 printk("[%02x] ", i * 4);
413 printk(" %08x", be32_to_cpu(((u32 *) mpt_entry)[i]));
414 if ((i + 1) % 4 == 0)
415 printk("\n");
416 }
417 }
418
419 err = mthca_SW2HW_MPT(dev, mpt_entry,
420 key & (dev->limits.num_mpts - 1),
421 &status);
422 if (err)
423 mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err);
424 else if (status) {
425 mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n",
426 status);
427 err = -EINVAL;
428 }
429
430 kfree(mailbox);
431 return err;
432
433 err_out_mailbox_free:
434 kfree(mailbox);
435
436 err_out_free_mtt:
437 mthca_free_mtt(dev, mr->first_seg, mr->order, &dev->mr_table.mtt_buddy);
438
439 err_out_table:
440 if (mthca_is_memfree(dev))
441 mthca_table_put(dev, dev->mr_table.mpt_table, key);
442
443 err_out_mpt_free:
444 mthca_free(&dev->mr_table.mpt_alloc, key);
445 return err;
446 }
447
448 /* Free mr or fmr */
449 static void mthca_free_region(struct mthca_dev *dev, u32 lkey, int order,
450 u32 first_seg, struct mthca_buddy *buddy)
451 {
452 if (order >= 0)
453 mthca_free_mtt(dev, first_seg, order, buddy);
454
455 if (mthca_is_memfree(dev))
456 mthca_table_put(dev, dev->mr_table.mpt_table,
457 arbel_key_to_hw_index(lkey));
458
459 mthca_free(&dev->mr_table.mpt_alloc, key_to_hw_index(dev, lkey));
460 }
461
462 void mthca_free_mr(struct mthca_dev *dev, struct mthca_mr *mr)
463 {
464 int err;
465 u8 status;
466
467 might_sleep();
468
469 err = mthca_HW2SW_MPT(dev, NULL,
470 key_to_hw_index(dev, mr->ibmr.lkey) &
471 (dev->limits.num_mpts - 1),
472 &status);
473 if (err)
474 mthca_warn(dev, "HW2SW_MPT failed (%d)\n", err);
475 else if (status)
476 mthca_warn(dev, "HW2SW_MPT returned status 0x%02x\n",
477 status);
478
479 mthca_free_region(dev, mr->ibmr.lkey, mr->order, mr->first_seg,
480 &dev->mr_table.mtt_buddy);
481 }
482
483 int mthca_fmr_alloc(struct mthca_dev *dev, u32 pd,
484 u32 access, struct mthca_fmr *mr)
485 {
486 struct mthca_mpt_entry *mpt_entry;
487 void *mailbox;
488 u64 mtt_seg;
489 u32 key, idx;
490 u8 status;
491 int list_len = mr->attr.max_pages;
492 int err = -ENOMEM;
493 int i;
494
495 might_sleep();
496
497 if (mr->attr.page_size < 12 || mr->attr.page_size >= 32)
498 return -EINVAL;
499
500 /* For Arbel, all MTTs must fit in the same page. */
501 if (mthca_is_memfree(dev) &&
502 mr->attr.max_pages * sizeof *mr->mem.arbel.mtts > PAGE_SIZE)
503 return -EINVAL;
504
505 mr->maps = 0;
506
507 key = mthca_alloc(&dev->mr_table.mpt_alloc);
508 if (key == -1)
509 return -ENOMEM;
510
511 idx = key & (dev->limits.num_mpts - 1);
512 mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key);
513
514 if (mthca_is_memfree(dev)) {
515 err = mthca_table_get(dev, dev->mr_table.mpt_table, key);
516 if (err)
517 goto err_out_mpt_free;
518
519 mr->mem.arbel.mpt = mthca_table_find(dev->mr_table.mpt_table, key);
520 BUG_ON(!mr->mem.arbel.mpt);
521 } else
522 mr->mem.tavor.mpt = dev->mr_table.tavor_fmr.mpt_base +
523 sizeof *(mr->mem.tavor.mpt) * idx;
524
525 for (i = MTHCA_MTT_SEG_SIZE / 8, mr->order = 0;
526 i < list_len;
527 i <<= 1, ++mr->order)
528 ; /* nothing */
529
530 mr->first_seg = mthca_alloc_mtt(dev, mr->order,
531 dev->mr_table.fmr_mtt_buddy);
532 if (mr->first_seg == -1)
533 goto err_out_table;
534
535 mtt_seg = mr->first_seg * MTHCA_MTT_SEG_SIZE;
536
537 if (mthca_is_memfree(dev)) {
538 mr->mem.arbel.mtts = mthca_table_find(dev->mr_table.mtt_table,
539 mr->first_seg);
540 BUG_ON(!mr->mem.arbel.mtts);
541 } else
542 mr->mem.tavor.mtts = dev->mr_table.tavor_fmr.mtt_base + mtt_seg;
543
544 mailbox = kmalloc(sizeof *mpt_entry + MTHCA_CMD_MAILBOX_EXTRA,
545 GFP_KERNEL);
546 if (!mailbox)
547 goto err_out_free_mtt;
548
549 mpt_entry = MAILBOX_ALIGN(mailbox);
550
551 mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS |
552 MTHCA_MPT_FLAG_MIO |
553 MTHCA_MPT_FLAG_REGION |
554 access);
555
556 mpt_entry->page_size = cpu_to_be32(mr->attr.page_size - 12);
557 mpt_entry->key = cpu_to_be32(key);
558 mpt_entry->pd = cpu_to_be32(pd);
559 memset(&mpt_entry->start, 0,
560 sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, start));
561 mpt_entry->mtt_seg = cpu_to_be64(dev->mr_table.mtt_base + mtt_seg);
562
563 if (0) {
564 mthca_dbg(dev, "Dumping MPT entry %08x:\n", mr->ibmr.lkey);
565 for (i = 0; i < sizeof (struct mthca_mpt_entry) / 4; ++i) {
566 if (i % 4 == 0)
567 printk("[%02x] ", i * 4);
568 printk(" %08x", be32_to_cpu(((u32 *) mpt_entry)[i]));
569 if ((i + 1) % 4 == 0)
570 printk("\n");
571 }
572 }
573
574 err = mthca_SW2HW_MPT(dev, mpt_entry,
575 key & (dev->limits.num_mpts - 1),
576 &status);
577 if (err) {
578 mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err);
579 goto err_out_mailbox_free;
580 }
581 if (status) {
582 mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n",
583 status);
584 err = -EINVAL;
585 goto err_out_mailbox_free;
586 }
587
588 kfree(mailbox);
589 return 0;
590
591 err_out_mailbox_free:
592 kfree(mailbox);
593
594 err_out_free_mtt:
595 mthca_free_mtt(dev, mr->first_seg, mr->order,
596 dev->mr_table.fmr_mtt_buddy);
597
598 err_out_table:
599 if (mthca_is_memfree(dev))
600 mthca_table_put(dev, dev->mr_table.mpt_table, key);
601
602 err_out_mpt_free:
603 mthca_free(&dev->mr_table.mpt_alloc, mr->ibmr.lkey);
604 return err;
605 }
606
607 int mthca_free_fmr(struct mthca_dev *dev, struct mthca_fmr *fmr)
608 {
609 if (fmr->maps)
610 return -EBUSY;
611
612 mthca_free_region(dev, fmr->ibmr.lkey, fmr->order, fmr->first_seg,
613 dev->mr_table.fmr_mtt_buddy);
614 return 0;
615 }
616
617 static inline int mthca_check_fmr(struct mthca_fmr *fmr, u64 *page_list,
618 int list_len, u64 iova)
619 {
620 int i, page_mask;
621
622 if (list_len > fmr->attr.max_pages)
623 return -EINVAL;
624
625 page_mask = (1 << fmr->attr.page_size) - 1;
626
627 /* We are getting page lists, so va must be page aligned. */
628 if (iova & page_mask)
629 return -EINVAL;
630
631 /* Trust the user not to pass misaligned data in page_list */
632 if (0)
633 for (i = 0; i < list_len; ++i) {
634 if (page_list[i] & ~page_mask)
635 return -EINVAL;
636 }
637
638 if (fmr->maps >= fmr->attr.max_maps)
639 return -EINVAL;
640
641 return 0;
642 }
643
644
645 int mthca_tavor_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
646 int list_len, u64 iova)
647 {
648 struct mthca_fmr *fmr = to_mfmr(ibfmr);
649 struct mthca_dev *dev = to_mdev(ibfmr->device);
650 struct mthca_mpt_entry mpt_entry;
651 u32 key;
652 int i, err;
653
654 err = mthca_check_fmr(fmr, page_list, list_len, iova);
655 if (err)
656 return err;
657
658 ++fmr->maps;
659
660 key = tavor_key_to_hw_index(fmr->ibmr.lkey);
661 key += dev->limits.num_mpts;
662 fmr->ibmr.lkey = fmr->ibmr.rkey = tavor_hw_index_to_key(key);
663
664 writeb(MTHCA_MPT_STATUS_SW, fmr->mem.tavor.mpt);
665
666 for (i = 0; i < list_len; ++i) {
667 __be64 mtt_entry = cpu_to_be64(page_list[i] |
668 MTHCA_MTT_FLAG_PRESENT);
669 mthca_write64_raw(mtt_entry, fmr->mem.tavor.mtts + i);
670 }
671
672 mpt_entry.lkey = cpu_to_be32(key);
673 mpt_entry.length = cpu_to_be64(list_len * (1ull << fmr->attr.page_size));
674 mpt_entry.start = cpu_to_be64(iova);
675
676 writel(mpt_entry.lkey, &fmr->mem.tavor.mpt->key);
677 memcpy_toio(&fmr->mem.tavor.mpt->start, &mpt_entry.start,
678 offsetof(struct mthca_mpt_entry, window_count) -
679 offsetof(struct mthca_mpt_entry, start));
680
681 writeb(MTHCA_MPT_STATUS_HW, fmr->mem.tavor.mpt);
682
683 return 0;
684 }
685
686 int mthca_arbel_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
687 int list_len, u64 iova)
688 {
689 struct mthca_fmr *fmr = to_mfmr(ibfmr);
690 struct mthca_dev *dev = to_mdev(ibfmr->device);
691 u32 key;
692 int i, err;
693
694 err = mthca_check_fmr(fmr, page_list, list_len, iova);
695 if (err)
696 return err;
697
698 ++fmr->maps;
699
700 key = arbel_key_to_hw_index(fmr->ibmr.lkey);
701 key += dev->limits.num_mpts;
702 fmr->ibmr.lkey = fmr->ibmr.rkey = arbel_hw_index_to_key(key);
703
704 *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_SW;
705
706 wmb();
707
708 for (i = 0; i < list_len; ++i)
709 fmr->mem.arbel.mtts[i] = cpu_to_be64(page_list[i] |
710 MTHCA_MTT_FLAG_PRESENT);
711
712 fmr->mem.arbel.mpt->key = cpu_to_be32(key);
713 fmr->mem.arbel.mpt->lkey = cpu_to_be32(key);
714 fmr->mem.arbel.mpt->length = cpu_to_be64(list_len * (1ull << fmr->attr.page_size));
715 fmr->mem.arbel.mpt->start = cpu_to_be64(iova);
716
717 wmb();
718
719 *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_HW;
720
721 wmb();
722
723 return 0;
724 }
725
726 void mthca_tavor_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr)
727 {
728 u32 key;
729
730 if (!fmr->maps)
731 return;
732
733 key = tavor_key_to_hw_index(fmr->ibmr.lkey);
734 key &= dev->limits.num_mpts - 1;
735 fmr->ibmr.lkey = fmr->ibmr.rkey = tavor_hw_index_to_key(key);
736
737 fmr->maps = 0;
738
739 writeb(MTHCA_MPT_STATUS_SW, fmr->mem.tavor.mpt);
740 }
741
742 void mthca_arbel_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr)
743 {
744 u32 key;
745
746 if (!fmr->maps)
747 return;
748
749 key = arbel_key_to_hw_index(fmr->ibmr.lkey);
750 key &= dev->limits.num_mpts - 1;
751 fmr->ibmr.lkey = fmr->ibmr.rkey = arbel_hw_index_to_key(key);
752
753 fmr->maps = 0;
754
755 *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_SW;
756 }
757
758 int __devinit mthca_init_mr_table(struct mthca_dev *dev)
759 {
760 int err, i;
761
762 err = mthca_alloc_init(&dev->mr_table.mpt_alloc,
763 dev->limits.num_mpts,
764 ~0, dev->limits.reserved_mrws);
765 if (err)
766 return err;
767
768 if (!mthca_is_memfree(dev) &&
769 (dev->mthca_flags & MTHCA_FLAG_DDR_HIDDEN))
770 dev->limits.fmr_reserved_mtts = 0;
771 else
772 dev->mthca_flags |= MTHCA_FLAG_FMR;
773
774 err = mthca_buddy_init(&dev->mr_table.mtt_buddy,
775 fls(dev->limits.num_mtt_segs - 1));
776
777 if (err)
778 goto err_mtt_buddy;
779
780 dev->mr_table.tavor_fmr.mpt_base = NULL;
781 dev->mr_table.tavor_fmr.mtt_base = NULL;
782
783 if (dev->limits.fmr_reserved_mtts) {
784 i = fls(dev->limits.fmr_reserved_mtts - 1);
785
786 if (i >= 31) {
787 mthca_warn(dev, "Unable to reserve 2^31 FMR MTTs.\n");
788 err = -EINVAL;
789 goto err_fmr_mpt;
790 }
791
792 dev->mr_table.tavor_fmr.mpt_base =
793 ioremap(dev->mr_table.mpt_base,
794 (1 << i) * sizeof (struct mthca_mpt_entry));
795
796 if (!dev->mr_table.tavor_fmr.mpt_base) {
797 mthca_warn(dev, "MPT ioremap for FMR failed.\n");
798 err = -ENOMEM;
799 goto err_fmr_mpt;
800 }
801
802 dev->mr_table.tavor_fmr.mtt_base =
803 ioremap(dev->mr_table.mtt_base,
804 (1 << i) * MTHCA_MTT_SEG_SIZE);
805 if (!dev->mr_table.tavor_fmr.mtt_base) {
806 mthca_warn(dev, "MTT ioremap for FMR failed.\n");
807 err = -ENOMEM;
808 goto err_fmr_mtt;
809 }
810
811 err = mthca_buddy_init(&dev->mr_table.tavor_fmr.mtt_buddy, i);
812 if (err)
813 goto err_fmr_mtt_buddy;
814
815 /* Prevent regular MRs from using FMR keys */
816 err = mthca_buddy_alloc(&dev->mr_table.mtt_buddy, i);
817 if (err)
818 goto err_reserve_fmr;
819
820 dev->mr_table.fmr_mtt_buddy =
821 &dev->mr_table.tavor_fmr.mtt_buddy;
822 } else
823 dev->mr_table.fmr_mtt_buddy = &dev->mr_table.mtt_buddy;
824
825 /* FMR table is always the first, take reserved MTTs out of there */
826 if (dev->limits.reserved_mtts) {
827 i = fls(dev->limits.reserved_mtts - 1);
828
829 if (mthca_alloc_mtt(dev, i, dev->mr_table.fmr_mtt_buddy) == -1) {
830 mthca_warn(dev, "MTT table of order %d is too small.\n",
831 dev->mr_table.fmr_mtt_buddy->max_order);
832 err = -ENOMEM;
833 goto err_reserve_mtts;
834 }
835 }
836
837 return 0;
838
839 err_reserve_mtts:
840 err_reserve_fmr:
841 if (dev->limits.fmr_reserved_mtts)
842 mthca_buddy_cleanup(&dev->mr_table.tavor_fmr.mtt_buddy);
843
844 err_fmr_mtt_buddy:
845 if (dev->mr_table.tavor_fmr.mtt_base)
846 iounmap(dev->mr_table.tavor_fmr.mtt_base);
847
848 err_fmr_mtt:
849 if (dev->mr_table.tavor_fmr.mpt_base)
850 iounmap(dev->mr_table.tavor_fmr.mpt_base);
851
852 err_fmr_mpt:
853 mthca_buddy_cleanup(&dev->mr_table.mtt_buddy);
854
855 err_mtt_buddy:
856 mthca_alloc_cleanup(&dev->mr_table.mpt_alloc);
857
858 return err;
859 }
860
861 void __devexit mthca_cleanup_mr_table(struct mthca_dev *dev)
862 {
863 /* XXX check if any MRs are still allocated? */
864 if (dev->limits.fmr_reserved_mtts)
865 mthca_buddy_cleanup(&dev->mr_table.tavor_fmr.mtt_buddy);
866
867 mthca_buddy_cleanup(&dev->mr_table.mtt_buddy);
868
869 if (dev->mr_table.tavor_fmr.mtt_base)
870 iounmap(dev->mr_table.tavor_fmr.mtt_base);
871 if (dev->mr_table.tavor_fmr.mpt_base)
872 iounmap(dev->mr_table.tavor_fmr.mpt_base);
873
874 mthca_alloc_cleanup(&dev->mr_table.mpt_alloc);
875 }
Verdex Pro support