2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/slab.h>
35 #include <linux/errno.h>
37 #include "mthca_dev.h"
38 #include "mthca_cmd.h"
39 #include "mthca_memfree.h"
42 struct mthca_buddy
*buddy
;
48 * Must be packed because mtt_seg is 64 bits but only aligned to 32 bits.
50 struct mthca_mpt_entry
{
59 __be32 window_count_limit
;
61 __be32 mtt_sz
; /* Arbel only */
63 } __attribute__((packed
));
65 #define MTHCA_MPT_FLAG_SW_OWNS (0xfUL << 28)
66 #define MTHCA_MPT_FLAG_MIO (1 << 17)
67 #define MTHCA_MPT_FLAG_BIND_ENABLE (1 << 15)
68 #define MTHCA_MPT_FLAG_PHYSICAL (1 << 9)
69 #define MTHCA_MPT_FLAG_REGION (1 << 8)
71 #define MTHCA_MTT_FLAG_PRESENT 1
73 #define MTHCA_MPT_STATUS_SW 0xF0
74 #define MTHCA_MPT_STATUS_HW 0x00
76 #define SINAI_FMR_KEY_INC 0x1000000
79 * Buddy allocator for MTT segments (currently not very efficient
80 * since it doesn't keep a free list and just searches linearly
81 * through the bitmaps)
84 static u32
mthca_buddy_alloc(struct mthca_buddy
*buddy
, int order
)
90 spin_lock(&buddy
->lock
);
92 for (o
= order
; o
<= buddy
->max_order
; ++o
)
93 if (buddy
->num_free
[o
]) {
94 m
= 1 << (buddy
->max_order
- o
);
95 seg
= find_first_bit(buddy
->bits
[o
], m
);
100 spin_unlock(&buddy
->lock
);
104 clear_bit(seg
, buddy
->bits
[o
]);
105 --buddy
->num_free
[o
];
110 set_bit(seg
^ 1, buddy
->bits
[o
]);
111 ++buddy
->num_free
[o
];
114 spin_unlock(&buddy
->lock
);
121 static void mthca_buddy_free(struct mthca_buddy
*buddy
, u32 seg
, int order
)
125 spin_lock(&buddy
->lock
);
127 while (test_bit(seg
^ 1, buddy
->bits
[order
])) {
128 clear_bit(seg
^ 1, buddy
->bits
[order
]);
129 --buddy
->num_free
[order
];
134 set_bit(seg
, buddy
->bits
[order
]);
135 ++buddy
->num_free
[order
];
137 spin_unlock(&buddy
->lock
);
140 static int mthca_buddy_init(struct mthca_buddy
*buddy
, int max_order
)
144 buddy
->max_order
= max_order
;
145 spin_lock_init(&buddy
->lock
);
147 buddy
->bits
= kzalloc((buddy
->max_order
+ 1) * sizeof (long *),
149 buddy
->num_free
= kzalloc((buddy
->max_order
+ 1) * sizeof (int *),
151 if (!buddy
->bits
|| !buddy
->num_free
)
154 for (i
= 0; i
<= buddy
->max_order
; ++i
) {
155 s
= BITS_TO_LONGS(1 << (buddy
->max_order
- i
));
156 buddy
->bits
[i
] = kmalloc(s
* sizeof (long), GFP_KERNEL
);
159 bitmap_zero(buddy
->bits
[i
],
160 1 << (buddy
->max_order
- i
));
163 set_bit(0, buddy
->bits
[buddy
->max_order
]);
164 buddy
->num_free
[buddy
->max_order
] = 1;
169 for (i
= 0; i
<= buddy
->max_order
; ++i
)
170 kfree(buddy
->bits
[i
]);
174 kfree(buddy
->num_free
);
179 static void mthca_buddy_cleanup(struct mthca_buddy
*buddy
)
183 for (i
= 0; i
<= buddy
->max_order
; ++i
)
184 kfree(buddy
->bits
[i
]);
187 kfree(buddy
->num_free
);
190 static u32
mthca_alloc_mtt_range(struct mthca_dev
*dev
, int order
,
191 struct mthca_buddy
*buddy
)
193 u32 seg
= mthca_buddy_alloc(buddy
, order
);
198 if (mthca_is_memfree(dev
))
199 if (mthca_table_get_range(dev
, dev
->mr_table
.mtt_table
, seg
,
200 seg
+ (1 << order
) - 1)) {
201 mthca_buddy_free(buddy
, seg
, order
);
208 static struct mthca_mtt
*__mthca_alloc_mtt(struct mthca_dev
*dev
, int size
,
209 struct mthca_buddy
*buddy
)
211 struct mthca_mtt
*mtt
;
215 return ERR_PTR(-EINVAL
);
217 mtt
= kmalloc(sizeof *mtt
, GFP_KERNEL
);
219 return ERR_PTR(-ENOMEM
);
223 for (i
= dev
->limits
.mtt_seg_size
/ 8; i
< size
; i
<<= 1)
226 mtt
->first_seg
= mthca_alloc_mtt_range(dev
, mtt
->order
, buddy
);
227 if (mtt
->first_seg
== -1) {
229 return ERR_PTR(-ENOMEM
);
235 struct mthca_mtt
*mthca_alloc_mtt(struct mthca_dev
*dev
, int size
)
237 return __mthca_alloc_mtt(dev
, size
, &dev
->mr_table
.mtt_buddy
);
240 void mthca_free_mtt(struct mthca_dev
*dev
, struct mthca_mtt
*mtt
)
245 mthca_buddy_free(mtt
->buddy
, mtt
->first_seg
, mtt
->order
);
247 mthca_table_put_range(dev
, dev
->mr_table
.mtt_table
,
249 mtt
->first_seg
+ (1 << mtt
->order
) - 1);
254 static int __mthca_write_mtt(struct mthca_dev
*dev
, struct mthca_mtt
*mtt
,
255 int start_index
, u64
*buffer_list
, int list_len
)
257 struct mthca_mailbox
*mailbox
;
262 mailbox
= mthca_alloc_mailbox(dev
, GFP_KERNEL
);
264 return PTR_ERR(mailbox
);
265 mtt_entry
= mailbox
->buf
;
267 while (list_len
> 0) {
268 mtt_entry
[0] = cpu_to_be64(dev
->mr_table
.mtt_base
+
269 mtt
->first_seg
* dev
->limits
.mtt_seg_size
+
272 for (i
= 0; i
< list_len
&& i
< MTHCA_MAILBOX_SIZE
/ 8 - 2; ++i
)
273 mtt_entry
[i
+ 2] = cpu_to_be64(buffer_list
[i
] |
274 MTHCA_MTT_FLAG_PRESENT
);
277 * If we have an odd number of entries to write, add
278 * one more dummy entry for firmware efficiency.
281 mtt_entry
[i
+ 2] = 0;
283 err
= mthca_WRITE_MTT(dev
, mailbox
, (i
+ 1) & ~1);
285 mthca_warn(dev
, "WRITE_MTT failed (%d)\n", err
);
295 mthca_free_mailbox(dev
, mailbox
);
299 int mthca_write_mtt_size(struct mthca_dev
*dev
)
301 if (dev
->mr_table
.fmr_mtt_buddy
!= &dev
->mr_table
.mtt_buddy
||
302 !(dev
->mthca_flags
& MTHCA_FLAG_FMR
))
304 * Be friendly to WRITE_MTT command
305 * and leave two empty slots for the
306 * index and reserved fields of the
309 return PAGE_SIZE
/ sizeof (u64
) - 2;
311 /* For Arbel, all MTTs must fit in the same page. */
312 return mthca_is_memfree(dev
) ? (PAGE_SIZE
/ sizeof (u64
)) : 0x7ffffff;
315 static void mthca_tavor_write_mtt_seg(struct mthca_dev
*dev
,
316 struct mthca_mtt
*mtt
, int start_index
,
317 u64
*buffer_list
, int list_len
)
322 mtts
= dev
->mr_table
.tavor_fmr
.mtt_base
+ mtt
->first_seg
* dev
->limits
.mtt_seg_size
+
323 start_index
* sizeof (u64
);
324 for (i
= 0; i
< list_len
; ++i
)
325 mthca_write64_raw(cpu_to_be64(buffer_list
[i
] | MTHCA_MTT_FLAG_PRESENT
),
329 static void mthca_arbel_write_mtt_seg(struct mthca_dev
*dev
,
330 struct mthca_mtt
*mtt
, int start_index
,
331 u64
*buffer_list
, int list_len
)
334 dma_addr_t dma_handle
;
336 int s
= start_index
* sizeof (u64
);
338 /* For Arbel, all MTTs must fit in the same page. */
339 BUG_ON(s
/ PAGE_SIZE
!= (s
+ list_len
* sizeof(u64
) - 1) / PAGE_SIZE
);
340 /* Require full segments */
341 BUG_ON(s
% dev
->limits
.mtt_seg_size
);
343 mtts
= mthca_table_find(dev
->mr_table
.mtt_table
, mtt
->first_seg
+
344 s
/ dev
->limits
.mtt_seg_size
, &dma_handle
);
348 dma_sync_single_for_cpu(&dev
->pdev
->dev
, dma_handle
,
349 list_len
* sizeof (u64
), DMA_TO_DEVICE
);
351 for (i
= 0; i
< list_len
; ++i
)
352 mtts
[i
] = cpu_to_be64(buffer_list
[i
] | MTHCA_MTT_FLAG_PRESENT
);
354 dma_sync_single_for_device(&dev
->pdev
->dev
, dma_handle
,
355 list_len
* sizeof (u64
), DMA_TO_DEVICE
);
358 int mthca_write_mtt(struct mthca_dev
*dev
, struct mthca_mtt
*mtt
,
359 int start_index
, u64
*buffer_list
, int list_len
)
361 int size
= mthca_write_mtt_size(dev
);
364 if (dev
->mr_table
.fmr_mtt_buddy
!= &dev
->mr_table
.mtt_buddy
||
365 !(dev
->mthca_flags
& MTHCA_FLAG_FMR
))
366 return __mthca_write_mtt(dev
, mtt
, start_index
, buffer_list
, list_len
);
368 while (list_len
> 0) {
369 chunk
= min(size
, list_len
);
370 if (mthca_is_memfree(dev
))
371 mthca_arbel_write_mtt_seg(dev
, mtt
, start_index
,
374 mthca_tavor_write_mtt_seg(dev
, mtt
, start_index
,
378 start_index
+= chunk
;
379 buffer_list
+= chunk
;
385 static inline u32
tavor_hw_index_to_key(u32 ind
)
390 static inline u32
tavor_key_to_hw_index(u32 key
)
395 static inline u32
arbel_hw_index_to_key(u32 ind
)
397 return (ind
>> 24) | (ind
<< 8);
400 static inline u32
arbel_key_to_hw_index(u32 key
)
402 return (key
<< 24) | (key
>> 8);
405 static inline u32
hw_index_to_key(struct mthca_dev
*dev
, u32 ind
)
407 if (mthca_is_memfree(dev
))
408 return arbel_hw_index_to_key(ind
);
410 return tavor_hw_index_to_key(ind
);
413 static inline u32
key_to_hw_index(struct mthca_dev
*dev
, u32 key
)
415 if (mthca_is_memfree(dev
))
416 return arbel_key_to_hw_index(key
);
418 return tavor_key_to_hw_index(key
);
421 static inline u32
adjust_key(struct mthca_dev
*dev
, u32 key
)
423 if (dev
->mthca_flags
& MTHCA_FLAG_SINAI_OPT
)
424 return ((key
<< 20) & 0x800000) | (key
& 0x7fffff);
429 int mthca_mr_alloc(struct mthca_dev
*dev
, u32 pd
, int buffer_size_shift
,
430 u64 iova
, u64 total_size
, u32 access
, struct mthca_mr
*mr
)
432 struct mthca_mailbox
*mailbox
;
433 struct mthca_mpt_entry
*mpt_entry
;
438 WARN_ON(buffer_size_shift
>= 32);
440 key
= mthca_alloc(&dev
->mr_table
.mpt_alloc
);
443 key
= adjust_key(dev
, key
);
444 mr
->ibmr
.rkey
= mr
->ibmr
.lkey
= hw_index_to_key(dev
, key
);
446 if (mthca_is_memfree(dev
)) {
447 err
= mthca_table_get(dev
, dev
->mr_table
.mpt_table
, key
);
449 goto err_out_mpt_free
;
452 mailbox
= mthca_alloc_mailbox(dev
, GFP_KERNEL
);
453 if (IS_ERR(mailbox
)) {
454 err
= PTR_ERR(mailbox
);
457 mpt_entry
= mailbox
->buf
;
459 mpt_entry
->flags
= cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS
|
461 MTHCA_MPT_FLAG_REGION
|
464 mpt_entry
->flags
|= cpu_to_be32(MTHCA_MPT_FLAG_PHYSICAL
);
466 mpt_entry
->page_size
= cpu_to_be32(buffer_size_shift
- 12);
467 mpt_entry
->key
= cpu_to_be32(key
);
468 mpt_entry
->pd
= cpu_to_be32(pd
);
469 mpt_entry
->start
= cpu_to_be64(iova
);
470 mpt_entry
->length
= cpu_to_be64(total_size
);
472 memset(&mpt_entry
->lkey
, 0,
473 sizeof *mpt_entry
- offsetof(struct mthca_mpt_entry
, lkey
));
477 cpu_to_be64(dev
->mr_table
.mtt_base
+
478 mr
->mtt
->first_seg
* dev
->limits
.mtt_seg_size
);
481 mthca_dbg(dev
, "Dumping MPT entry %08x:\n", mr
->ibmr
.lkey
);
482 for (i
= 0; i
< sizeof (struct mthca_mpt_entry
) / 4; ++i
) {
484 printk("[%02x] ", i
* 4);
485 printk(" %08x", be32_to_cpu(((__be32
*) mpt_entry
)[i
]));
486 if ((i
+ 1) % 4 == 0)
491 err
= mthca_SW2HW_MPT(dev
, mailbox
,
492 key
& (dev
->limits
.num_mpts
- 1));
494 mthca_warn(dev
, "SW2HW_MPT failed (%d)\n", err
);
495 goto err_out_mailbox
;
498 mthca_free_mailbox(dev
, mailbox
);
502 mthca_free_mailbox(dev
, mailbox
);
505 mthca_table_put(dev
, dev
->mr_table
.mpt_table
, key
);
508 mthca_free(&dev
->mr_table
.mpt_alloc
, key
);
512 int mthca_mr_alloc_notrans(struct mthca_dev
*dev
, u32 pd
,
513 u32 access
, struct mthca_mr
*mr
)
516 return mthca_mr_alloc(dev
, pd
, 12, 0, ~0ULL, access
, mr
);
519 int mthca_mr_alloc_phys(struct mthca_dev
*dev
, u32 pd
,
520 u64
*buffer_list
, int buffer_size_shift
,
521 int list_len
, u64 iova
, u64 total_size
,
522 u32 access
, struct mthca_mr
*mr
)
526 mr
->mtt
= mthca_alloc_mtt(dev
, list_len
);
528 return PTR_ERR(mr
->mtt
);
530 err
= mthca_write_mtt(dev
, mr
->mtt
, 0, buffer_list
, list_len
);
532 mthca_free_mtt(dev
, mr
->mtt
);
536 err
= mthca_mr_alloc(dev
, pd
, buffer_size_shift
, iova
,
537 total_size
, access
, mr
);
539 mthca_free_mtt(dev
, mr
->mtt
);
545 static void mthca_free_region(struct mthca_dev
*dev
, u32 lkey
)
547 mthca_table_put(dev
, dev
->mr_table
.mpt_table
,
548 key_to_hw_index(dev
, lkey
));
550 mthca_free(&dev
->mr_table
.mpt_alloc
, key_to_hw_index(dev
, lkey
));
553 void mthca_free_mr(struct mthca_dev
*dev
, struct mthca_mr
*mr
)
557 err
= mthca_HW2SW_MPT(dev
, NULL
,
558 key_to_hw_index(dev
, mr
->ibmr
.lkey
) &
559 (dev
->limits
.num_mpts
- 1));
561 mthca_warn(dev
, "HW2SW_MPT failed (%d)\n", err
);
563 mthca_free_region(dev
, mr
->ibmr
.lkey
);
564 mthca_free_mtt(dev
, mr
->mtt
);
567 int mthca_fmr_alloc(struct mthca_dev
*dev
, u32 pd
,
568 u32 access
, struct mthca_fmr
*mr
)
570 struct mthca_mpt_entry
*mpt_entry
;
571 struct mthca_mailbox
*mailbox
;
574 int list_len
= mr
->attr
.max_pages
;
578 if (mr
->attr
.page_shift
< 12 || mr
->attr
.page_shift
>= 32)
581 /* For Arbel, all MTTs must fit in the same page. */
582 if (mthca_is_memfree(dev
) &&
583 mr
->attr
.max_pages
* sizeof *mr
->mem
.arbel
.mtts
> PAGE_SIZE
)
588 key
= mthca_alloc(&dev
->mr_table
.mpt_alloc
);
591 key
= adjust_key(dev
, key
);
593 idx
= key
& (dev
->limits
.num_mpts
- 1);
594 mr
->ibmr
.rkey
= mr
->ibmr
.lkey
= hw_index_to_key(dev
, key
);
596 if (mthca_is_memfree(dev
)) {
597 err
= mthca_table_get(dev
, dev
->mr_table
.mpt_table
, key
);
599 goto err_out_mpt_free
;
601 mr
->mem
.arbel
.mpt
= mthca_table_find(dev
->mr_table
.mpt_table
, key
, NULL
);
602 BUG_ON(!mr
->mem
.arbel
.mpt
);
604 mr
->mem
.tavor
.mpt
= dev
->mr_table
.tavor_fmr
.mpt_base
+
605 sizeof *(mr
->mem
.tavor
.mpt
) * idx
;
607 mr
->mtt
= __mthca_alloc_mtt(dev
, list_len
, dev
->mr_table
.fmr_mtt_buddy
);
608 if (IS_ERR(mr
->mtt
)) {
609 err
= PTR_ERR(mr
->mtt
);
613 mtt_seg
= mr
->mtt
->first_seg
* dev
->limits
.mtt_seg_size
;
615 if (mthca_is_memfree(dev
)) {
616 mr
->mem
.arbel
.mtts
= mthca_table_find(dev
->mr_table
.mtt_table
,
618 &mr
->mem
.arbel
.dma_handle
);
619 BUG_ON(!mr
->mem
.arbel
.mtts
);
621 mr
->mem
.tavor
.mtts
= dev
->mr_table
.tavor_fmr
.mtt_base
+ mtt_seg
;
623 mailbox
= mthca_alloc_mailbox(dev
, GFP_KERNEL
);
624 if (IS_ERR(mailbox
)) {
625 err
= PTR_ERR(mailbox
);
626 goto err_out_free_mtt
;
629 mpt_entry
= mailbox
->buf
;
631 mpt_entry
->flags
= cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS
|
633 MTHCA_MPT_FLAG_REGION
|
636 mpt_entry
->page_size
= cpu_to_be32(mr
->attr
.page_shift
- 12);
637 mpt_entry
->key
= cpu_to_be32(key
);
638 mpt_entry
->pd
= cpu_to_be32(pd
);
639 memset(&mpt_entry
->start
, 0,
640 sizeof *mpt_entry
- offsetof(struct mthca_mpt_entry
, start
));
641 mpt_entry
->mtt_seg
= cpu_to_be64(dev
->mr_table
.mtt_base
+ mtt_seg
);
644 mthca_dbg(dev
, "Dumping MPT entry %08x:\n", mr
->ibmr
.lkey
);
645 for (i
= 0; i
< sizeof (struct mthca_mpt_entry
) / 4; ++i
) {
647 printk("[%02x] ", i
* 4);
648 printk(" %08x", be32_to_cpu(((__be32
*) mpt_entry
)[i
]));
649 if ((i
+ 1) % 4 == 0)
654 err
= mthca_SW2HW_MPT(dev
, mailbox
,
655 key
& (dev
->limits
.num_mpts
- 1));
657 mthca_warn(dev
, "SW2HW_MPT failed (%d)\n", err
);
658 goto err_out_mailbox_free
;
661 mthca_free_mailbox(dev
, mailbox
);
664 err_out_mailbox_free
:
665 mthca_free_mailbox(dev
, mailbox
);
668 mthca_free_mtt(dev
, mr
->mtt
);
671 mthca_table_put(dev
, dev
->mr_table
.mpt_table
, key
);
674 mthca_free(&dev
->mr_table
.mpt_alloc
, key
);
678 int mthca_free_fmr(struct mthca_dev
*dev
, struct mthca_fmr
*fmr
)
683 mthca_free_region(dev
, fmr
->ibmr
.lkey
);
684 mthca_free_mtt(dev
, fmr
->mtt
);
689 static inline int mthca_check_fmr(struct mthca_fmr
*fmr
, u64
*page_list
,
690 int list_len
, u64 iova
)
694 if (list_len
> fmr
->attr
.max_pages
)
697 page_mask
= (1 << fmr
->attr
.page_shift
) - 1;
699 /* We are getting page lists, so va must be page aligned. */
700 if (iova
& page_mask
)
703 /* Trust the user not to pass misaligned data in page_list */
705 for (i
= 0; i
< list_len
; ++i
) {
706 if (page_list
[i
] & ~page_mask
)
710 if (fmr
->maps
>= fmr
->attr
.max_maps
)
717 int mthca_tavor_map_phys_fmr(struct ib_fmr
*ibfmr
, u64
*page_list
,
718 int list_len
, u64 iova
)
720 struct mthca_fmr
*fmr
= to_mfmr(ibfmr
);
721 struct mthca_dev
*dev
= to_mdev(ibfmr
->device
);
722 struct mthca_mpt_entry mpt_entry
;
726 err
= mthca_check_fmr(fmr
, page_list
, list_len
, iova
);
732 key
= tavor_key_to_hw_index(fmr
->ibmr
.lkey
);
733 key
+= dev
->limits
.num_mpts
;
734 fmr
->ibmr
.lkey
= fmr
->ibmr
.rkey
= tavor_hw_index_to_key(key
);
736 writeb(MTHCA_MPT_STATUS_SW
, fmr
->mem
.tavor
.mpt
);
738 for (i
= 0; i
< list_len
; ++i
) {
739 __be64 mtt_entry
= cpu_to_be64(page_list
[i
] |
740 MTHCA_MTT_FLAG_PRESENT
);
741 mthca_write64_raw(mtt_entry
, fmr
->mem
.tavor
.mtts
+ i
);
744 mpt_entry
.lkey
= cpu_to_be32(key
);
745 mpt_entry
.length
= cpu_to_be64(list_len
* (1ull << fmr
->attr
.page_shift
));
746 mpt_entry
.start
= cpu_to_be64(iova
);
748 __raw_writel((__force u32
) mpt_entry
.lkey
, &fmr
->mem
.tavor
.mpt
->key
);
749 memcpy_toio(&fmr
->mem
.tavor
.mpt
->start
, &mpt_entry
.start
,
750 offsetof(struct mthca_mpt_entry
, window_count
) -
751 offsetof(struct mthca_mpt_entry
, start
));
753 writeb(MTHCA_MPT_STATUS_HW
, fmr
->mem
.tavor
.mpt
);
758 int mthca_arbel_map_phys_fmr(struct ib_fmr
*ibfmr
, u64
*page_list
,
759 int list_len
, u64 iova
)
761 struct mthca_fmr
*fmr
= to_mfmr(ibfmr
);
762 struct mthca_dev
*dev
= to_mdev(ibfmr
->device
);
766 err
= mthca_check_fmr(fmr
, page_list
, list_len
, iova
);
772 key
= arbel_key_to_hw_index(fmr
->ibmr
.lkey
);
773 if (dev
->mthca_flags
& MTHCA_FLAG_SINAI_OPT
)
774 key
+= SINAI_FMR_KEY_INC
;
776 key
+= dev
->limits
.num_mpts
;
777 fmr
->ibmr
.lkey
= fmr
->ibmr
.rkey
= arbel_hw_index_to_key(key
);
779 *(u8
*) fmr
->mem
.arbel
.mpt
= MTHCA_MPT_STATUS_SW
;
783 dma_sync_single_for_cpu(&dev
->pdev
->dev
, fmr
->mem
.arbel
.dma_handle
,
784 list_len
* sizeof(u64
), DMA_TO_DEVICE
);
786 for (i
= 0; i
< list_len
; ++i
)
787 fmr
->mem
.arbel
.mtts
[i
] = cpu_to_be64(page_list
[i
] |
788 MTHCA_MTT_FLAG_PRESENT
);
790 dma_sync_single_for_device(&dev
->pdev
->dev
, fmr
->mem
.arbel
.dma_handle
,
791 list_len
* sizeof(u64
), DMA_TO_DEVICE
);
793 fmr
->mem
.arbel
.mpt
->key
= cpu_to_be32(key
);
794 fmr
->mem
.arbel
.mpt
->lkey
= cpu_to_be32(key
);
795 fmr
->mem
.arbel
.mpt
->length
= cpu_to_be64(list_len
* (1ull << fmr
->attr
.page_shift
));
796 fmr
->mem
.arbel
.mpt
->start
= cpu_to_be64(iova
);
800 *(u8
*) fmr
->mem
.arbel
.mpt
= MTHCA_MPT_STATUS_HW
;
807 void mthca_tavor_fmr_unmap(struct mthca_dev
*dev
, struct mthca_fmr
*fmr
)
814 writeb(MTHCA_MPT_STATUS_SW
, fmr
->mem
.tavor
.mpt
);
817 void mthca_arbel_fmr_unmap(struct mthca_dev
*dev
, struct mthca_fmr
*fmr
)
824 *(u8
*) fmr
->mem
.arbel
.mpt
= MTHCA_MPT_STATUS_SW
;
827 int mthca_init_mr_table(struct mthca_dev
*dev
)
830 int mpts
, mtts
, err
, i
;
832 err
= mthca_alloc_init(&dev
->mr_table
.mpt_alloc
,
833 dev
->limits
.num_mpts
,
834 ~0, dev
->limits
.reserved_mrws
);
838 if (!mthca_is_memfree(dev
) &&
839 (dev
->mthca_flags
& MTHCA_FLAG_DDR_HIDDEN
))
840 dev
->limits
.fmr_reserved_mtts
= 0;
842 dev
->mthca_flags
|= MTHCA_FLAG_FMR
;
844 if (dev
->mthca_flags
& MTHCA_FLAG_SINAI_OPT
)
845 mthca_dbg(dev
, "Memory key throughput optimization activated.\n");
847 err
= mthca_buddy_init(&dev
->mr_table
.mtt_buddy
,
848 fls(dev
->limits
.num_mtt_segs
- 1));
853 dev
->mr_table
.tavor_fmr
.mpt_base
= NULL
;
854 dev
->mr_table
.tavor_fmr
.mtt_base
= NULL
;
856 if (dev
->limits
.fmr_reserved_mtts
) {
857 i
= fls(dev
->limits
.fmr_reserved_mtts
- 1);
860 mthca_warn(dev
, "Unable to reserve 2^31 FMR MTTs.\n");
864 mpts
= mtts
= 1 << i
;
866 mtts
= dev
->limits
.num_mtt_segs
;
867 mpts
= dev
->limits
.num_mpts
;
870 if (!mthca_is_memfree(dev
) &&
871 (dev
->mthca_flags
& MTHCA_FLAG_FMR
)) {
873 addr
= pci_resource_start(dev
->pdev
, 4) +
874 ((pci_resource_len(dev
->pdev
, 4) - 1) &
875 dev
->mr_table
.mpt_base
);
877 dev
->mr_table
.tavor_fmr
.mpt_base
=
878 ioremap(addr
, mpts
* sizeof(struct mthca_mpt_entry
));
880 if (!dev
->mr_table
.tavor_fmr
.mpt_base
) {
881 mthca_warn(dev
, "MPT ioremap for FMR failed.\n");
886 addr
= pci_resource_start(dev
->pdev
, 4) +
887 ((pci_resource_len(dev
->pdev
, 4) - 1) &
888 dev
->mr_table
.mtt_base
);
890 dev
->mr_table
.tavor_fmr
.mtt_base
=
891 ioremap(addr
, mtts
* dev
->limits
.mtt_seg_size
);
892 if (!dev
->mr_table
.tavor_fmr
.mtt_base
) {
893 mthca_warn(dev
, "MTT ioremap for FMR failed.\n");
899 if (dev
->limits
.fmr_reserved_mtts
) {
900 err
= mthca_buddy_init(&dev
->mr_table
.tavor_fmr
.mtt_buddy
, fls(mtts
- 1));
902 goto err_fmr_mtt_buddy
;
904 /* Prevent regular MRs from using FMR keys */
905 err
= mthca_buddy_alloc(&dev
->mr_table
.mtt_buddy
, fls(mtts
- 1));
907 goto err_reserve_fmr
;
909 dev
->mr_table
.fmr_mtt_buddy
=
910 &dev
->mr_table
.tavor_fmr
.mtt_buddy
;
912 dev
->mr_table
.fmr_mtt_buddy
= &dev
->mr_table
.mtt_buddy
;
914 /* FMR table is always the first, take reserved MTTs out of there */
915 if (dev
->limits
.reserved_mtts
) {
916 i
= fls(dev
->limits
.reserved_mtts
- 1);
918 if (mthca_alloc_mtt_range(dev
, i
,
919 dev
->mr_table
.fmr_mtt_buddy
) == -1) {
920 mthca_warn(dev
, "MTT table of order %d is too small.\n",
921 dev
->mr_table
.fmr_mtt_buddy
->max_order
);
923 goto err_reserve_mtts
;
931 if (dev
->limits
.fmr_reserved_mtts
)
932 mthca_buddy_cleanup(&dev
->mr_table
.tavor_fmr
.mtt_buddy
);
935 if (dev
->mr_table
.tavor_fmr
.mtt_base
)
936 iounmap(dev
->mr_table
.tavor_fmr
.mtt_base
);
939 if (dev
->mr_table
.tavor_fmr
.mpt_base
)
940 iounmap(dev
->mr_table
.tavor_fmr
.mpt_base
);
943 mthca_buddy_cleanup(&dev
->mr_table
.mtt_buddy
);
946 mthca_alloc_cleanup(&dev
->mr_table
.mpt_alloc
);
951 void mthca_cleanup_mr_table(struct mthca_dev
*dev
)
953 /* XXX check if any MRs are still allocated? */
954 if (dev
->limits
.fmr_reserved_mtts
)
955 mthca_buddy_cleanup(&dev
->mr_table
.tavor_fmr
.mtt_buddy
);
957 mthca_buddy_cleanup(&dev
->mr_table
.mtt_buddy
);
959 if (dev
->mr_table
.tavor_fmr
.mtt_base
)
960 iounmap(dev
->mr_table
.tavor_fmr
.mtt_base
);
961 if (dev
->mr_table
.tavor_fmr
.mpt_base
)
962 iounmap(dev
->mr_table
.tavor_fmr
.mpt_base
);
964 mthca_alloc_cleanup(&dev
->mr_table
.mpt_alloc
);