2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved.
4 * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35 #include <linux/init.h>
36 #include <linux/errno.h>
38 #include <linux/mlx4/cmd.h>
44 * Must be packed because mtt_seg is 64 bits but only aligned to 32 bits.
46 struct mlx4_mpt_entry
{
60 __be32 first_byte_offset
;
61 } __attribute__((packed
));
63 #define MLX4_MPT_FLAG_SW_OWNS (0xfUL << 28)
64 #define MLX4_MPT_FLAG_FREE (0x3UL << 28)
65 #define MLX4_MPT_FLAG_MIO (1 << 17)
66 #define MLX4_MPT_FLAG_BIND_ENABLE (1 << 15)
67 #define MLX4_MPT_FLAG_PHYSICAL (1 << 9)
68 #define MLX4_MPT_FLAG_REGION (1 << 8)
70 #define MLX4_MPT_PD_FLAG_FAST_REG (1 << 27)
71 #define MLX4_MPT_PD_FLAG_RAE (1 << 28)
72 #define MLX4_MPT_PD_FLAG_EN_INV (3 << 24)
74 #define MLX4_MPT_STATUS_SW 0xF0
75 #define MLX4_MPT_STATUS_HW 0x00
77 static u32
mlx4_buddy_alloc(struct mlx4_buddy
*buddy
, int order
)
83 spin_lock(&buddy
->lock
);
85 for (o
= order
; o
<= buddy
->max_order
; ++o
)
86 if (buddy
->num_free
[o
]) {
87 m
= 1 << (buddy
->max_order
- o
);
88 seg
= find_first_bit(buddy
->bits
[o
], m
);
93 spin_unlock(&buddy
->lock
);
97 clear_bit(seg
, buddy
->bits
[o
]);
103 set_bit(seg
^ 1, buddy
->bits
[o
]);
104 ++buddy
->num_free
[o
];
107 spin_unlock(&buddy
->lock
);
114 static void mlx4_buddy_free(struct mlx4_buddy
*buddy
, u32 seg
, int order
)
118 spin_lock(&buddy
->lock
);
120 while (test_bit(seg
^ 1, buddy
->bits
[order
])) {
121 clear_bit(seg
^ 1, buddy
->bits
[order
]);
122 --buddy
->num_free
[order
];
127 set_bit(seg
, buddy
->bits
[order
]);
128 ++buddy
->num_free
[order
];
130 spin_unlock(&buddy
->lock
);
133 static int mlx4_buddy_init(struct mlx4_buddy
*buddy
, int max_order
)
137 buddy
->max_order
= max_order
;
138 spin_lock_init(&buddy
->lock
);
140 buddy
->bits
= kzalloc((buddy
->max_order
+ 1) * sizeof (long *),
142 buddy
->num_free
= kzalloc((buddy
->max_order
+ 1) * sizeof (int *),
144 if (!buddy
->bits
|| !buddy
->num_free
)
147 for (i
= 0; i
<= buddy
->max_order
; ++i
) {
148 s
= BITS_TO_LONGS(1 << (buddy
->max_order
- i
));
149 buddy
->bits
[i
] = kmalloc(s
* sizeof (long), GFP_KERNEL
);
152 bitmap_zero(buddy
->bits
[i
], 1 << (buddy
->max_order
- i
));
155 set_bit(0, buddy
->bits
[buddy
->max_order
]);
156 buddy
->num_free
[buddy
->max_order
] = 1;
161 for (i
= 0; i
<= buddy
->max_order
; ++i
)
162 kfree(buddy
->bits
[i
]);
166 kfree(buddy
->num_free
);
171 static void mlx4_buddy_cleanup(struct mlx4_buddy
*buddy
)
175 for (i
= 0; i
<= buddy
->max_order
; ++i
)
176 kfree(buddy
->bits
[i
]);
179 kfree(buddy
->num_free
);
182 static u32
mlx4_alloc_mtt_range(struct mlx4_dev
*dev
, int order
)
184 struct mlx4_mr_table
*mr_table
= &mlx4_priv(dev
)->mr_table
;
187 seg
= mlx4_buddy_alloc(&mr_table
->mtt_buddy
, order
);
191 if (mlx4_table_get_range(dev
, &mr_table
->mtt_table
, seg
,
192 seg
+ (1 << order
) - 1)) {
193 mlx4_buddy_free(&mr_table
->mtt_buddy
, seg
, order
);
200 int mlx4_mtt_init(struct mlx4_dev
*dev
, int npages
, int page_shift
,
201 struct mlx4_mtt
*mtt
)
207 mtt
->page_shift
= MLX4_ICM_PAGE_SHIFT
;
210 mtt
->page_shift
= page_shift
;
212 for (mtt
->order
= 0, i
= dev
->caps
.mtts_per_seg
; i
< npages
; i
<<= 1)
215 mtt
->first_seg
= mlx4_alloc_mtt_range(dev
, mtt
->order
);
216 if (mtt
->first_seg
== -1)
221 EXPORT_SYMBOL_GPL(mlx4_mtt_init
);
223 void mlx4_mtt_cleanup(struct mlx4_dev
*dev
, struct mlx4_mtt
*mtt
)
225 struct mlx4_mr_table
*mr_table
= &mlx4_priv(dev
)->mr_table
;
230 mlx4_buddy_free(&mr_table
->mtt_buddy
, mtt
->first_seg
, mtt
->order
);
231 mlx4_table_put_range(dev
, &mr_table
->mtt_table
, mtt
->first_seg
,
232 mtt
->first_seg
+ (1 << mtt
->order
) - 1);
234 EXPORT_SYMBOL_GPL(mlx4_mtt_cleanup
);
236 u64
mlx4_mtt_addr(struct mlx4_dev
*dev
, struct mlx4_mtt
*mtt
)
238 return (u64
) mtt
->first_seg
* dev
->caps
.mtt_entry_sz
;
240 EXPORT_SYMBOL_GPL(mlx4_mtt_addr
);
242 static u32
hw_index_to_key(u32 ind
)
244 return (ind
>> 24) | (ind
<< 8);
247 static u32
key_to_hw_index(u32 key
)
249 return (key
<< 24) | (key
>> 8);
252 static int mlx4_SW2HW_MPT(struct mlx4_dev
*dev
, struct mlx4_cmd_mailbox
*mailbox
,
255 return mlx4_cmd(dev
, mailbox
->dma
, mpt_index
, 0, MLX4_CMD_SW2HW_MPT
,
256 MLX4_CMD_TIME_CLASS_B
);
259 static int mlx4_HW2SW_MPT(struct mlx4_dev
*dev
, struct mlx4_cmd_mailbox
*mailbox
,
262 return mlx4_cmd_box(dev
, 0, mailbox
? mailbox
->dma
: 0, mpt_index
,
263 !mailbox
, MLX4_CMD_HW2SW_MPT
, MLX4_CMD_TIME_CLASS_B
);
266 int mlx4_mr_alloc(struct mlx4_dev
*dev
, u32 pd
, u64 iova
, u64 size
, u32 access
,
267 int npages
, int page_shift
, struct mlx4_mr
*mr
)
269 struct mlx4_priv
*priv
= mlx4_priv(dev
);
273 index
= mlx4_bitmap_alloc(&priv
->mr_table
.mpt_bitmap
);
282 mr
->key
= hw_index_to_key(index
);
284 err
= mlx4_mtt_init(dev
, npages
, page_shift
, &mr
->mtt
);
286 mlx4_bitmap_free(&priv
->mr_table
.mpt_bitmap
, index
);
290 EXPORT_SYMBOL_GPL(mlx4_mr_alloc
);
292 void mlx4_mr_free(struct mlx4_dev
*dev
, struct mlx4_mr
*mr
)
294 struct mlx4_priv
*priv
= mlx4_priv(dev
);
298 err
= mlx4_HW2SW_MPT(dev
, NULL
,
299 key_to_hw_index(mr
->key
) &
300 (dev
->caps
.num_mpts
- 1));
302 mlx4_warn(dev
, "HW2SW_MPT failed (%d)\n", err
);
305 mlx4_mtt_cleanup(dev
, &mr
->mtt
);
306 mlx4_bitmap_free(&priv
->mr_table
.mpt_bitmap
, key_to_hw_index(mr
->key
));
308 EXPORT_SYMBOL_GPL(mlx4_mr_free
);
310 int mlx4_mr_enable(struct mlx4_dev
*dev
, struct mlx4_mr
*mr
)
312 struct mlx4_mr_table
*mr_table
= &mlx4_priv(dev
)->mr_table
;
313 struct mlx4_cmd_mailbox
*mailbox
;
314 struct mlx4_mpt_entry
*mpt_entry
;
317 err
= mlx4_table_get(dev
, &mr_table
->dmpt_table
, key_to_hw_index(mr
->key
));
321 mailbox
= mlx4_alloc_cmd_mailbox(dev
);
322 if (IS_ERR(mailbox
)) {
323 err
= PTR_ERR(mailbox
);
326 mpt_entry
= mailbox
->buf
;
328 memset(mpt_entry
, 0, sizeof *mpt_entry
);
330 mpt_entry
->flags
= cpu_to_be32(MLX4_MPT_FLAG_MIO
|
331 MLX4_MPT_FLAG_REGION
|
334 mpt_entry
->key
= cpu_to_be32(key_to_hw_index(mr
->key
));
335 mpt_entry
->pd_flags
= cpu_to_be32(mr
->pd
| MLX4_MPT_PD_FLAG_EN_INV
);
336 mpt_entry
->start
= cpu_to_be64(mr
->iova
);
337 mpt_entry
->length
= cpu_to_be64(mr
->size
);
338 mpt_entry
->entity_size
= cpu_to_be32(mr
->mtt
.page_shift
);
340 if (mr
->mtt
.order
< 0) {
341 mpt_entry
->flags
|= cpu_to_be32(MLX4_MPT_FLAG_PHYSICAL
);
342 mpt_entry
->mtt_seg
= 0;
344 mpt_entry
->mtt_seg
= cpu_to_be64(mlx4_mtt_addr(dev
, &mr
->mtt
));
347 if (mr
->mtt
.order
>= 0 && mr
->mtt
.page_shift
== 0) {
348 /* fast register MR in free state */
349 mpt_entry
->flags
|= cpu_to_be32(MLX4_MPT_FLAG_FREE
);
350 mpt_entry
->pd_flags
|= cpu_to_be32(MLX4_MPT_PD_FLAG_FAST_REG
|
351 MLX4_MPT_PD_FLAG_RAE
);
352 mpt_entry
->mtt_sz
= cpu_to_be32((1 << mr
->mtt
.order
) *
353 dev
->caps
.mtts_per_seg
);
355 mpt_entry
->flags
|= cpu_to_be32(MLX4_MPT_FLAG_SW_OWNS
);
358 err
= mlx4_SW2HW_MPT(dev
, mailbox
,
359 key_to_hw_index(mr
->key
) & (dev
->caps
.num_mpts
- 1));
361 mlx4_warn(dev
, "SW2HW_MPT failed (%d)\n", err
);
367 mlx4_free_cmd_mailbox(dev
, mailbox
);
372 mlx4_free_cmd_mailbox(dev
, mailbox
);
375 mlx4_table_put(dev
, &mr_table
->dmpt_table
, key_to_hw_index(mr
->key
));
378 EXPORT_SYMBOL_GPL(mlx4_mr_enable
);
380 static int mlx4_write_mtt_chunk(struct mlx4_dev
*dev
, struct mlx4_mtt
*mtt
,
381 int start_index
, int npages
, u64
*page_list
)
383 struct mlx4_priv
*priv
= mlx4_priv(dev
);
385 dma_addr_t dma_handle
;
387 int s
= start_index
* sizeof (u64
);
389 /* All MTTs must fit in the same page */
390 if (start_index
/ (PAGE_SIZE
/ sizeof (u64
)) !=
391 (start_index
+ npages
- 1) / (PAGE_SIZE
/ sizeof (u64
)))
394 if (start_index
& (dev
->caps
.mtts_per_seg
- 1))
397 mtts
= mlx4_table_find(&priv
->mr_table
.mtt_table
, mtt
->first_seg
+
398 s
/ dev
->caps
.mtt_entry_sz
, &dma_handle
);
402 dma_sync_single_for_cpu(&dev
->pdev
->dev
, dma_handle
,
403 npages
* sizeof (u64
), DMA_TO_DEVICE
);
405 for (i
= 0; i
< npages
; ++i
)
406 mtts
[i
] = cpu_to_be64(page_list
[i
] | MLX4_MTT_FLAG_PRESENT
);
408 dma_sync_single_for_device(&dev
->pdev
->dev
, dma_handle
,
409 npages
* sizeof (u64
), DMA_TO_DEVICE
);
414 int mlx4_write_mtt(struct mlx4_dev
*dev
, struct mlx4_mtt
*mtt
,
415 int start_index
, int npages
, u64
*page_list
)
424 chunk
= min_t(int, PAGE_SIZE
/ sizeof(u64
), npages
);
425 err
= mlx4_write_mtt_chunk(dev
, mtt
, start_index
, chunk
, page_list
);
430 start_index
+= chunk
;
436 EXPORT_SYMBOL_GPL(mlx4_write_mtt
);
438 int mlx4_buf_write_mtt(struct mlx4_dev
*dev
, struct mlx4_mtt
*mtt
,
439 struct mlx4_buf
*buf
)
445 page_list
= kmalloc(buf
->npages
* sizeof *page_list
, GFP_KERNEL
);
449 for (i
= 0; i
< buf
->npages
; ++i
)
451 page_list
[i
] = buf
->direct
.map
+ (i
<< buf
->page_shift
);
453 page_list
[i
] = buf
->page_list
[i
].map
;
455 err
= mlx4_write_mtt(dev
, mtt
, 0, buf
->npages
, page_list
);
460 EXPORT_SYMBOL_GPL(mlx4_buf_write_mtt
);
462 int mlx4_init_mr_table(struct mlx4_dev
*dev
)
464 struct mlx4_mr_table
*mr_table
= &mlx4_priv(dev
)->mr_table
;
467 err
= mlx4_bitmap_init(&mr_table
->mpt_bitmap
, dev
->caps
.num_mpts
,
468 ~0, dev
->caps
.reserved_mrws
, 0);
472 err
= mlx4_buddy_init(&mr_table
->mtt_buddy
,
473 ilog2(dev
->caps
.num_mtt_segs
));
477 if (dev
->caps
.reserved_mtts
) {
478 if (mlx4_alloc_mtt_range(dev
, fls(dev
->caps
.reserved_mtts
- 1)) == -1) {
479 mlx4_warn(dev
, "MTT table of order %d is too small.\n",
480 mr_table
->mtt_buddy
.max_order
);
482 goto err_reserve_mtts
;
489 mlx4_buddy_cleanup(&mr_table
->mtt_buddy
);
492 mlx4_bitmap_cleanup(&mr_table
->mpt_bitmap
);
497 void mlx4_cleanup_mr_table(struct mlx4_dev
*dev
)
499 struct mlx4_mr_table
*mr_table
= &mlx4_priv(dev
)->mr_table
;
501 mlx4_buddy_cleanup(&mr_table
->mtt_buddy
);
502 mlx4_bitmap_cleanup(&mr_table
->mpt_bitmap
);
505 static inline int mlx4_check_fmr(struct mlx4_fmr
*fmr
, u64
*page_list
,
506 int npages
, u64 iova
)
510 if (npages
> fmr
->max_pages
)
513 page_mask
= (1 << fmr
->page_shift
) - 1;
515 /* We are getting page lists, so va must be page aligned. */
516 if (iova
& page_mask
)
519 /* Trust the user not to pass misaligned data in page_list */
521 for (i
= 0; i
< npages
; ++i
) {
522 if (page_list
[i
] & ~page_mask
)
526 if (fmr
->maps
>= fmr
->max_maps
)
532 int mlx4_map_phys_fmr(struct mlx4_dev
*dev
, struct mlx4_fmr
*fmr
, u64
*page_list
,
533 int npages
, u64 iova
, u32
*lkey
, u32
*rkey
)
538 err
= mlx4_check_fmr(fmr
, page_list
, npages
, iova
);
544 key
= key_to_hw_index(fmr
->mr
.key
);
545 key
+= dev
->caps
.num_mpts
;
546 *lkey
= *rkey
= fmr
->mr
.key
= hw_index_to_key(key
);
548 *(u8
*) fmr
->mpt
= MLX4_MPT_STATUS_SW
;
550 /* Make sure MPT status is visible before writing MTT entries */
553 dma_sync_single_for_cpu(&dev
->pdev
->dev
, fmr
->dma_handle
,
554 npages
* sizeof(u64
), DMA_TO_DEVICE
);
556 for (i
= 0; i
< npages
; ++i
)
557 fmr
->mtts
[i
] = cpu_to_be64(page_list
[i
] | MLX4_MTT_FLAG_PRESENT
);
559 dma_sync_single_for_device(&dev
->pdev
->dev
, fmr
->dma_handle
,
560 npages
* sizeof(u64
), DMA_TO_DEVICE
);
562 fmr
->mpt
->key
= cpu_to_be32(key
);
563 fmr
->mpt
->lkey
= cpu_to_be32(key
);
564 fmr
->mpt
->length
= cpu_to_be64(npages
* (1ull << fmr
->page_shift
));
565 fmr
->mpt
->start
= cpu_to_be64(iova
);
567 /* Make MTT entries are visible before setting MPT status */
570 *(u8
*) fmr
->mpt
= MLX4_MPT_STATUS_HW
;
572 /* Make sure MPT status is visible before consumer can use FMR */
577 EXPORT_SYMBOL_GPL(mlx4_map_phys_fmr
);
579 int mlx4_fmr_alloc(struct mlx4_dev
*dev
, u32 pd
, u32 access
, int max_pages
,
580 int max_maps
, u8 page_shift
, struct mlx4_fmr
*fmr
)
582 struct mlx4_priv
*priv
= mlx4_priv(dev
);
586 if (page_shift
< (ffs(dev
->caps
.page_size_cap
) - 1) || page_shift
>= 32)
589 /* All MTTs must fit in the same page */
590 if (max_pages
* sizeof *fmr
->mtts
> PAGE_SIZE
)
593 fmr
->page_shift
= page_shift
;
594 fmr
->max_pages
= max_pages
;
595 fmr
->max_maps
= max_maps
;
598 err
= mlx4_mr_alloc(dev
, pd
, 0, 0, access
, max_pages
,
599 page_shift
, &fmr
->mr
);
603 mtt_seg
= fmr
->mr
.mtt
.first_seg
* dev
->caps
.mtt_entry_sz
;
605 fmr
->mtts
= mlx4_table_find(&priv
->mr_table
.mtt_table
,
606 fmr
->mr
.mtt
.first_seg
,
616 mlx4_mr_free(dev
, &fmr
->mr
);
619 EXPORT_SYMBOL_GPL(mlx4_fmr_alloc
);
621 int mlx4_fmr_enable(struct mlx4_dev
*dev
, struct mlx4_fmr
*fmr
)
623 struct mlx4_priv
*priv
= mlx4_priv(dev
);
626 err
= mlx4_mr_enable(dev
, &fmr
->mr
);
630 fmr
->mpt
= mlx4_table_find(&priv
->mr_table
.dmpt_table
,
631 key_to_hw_index(fmr
->mr
.key
), NULL
);
637 EXPORT_SYMBOL_GPL(mlx4_fmr_enable
);
639 void mlx4_fmr_unmap(struct mlx4_dev
*dev
, struct mlx4_fmr
*fmr
,
640 u32
*lkey
, u32
*rkey
)
647 *(u8
*) fmr
->mpt
= MLX4_MPT_STATUS_SW
;
649 EXPORT_SYMBOL_GPL(mlx4_fmr_unmap
);
651 int mlx4_fmr_free(struct mlx4_dev
*dev
, struct mlx4_fmr
*fmr
)
657 mlx4_mr_free(dev
, &fmr
->mr
);
661 EXPORT_SYMBOL_GPL(mlx4_fmr_free
);
663 int mlx4_SYNC_TPT(struct mlx4_dev
*dev
)
665 return mlx4_cmd(dev
, 0, 0, 0, MLX4_CMD_SYNC_TPT
, 1000);
667 EXPORT_SYMBOL_GPL(mlx4_SYNC_TPT
);