2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/module.h>
35 #include <linux/moduleparam.h>
36 #include <linux/string.h>
37 #include <linux/slab.h>
39 #include "mthca_profile.h"
61 MTHCA_NUM_PDS
= 1 << 15
64 s64
mthca_make_profile(struct mthca_dev
*dev
,
65 struct mthca_profile
*request
,
66 struct mthca_dev_lim
*dev_lim
,
67 struct mthca_init_hca_param
*init_hca
)
69 struct mthca_resource
{
77 u64 mem_base
, mem_avail
;
79 struct mthca_resource
*profile
;
82 profile
= kzalloc(MTHCA_RES_NUM
* sizeof *profile
, GFP_KERNEL
);
86 profile
[MTHCA_RES_QP
].size
= dev_lim
->qpc_entry_sz
;
87 profile
[MTHCA_RES_EEC
].size
= dev_lim
->eec_entry_sz
;
88 profile
[MTHCA_RES_SRQ
].size
= dev_lim
->srq_entry_sz
;
89 profile
[MTHCA_RES_CQ
].size
= dev_lim
->cqc_entry_sz
;
90 profile
[MTHCA_RES_EQP
].size
= dev_lim
->eqpc_entry_sz
;
91 profile
[MTHCA_RES_EEEC
].size
= dev_lim
->eeec_entry_sz
;
92 profile
[MTHCA_RES_EQ
].size
= dev_lim
->eqc_entry_sz
;
93 profile
[MTHCA_RES_RDB
].size
= MTHCA_RDB_ENTRY_SIZE
;
94 profile
[MTHCA_RES_MCG
].size
= MTHCA_MGM_ENTRY_SIZE
;
95 profile
[MTHCA_RES_MPT
].size
= dev_lim
->mpt_entry_sz
;
96 profile
[MTHCA_RES_MTT
].size
= dev
->limits
.mtt_seg_size
;
97 profile
[MTHCA_RES_UAR
].size
= dev_lim
->uar_scratch_entry_sz
;
98 profile
[MTHCA_RES_UDAV
].size
= MTHCA_AV_SIZE
;
99 profile
[MTHCA_RES_UARC
].size
= request
->uarc_size
;
101 profile
[MTHCA_RES_QP
].num
= request
->num_qp
;
102 profile
[MTHCA_RES_SRQ
].num
= request
->num_srq
;
103 profile
[MTHCA_RES_EQP
].num
= request
->num_qp
;
104 profile
[MTHCA_RES_RDB
].num
= request
->num_qp
* request
->rdb_per_qp
;
105 profile
[MTHCA_RES_CQ
].num
= request
->num_cq
;
106 profile
[MTHCA_RES_EQ
].num
= MTHCA_NUM_EQS
;
107 profile
[MTHCA_RES_MCG
].num
= request
->num_mcg
;
108 profile
[MTHCA_RES_MPT
].num
= request
->num_mpt
;
109 profile
[MTHCA_RES_MTT
].num
= request
->num_mtt
;
110 profile
[MTHCA_RES_UAR
].num
= request
->num_uar
;
111 profile
[MTHCA_RES_UARC
].num
= request
->num_uar
;
112 profile
[MTHCA_RES_UDAV
].num
= request
->num_udav
;
114 for (i
= 0; i
< MTHCA_RES_NUM
; ++i
) {
116 profile
[i
].log_num
= max(ffs(profile
[i
].num
) - 1, 0);
117 profile
[i
].size
*= profile
[i
].num
;
118 if (mthca_is_memfree(dev
))
119 profile
[i
].size
= max(profile
[i
].size
, (u64
) PAGE_SIZE
);
122 if (mthca_is_memfree(dev
)) {
124 mem_avail
= dev_lim
->hca
.arbel
.max_icm_sz
;
126 mem_base
= dev
->ddr_start
;
127 mem_avail
= dev
->fw
.tavor
.fw_start
- dev
->ddr_start
;
131 * Sort the resources in decreasing order of size. Since they
132 * all have sizes that are powers of 2, we'll be able to keep
133 * resources aligned to their size and pack them without gaps
134 * using the sorted order.
136 for (i
= MTHCA_RES_NUM
; i
> 0; --i
)
137 for (j
= 1; j
< i
; ++j
) {
138 if (profile
[j
].size
> profile
[j
- 1].size
)
139 swap(profile
[j
], profile
[j
- 1]);
142 for (i
= 0; i
< MTHCA_RES_NUM
; ++i
) {
143 if (profile
[i
].size
) {
144 profile
[i
].start
= mem_base
+ total_size
;
145 total_size
+= profile
[i
].size
;
147 if (total_size
> mem_avail
) {
148 mthca_err(dev
, "Profile requires 0x%llx bytes; "
149 "won't fit in 0x%llx bytes of context memory.\n",
150 (unsigned long long) total_size
,
151 (unsigned long long) mem_avail
);
157 mthca_dbg(dev
, "profile[%2d]--%2d/%2d @ 0x%16llx "
159 i
, profile
[i
].type
, profile
[i
].log_num
,
160 (unsigned long long) profile
[i
].start
,
161 (unsigned long long) profile
[i
].size
);
164 if (mthca_is_memfree(dev
))
165 mthca_dbg(dev
, "HCA context memory: reserving %d KB\n",
166 (int) (total_size
>> 10));
168 mthca_dbg(dev
, "HCA memory: allocated %d KB/%d KB (%d KB free)\n",
169 (int) (total_size
>> 10), (int) (mem_avail
>> 10),
170 (int) ((mem_avail
- total_size
) >> 10));
172 for (i
= 0; i
< MTHCA_RES_NUM
; ++i
) {
173 switch (profile
[i
].type
) {
175 dev
->limits
.num_qps
= profile
[i
].num
;
176 init_hca
->qpc_base
= profile
[i
].start
;
177 init_hca
->log_num_qps
= profile
[i
].log_num
;
180 dev
->limits
.num_eecs
= profile
[i
].num
;
181 init_hca
->eec_base
= profile
[i
].start
;
182 init_hca
->log_num_eecs
= profile
[i
].log_num
;
185 dev
->limits
.num_srqs
= profile
[i
].num
;
186 init_hca
->srqc_base
= profile
[i
].start
;
187 init_hca
->log_num_srqs
= profile
[i
].log_num
;
190 dev
->limits
.num_cqs
= profile
[i
].num
;
191 init_hca
->cqc_base
= profile
[i
].start
;
192 init_hca
->log_num_cqs
= profile
[i
].log_num
;
195 init_hca
->eqpc_base
= profile
[i
].start
;
198 init_hca
->eeec_base
= profile
[i
].start
;
201 dev
->limits
.num_eqs
= profile
[i
].num
;
202 init_hca
->eqc_base
= profile
[i
].start
;
203 init_hca
->log_num_eqs
= profile
[i
].log_num
;
206 for (dev
->qp_table
.rdb_shift
= 0;
207 request
->num_qp
<< dev
->qp_table
.rdb_shift
< profile
[i
].num
;
208 ++dev
->qp_table
.rdb_shift
)
210 dev
->qp_table
.rdb_base
= (u32
) profile
[i
].start
;
211 init_hca
->rdb_base
= profile
[i
].start
;
214 dev
->limits
.num_mgms
= profile
[i
].num
>> 1;
215 dev
->limits
.num_amgms
= profile
[i
].num
>> 1;
216 init_hca
->mc_base
= profile
[i
].start
;
217 init_hca
->log_mc_entry_sz
= ffs(MTHCA_MGM_ENTRY_SIZE
) - 1;
218 init_hca
->log_mc_table_sz
= profile
[i
].log_num
;
219 init_hca
->mc_hash_sz
= 1 << (profile
[i
].log_num
- 1);
222 dev
->limits
.num_mpts
= profile
[i
].num
;
223 dev
->mr_table
.mpt_base
= profile
[i
].start
;
224 init_hca
->mpt_base
= profile
[i
].start
;
225 init_hca
->log_mpt_sz
= profile
[i
].log_num
;
228 dev
->limits
.num_mtt_segs
= profile
[i
].num
;
229 dev
->mr_table
.mtt_base
= profile
[i
].start
;
230 init_hca
->mtt_base
= profile
[i
].start
;
231 init_hca
->mtt_seg_sz
= ffs(dev
->limits
.mtt_seg_size
) - 7;
234 dev
->limits
.num_uars
= profile
[i
].num
;
235 init_hca
->uar_scratch_base
= profile
[i
].start
;
238 dev
->av_table
.ddr_av_base
= profile
[i
].start
;
239 dev
->av_table
.num_ddr_avs
= profile
[i
].num
;
242 dev
->uar_table
.uarc_size
= request
->uarc_size
;
243 dev
->uar_table
.uarc_base
= profile
[i
].start
;
244 init_hca
->uarc_base
= profile
[i
].start
;
245 init_hca
->log_uarc_sz
= ffs(request
->uarc_size
) - 13;
246 init_hca
->log_uar_sz
= ffs(request
->num_uar
) - 1;
254 * PDs don't take any HCA memory, but we assign them as part
255 * of the HCA profile anyway.
257 dev
->limits
.num_pds
= MTHCA_NUM_PDS
;
259 if (dev
->mthca_flags
& MTHCA_FLAG_SINAI_OPT
&&
260 init_hca
->log_mpt_sz
> 23) {
261 mthca_warn(dev
, "MPT table too large (requested size 2^%d >= 2^24)\n",
262 init_hca
->log_mpt_sz
);
263 mthca_warn(dev
, "Disabling memory key throughput optimization.\n");
264 dev
->mthca_flags
&= ~MTHCA_FLAG_SINAI_OPT
;
268 * For Tavor, FMRs use ioremapped PCI memory. For 32 bit
269 * systems it may use too much vmalloc space to map all MTT
270 * memory, so we reserve some MTTs for FMR access, taking them
271 * out of the MR pool. They don't use additional memory, but
272 * we assign them as part of the HCA profile anyway.
274 if (mthca_is_memfree(dev
) || BITS_PER_LONG
== 64)
275 dev
->limits
.fmr_reserved_mtts
= 0;
277 dev
->limits
.fmr_reserved_mtts
= request
->fmr_reserved_mtts
;