?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
gpio: rcar: Fix runtime PM imbalance on error
[linux/fpc-iii.git] / drivers / infiniband / hw / hns / hns_roce_hem.c
blob263338b90d7ac3bf2428fe93690ee2d0a2f9bb12
1 /*
2 * Copyright (c) 2016 Hisilicon Limited.
3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34 #include <linux/platform_device.h>
35 #include "hns_roce_device.h"
36 #include "hns_roce_hem.h"
37 #include "hns_roce_common.h"
38
39 #define DMA_ADDR_T_SHIFT 12
40 #define BT_BA_SHIFT 32
41
42 #define HEM_INDEX_BUF BIT(0)
43 #define HEM_INDEX_L0 BIT(1)
44 #define HEM_INDEX_L1 BIT(2)
45 struct hns_roce_hem_index {
46 u64 buf;
47 u64 l0;
48 u64 l1;
49 u32 inited; /* indicate which index is available */
50 };
51
52 bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type)
53 {
54 int hop_num = 0;
55
56 switch (type) {
57 case HEM_TYPE_QPC:
58 hop_num = hr_dev->caps.qpc_hop_num;
59 break;
60 case HEM_TYPE_MTPT:
61 hop_num = hr_dev->caps.mpt_hop_num;
62 break;
63 case HEM_TYPE_CQC:
64 hop_num = hr_dev->caps.cqc_hop_num;
65 break;
66 case HEM_TYPE_SRQC:
67 hop_num = hr_dev->caps.srqc_hop_num;
68 break;
69 case HEM_TYPE_SCCC:
70 hop_num = hr_dev->caps.sccc_hop_num;
71 break;
72 case HEM_TYPE_QPC_TIMER:
73 hop_num = hr_dev->caps.qpc_timer_hop_num;
74 break;
75 case HEM_TYPE_CQC_TIMER:
76 hop_num = hr_dev->caps.cqc_timer_hop_num;
77 break;
78 case HEM_TYPE_CQE:
79 hop_num = hr_dev->caps.cqe_hop_num;
80 break;
81 case HEM_TYPE_MTT:
82 hop_num = hr_dev->caps.mtt_hop_num;
83 break;
84 case HEM_TYPE_SRQWQE:
85 hop_num = hr_dev->caps.srqwqe_hop_num;
86 break;
87 case HEM_TYPE_IDX:
88 hop_num = hr_dev->caps.idx_hop_num;
89 break;
90 default:
91 return false;
92 }
93
94 return hop_num ? true : false;
95 }
96
97 static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 hem_idx,
98 u32 bt_chunk_num, u64 hem_max_num)
99 {
100 u64 start_idx = round_down(hem_idx, bt_chunk_num);
101 u64 check_max_num = start_idx + bt_chunk_num;
102 u64 i;
103
104 for (i = start_idx; (i < check_max_num) && (i < hem_max_num); i++)
105 if (i != hem_idx && hem[i])
106 return false;
107
108 return true;
109 }
110
111 static bool hns_roce_check_bt_null(u64 **bt, u64 ba_idx, u32 bt_chunk_num)
112 {
113 u64 start_idx = round_down(ba_idx, bt_chunk_num);
114 int i;
115
116 for (i = 0; i < bt_chunk_num; i++)
117 if (i != ba_idx && bt[start_idx + i])
118 return false;
119
120 return true;
121 }
122
123 static int hns_roce_get_bt_num(u32 table_type, u32 hop_num)
124 {
125 if (check_whether_bt_num_3(table_type, hop_num))
126 return 3;
127 else if (check_whether_bt_num_2(table_type, hop_num))
128 return 2;
129 else if (check_whether_bt_num_1(table_type, hop_num))
130 return 1;
131 else
132 return 0;
133 }
134
135 static int get_hem_table_config(struct hns_roce_dev *hr_dev,
136 struct hns_roce_hem_mhop *mhop,
137 u32 type)
138 {
139 struct device *dev = hr_dev->dev;
140
141 switch (type) {
142 case HEM_TYPE_QPC:
143 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz
144 + PAGE_SHIFT);
145 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz
146 + PAGE_SHIFT);
147 mhop->ba_l0_num = hr_dev->caps.qpc_bt_num;
148 mhop->hop_num = hr_dev->caps.qpc_hop_num;
149 break;
150 case HEM_TYPE_MTPT:
151 mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz
152 + PAGE_SHIFT);
153 mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz
154 + PAGE_SHIFT);
155 mhop->ba_l0_num = hr_dev->caps.mpt_bt_num;
156 mhop->hop_num = hr_dev->caps.mpt_hop_num;
157 break;
158 case HEM_TYPE_CQC:
159 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz
160 + PAGE_SHIFT);
161 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz
162 + PAGE_SHIFT);
163 mhop->ba_l0_num = hr_dev->caps.cqc_bt_num;
164 mhop->hop_num = hr_dev->caps.cqc_hop_num;
165 break;
166 case HEM_TYPE_SCCC:
167 mhop->buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz
168 + PAGE_SHIFT);
169 mhop->bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz
170 + PAGE_SHIFT);
171 mhop->ba_l0_num = hr_dev->caps.sccc_bt_num;
172 mhop->hop_num = hr_dev->caps.sccc_hop_num;
173 break;
174 case HEM_TYPE_QPC_TIMER:
175 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz
176 + PAGE_SHIFT);
177 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz
178 + PAGE_SHIFT);
179 mhop->ba_l0_num = hr_dev->caps.qpc_timer_bt_num;
180 mhop->hop_num = hr_dev->caps.qpc_timer_hop_num;
181 break;
182 case HEM_TYPE_CQC_TIMER:
183 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz
184 + PAGE_SHIFT);
185 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz
186 + PAGE_SHIFT);
187 mhop->ba_l0_num = hr_dev->caps.cqc_timer_bt_num;
188 mhop->hop_num = hr_dev->caps.cqc_timer_hop_num;
189 break;
190 case HEM_TYPE_SRQC:
191 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz
192 + PAGE_SHIFT);
193 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz
194 + PAGE_SHIFT);
195 mhop->ba_l0_num = hr_dev->caps.srqc_bt_num;
196 mhop->hop_num = hr_dev->caps.srqc_hop_num;
197 break;
198 case HEM_TYPE_MTT:
199 mhop->buf_chunk_size = 1 << (hr_dev->caps.mtt_buf_pg_sz
200 + PAGE_SHIFT);
201 mhop->bt_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz
202 + PAGE_SHIFT);
203 mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN;
204 mhop->hop_num = hr_dev->caps.mtt_hop_num;
205 break;
206 case HEM_TYPE_CQE:
207 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqe_buf_pg_sz
208 + PAGE_SHIFT);
209 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz
210 + PAGE_SHIFT);
211 mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN;
212 mhop->hop_num = hr_dev->caps.cqe_hop_num;
213 break;
214 case HEM_TYPE_SRQWQE:
215 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqwqe_buf_pg_sz
216 + PAGE_SHIFT);
217 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz
218 + PAGE_SHIFT);
219 mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN;
220 mhop->hop_num = hr_dev->caps.srqwqe_hop_num;
221 break;
222 case HEM_TYPE_IDX:
223 mhop->buf_chunk_size = 1 << (hr_dev->caps.idx_buf_pg_sz
224 + PAGE_SHIFT);
225 mhop->bt_chunk_size = 1 << (hr_dev->caps.idx_ba_pg_sz
226 + PAGE_SHIFT);
227 mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN;
228 mhop->hop_num = hr_dev->caps.idx_hop_num;
229 break;
230 default:
231 dev_err(dev, "Table %d not support multi-hop addressing!\n",
232 type);
233 return -EINVAL;
234 }
235
236 return 0;
237 }
238
239 int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
240 struct hns_roce_hem_table *table, unsigned long *obj,
241 struct hns_roce_hem_mhop *mhop)
242 {
243 struct device *dev = hr_dev->dev;
244 u32 chunk_ba_num;
245 u32 table_idx;
246 u32 bt_num;
247 u32 chunk_size;
248
249 if (get_hem_table_config(hr_dev, mhop, table->type))
250 return -EINVAL;
251
252 if (!obj)
253 return 0;
254
255 /*
256 * QPC/MTPT/CQC/SRQC/SCCC alloc hem for buffer pages.
257 * MTT/CQE alloc hem for bt pages.
258 */
259 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
260 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
261 chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size :
262 mhop->bt_chunk_size;
263 table_idx = (*obj & (table->num_obj - 1)) /
264 (chunk_size / table->obj_size);
265 switch (bt_num) {
266 case 3:
267 mhop->l2_idx = table_idx & (chunk_ba_num - 1);
268 mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1);
269 mhop->l0_idx = (table_idx / chunk_ba_num) / chunk_ba_num;
270 break;
271 case 2:
272 mhop->l1_idx = table_idx & (chunk_ba_num - 1);
273 mhop->l0_idx = table_idx / chunk_ba_num;
274 break;
275 case 1:
276 mhop->l0_idx = table_idx;
277 break;
278 default:
279 dev_err(dev, "Table %d not support hop_num = %d!\n",
280 table->type, mhop->hop_num);
281 return -EINVAL;
282 }
283 if (mhop->l0_idx >= mhop->ba_l0_num)
284 mhop->l0_idx %= mhop->ba_l0_num;
285
286 return 0;
287 }
288
289 static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev,
290 int npages,
291 unsigned long hem_alloc_size,
292 gfp_t gfp_mask)
293 {
294 struct hns_roce_hem_chunk *chunk = NULL;
295 struct hns_roce_hem *hem;
296 struct scatterlist *mem;
297 int order;
298 void *buf;
299
300 WARN_ON(gfp_mask & __GFP_HIGHMEM);
301
302 hem = kmalloc(sizeof(*hem),
303 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
304 if (!hem)
305 return NULL;
306
307 hem->refcount = 0;
308 INIT_LIST_HEAD(&hem->chunk_list);
309
310 order = get_order(hem_alloc_size);
311
312 while (npages > 0) {
313 if (!chunk) {
314 chunk = kmalloc(sizeof(*chunk),
315 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
316 if (!chunk)
317 goto fail;
318
319 sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN);
320 chunk->npages = 0;
321 chunk->nsg = 0;
322 memset(chunk->buf, 0, sizeof(chunk->buf));
323 list_add_tail(&chunk->list, &hem->chunk_list);
324 }
325
326 while (1 << order > npages)
327 --order;
328
329 /*
330 * Alloc memory one time. If failed, don't alloc small block
331 * memory, directly return fail.
332 */
333 mem = &chunk->mem[chunk->npages];
334 buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order,
335 &sg_dma_address(mem), gfp_mask);
336 if (!buf)
337 goto fail;
338
339 chunk->buf[chunk->npages] = buf;
340 sg_dma_len(mem) = PAGE_SIZE << order;
341
342 ++chunk->npages;
343 ++chunk->nsg;
344 npages -= 1 << order;
345 }
346
347 return hem;
348
349 fail:
350 hns_roce_free_hem(hr_dev, hem);
351 return NULL;
352 }
353
354 void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem)
355 {
356 struct hns_roce_hem_chunk *chunk, *tmp;
357 int i;
358
359 if (!hem)
360 return;
361
362 list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) {
363 for (i = 0; i < chunk->npages; ++i)
364 dma_free_coherent(hr_dev->dev,
365 sg_dma_len(&chunk->mem[i]),
366 chunk->buf[i],
367 sg_dma_address(&chunk->mem[i]));
368 kfree(chunk);
369 }
370
371 kfree(hem);
372 }
373
374 static int hns_roce_set_hem(struct hns_roce_dev *hr_dev,
375 struct hns_roce_hem_table *table, unsigned long obj)
376 {
377 spinlock_t *lock = &hr_dev->bt_cmd_lock;
378 struct device *dev = hr_dev->dev;
379 long end;
380 unsigned long flags;
381 struct hns_roce_hem_iter iter;
382 void __iomem *bt_cmd;
383 __le32 bt_cmd_val[2];
384 __le32 bt_cmd_h = 0;
385 __le32 bt_cmd_l = 0;
386 u64 bt_ba = 0;
387 int ret = 0;
388
389 /* Find the HEM(Hardware Entry Memory) entry */
390 unsigned long i = (obj & (table->num_obj - 1)) /
391 (table->table_chunk_size / table->obj_size);
392
393 switch (table->type) {
394 case HEM_TYPE_QPC:
395 case HEM_TYPE_MTPT:
396 case HEM_TYPE_CQC:
397 case HEM_TYPE_SRQC:
398 roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
399 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, table->type);
400 break;
401 default:
402 return ret;
403 }
404
405 roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M,
406 ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj);
407 roce_set_bit(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0);
408 roce_set_bit(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1);
409
410 /* Currently iter only a chunk */
411 for (hns_roce_hem_first(table->hem[i], &iter);
412 !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
413 bt_ba = hns_roce_hem_addr(&iter) >> DMA_ADDR_T_SHIFT;
414
415 spin_lock_irqsave(lock, flags);
416
417 bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG;
418
419 end = HW_SYNC_TIMEOUT_MSECS;
420 while (end > 0) {
421 if (!(readl(bt_cmd) >> BT_CMD_SYNC_SHIFT))
422 break;
423
424 mdelay(HW_SYNC_SLEEP_TIME_INTERVAL);
425 end -= HW_SYNC_SLEEP_TIME_INTERVAL;
426 }
427
428 if (end <= 0) {
429 dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
430 spin_unlock_irqrestore(lock, flags);
431 return -EBUSY;
432 }
433
434 bt_cmd_l = cpu_to_le32(bt_ba);
435 roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M,
436 ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S,
437 bt_ba >> BT_BA_SHIFT);
438
439 bt_cmd_val[0] = bt_cmd_l;
440 bt_cmd_val[1] = bt_cmd_h;
441 hns_roce_write64_k(bt_cmd_val,
442 hr_dev->reg_base + ROCEE_BT_CMD_L_REG);
443 spin_unlock_irqrestore(lock, flags);
444 }
445
446 return ret;
447 }
448
449 static int calc_hem_config(struct hns_roce_dev *hr_dev,
450 struct hns_roce_hem_table *table, unsigned long obj,
451 struct hns_roce_hem_mhop *mhop,
452 struct hns_roce_hem_index *index)
453 {
454 struct ib_device *ibdev = &hr_dev->ib_dev;
455 unsigned long mhop_obj = obj;
456 u32 l0_idx, l1_idx, l2_idx;
457 u32 chunk_ba_num;
458 u32 bt_num;
459 int ret;
460
461 ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, mhop);
462 if (ret)
463 return ret;
464
465 l0_idx = mhop->l0_idx;
466 l1_idx = mhop->l1_idx;
467 l2_idx = mhop->l2_idx;
468 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
469 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
470 switch (bt_num) {
471 case 3:
472 index->l1 = l0_idx * chunk_ba_num + l1_idx;
473 index->l0 = l0_idx;
474 index->buf = l0_idx * chunk_ba_num * chunk_ba_num +
475 l1_idx * chunk_ba_num + l2_idx;
476 break;
477 case 2:
478 index->l0 = l0_idx;
479 index->buf = l0_idx * chunk_ba_num + l1_idx;
480 break;
481 case 1:
482 index->buf = l0_idx;
483 break;
484 default:
485 ibdev_err(ibdev, "Table %d not support mhop.hop_num = %d!\n",
486 table->type, mhop->hop_num);
487 return -EINVAL;
488 }
489
490 if (unlikely(index->buf >= table->num_hem)) {
491 ibdev_err(ibdev, "Table %d exceed hem limt idx %llu,max %lu!\n",
492 table->type, index->buf, table->num_hem);
493 return -EINVAL;
494 }
495
496 return 0;
497 }
498
499 static void free_mhop_hem(struct hns_roce_dev *hr_dev,
500 struct hns_roce_hem_table *table,
501 struct hns_roce_hem_mhop *mhop,
502 struct hns_roce_hem_index *index)
503 {
504 u32 bt_size = mhop->bt_chunk_size;
505 struct device *dev = hr_dev->dev;
506
507 if (index->inited & HEM_INDEX_BUF) {
508 hns_roce_free_hem(hr_dev, table->hem[index->buf]);
509 table->hem[index->buf] = NULL;
510 }
511
512 if (index->inited & HEM_INDEX_L1) {
513 dma_free_coherent(dev, bt_size, table->bt_l1[index->l1],
514 table->bt_l1_dma_addr[index->l1]);
515 table->bt_l1[index->l1] = NULL;
516 }
517
518 if (index->inited & HEM_INDEX_L0) {
519 dma_free_coherent(dev, bt_size, table->bt_l0[index->l0],
520 table->bt_l0_dma_addr[index->l0]);
521 table->bt_l0[index->l0] = NULL;
522 }
523 }
524
525 static int alloc_mhop_hem(struct hns_roce_dev *hr_dev,
526 struct hns_roce_hem_table *table,
527 struct hns_roce_hem_mhop *mhop,
528 struct hns_roce_hem_index *index)
529 {
530 u32 bt_size = mhop->bt_chunk_size;
531 struct device *dev = hr_dev->dev;
532 struct hns_roce_hem_iter iter;
533 gfp_t flag;
534 u64 bt_ba;
535 u32 size;
536 int ret;
537
538 /* alloc L1 BA's chunk */
539 if ((check_whether_bt_num_3(table->type, mhop->hop_num) ||
540 check_whether_bt_num_2(table->type, mhop->hop_num)) &&
541 !table->bt_l0[index->l0]) {
542 table->bt_l0[index->l0] = dma_alloc_coherent(dev, bt_size,
543 &table->bt_l0_dma_addr[index->l0],
544 GFP_KERNEL);
545 if (!table->bt_l0[index->l0]) {
546 ret = -ENOMEM;
547 goto out;
548 }
549 index->inited |= HEM_INDEX_L0;
550 }
551
552 /* alloc L2 BA's chunk */
553 if (check_whether_bt_num_3(table->type, mhop->hop_num) &&
554 !table->bt_l1[index->l1]) {
555 table->bt_l1[index->l1] = dma_alloc_coherent(dev, bt_size,
556 &table->bt_l1_dma_addr[index->l1],
557 GFP_KERNEL);
558 if (!table->bt_l1[index->l1]) {
559 ret = -ENOMEM;
560 goto err_alloc_hem;
561 }
562 index->inited |= HEM_INDEX_L1;
563 *(table->bt_l0[index->l0] + mhop->l1_idx) =
564 table->bt_l1_dma_addr[index->l1];
565 }
566
567 /*
568 * alloc buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC.
569 * alloc bt space chunk for MTT/CQE.
570 */
571 size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size;
572 flag = (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) | __GFP_NOWARN;
573 table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size >> PAGE_SHIFT,
574 size, flag);
575 if (!table->hem[index->buf]) {
576 ret = -ENOMEM;
577 goto err_alloc_hem;
578 }
579
580 index->inited |= HEM_INDEX_BUF;
581 hns_roce_hem_first(table->hem[index->buf], &iter);
582 bt_ba = hns_roce_hem_addr(&iter);
583 if (table->type < HEM_TYPE_MTT) {
584 if (mhop->hop_num == 2)
585 *(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba;
586 else if (mhop->hop_num == 1)
587 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba;
588 } else if (mhop->hop_num == 2) {
589 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba;
590 }
591
592 return 0;
593 err_alloc_hem:
594 free_mhop_hem(hr_dev, table, mhop, index);
595 out:
596 return ret;
597 }
598
599 static int set_mhop_hem(struct hns_roce_dev *hr_dev,
600 struct hns_roce_hem_table *table, unsigned long obj,
601 struct hns_roce_hem_mhop *mhop,
602 struct hns_roce_hem_index *index)
603 {
604 struct ib_device *ibdev = &hr_dev->ib_dev;
605 int step_idx;
606 int ret = 0;
607
608 if (index->inited & HEM_INDEX_L0) {
609 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 0);
610 if (ret) {
611 ibdev_err(ibdev, "set HEM step 0 failed!\n");
612 goto out;
613 }
614 }
615
616 if (index->inited & HEM_INDEX_L1) {
617 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 1);
618 if (ret) {
619 ibdev_err(ibdev, "set HEM step 1 failed!\n");
620 goto out;
621 }
622 }
623
624 if (index->inited & HEM_INDEX_BUF) {
625 if (mhop->hop_num == HNS_ROCE_HOP_NUM_0)
626 step_idx = 0;
627 else
628 step_idx = mhop->hop_num;
629 ret = hr_dev->hw->set_hem(hr_dev, table, obj, step_idx);
630 if (ret)
631 ibdev_err(ibdev, "set HEM step last failed!\n");
632 }
633 out:
634 return ret;
635 }
636
637 static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev,
638 struct hns_roce_hem_table *table,
639 unsigned long obj)
640 {
641 struct ib_device *ibdev = &hr_dev->ib_dev;
642 struct hns_roce_hem_index index = {};
643 struct hns_roce_hem_mhop mhop = {};
644 int ret;
645
646 ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);
647 if (ret) {
648 ibdev_err(ibdev, "calc hem config failed!\n");
649 return ret;
650 }
651
652 mutex_lock(&table->mutex);
653 if (table->hem[index.buf]) {
654 ++table->hem[index.buf]->refcount;
655 goto out;
656 }
657
658 ret = alloc_mhop_hem(hr_dev, table, &mhop, &index);
659 if (ret) {
660 ibdev_err(ibdev, "alloc mhop hem failed!\n");
661 goto out;
662 }
663
664 /* set HEM base address to hardware */
665 if (table->type < HEM_TYPE_MTT) {
666 ret = set_mhop_hem(hr_dev, table, obj, &mhop, &index);
667 if (ret) {
668 ibdev_err(ibdev, "set HEM address to HW failed!\n");
669 goto err_alloc;
670 }
671 }
672
673 ++table->hem[index.buf]->refcount;
674 goto out;
675
676 err_alloc:
677 free_mhop_hem(hr_dev, table, &mhop, &index);
678 out:
679 mutex_unlock(&table->mutex);
680 return ret;
681 }
682
683 int hns_roce_table_get(struct hns_roce_dev *hr_dev,
684 struct hns_roce_hem_table *table, unsigned long obj)
685 {
686 struct device *dev = hr_dev->dev;
687 int ret = 0;
688 unsigned long i;
689
690 if (hns_roce_check_whether_mhop(hr_dev, table->type))
691 return hns_roce_table_mhop_get(hr_dev, table, obj);
692
693 i = (obj & (table->num_obj - 1)) / (table->table_chunk_size /
694 table->obj_size);
695
696 mutex_lock(&table->mutex);
697
698 if (table->hem[i]) {
699 ++table->hem[i]->refcount;
700 goto out;
701 }
702
703 table->hem[i] = hns_roce_alloc_hem(hr_dev,
704 table->table_chunk_size >> PAGE_SHIFT,
705 table->table_chunk_size,
706 (table->lowmem ? GFP_KERNEL :
707 GFP_HIGHUSER) | __GFP_NOWARN);
708 if (!table->hem[i]) {
709 ret = -ENOMEM;
710 goto out;
711 }
712
713 /* Set HEM base address(128K/page, pa) to Hardware */
714 if (hns_roce_set_hem(hr_dev, table, obj)) {
715 hns_roce_free_hem(hr_dev, table->hem[i]);
716 table->hem[i] = NULL;
717 ret = -ENODEV;
718 dev_err(dev, "set HEM base address to HW failed.\n");
719 goto out;
720 }
721
722 ++table->hem[i]->refcount;
723 out:
724 mutex_unlock(&table->mutex);
725 return ret;
726 }
727
728 static void clear_mhop_hem(struct hns_roce_dev *hr_dev,
729 struct hns_roce_hem_table *table, unsigned long obj,
730 struct hns_roce_hem_mhop *mhop,
731 struct hns_roce_hem_index *index)
732 {
733 struct ib_device *ibdev = &hr_dev->ib_dev;
734 u32 hop_num = mhop->hop_num;
735 u32 chunk_ba_num;
736 int step_idx;
737
738 index->inited = HEM_INDEX_BUF;
739 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
740 if (check_whether_bt_num_2(table->type, hop_num)) {
741 if (hns_roce_check_hem_null(table->hem, index->buf,
742 chunk_ba_num, table->num_hem))
743 index->inited |= HEM_INDEX_L0;
744 } else if (check_whether_bt_num_3(table->type, hop_num)) {
745 if (hns_roce_check_hem_null(table->hem, index->buf,
746 chunk_ba_num, table->num_hem)) {
747 index->inited |= HEM_INDEX_L1;
748 if (hns_roce_check_bt_null(table->bt_l1, index->l1,
749 chunk_ba_num))
750 index->inited |= HEM_INDEX_L0;
751 }
752 }
753
754 if (table->type < HEM_TYPE_MTT) {
755 if (hop_num == HNS_ROCE_HOP_NUM_0)
756 step_idx = 0;
757 else
758 step_idx = hop_num;
759
760 if (hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx))
761 ibdev_warn(ibdev, "Clear hop%d HEM failed.\n", hop_num);
762
763 if (index->inited & HEM_INDEX_L1)
764 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
765 ibdev_warn(ibdev, "Clear HEM step 1 failed.\n");
766
767 if (index->inited & HEM_INDEX_L0)
768 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
769 ibdev_warn(ibdev, "Clear HEM step 0 failed.\n");
770 }
771 }
772
773 static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev,
774 struct hns_roce_hem_table *table,
775 unsigned long obj,
776 int check_refcount)
777 {
778 struct ib_device *ibdev = &hr_dev->ib_dev;
779 struct hns_roce_hem_index index = {};
780 struct hns_roce_hem_mhop mhop = {};
781 int ret;
782
783 ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);
784 if (ret) {
785 ibdev_err(ibdev, "calc hem config failed!\n");
786 return;
787 }
788
789 mutex_lock(&table->mutex);
790 if (check_refcount && (--table->hem[index.buf]->refcount > 0)) {
791 mutex_unlock(&table->mutex);
792 return;
793 }
794
795 clear_mhop_hem(hr_dev, table, obj, &mhop, &index);
796 free_mhop_hem(hr_dev, table, &mhop, &index);
797
798 mutex_unlock(&table->mutex);
799 }
800
801 void hns_roce_table_put(struct hns_roce_dev *hr_dev,
802 struct hns_roce_hem_table *table, unsigned long obj)
803 {
804 struct device *dev = hr_dev->dev;
805 unsigned long i;
806
807 if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
808 hns_roce_table_mhop_put(hr_dev, table, obj, 1);
809 return;
810 }
811
812 i = (obj & (table->num_obj - 1)) /
813 (table->table_chunk_size / table->obj_size);
814
815 mutex_lock(&table->mutex);
816
817 if (--table->hem[i]->refcount == 0) {
818 /* Clear HEM base address */
819 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
820 dev_warn(dev, "Clear HEM base address failed.\n");
821
822 hns_roce_free_hem(hr_dev, table->hem[i]);
823 table->hem[i] = NULL;
824 }
825
826 mutex_unlock(&table->mutex);
827 }
828
829 void *hns_roce_table_find(struct hns_roce_dev *hr_dev,
830 struct hns_roce_hem_table *table,
831 unsigned long obj, dma_addr_t *dma_handle)
832 {
833 struct hns_roce_hem_chunk *chunk;
834 struct hns_roce_hem_mhop mhop;
835 struct hns_roce_hem *hem;
836 void *addr = NULL;
837 unsigned long mhop_obj = obj;
838 unsigned long obj_per_chunk;
839 unsigned long idx_offset;
840 int offset, dma_offset;
841 int length;
842 int i, j;
843 u32 hem_idx = 0;
844
845 if (!table->lowmem)
846 return NULL;
847
848 mutex_lock(&table->mutex);
849
850 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) {
851 obj_per_chunk = table->table_chunk_size / table->obj_size;
852 hem = table->hem[(obj & (table->num_obj - 1)) / obj_per_chunk];
853 idx_offset = (obj & (table->num_obj - 1)) % obj_per_chunk;
854 dma_offset = offset = idx_offset * table->obj_size;
855 } else {
856 u32 seg_size = 64; /* 8 bytes per BA and 8 BA per segment */
857
858 if (hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop))
859 goto out;
860 /* mtt mhop */
861 i = mhop.l0_idx;
862 j = mhop.l1_idx;
863 if (mhop.hop_num == 2)
864 hem_idx = i * (mhop.bt_chunk_size / BA_BYTE_LEN) + j;
865 else if (mhop.hop_num == 1 ||
866 mhop.hop_num == HNS_ROCE_HOP_NUM_0)
867 hem_idx = i;
868
869 hem = table->hem[hem_idx];
870 dma_offset = offset = (obj & (table->num_obj - 1)) * seg_size %
871 mhop.bt_chunk_size;
872 if (mhop.hop_num == 2)
873 dma_offset = offset = 0;
874 }
875
876 if (!hem)
877 goto out;
878
879 list_for_each_entry(chunk, &hem->chunk_list, list) {
880 for (i = 0; i < chunk->npages; ++i) {
881 length = sg_dma_len(&chunk->mem[i]);
882 if (dma_handle && dma_offset >= 0) {
883 if (length > (u32)dma_offset)
884 *dma_handle = sg_dma_address(
885 &chunk->mem[i]) + dma_offset;
886 dma_offset -= length;
887 }
888
889 if (length > (u32)offset) {
890 addr = chunk->buf[i] + offset;
891 goto out;
892 }
893 offset -= length;
894 }
895 }
896
897 out:
898 mutex_unlock(&table->mutex);
899 return addr;
900 }
901
902 int hns_roce_table_get_range(struct hns_roce_dev *hr_dev,
903 struct hns_roce_hem_table *table,
904 unsigned long start, unsigned long end)
905 {
906 struct hns_roce_hem_mhop mhop;
907 unsigned long inc = table->table_chunk_size / table->obj_size;
908 unsigned long i = 0;
909 int ret;
910
911 if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
912 ret = hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
913 if (ret)
914 goto fail;
915 inc = mhop.bt_chunk_size / table->obj_size;
916 }
917
918 /* Allocate MTT entry memory according to chunk(128K) */
919 for (i = start; i <= end; i += inc) {
920 ret = hns_roce_table_get(hr_dev, table, i);
921 if (ret)
922 goto fail;
923 }
924
925 return 0;
926
927 fail:
928 while (i > start) {
929 i -= inc;
930 hns_roce_table_put(hr_dev, table, i);
931 }
932 return ret;
933 }
934
935 void hns_roce_table_put_range(struct hns_roce_dev *hr_dev,
936 struct hns_roce_hem_table *table,
937 unsigned long start, unsigned long end)
938 {
939 struct hns_roce_hem_mhop mhop;
940 unsigned long inc = table->table_chunk_size / table->obj_size;
941 unsigned long i;
942
943 if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
944 if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop))
945 return;
946 inc = mhop.bt_chunk_size / table->obj_size;
947 }
948
949 for (i = start; i <= end; i += inc)
950 hns_roce_table_put(hr_dev, table, i);
951 }
952
953 int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
954 struct hns_roce_hem_table *table, u32 type,
955 unsigned long obj_size, unsigned long nobj,
956 int use_lowmem)
957 {
958 unsigned long obj_per_chunk;
959 unsigned long num_hem;
960
961 if (!hns_roce_check_whether_mhop(hr_dev, type)) {
962 table->table_chunk_size = hr_dev->caps.chunk_sz;
963 obj_per_chunk = table->table_chunk_size / obj_size;
964 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;
965
966 table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL);
967 if (!table->hem)
968 return -ENOMEM;
969 } else {
970 struct hns_roce_hem_mhop mhop = {};
971 unsigned long buf_chunk_size;
972 unsigned long bt_chunk_size;
973 unsigned long bt_chunk_num;
974 unsigned long num_bt_l0 = 0;
975 u32 hop_num;
976
977 if (get_hem_table_config(hr_dev, &mhop, type))
978 return -EINVAL;
979
980 buf_chunk_size = mhop.buf_chunk_size;
981 bt_chunk_size = mhop.bt_chunk_size;
982 num_bt_l0 = mhop.ba_l0_num;
983 hop_num = mhop.hop_num;
984
985 obj_per_chunk = buf_chunk_size / obj_size;
986 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;
987 bt_chunk_num = bt_chunk_size / BA_BYTE_LEN;
988 if (type >= HEM_TYPE_MTT)
989 num_bt_l0 = bt_chunk_num;
990
991 table->hem = kcalloc(num_hem, sizeof(*table->hem),
992 GFP_KERNEL);
993 if (!table->hem)
994 goto err_kcalloc_hem_buf;
995
996 if (check_whether_bt_num_3(type, hop_num)) {
997 unsigned long num_bt_l1;
998
999 num_bt_l1 = (num_hem + bt_chunk_num - 1) /
1000 bt_chunk_num;
1001 table->bt_l1 = kcalloc(num_bt_l1,
1002 sizeof(*table->bt_l1),
1003 GFP_KERNEL);
1004 if (!table->bt_l1)
1005 goto err_kcalloc_bt_l1;
1006
1007 table->bt_l1_dma_addr = kcalloc(num_bt_l1,
1008 sizeof(*table->bt_l1_dma_addr),
1009 GFP_KERNEL);
1010
1011 if (!table->bt_l1_dma_addr)
1012 goto err_kcalloc_l1_dma;
1013 }
1014
1015 if (check_whether_bt_num_2(type, hop_num) ||
1016 check_whether_bt_num_3(type, hop_num)) {
1017 table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0),
1018 GFP_KERNEL);
1019 if (!table->bt_l0)
1020 goto err_kcalloc_bt_l0;
1021
1022 table->bt_l0_dma_addr = kcalloc(num_bt_l0,
1023 sizeof(*table->bt_l0_dma_addr),
1024 GFP_KERNEL);
1025 if (!table->bt_l0_dma_addr)
1026 goto err_kcalloc_l0_dma;
1027 }
1028 }
1029
1030 table->type = type;
1031 table->num_hem = num_hem;
1032 table->num_obj = nobj;
1033 table->obj_size = obj_size;
1034 table->lowmem = use_lowmem;
1035 mutex_init(&table->mutex);
1036
1037 return 0;
1038
1039 err_kcalloc_l0_dma:
1040 kfree(table->bt_l0);
1041 table->bt_l0 = NULL;
1042
1043 err_kcalloc_bt_l0:
1044 kfree(table->bt_l1_dma_addr);
1045 table->bt_l1_dma_addr = NULL;
1046
1047 err_kcalloc_l1_dma:
1048 kfree(table->bt_l1);
1049 table->bt_l1 = NULL;
1050
1051 err_kcalloc_bt_l1:
1052 kfree(table->hem);
1053 table->hem = NULL;
1054
1055 err_kcalloc_hem_buf:
1056 return -ENOMEM;
1057 }
1058
1059 static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev,
1060 struct hns_roce_hem_table *table)
1061 {
1062 struct hns_roce_hem_mhop mhop;
1063 u32 buf_chunk_size;
1064 int i;
1065 u64 obj;
1066
1067 if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop))
1068 return;
1069 buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size :
1070 mhop.bt_chunk_size;
1071
1072 for (i = 0; i < table->num_hem; ++i) {
1073 obj = i * buf_chunk_size / table->obj_size;
1074 if (table->hem[i])
1075 hns_roce_table_mhop_put(hr_dev, table, obj, 0);
1076 }
1077
1078 kfree(table->hem);
1079 table->hem = NULL;
1080 kfree(table->bt_l1);
1081 table->bt_l1 = NULL;
1082 kfree(table->bt_l1_dma_addr);
1083 table->bt_l1_dma_addr = NULL;
1084 kfree(table->bt_l0);
1085 table->bt_l0 = NULL;
1086 kfree(table->bt_l0_dma_addr);
1087 table->bt_l0_dma_addr = NULL;
1088 }
1089
1090 void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev,
1091 struct hns_roce_hem_table *table)
1092 {
1093 struct device *dev = hr_dev->dev;
1094 unsigned long i;
1095
1096 if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
1097 hns_roce_cleanup_mhop_hem_table(hr_dev, table);
1098 return;
1099 }
1100
1101 for (i = 0; i < table->num_hem; ++i)
1102 if (table->hem[i]) {
1103 if (hr_dev->hw->clear_hem(hr_dev, table,
1104 i * table->table_chunk_size / table->obj_size, 0))
1105 dev_err(dev, "Clear HEM base address failed.\n");
1106
1107 hns_roce_free_hem(hr_dev, table->hem[i]);
1108 }
1109
1110 kfree(table->hem);
1111 }
1112
1113 void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)
1114 {
1115 if ((hr_dev->caps.num_idx_segs))
1116 hns_roce_cleanup_hem_table(hr_dev,
1117 &hr_dev->mr_table.mtt_idx_table);
1118 if (hr_dev->caps.num_srqwqe_segs)
1119 hns_roce_cleanup_hem_table(hr_dev,
1120 &hr_dev->mr_table.mtt_srqwqe_table);
1121 if (hr_dev->caps.srqc_entry_sz)
1122 hns_roce_cleanup_hem_table(hr_dev,
1123 &hr_dev->srq_table.table);
1124 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);
1125 if (hr_dev->caps.qpc_timer_entry_sz)
1126 hns_roce_cleanup_hem_table(hr_dev,
1127 &hr_dev->qpc_timer_table);
1128 if (hr_dev->caps.cqc_timer_entry_sz)
1129 hns_roce_cleanup_hem_table(hr_dev,
1130 &hr_dev->cqc_timer_table);
1131 if (hr_dev->caps.sccc_entry_sz)
1132 hns_roce_cleanup_hem_table(hr_dev,
1133 &hr_dev->qp_table.sccc_table);
1134 if (hr_dev->caps.trrl_entry_sz)
1135 hns_roce_cleanup_hem_table(hr_dev,
1136 &hr_dev->qp_table.trrl_table);
1137 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);
1138 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);
1139 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
1140 if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
1141 hns_roce_cleanup_hem_table(hr_dev,
1142 &hr_dev->mr_table.mtt_cqe_table);
1143 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table);
1144 }
1145
1146 struct roce_hem_item {
1147 struct list_head list; /* link all hems in the same bt level */
1148 struct list_head sibling; /* link all hems in last hop for mtt */
1149 void *addr;
1150 dma_addr_t dma_addr;
1151 size_t count; /* max ba numbers */
1152 int start; /* start buf offset in this hem */
1153 int end; /* end buf offset in this hem */
1154 };
1155
1156 static struct roce_hem_item *hem_list_alloc_item(struct hns_roce_dev *hr_dev,
1157 int start, int end,
1158 int count, bool exist_bt,
1159 int bt_level)
1160 {
1161 struct roce_hem_item *hem;
1162
1163 hem = kzalloc(sizeof(*hem), GFP_KERNEL);
1164 if (!hem)
1165 return NULL;
1166
1167 if (exist_bt) {
1168 hem->addr = dma_alloc_coherent(hr_dev->dev,
1169 count * BA_BYTE_LEN,
1170 &hem->dma_addr, GFP_KERNEL);
1171 if (!hem->addr) {
1172 kfree(hem);
1173 return NULL;
1174 }
1175 }
1176
1177 hem->count = count;
1178 hem->start = start;
1179 hem->end = end;
1180 INIT_LIST_HEAD(&hem->list);
1181 INIT_LIST_HEAD(&hem->sibling);
1182
1183 return hem;
1184 }
1185
1186 static void hem_list_free_item(struct hns_roce_dev *hr_dev,
1187 struct roce_hem_item *hem, bool exist_bt)
1188 {
1189 if (exist_bt)
1190 dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN,
1191 hem->addr, hem->dma_addr);
1192 kfree(hem);
1193 }
1194
1195 static void hem_list_free_all(struct hns_roce_dev *hr_dev,
1196 struct list_head *head, bool exist_bt)
1197 {
1198 struct roce_hem_item *hem, *temp_hem;
1199
1200 list_for_each_entry_safe(hem, temp_hem, head, list) {
1201 list_del(&hem->list);
1202 hem_list_free_item(hr_dev, hem, exist_bt);
1203 }
1204 }
1205
1206 static void hem_list_link_bt(struct hns_roce_dev *hr_dev, void *base_addr,
1207 u64 table_addr)
1208 {
1209 *(u64 *)(base_addr) = table_addr;
1210 }
1211
1212 /* assign L0 table address to hem from root bt */
1213 static void hem_list_assign_bt(struct hns_roce_dev *hr_dev,
1214 struct roce_hem_item *hem, void *cpu_addr,
1215 u64 phy_addr)
1216 {
1217 hem->addr = cpu_addr;
1218 hem->dma_addr = (dma_addr_t)phy_addr;
1219 }
1220
1221 static inline bool hem_list_page_is_in_range(struct roce_hem_item *hem,
1222 int offset)
1223 {
1224 return (hem->start <= offset && offset <= hem->end);
1225 }
1226
1227 static struct roce_hem_item *hem_list_search_item(struct list_head *ba_list,
1228 int page_offset)
1229 {
1230 struct roce_hem_item *hem, *temp_hem;
1231 struct roce_hem_item *found = NULL;
1232
1233 list_for_each_entry_safe(hem, temp_hem, ba_list, list) {
1234 if (hem_list_page_is_in_range(hem, page_offset)) {
1235 found = hem;
1236 break;
1237 }
1238 }
1239
1240 return found;
1241 }
1242
1243 static bool hem_list_is_bottom_bt(int hopnum, int bt_level)
1244 {
1245 /*
1246 * hopnum base address table levels
1247 * 0 L0(buf)
1248 * 1 L0 -> buf
1249 * 2 L0 -> L1 -> buf
1250 * 3 L0 -> L1 -> L2 -> buf
1251 */
1252 return bt_level >= (hopnum ? hopnum - 1 : hopnum);
1253 }
1254
1255 /**
1256 * calc base address entries num
1257 * @hopnum: num of mutihop addressing
1258 * @bt_level: base address table level
1259 * @unit: ba entries per bt page
1260 */
1261 static u32 hem_list_calc_ba_range(int hopnum, int bt_level, int unit)
1262 {
1263 u32 step;
1264 int max;
1265 int i;
1266
1267 if (hopnum <= bt_level)
1268 return 0;
1269 /*
1270 * hopnum bt_level range
1271 * 1 0 unit
1272 * ------------
1273 * 2 0 unit * unit
1274 * 2 1 unit
1275 * ------------
1276 * 3 0 unit * unit * unit
1277 * 3 1 unit * unit
1278 * 3 2 unit
1279 */
1280 step = 1;
1281 max = hopnum - bt_level;
1282 for (i = 0; i < max; i++)
1283 step = step * unit;
1284
1285 return step;
1286 }
1287
1288 /**
1289 * calc the root ba entries which could cover all regions
1290 * @regions: buf region array
1291 * @region_cnt: array size of @regions
1292 * @unit: ba entries per bt page
1293 */
1294 int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions,
1295 int region_cnt, int unit)
1296 {
1297 struct hns_roce_buf_region *r;
1298 int total = 0;
1299 int step;
1300 int i;
1301
1302 for (i = 0; i < region_cnt; i++) {
1303 r = (struct hns_roce_buf_region *)&regions[i];
1304 if (r->hopnum > 1) {
1305 step = hem_list_calc_ba_range(r->hopnum, 1, unit);
1306 if (step > 0)
1307 total += (r->count + step - 1) / step;
1308 } else {
1309 total += r->count;
1310 }
1311 }
1312
1313 return total;
1314 }
1315
1316 static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev,
1317 const struct hns_roce_buf_region *r, int unit,
1318 int offset, struct list_head *mid_bt,
1319 struct list_head *btm_bt)
1320 {
1321 struct roce_hem_item *hem_ptrs[HNS_ROCE_MAX_BT_LEVEL] = { NULL };
1322 struct list_head temp_list[HNS_ROCE_MAX_BT_LEVEL];
1323 struct roce_hem_item *cur, *pre;
1324 const int hopnum = r->hopnum;
1325 int start_aligned;
1326 int distance;
1327 int ret = 0;
1328 int max_ofs;
1329 int level;
1330 u32 step;
1331 int end;
1332
1333 if (hopnum <= 1)
1334 return 0;
1335
1336 if (hopnum > HNS_ROCE_MAX_BT_LEVEL) {
1337 dev_err(hr_dev->dev, "invalid hopnum %d!\n", hopnum);
1338 return -EINVAL;
1339 }
1340
1341 if (offset < r->offset) {
1342 dev_err(hr_dev->dev, "invalid offset %d,min %d!\n",
1343 offset, r->offset);
1344 return -EINVAL;
1345 }
1346
1347 distance = offset - r->offset;
1348 max_ofs = r->offset + r->count - 1;
1349 for (level = 0; level < hopnum; level++)
1350 INIT_LIST_HEAD(&temp_list[level]);
1351
1352 /* config L1 bt to last bt and link them to corresponding parent */
1353 for (level = 1; level < hopnum; level++) {
1354 cur = hem_list_search_item(&mid_bt[level], offset);
1355 if (cur) {
1356 hem_ptrs[level] = cur;
1357 continue;
1358 }
1359
1360 step = hem_list_calc_ba_range(hopnum, level, unit);
1361 if (step < 1) {
1362 ret = -EINVAL;
1363 goto err_exit;
1364 }
1365
1366 start_aligned = (distance / step) * step + r->offset;
1367 end = min_t(int, start_aligned + step - 1, max_ofs);
1368 cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit,
1369 true, level);
1370 if (!cur) {
1371 ret = -ENOMEM;
1372 goto err_exit;
1373 }
1374 hem_ptrs[level] = cur;
1375 list_add(&cur->list, &temp_list[level]);
1376 if (hem_list_is_bottom_bt(hopnum, level))
1377 list_add(&cur->sibling, &temp_list[0]);
1378
1379 /* link bt to parent bt */
1380 if (level > 1) {
1381 pre = hem_ptrs[level - 1];
1382 step = (cur->start - pre->start) / step * BA_BYTE_LEN;
1383 hem_list_link_bt(hr_dev, pre->addr + step,
1384 cur->dma_addr);
1385 }
1386 }
1387
1388 list_splice(&temp_list[0], btm_bt);
1389 for (level = 1; level < hopnum; level++)
1390 list_splice(&temp_list[level], &mid_bt[level]);
1391
1392 return 0;
1393
1394 err_exit:
1395 for (level = 1; level < hopnum; level++)
1396 hem_list_free_all(hr_dev, &temp_list[level], true);
1397
1398 return ret;
1399 }
1400
1401 static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev,
1402 struct hns_roce_hem_list *hem_list, int unit,
1403 const struct hns_roce_buf_region *regions,
1404 int region_cnt)
1405 {
1406 struct roce_hem_item *hem, *temp_hem, *root_hem;
1407 struct list_head temp_list[HNS_ROCE_MAX_BT_REGION];
1408 const struct hns_roce_buf_region *r;
1409 struct list_head temp_root;
1410 struct list_head temp_btm;
1411 void *cpu_base;
1412 u64 phy_base;
1413 int ret = 0;
1414 int ba_num;
1415 int offset;
1416 int total;
1417 int step;
1418 int i;
1419
1420 r = &regions[0];
1421 root_hem = hem_list_search_item(&hem_list->root_bt, r->offset);
1422 if (root_hem)
1423 return 0;
1424
1425 ba_num = hns_roce_hem_list_calc_root_ba(regions, region_cnt, unit);
1426 if (ba_num < 1)
1427 return -ENOMEM;
1428
1429 INIT_LIST_HEAD(&temp_root);
1430 offset = r->offset;
1431 /* indicate to last region */
1432 r = &regions[region_cnt - 1];
1433 root_hem = hem_list_alloc_item(hr_dev, offset, r->offset + r->count - 1,
1434 ba_num, true, 0);
1435 if (!root_hem)
1436 return -ENOMEM;
1437 list_add(&root_hem->list, &temp_root);
1438
1439 hem_list->root_ba = root_hem->dma_addr;
1440
1441 INIT_LIST_HEAD(&temp_btm);
1442 for (i = 0; i < region_cnt; i++)
1443 INIT_LIST_HEAD(&temp_list[i]);
1444
1445 total = 0;
1446 for (i = 0; i < region_cnt && total < ba_num; i++) {
1447 r = &regions[i];
1448 if (!r->count)
1449 continue;
1450
1451 /* all regions's mid[x][0] shared the root_bt's trunk */
1452 cpu_base = root_hem->addr + total * BA_BYTE_LEN;
1453 phy_base = root_hem->dma_addr + total * BA_BYTE_LEN;
1454
1455 /* if hopnum is 0 or 1, cut a new fake hem from the root bt
1456 * which's address share to all regions.
1457 */
1458 if (hem_list_is_bottom_bt(r->hopnum, 0)) {
1459 hem = hem_list_alloc_item(hr_dev, r->offset,
1460 r->offset + r->count - 1,
1461 r->count, false, 0);
1462 if (!hem) {
1463 ret = -ENOMEM;
1464 goto err_exit;
1465 }
1466 hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base);
1467 list_add(&hem->list, &temp_list[i]);
1468 list_add(&hem->sibling, &temp_btm);
1469 total += r->count;
1470 } else {
1471 step = hem_list_calc_ba_range(r->hopnum, 1, unit);
1472 if (step < 1) {
1473 ret = -EINVAL;
1474 goto err_exit;
1475 }
1476 /* if exist mid bt, link L1 to L0 */
1477 list_for_each_entry_safe(hem, temp_hem,
1478 &hem_list->mid_bt[i][1], list) {
1479 offset = (hem->start - r->offset) / step *
1480 BA_BYTE_LEN;
1481 hem_list_link_bt(hr_dev, cpu_base + offset,
1482 hem->dma_addr);
1483 total++;
1484 }
1485 }
1486 }
1487
1488 list_splice(&temp_btm, &hem_list->btm_bt);
1489 list_splice(&temp_root, &hem_list->root_bt);
1490 for (i = 0; i < region_cnt; i++)
1491 list_splice(&temp_list[i], &hem_list->mid_bt[i][0]);
1492
1493 return 0;
1494
1495 err_exit:
1496 for (i = 0; i < region_cnt; i++)
1497 hem_list_free_all(hr_dev, &temp_list[i], false);
1498
1499 hem_list_free_all(hr_dev, &temp_root, true);
1500
1501 return ret;
1502 }
1503
1504 /* construct the base address table and link them by address hop config */
1505 int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev,
1506 struct hns_roce_hem_list *hem_list,
1507 const struct hns_roce_buf_region *regions,
1508 int region_cnt)
1509 {
1510 const struct hns_roce_buf_region *r;
1511 int ofs, end;
1512 int ret = 0;
1513 int unit;
1514 int i;
1515
1516 if (region_cnt > HNS_ROCE_MAX_BT_REGION) {
1517 dev_err(hr_dev->dev, "invalid region region_cnt %d!\n",
1518 region_cnt);
1519 return -EINVAL;
1520 }
1521
1522 unit = (1 << hem_list->bt_pg_shift) / BA_BYTE_LEN;
1523 for (i = 0; i < region_cnt; i++) {
1524 r = &regions[i];
1525 if (!r->count)
1526 continue;
1527
1528 end = r->offset + r->count;
1529 for (ofs = r->offset; ofs < end; ofs += unit) {
1530 ret = hem_list_alloc_mid_bt(hr_dev, r, unit, ofs,
1531 hem_list->mid_bt[i],
1532 &hem_list->btm_bt);
1533 if (ret) {
1534 dev_err(hr_dev->dev,
1535 "alloc hem trunk fail ret=%d!\n", ret);
1536 goto err_alloc;
1537 }
1538 }
1539 }
1540
1541 ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions,
1542 region_cnt);
1543 if (ret)
1544 dev_err(hr_dev->dev, "alloc hem root fail ret=%d!\n", ret);
1545 else
1546 return 0;
1547
1548 err_alloc:
1549 hns_roce_hem_list_release(hr_dev, hem_list);
1550
1551 return ret;
1552 }
1553
1554 void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev,
1555 struct hns_roce_hem_list *hem_list)
1556 {
1557 int i, j;
1558
1559 for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++)
1560 for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++)
1561 hem_list_free_all(hr_dev, &hem_list->mid_bt[i][j],
1562 j != 0);
1563
1564 hem_list_free_all(hr_dev, &hem_list->root_bt, true);
1565 INIT_LIST_HEAD(&hem_list->btm_bt);
1566 hem_list->root_ba = 0;
1567 }
1568
1569 void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list,
1570 int bt_page_order)
1571 {
1572 int i, j;
1573
1574 INIT_LIST_HEAD(&hem_list->root_bt);
1575 INIT_LIST_HEAD(&hem_list->btm_bt);
1576 for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++)
1577 for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++)
1578 INIT_LIST_HEAD(&hem_list->mid_bt[i][j]);
1579
1580 hem_list->bt_pg_shift = bt_page_order;
1581 }
1582
1583 void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev,
1584 struct hns_roce_hem_list *hem_list,
1585 int offset, int *mtt_cnt, u64 *phy_addr)
1586 {
1587 struct list_head *head = &hem_list->btm_bt;
1588 struct roce_hem_item *hem, *temp_hem;
1589 void *cpu_base = NULL;
1590 u64 phy_base = 0;
1591 int nr = 0;
1592
1593 list_for_each_entry_safe(hem, temp_hem, head, sibling) {
1594 if (hem_list_page_is_in_range(hem, offset)) {
1595 nr = offset - hem->start;
1596 cpu_base = hem->addr + nr * BA_BYTE_LEN;
1597 phy_base = hem->dma_addr + nr * BA_BYTE_LEN;
1598 nr = hem->end + 1 - offset;
1599 break;
1600 }
1601 }
1602
1603 if (mtt_cnt)
1604 *mtt_cnt = nr;
1605
1606 if (phy_addr)
1607 *phy_addr = phy_base;
1608
1609 return cpu_base;
1610 }
Linux with added FPC-III support