2 * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved.
3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/module.h>
35 #include <linux/init.h>
36 #include <linux/slab.h>
37 #include <linux/errno.h>
38 #include <linux/netdevice.h>
39 #include <linux/inetdevice.h>
40 #include <linux/rtnetlink.h>
41 #include <linux/if_vlan.h>
42 #include <linux/sched/mm.h>
43 #include <linux/sched/task.h>
46 #include <net/addrconf.h>
47 #include <net/devlink.h>
49 #include <rdma/ib_smi.h>
50 #include <rdma/ib_user_verbs.h>
51 #include <rdma/ib_addr.h>
52 #include <rdma/ib_cache.h>
54 #include <net/bonding.h>
56 #include <linux/mlx4/driver.h>
57 #include <linux/mlx4/cmd.h>
58 #include <linux/mlx4/qp.h>
61 #include <rdma/mlx4-abi.h>
63 #define DRV_NAME MLX4_IB_DRV_NAME
64 #define DRV_VERSION "4.0-0"
66 #define MLX4_IB_FLOW_MAX_PRIO 0xFFF
67 #define MLX4_IB_FLOW_QPN_MASK 0xFFFFFF
68 #define MLX4_IB_CARD_REV_A0 0xA0
70 MODULE_AUTHOR("Roland Dreier");
71 MODULE_DESCRIPTION("Mellanox ConnectX HCA InfiniBand driver");
72 MODULE_LICENSE("Dual BSD/GPL");
74 int mlx4_ib_sm_guid_assign
= 0;
75 module_param_named(sm_guid_assign
, mlx4_ib_sm_guid_assign
, int, 0444);
76 MODULE_PARM_DESC(sm_guid_assign
, "Enable SM alias_GUID assignment if sm_guid_assign > 0 (Default: 0)");
78 static const char mlx4_ib_version
[] =
79 DRV_NAME
": Mellanox ConnectX InfiniBand driver v"
82 static void do_slave_init(struct mlx4_ib_dev
*ibdev
, int slave
, int do_init
);
83 static enum rdma_link_layer
mlx4_ib_port_link_layer(struct ib_device
*device
,
85 static int mlx4_ib_event(struct notifier_block
*this, unsigned long event
,
88 static struct workqueue_struct
*wq
;
90 static int check_flow_steering_support(struct mlx4_dev
*dev
)
92 int eth_num_ports
= 0;
95 int dmfs
= dev
->caps
.steering_mode
== MLX4_STEERING_MODE_DEVICE_MANAGED
;
99 mlx4_foreach_port(i
, dev
, MLX4_PORT_TYPE_ETH
)
101 mlx4_foreach_port(i
, dev
, MLX4_PORT_TYPE_IB
)
103 dmfs
&= (!ib_num_ports
||
104 (dev
->caps
.flags2
& MLX4_DEV_CAP_FLAG2_DMFS_IPOIB
)) &&
106 (dev
->caps
.flags2
& MLX4_DEV_CAP_FLAG2_FS_EN
));
107 if (ib_num_ports
&& mlx4_is_mfunc(dev
)) {
108 pr_warn("Device managed flow steering is unavailable for IB port in multifunction env.\n");
115 static int num_ib_ports(struct mlx4_dev
*dev
)
120 mlx4_foreach_port(i
, dev
, MLX4_PORT_TYPE_IB
)
126 static struct net_device
*mlx4_ib_get_netdev(struct ib_device
*device
,
129 struct mlx4_ib_dev
*ibdev
= to_mdev(device
);
130 struct net_device
*dev
, *ret
= NULL
;
133 for_each_netdev_rcu(&init_net
, dev
) {
134 if (dev
->dev
.parent
!= ibdev
->ib_dev
.dev
.parent
||
135 dev
->dev_port
+ 1 != port_num
)
138 if (mlx4_is_bonded(ibdev
->dev
)) {
139 struct net_device
*upper
;
141 upper
= netdev_master_upper_dev_get_rcu(dev
);
143 struct net_device
*active
;
145 active
= bond_option_active_slave_get_rcu(netdev_priv(upper
));
160 static int mlx4_ib_update_gids_v1(struct gid_entry
*gids
,
161 struct mlx4_ib_dev
*ibdev
,
164 struct mlx4_cmd_mailbox
*mailbox
;
166 struct mlx4_dev
*dev
= ibdev
->dev
;
168 union ib_gid
*gid_tbl
;
170 mailbox
= mlx4_alloc_cmd_mailbox(dev
);
174 gid_tbl
= mailbox
->buf
;
176 for (i
= 0; i
< MLX4_MAX_PORT_GIDS
; ++i
)
177 memcpy(&gid_tbl
[i
], &gids
[i
].gid
, sizeof(union ib_gid
));
179 err
= mlx4_cmd(dev
, mailbox
->dma
,
180 MLX4_SET_PORT_GID_TABLE
<< 8 | port_num
,
181 1, MLX4_CMD_SET_PORT
, MLX4_CMD_TIME_CLASS_B
,
183 if (mlx4_is_bonded(dev
))
184 err
+= mlx4_cmd(dev
, mailbox
->dma
,
185 MLX4_SET_PORT_GID_TABLE
<< 8 | 2,
186 1, MLX4_CMD_SET_PORT
, MLX4_CMD_TIME_CLASS_B
,
189 mlx4_free_cmd_mailbox(dev
, mailbox
);
193 static int mlx4_ib_update_gids_v1_v2(struct gid_entry
*gids
,
194 struct mlx4_ib_dev
*ibdev
,
197 struct mlx4_cmd_mailbox
*mailbox
;
199 struct mlx4_dev
*dev
= ibdev
->dev
;
210 mailbox
= mlx4_alloc_cmd_mailbox(dev
);
214 gid_tbl
= mailbox
->buf
;
215 for (i
= 0; i
< MLX4_MAX_PORT_GIDS
; ++i
) {
216 memcpy(&gid_tbl
[i
].gid
, &gids
[i
].gid
, sizeof(union ib_gid
));
217 if (gids
[i
].gid_type
== IB_GID_TYPE_ROCE_UDP_ENCAP
) {
218 gid_tbl
[i
].version
= 2;
219 if (!ipv6_addr_v4mapped((struct in6_addr
*)&gids
[i
].gid
))
224 err
= mlx4_cmd(dev
, mailbox
->dma
,
225 MLX4_SET_PORT_ROCE_ADDR
<< 8 | port_num
,
226 1, MLX4_CMD_SET_PORT
, MLX4_CMD_TIME_CLASS_B
,
228 if (mlx4_is_bonded(dev
))
229 err
+= mlx4_cmd(dev
, mailbox
->dma
,
230 MLX4_SET_PORT_ROCE_ADDR
<< 8 | 2,
231 1, MLX4_CMD_SET_PORT
, MLX4_CMD_TIME_CLASS_B
,
234 mlx4_free_cmd_mailbox(dev
, mailbox
);
238 static int mlx4_ib_update_gids(struct gid_entry
*gids
,
239 struct mlx4_ib_dev
*ibdev
,
242 if (ibdev
->dev
->caps
.flags2
& MLX4_DEV_CAP_FLAG2_ROCE_V1_V2
)
243 return mlx4_ib_update_gids_v1_v2(gids
, ibdev
, port_num
);
245 return mlx4_ib_update_gids_v1(gids
, ibdev
, port_num
);
248 static void free_gid_entry(struct gid_entry
*entry
)
250 memset(&entry
->gid
, 0, sizeof(entry
->gid
));
255 static int mlx4_ib_add_gid(const struct ib_gid_attr
*attr
, void **context
)
257 struct mlx4_ib_dev
*ibdev
= to_mdev(attr
->device
);
258 struct mlx4_ib_iboe
*iboe
= &ibdev
->iboe
;
259 struct mlx4_port_gid_table
*port_gid_table
;
260 int free
= -1, found
= -1;
264 struct gid_entry
*gids
;
265 u16 vlan_id
= 0xffff;
268 if (!rdma_cap_roce_gid_table(attr
->device
, attr
->port_num
))
271 if (attr
->port_num
> MLX4_MAX_PORTS
)
277 ret
= rdma_read_gid_l2_fields(attr
, &vlan_id
, &mac
[0]);
280 port_gid_table
= &iboe
->gids
[attr
->port_num
- 1];
281 spin_lock_bh(&iboe
->lock
);
282 for (i
= 0; i
< MLX4_MAX_PORT_GIDS
; ++i
) {
283 if (!memcmp(&port_gid_table
->gids
[i
].gid
,
284 &attr
->gid
, sizeof(attr
->gid
)) &&
285 port_gid_table
->gids
[i
].gid_type
== attr
->gid_type
&&
286 port_gid_table
->gids
[i
].vlan_id
== vlan_id
) {
290 if (free
< 0 && rdma_is_zero_gid(&port_gid_table
->gids
[i
].gid
))
291 free
= i
; /* HW has space */
298 port_gid_table
->gids
[free
].ctx
= kmalloc(sizeof(*port_gid_table
->gids
[free
].ctx
), GFP_ATOMIC
);
299 if (!port_gid_table
->gids
[free
].ctx
) {
302 *context
= port_gid_table
->gids
[free
].ctx
;
303 port_gid_table
->gids
[free
].gid
= attr
->gid
;
304 port_gid_table
->gids
[free
].gid_type
= attr
->gid_type
;
305 port_gid_table
->gids
[free
].vlan_id
= vlan_id
;
306 port_gid_table
->gids
[free
].ctx
->real_index
= free
;
307 port_gid_table
->gids
[free
].ctx
->refcount
= 1;
312 struct gid_cache_context
*ctx
= port_gid_table
->gids
[found
].ctx
;
316 if (!ret
&& hw_update
) {
317 gids
= kmalloc_array(MLX4_MAX_PORT_GIDS
, sizeof(*gids
),
322 free_gid_entry(&port_gid_table
->gids
[free
]);
324 for (i
= 0; i
< MLX4_MAX_PORT_GIDS
; i
++) {
325 memcpy(&gids
[i
].gid
, &port_gid_table
->gids
[i
].gid
, sizeof(union ib_gid
));
326 gids
[i
].gid_type
= port_gid_table
->gids
[i
].gid_type
;
330 spin_unlock_bh(&iboe
->lock
);
332 if (!ret
&& hw_update
) {
333 ret
= mlx4_ib_update_gids(gids
, ibdev
, attr
->port_num
);
335 spin_lock_bh(&iboe
->lock
);
337 free_gid_entry(&port_gid_table
->gids
[free
]);
338 spin_unlock_bh(&iboe
->lock
);
346 static int mlx4_ib_del_gid(const struct ib_gid_attr
*attr
, void **context
)
348 struct gid_cache_context
*ctx
= *context
;
349 struct mlx4_ib_dev
*ibdev
= to_mdev(attr
->device
);
350 struct mlx4_ib_iboe
*iboe
= &ibdev
->iboe
;
351 struct mlx4_port_gid_table
*port_gid_table
;
354 struct gid_entry
*gids
;
356 if (!rdma_cap_roce_gid_table(attr
->device
, attr
->port_num
))
359 if (attr
->port_num
> MLX4_MAX_PORTS
)
362 port_gid_table
= &iboe
->gids
[attr
->port_num
- 1];
363 spin_lock_bh(&iboe
->lock
);
366 if (!ctx
->refcount
) {
367 unsigned int real_index
= ctx
->real_index
;
369 free_gid_entry(&port_gid_table
->gids
[real_index
]);
373 if (!ret
&& hw_update
) {
376 gids
= kmalloc_array(MLX4_MAX_PORT_GIDS
, sizeof(*gids
),
381 for (i
= 0; i
< MLX4_MAX_PORT_GIDS
; i
++) {
383 &port_gid_table
->gids
[i
].gid
,
384 sizeof(union ib_gid
));
386 port_gid_table
->gids
[i
].gid_type
;
390 spin_unlock_bh(&iboe
->lock
);
392 if (!ret
&& hw_update
) {
393 ret
= mlx4_ib_update_gids(gids
, ibdev
, attr
->port_num
);
399 int mlx4_ib_gid_index_to_real_index(struct mlx4_ib_dev
*ibdev
,
400 const struct ib_gid_attr
*attr
)
402 struct mlx4_ib_iboe
*iboe
= &ibdev
->iboe
;
403 struct gid_cache_context
*ctx
= NULL
;
404 struct mlx4_port_gid_table
*port_gid_table
;
405 int real_index
= -EINVAL
;
408 u32 port_num
= attr
->port_num
;
410 if (port_num
> MLX4_MAX_PORTS
)
413 if (mlx4_is_bonded(ibdev
->dev
))
416 if (!rdma_cap_roce_gid_table(&ibdev
->ib_dev
, port_num
))
419 spin_lock_irqsave(&iboe
->lock
, flags
);
420 port_gid_table
= &iboe
->gids
[port_num
- 1];
422 for (i
= 0; i
< MLX4_MAX_PORT_GIDS
; ++i
)
423 if (!memcmp(&port_gid_table
->gids
[i
].gid
,
424 &attr
->gid
, sizeof(attr
->gid
)) &&
425 attr
->gid_type
== port_gid_table
->gids
[i
].gid_type
) {
426 ctx
= port_gid_table
->gids
[i
].ctx
;
430 real_index
= ctx
->real_index
;
431 spin_unlock_irqrestore(&iboe
->lock
, flags
);
435 static int mlx4_ib_query_device(struct ib_device
*ibdev
,
436 struct ib_device_attr
*props
,
437 struct ib_udata
*uhw
)
439 struct mlx4_ib_dev
*dev
= to_mdev(ibdev
);
440 struct ib_smp
*in_mad
;
441 struct ib_smp
*out_mad
;
444 struct mlx4_uverbs_ex_query_device cmd
;
445 struct mlx4_uverbs_ex_query_device_resp resp
= {};
446 struct mlx4_clock_params clock_params
;
449 if (uhw
->inlen
< sizeof(cmd
))
452 err
= ib_copy_from_udata(&cmd
, uhw
, sizeof(cmd
));
463 resp
.response_length
= offsetof(typeof(resp
), response_length
) +
464 sizeof(resp
.response_length
);
465 in_mad
= kzalloc(sizeof *in_mad
, GFP_KERNEL
);
466 out_mad
= kmalloc(sizeof *out_mad
, GFP_KERNEL
);
468 if (!in_mad
|| !out_mad
)
471 ib_init_query_mad(in_mad
);
472 in_mad
->attr_id
= IB_SMP_ATTR_NODE_INFO
;
474 err
= mlx4_MAD_IFC(to_mdev(ibdev
), MLX4_MAD_IFC_IGNORE_KEYS
,
475 1, NULL
, NULL
, in_mad
, out_mad
);
479 memset(props
, 0, sizeof *props
);
481 have_ib_ports
= num_ib_ports(dev
->dev
);
483 props
->fw_ver
= dev
->dev
->caps
.fw_ver
;
484 props
->device_cap_flags
= IB_DEVICE_CHANGE_PHY_PORT
|
485 IB_DEVICE_PORT_ACTIVE_EVENT
|
486 IB_DEVICE_SYS_IMAGE_GUID
|
487 IB_DEVICE_RC_RNR_NAK_GEN
;
488 props
->kernel_cap_flags
= IBK_BLOCK_MULTICAST_LOOPBACK
;
489 if (dev
->dev
->caps
.flags
& MLX4_DEV_CAP_FLAG_BAD_PKEY_CNTR
)
490 props
->device_cap_flags
|= IB_DEVICE_BAD_PKEY_CNTR
;
491 if (dev
->dev
->caps
.flags
& MLX4_DEV_CAP_FLAG_BAD_QKEY_CNTR
)
492 props
->device_cap_flags
|= IB_DEVICE_BAD_QKEY_CNTR
;
493 if (dev
->dev
->caps
.flags
& MLX4_DEV_CAP_FLAG_APM
&& have_ib_ports
)
494 props
->device_cap_flags
|= IB_DEVICE_AUTO_PATH_MIG
;
495 if (dev
->dev
->caps
.flags
& MLX4_DEV_CAP_FLAG_UD_AV_PORT
)
496 props
->device_cap_flags
|= IB_DEVICE_UD_AV_PORT_ENFORCE
;
497 if (dev
->dev
->caps
.flags
& MLX4_DEV_CAP_FLAG_IPOIB_CSUM
)
498 props
->device_cap_flags
|= IB_DEVICE_UD_IP_CSUM
;
499 if (dev
->dev
->caps
.max_gso_sz
&&
500 (dev
->dev
->rev_id
!= MLX4_IB_CARD_REV_A0
) &&
501 (dev
->dev
->caps
.flags
& MLX4_DEV_CAP_FLAG_BLH
))
502 props
->kernel_cap_flags
|= IBK_UD_TSO
;
503 if (dev
->dev
->caps
.bmme_flags
& MLX4_BMME_FLAG_RESERVED_LKEY
)
504 props
->kernel_cap_flags
|= IBK_LOCAL_DMA_LKEY
;
505 if ((dev
->dev
->caps
.bmme_flags
& MLX4_BMME_FLAG_LOCAL_INV
) &&
506 (dev
->dev
->caps
.bmme_flags
& MLX4_BMME_FLAG_REMOTE_INV
) &&
507 (dev
->dev
->caps
.bmme_flags
& MLX4_BMME_FLAG_FAST_REG_WR
))
508 props
->device_cap_flags
|= IB_DEVICE_MEM_MGT_EXTENSIONS
;
509 if (dev
->dev
->caps
.flags
& MLX4_DEV_CAP_FLAG_XRC
)
510 props
->device_cap_flags
|= IB_DEVICE_XRC
;
511 if (dev
->dev
->caps
.flags
& MLX4_DEV_CAP_FLAG_MEM_WINDOW
)
512 props
->device_cap_flags
|= IB_DEVICE_MEM_WINDOW
;
513 if (dev
->dev
->caps
.bmme_flags
& MLX4_BMME_FLAG_TYPE_2_WIN
) {
514 if (dev
->dev
->caps
.bmme_flags
& MLX4_BMME_FLAG_WIN_TYPE_2B
)
515 props
->device_cap_flags
|= IB_DEVICE_MEM_WINDOW_TYPE_2B
;
517 props
->device_cap_flags
|= IB_DEVICE_MEM_WINDOW_TYPE_2A
;
519 if (dev
->steering_support
== MLX4_STEERING_MODE_DEVICE_MANAGED
)
520 props
->device_cap_flags
|= IB_DEVICE_MANAGED_FLOW_STEERING
;
522 props
->device_cap_flags
|= IB_DEVICE_RAW_IP_CSUM
;
524 props
->vendor_id
= be32_to_cpup((__be32
*) (out_mad
->data
+ 36)) &
526 props
->vendor_part_id
= dev
->dev
->persist
->pdev
->device
;
527 props
->hw_ver
= be32_to_cpup((__be32
*) (out_mad
->data
+ 32));
528 memcpy(&props
->sys_image_guid
, out_mad
->data
+ 4, 8);
530 props
->max_mr_size
= ~0ull;
531 props
->page_size_cap
= dev
->dev
->caps
.page_size_cap
;
532 props
->max_qp
= dev
->dev
->quotas
.qp
;
533 props
->max_qp_wr
= dev
->dev
->caps
.max_wqes
- MLX4_IB_SQ_MAX_SPARE
;
534 props
->max_send_sge
=
535 min(dev
->dev
->caps
.max_sq_sg
, dev
->dev
->caps
.max_rq_sg
);
536 props
->max_recv_sge
=
537 min(dev
->dev
->caps
.max_sq_sg
, dev
->dev
->caps
.max_rq_sg
);
538 props
->max_sge_rd
= MLX4_MAX_SGE_RD
;
539 props
->max_cq
= dev
->dev
->quotas
.cq
;
540 props
->max_cqe
= dev
->dev
->caps
.max_cqes
;
541 props
->max_mr
= dev
->dev
->quotas
.mpt
;
542 props
->max_pd
= dev
->dev
->caps
.num_pds
- dev
->dev
->caps
.reserved_pds
;
543 props
->max_qp_rd_atom
= dev
->dev
->caps
.max_qp_dest_rdma
;
544 props
->max_qp_init_rd_atom
= dev
->dev
->caps
.max_qp_init_rdma
;
545 props
->max_res_rd_atom
= props
->max_qp_rd_atom
* props
->max_qp
;
546 props
->max_srq
= dev
->dev
->quotas
.srq
;
547 props
->max_srq_wr
= dev
->dev
->caps
.max_srq_wqes
- 1;
548 props
->max_srq_sge
= dev
->dev
->caps
.max_srq_sge
;
549 props
->max_fast_reg_page_list_len
= MLX4_MAX_FAST_REG_PAGES
;
550 props
->local_ca_ack_delay
= dev
->dev
->caps
.local_ca_ack_delay
;
551 props
->atomic_cap
= dev
->dev
->caps
.flags
& MLX4_DEV_CAP_FLAG_ATOMIC
?
552 IB_ATOMIC_HCA
: IB_ATOMIC_NONE
;
553 props
->masked_atomic_cap
= props
->atomic_cap
;
554 props
->max_pkeys
= dev
->dev
->caps
.pkey_table_len
[1];
555 props
->max_mcast_grp
= dev
->dev
->caps
.num_mgms
+ dev
->dev
->caps
.num_amgms
;
556 props
->max_mcast_qp_attach
= dev
->dev
->caps
.num_qp_per_mgm
;
557 props
->max_total_mcast_qp_attach
= props
->max_mcast_qp_attach
*
558 props
->max_mcast_grp
;
559 props
->hca_core_clock
= dev
->dev
->caps
.hca_core_clock
* 1000UL;
560 props
->timestamp_mask
= 0xFFFFFFFFFFFFULL
;
561 props
->max_ah
= INT_MAX
;
563 if (mlx4_ib_port_link_layer(ibdev
, 1) == IB_LINK_LAYER_ETHERNET
||
564 mlx4_ib_port_link_layer(ibdev
, 2) == IB_LINK_LAYER_ETHERNET
) {
565 if (dev
->dev
->caps
.flags2
& MLX4_DEV_CAP_FLAG2_RSS
) {
566 props
->rss_caps
.max_rwq_indirection_tables
=
568 props
->rss_caps
.max_rwq_indirection_table_size
=
569 dev
->dev
->caps
.max_rss_tbl_sz
;
570 props
->rss_caps
.supported_qpts
= 1 << IB_QPT_RAW_PACKET
;
571 props
->max_wq_type_rq
= props
->max_qp
;
574 if (dev
->dev
->caps
.flags
& MLX4_DEV_CAP_FLAG_FCS_KEEP
)
575 props
->raw_packet_caps
|= IB_RAW_PACKET_CAP_SCATTER_FCS
;
578 props
->cq_caps
.max_cq_moderation_count
= MLX4_MAX_CQ_COUNT
;
579 props
->cq_caps
.max_cq_moderation_period
= MLX4_MAX_CQ_PERIOD
;
581 if (uhw
->outlen
>= resp
.response_length
+ sizeof(resp
.hca_core_clock_offset
)) {
582 resp
.response_length
+= sizeof(resp
.hca_core_clock_offset
);
583 if (!mlx4_get_internal_clock_params(dev
->dev
, &clock_params
)) {
584 resp
.comp_mask
|= MLX4_IB_QUERY_DEV_RESP_MASK_CORE_CLOCK_OFFSET
;
585 resp
.hca_core_clock_offset
= clock_params
.offset
% PAGE_SIZE
;
589 if (uhw
->outlen
>= resp
.response_length
+
590 sizeof(resp
.max_inl_recv_sz
)) {
591 resp
.response_length
+= sizeof(resp
.max_inl_recv_sz
);
592 resp
.max_inl_recv_sz
= dev
->dev
->caps
.max_rq_sg
*
593 sizeof(struct mlx4_wqe_data_seg
);
596 if (offsetofend(typeof(resp
), rss_caps
) <= uhw
->outlen
) {
597 if (props
->rss_caps
.supported_qpts
) {
598 resp
.rss_caps
.rx_hash_function
=
599 MLX4_IB_RX_HASH_FUNC_TOEPLITZ
;
601 resp
.rss_caps
.rx_hash_fields_mask
=
602 MLX4_IB_RX_HASH_SRC_IPV4
|
603 MLX4_IB_RX_HASH_DST_IPV4
|
604 MLX4_IB_RX_HASH_SRC_IPV6
|
605 MLX4_IB_RX_HASH_DST_IPV6
|
606 MLX4_IB_RX_HASH_SRC_PORT_TCP
|
607 MLX4_IB_RX_HASH_DST_PORT_TCP
|
608 MLX4_IB_RX_HASH_SRC_PORT_UDP
|
609 MLX4_IB_RX_HASH_DST_PORT_UDP
;
611 if (dev
->dev
->caps
.tunnel_offload_mode
==
612 MLX4_TUNNEL_OFFLOAD_MODE_VXLAN
)
613 resp
.rss_caps
.rx_hash_fields_mask
|=
614 MLX4_IB_RX_HASH_INNER
;
616 resp
.response_length
= offsetof(typeof(resp
), rss_caps
) +
617 sizeof(resp
.rss_caps
);
620 if (offsetofend(typeof(resp
), tso_caps
) <= uhw
->outlen
) {
621 if (dev
->dev
->caps
.max_gso_sz
&&
622 ((mlx4_ib_port_link_layer(ibdev
, 1) ==
623 IB_LINK_LAYER_ETHERNET
) ||
624 (mlx4_ib_port_link_layer(ibdev
, 2) ==
625 IB_LINK_LAYER_ETHERNET
))) {
626 resp
.tso_caps
.max_tso
= dev
->dev
->caps
.max_gso_sz
;
627 resp
.tso_caps
.supported_qpts
|=
628 1 << IB_QPT_RAW_PACKET
;
630 resp
.response_length
= offsetof(typeof(resp
), tso_caps
) +
631 sizeof(resp
.tso_caps
);
635 err
= ib_copy_to_udata(uhw
, &resp
, resp
.response_length
);
646 static enum rdma_link_layer
647 mlx4_ib_port_link_layer(struct ib_device
*device
, u32 port_num
)
649 struct mlx4_dev
*dev
= to_mdev(device
)->dev
;
651 return dev
->caps
.port_mask
[port_num
] == MLX4_PORT_TYPE_IB
?
652 IB_LINK_LAYER_INFINIBAND
: IB_LINK_LAYER_ETHERNET
;
655 static int ib_link_query_port(struct ib_device
*ibdev
, u32 port
,
656 struct ib_port_attr
*props
, int netw_view
)
658 struct ib_smp
*in_mad
;
659 struct ib_smp
*out_mad
;
660 int ext_active_speed
;
661 int mad_ifc_flags
= MLX4_MAD_IFC_IGNORE_KEYS
;
664 in_mad
= kzalloc(sizeof *in_mad
, GFP_KERNEL
);
665 out_mad
= kmalloc(sizeof *out_mad
, GFP_KERNEL
);
666 if (!in_mad
|| !out_mad
)
669 ib_init_query_mad(in_mad
);
670 in_mad
->attr_id
= IB_SMP_ATTR_PORT_INFO
;
671 in_mad
->attr_mod
= cpu_to_be32(port
);
673 if (mlx4_is_mfunc(to_mdev(ibdev
)->dev
) && netw_view
)
674 mad_ifc_flags
|= MLX4_MAD_IFC_NET_VIEW
;
676 err
= mlx4_MAD_IFC(to_mdev(ibdev
), mad_ifc_flags
, port
, NULL
, NULL
,
682 props
->lid
= be16_to_cpup((__be16
*) (out_mad
->data
+ 16));
683 props
->lmc
= out_mad
->data
[34] & 0x7;
684 props
->sm_lid
= be16_to_cpup((__be16
*) (out_mad
->data
+ 18));
685 props
->sm_sl
= out_mad
->data
[36] & 0xf;
686 props
->state
= out_mad
->data
[32] & 0xf;
687 props
->phys_state
= out_mad
->data
[33] >> 4;
688 props
->port_cap_flags
= be32_to_cpup((__be32
*) (out_mad
->data
+ 20));
690 props
->gid_tbl_len
= out_mad
->data
[50];
692 props
->gid_tbl_len
= to_mdev(ibdev
)->dev
->caps
.gid_table_len
[port
];
693 props
->max_msg_sz
= to_mdev(ibdev
)->dev
->caps
.max_msg_sz
;
694 props
->pkey_tbl_len
= to_mdev(ibdev
)->dev
->caps
.pkey_table_len
[port
];
695 props
->bad_pkey_cntr
= be16_to_cpup((__be16
*) (out_mad
->data
+ 46));
696 props
->qkey_viol_cntr
= be16_to_cpup((__be16
*) (out_mad
->data
+ 48));
697 props
->active_width
= out_mad
->data
[31] & 0xf;
698 props
->active_speed
= out_mad
->data
[35] >> 4;
699 props
->max_mtu
= out_mad
->data
[41] & 0xf;
700 props
->active_mtu
= out_mad
->data
[36] >> 4;
701 props
->subnet_timeout
= out_mad
->data
[51] & 0x1f;
702 props
->max_vl_num
= out_mad
->data
[37] >> 4;
703 props
->init_type_reply
= out_mad
->data
[41] >> 4;
705 /* Check if extended speeds (EDR/FDR/...) are supported */
706 if (props
->port_cap_flags
& IB_PORT_EXTENDED_SPEEDS_SUP
) {
707 ext_active_speed
= out_mad
->data
[62] >> 4;
709 switch (ext_active_speed
) {
711 props
->active_speed
= IB_SPEED_FDR
;
714 props
->active_speed
= IB_SPEED_EDR
;
719 /* If reported active speed is QDR, check if is FDR-10 */
720 if (props
->active_speed
== IB_SPEED_QDR
) {
721 ib_init_query_mad(in_mad
);
722 in_mad
->attr_id
= MLX4_ATTR_EXTENDED_PORT_INFO
;
723 in_mad
->attr_mod
= cpu_to_be32(port
);
725 err
= mlx4_MAD_IFC(to_mdev(ibdev
), mad_ifc_flags
, port
,
726 NULL
, NULL
, in_mad
, out_mad
);
730 /* Checking LinkSpeedActive for FDR-10 */
731 if (out_mad
->data
[15] & 0x1)
732 props
->active_speed
= IB_SPEED_FDR10
;
735 /* Avoid wrong speed value returned by FW if the IB link is down. */
736 if (props
->state
== IB_PORT_DOWN
)
737 props
->active_speed
= IB_SPEED_SDR
;
745 static u8
state_to_phys_state(enum ib_port_state state
)
747 return state
== IB_PORT_ACTIVE
?
748 IB_PORT_PHYS_STATE_LINK_UP
: IB_PORT_PHYS_STATE_DISABLED
;
751 static int eth_link_query_port(struct ib_device
*ibdev
, u32 port
,
752 struct ib_port_attr
*props
)
755 struct mlx4_ib_dev
*mdev
= to_mdev(ibdev
);
756 struct mlx4_ib_iboe
*iboe
= &mdev
->iboe
;
757 struct net_device
*ndev
;
759 struct mlx4_cmd_mailbox
*mailbox
;
761 int is_bonded
= mlx4_is_bonded(mdev
->dev
);
763 mailbox
= mlx4_alloc_cmd_mailbox(mdev
->dev
);
765 return PTR_ERR(mailbox
);
767 err
= mlx4_cmd_box(mdev
->dev
, 0, mailbox
->dma
, port
, 0,
768 MLX4_CMD_QUERY_PORT
, MLX4_CMD_TIME_CLASS_B
,
773 props
->active_width
= (((u8
*)mailbox
->buf
)[5] == 0x40) ||
774 (((u8
*)mailbox
->buf
)[5] == 0x20 /*56Gb*/) ?
775 IB_WIDTH_4X
: IB_WIDTH_1X
;
776 props
->active_speed
= (((u8
*)mailbox
->buf
)[5] == 0x20 /*56Gb*/) ?
777 IB_SPEED_FDR
: IB_SPEED_QDR
;
778 props
->port_cap_flags
= IB_PORT_CM_SUP
;
779 props
->ip_gids
= true;
780 props
->gid_tbl_len
= mdev
->dev
->caps
.gid_table_len
[port
];
781 props
->max_msg_sz
= mdev
->dev
->caps
.max_msg_sz
;
782 if (mdev
->dev
->caps
.pkey_table_len
[port
])
783 props
->pkey_tbl_len
= 1;
784 props
->max_mtu
= IB_MTU_4096
;
785 props
->max_vl_num
= 2;
786 props
->state
= IB_PORT_DOWN
;
787 props
->phys_state
= state_to_phys_state(props
->state
);
788 props
->active_mtu
= IB_MTU_256
;
789 spin_lock_bh(&iboe
->lock
);
790 ndev
= iboe
->netdevs
[port
- 1];
791 if (ndev
&& is_bonded
) {
792 rcu_read_lock(); /* required to get upper dev */
793 ndev
= netdev_master_upper_dev_get_rcu(ndev
);
799 tmp
= iboe_get_mtu(ndev
->mtu
);
800 props
->active_mtu
= tmp
? min(props
->max_mtu
, tmp
) : IB_MTU_256
;
802 props
->state
= (netif_running(ndev
) && netif_carrier_ok(ndev
)) ?
803 IB_PORT_ACTIVE
: IB_PORT_DOWN
;
804 props
->phys_state
= state_to_phys_state(props
->state
);
806 spin_unlock_bh(&iboe
->lock
);
808 mlx4_free_cmd_mailbox(mdev
->dev
, mailbox
);
812 int __mlx4_ib_query_port(struct ib_device
*ibdev
, u32 port
,
813 struct ib_port_attr
*props
, int netw_view
)
817 /* props being zeroed by the caller, avoid zeroing it here */
819 err
= mlx4_ib_port_link_layer(ibdev
, port
) == IB_LINK_LAYER_INFINIBAND
?
820 ib_link_query_port(ibdev
, port
, props
, netw_view
) :
821 eth_link_query_port(ibdev
, port
, props
);
826 static int mlx4_ib_query_port(struct ib_device
*ibdev
, u32 port
,
827 struct ib_port_attr
*props
)
829 /* returns host view */
830 return __mlx4_ib_query_port(ibdev
, port
, props
, 0);
833 int __mlx4_ib_query_gid(struct ib_device
*ibdev
, u32 port
, int index
,
834 union ib_gid
*gid
, int netw_view
)
836 struct ib_smp
*in_mad
;
837 struct ib_smp
*out_mad
;
839 struct mlx4_ib_dev
*dev
= to_mdev(ibdev
);
841 int mad_ifc_flags
= MLX4_MAD_IFC_IGNORE_KEYS
;
843 in_mad
= kzalloc(sizeof *in_mad
, GFP_KERNEL
);
844 out_mad
= kmalloc(sizeof *out_mad
, GFP_KERNEL
);
845 if (!in_mad
|| !out_mad
)
848 ib_init_query_mad(in_mad
);
849 in_mad
->attr_id
= IB_SMP_ATTR_PORT_INFO
;
850 in_mad
->attr_mod
= cpu_to_be32(port
);
852 if (mlx4_is_mfunc(dev
->dev
) && netw_view
)
853 mad_ifc_flags
|= MLX4_MAD_IFC_NET_VIEW
;
855 err
= mlx4_MAD_IFC(dev
, mad_ifc_flags
, port
, NULL
, NULL
, in_mad
, out_mad
);
859 memcpy(gid
->raw
, out_mad
->data
+ 8, 8);
861 if (mlx4_is_mfunc(dev
->dev
) && !netw_view
) {
863 /* For any index > 0, return the null guid */
870 ib_init_query_mad(in_mad
);
871 in_mad
->attr_id
= IB_SMP_ATTR_GUID_INFO
;
872 in_mad
->attr_mod
= cpu_to_be32(index
/ 8);
874 err
= mlx4_MAD_IFC(dev
, mad_ifc_flags
, port
,
875 NULL
, NULL
, in_mad
, out_mad
);
879 memcpy(gid
->raw
+ 8, out_mad
->data
+ (index
% 8) * 8, 8);
883 memset(gid
->raw
+ 8, 0, 8);
889 static int mlx4_ib_query_gid(struct ib_device
*ibdev
, u32 port
, int index
,
892 if (rdma_protocol_ib(ibdev
, port
))
893 return __mlx4_ib_query_gid(ibdev
, port
, index
, gid
, 0);
897 static int mlx4_ib_query_sl2vl(struct ib_device
*ibdev
, u32 port
,
900 union sl2vl_tbl_to_u64 sl2vl64
;
901 struct ib_smp
*in_mad
;
902 struct ib_smp
*out_mad
;
903 int mad_ifc_flags
= MLX4_MAD_IFC_IGNORE_KEYS
;
907 if (mlx4_is_slave(to_mdev(ibdev
)->dev
)) {
912 in_mad
= kzalloc(sizeof(*in_mad
), GFP_KERNEL
);
913 out_mad
= kmalloc(sizeof(*out_mad
), GFP_KERNEL
);
914 if (!in_mad
|| !out_mad
)
917 ib_init_query_mad(in_mad
);
918 in_mad
->attr_id
= IB_SMP_ATTR_SL_TO_VL_TABLE
;
919 in_mad
->attr_mod
= 0;
921 if (mlx4_is_mfunc(to_mdev(ibdev
)->dev
))
922 mad_ifc_flags
|= MLX4_MAD_IFC_NET_VIEW
;
924 err
= mlx4_MAD_IFC(to_mdev(ibdev
), mad_ifc_flags
, port
, NULL
, NULL
,
929 for (jj
= 0; jj
< 8; jj
++)
930 sl2vl64
.sl8
[jj
] = ((struct ib_smp
*)out_mad
)->data
[jj
];
931 *sl2vl_tbl
= sl2vl64
.sl64
;
939 static void mlx4_init_sl2vl_tbl(struct mlx4_ib_dev
*mdev
)
945 for (i
= 1; i
<= mdev
->dev
->caps
.num_ports
; i
++) {
946 if (mdev
->dev
->caps
.port_type
[i
] == MLX4_PORT_TYPE_ETH
)
948 err
= mlx4_ib_query_sl2vl(&mdev
->ib_dev
, i
, &sl2vl
);
950 pr_err("Unable to get default sl to vl mapping for port %d. Using all zeroes (%d)\n",
954 atomic64_set(&mdev
->sl2vl
[i
- 1], sl2vl
);
958 int __mlx4_ib_query_pkey(struct ib_device
*ibdev
, u32 port
, u16 index
,
959 u16
*pkey
, int netw_view
)
961 struct ib_smp
*in_mad
;
962 struct ib_smp
*out_mad
;
963 int mad_ifc_flags
= MLX4_MAD_IFC_IGNORE_KEYS
;
966 in_mad
= kzalloc(sizeof *in_mad
, GFP_KERNEL
);
967 out_mad
= kmalloc(sizeof *out_mad
, GFP_KERNEL
);
968 if (!in_mad
|| !out_mad
)
971 ib_init_query_mad(in_mad
);
972 in_mad
->attr_id
= IB_SMP_ATTR_PKEY_TABLE
;
973 in_mad
->attr_mod
= cpu_to_be32(index
/ 32);
975 if (mlx4_is_mfunc(to_mdev(ibdev
)->dev
) && netw_view
)
976 mad_ifc_flags
|= MLX4_MAD_IFC_NET_VIEW
;
978 err
= mlx4_MAD_IFC(to_mdev(ibdev
), mad_ifc_flags
, port
, NULL
, NULL
,
983 *pkey
= be16_to_cpu(((__be16
*) out_mad
->data
)[index
% 32]);
991 static int mlx4_ib_query_pkey(struct ib_device
*ibdev
, u32 port
, u16 index
,
994 return __mlx4_ib_query_pkey(ibdev
, port
, index
, pkey
, 0);
997 static int mlx4_ib_modify_device(struct ib_device
*ibdev
, int mask
,
998 struct ib_device_modify
*props
)
1000 struct mlx4_cmd_mailbox
*mailbox
;
1001 unsigned long flags
;
1003 if (mask
& ~IB_DEVICE_MODIFY_NODE_DESC
)
1006 if (!(mask
& IB_DEVICE_MODIFY_NODE_DESC
))
1009 if (mlx4_is_slave(to_mdev(ibdev
)->dev
))
1012 spin_lock_irqsave(&to_mdev(ibdev
)->sm_lock
, flags
);
1013 memcpy(ibdev
->node_desc
, props
->node_desc
, IB_DEVICE_NODE_DESC_MAX
);
1014 spin_unlock_irqrestore(&to_mdev(ibdev
)->sm_lock
, flags
);
1017 * If possible, pass node desc to FW, so it can generate
1018 * a 144 trap. If cmd fails, just ignore.
1020 mailbox
= mlx4_alloc_cmd_mailbox(to_mdev(ibdev
)->dev
);
1021 if (IS_ERR(mailbox
))
1024 memcpy(mailbox
->buf
, props
->node_desc
, IB_DEVICE_NODE_DESC_MAX
);
1025 mlx4_cmd(to_mdev(ibdev
)->dev
, mailbox
->dma
, 1, 0,
1026 MLX4_CMD_SET_NODE
, MLX4_CMD_TIME_CLASS_A
, MLX4_CMD_NATIVE
);
1028 mlx4_free_cmd_mailbox(to_mdev(ibdev
)->dev
, mailbox
);
1033 static int mlx4_ib_SET_PORT(struct mlx4_ib_dev
*dev
, u32 port
,
1034 int reset_qkey_viols
, u32 cap_mask
)
1036 struct mlx4_cmd_mailbox
*mailbox
;
1039 mailbox
= mlx4_alloc_cmd_mailbox(dev
->dev
);
1040 if (IS_ERR(mailbox
))
1041 return PTR_ERR(mailbox
);
1043 if (dev
->dev
->flags
& MLX4_FLAG_OLD_PORT_CMDS
) {
1044 *(u8
*) mailbox
->buf
= !!reset_qkey_viols
<< 6;
1045 ((__be32
*) mailbox
->buf
)[2] = cpu_to_be32(cap_mask
);
1047 ((u8
*) mailbox
->buf
)[3] = !!reset_qkey_viols
;
1048 ((__be32
*) mailbox
->buf
)[1] = cpu_to_be32(cap_mask
);
1051 err
= mlx4_cmd(dev
->dev
, mailbox
->dma
, port
, MLX4_SET_PORT_IB_OPCODE
,
1052 MLX4_CMD_SET_PORT
, MLX4_CMD_TIME_CLASS_B
,
1055 mlx4_free_cmd_mailbox(dev
->dev
, mailbox
);
1059 static int mlx4_ib_modify_port(struct ib_device
*ibdev
, u32 port
, int mask
,
1060 struct ib_port_modify
*props
)
1062 struct mlx4_ib_dev
*mdev
= to_mdev(ibdev
);
1063 u8 is_eth
= mdev
->dev
->caps
.port_type
[port
] == MLX4_PORT_TYPE_ETH
;
1064 struct ib_port_attr attr
;
1068 /* return OK if this is RoCE. CM calls ib_modify_port() regardless
1069 * of whether port link layer is ETH or IB. For ETH ports, qkey
1070 * violations and port capabilities are not meaningful.
1075 mutex_lock(&mdev
->cap_mask_mutex
);
1077 err
= ib_query_port(ibdev
, port
, &attr
);
1081 cap_mask
= (attr
.port_cap_flags
| props
->set_port_cap_mask
) &
1082 ~props
->clr_port_cap_mask
;
1084 err
= mlx4_ib_SET_PORT(mdev
, port
,
1085 !!(mask
& IB_PORT_RESET_QKEY_CNTR
),
1089 mutex_unlock(&to_mdev(ibdev
)->cap_mask_mutex
);
1093 static int mlx4_ib_alloc_ucontext(struct ib_ucontext
*uctx
,
1094 struct ib_udata
*udata
)
1096 struct ib_device
*ibdev
= uctx
->device
;
1097 struct mlx4_ib_dev
*dev
= to_mdev(ibdev
);
1098 struct mlx4_ib_ucontext
*context
= to_mucontext(uctx
);
1099 struct mlx4_ib_alloc_ucontext_resp_v3 resp_v3
;
1100 struct mlx4_ib_alloc_ucontext_resp resp
;
1103 if (!dev
->ib_active
)
1106 if (ibdev
->ops
.uverbs_abi_ver
==
1107 MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION
) {
1108 resp_v3
.qp_tab_size
= dev
->dev
->caps
.num_qps
;
1109 resp_v3
.bf_reg_size
= dev
->dev
->caps
.bf_reg_size
;
1110 resp_v3
.bf_regs_per_page
= dev
->dev
->caps
.bf_regs_per_page
;
1112 resp
.dev_caps
= dev
->dev
->caps
.userspace_caps
;
1113 resp
.qp_tab_size
= dev
->dev
->caps
.num_qps
;
1114 resp
.bf_reg_size
= dev
->dev
->caps
.bf_reg_size
;
1115 resp
.bf_regs_per_page
= dev
->dev
->caps
.bf_regs_per_page
;
1116 resp
.cqe_size
= dev
->dev
->caps
.cqe_size
;
1119 err
= mlx4_uar_alloc(to_mdev(ibdev
)->dev
, &context
->uar
);
1123 INIT_LIST_HEAD(&context
->db_page_list
);
1124 mutex_init(&context
->db_page_mutex
);
1126 INIT_LIST_HEAD(&context
->wqn_ranges_list
);
1127 mutex_init(&context
->wqn_ranges_mutex
);
1129 if (ibdev
->ops
.uverbs_abi_ver
== MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION
)
1130 err
= ib_copy_to_udata(udata
, &resp_v3
, sizeof(resp_v3
));
1132 err
= ib_copy_to_udata(udata
, &resp
, sizeof(resp
));
1135 mlx4_uar_free(to_mdev(ibdev
)->dev
, &context
->uar
);
1142 static void mlx4_ib_dealloc_ucontext(struct ib_ucontext
*ibcontext
)
1144 struct mlx4_ib_ucontext
*context
= to_mucontext(ibcontext
);
1146 mlx4_uar_free(to_mdev(ibcontext
->device
)->dev
, &context
->uar
);
1149 static void mlx4_ib_disassociate_ucontext(struct ib_ucontext
*ibcontext
)
1153 static int mlx4_ib_mmap(struct ib_ucontext
*context
, struct vm_area_struct
*vma
)
1155 struct mlx4_ib_dev
*dev
= to_mdev(context
->device
);
1157 switch (vma
->vm_pgoff
) {
1159 return rdma_user_mmap_io(context
, vma
,
1160 to_mucontext(context
)->uar
.pfn
,
1162 pgprot_noncached(vma
->vm_page_prot
),
1166 if (dev
->dev
->caps
.bf_reg_size
== 0)
1168 return rdma_user_mmap_io(
1170 to_mucontext(context
)->uar
.pfn
+
1171 dev
->dev
->caps
.num_uars
,
1172 PAGE_SIZE
, pgprot_writecombine(vma
->vm_page_prot
),
1176 struct mlx4_clock_params params
;
1179 ret
= mlx4_get_internal_clock_params(dev
->dev
, ¶ms
);
1183 return rdma_user_mmap_io(
1185 (pci_resource_start(dev
->dev
->persist
->pdev
,
1189 PAGE_SIZE
, pgprot_noncached(vma
->vm_page_prot
),
1198 static int mlx4_ib_alloc_pd(struct ib_pd
*ibpd
, struct ib_udata
*udata
)
1200 struct mlx4_ib_pd
*pd
= to_mpd(ibpd
);
1201 struct ib_device
*ibdev
= ibpd
->device
;
1204 err
= mlx4_pd_alloc(to_mdev(ibdev
)->dev
, &pd
->pdn
);
1208 if (udata
&& ib_copy_to_udata(udata
, &pd
->pdn
, sizeof(__u32
))) {
1209 mlx4_pd_free(to_mdev(ibdev
)->dev
, pd
->pdn
);
1215 static int mlx4_ib_dealloc_pd(struct ib_pd
*pd
, struct ib_udata
*udata
)
1217 mlx4_pd_free(to_mdev(pd
->device
)->dev
, to_mpd(pd
)->pdn
);
1221 static int mlx4_ib_alloc_xrcd(struct ib_xrcd
*ibxrcd
, struct ib_udata
*udata
)
1223 struct mlx4_ib_dev
*dev
= to_mdev(ibxrcd
->device
);
1224 struct mlx4_ib_xrcd
*xrcd
= to_mxrcd(ibxrcd
);
1225 struct ib_cq_init_attr cq_attr
= {};
1228 if (!(dev
->dev
->caps
.flags
& MLX4_DEV_CAP_FLAG_XRC
))
1231 err
= mlx4_xrcd_alloc(dev
->dev
, &xrcd
->xrcdn
);
1235 xrcd
->pd
= ib_alloc_pd(ibxrcd
->device
, 0);
1236 if (IS_ERR(xrcd
->pd
)) {
1237 err
= PTR_ERR(xrcd
->pd
);
1242 xrcd
->cq
= ib_create_cq(ibxrcd
->device
, NULL
, NULL
, xrcd
, &cq_attr
);
1243 if (IS_ERR(xrcd
->cq
)) {
1244 err
= PTR_ERR(xrcd
->cq
);
1251 ib_dealloc_pd(xrcd
->pd
);
1253 mlx4_xrcd_free(dev
->dev
, xrcd
->xrcdn
);
1257 static int mlx4_ib_dealloc_xrcd(struct ib_xrcd
*xrcd
, struct ib_udata
*udata
)
1259 ib_destroy_cq(to_mxrcd(xrcd
)->cq
);
1260 ib_dealloc_pd(to_mxrcd(xrcd
)->pd
);
1261 mlx4_xrcd_free(to_mdev(xrcd
->device
)->dev
, to_mxrcd(xrcd
)->xrcdn
);
1265 static int add_gid_entry(struct ib_qp
*ibqp
, union ib_gid
*gid
)
1267 struct mlx4_ib_qp
*mqp
= to_mqp(ibqp
);
1268 struct mlx4_ib_dev
*mdev
= to_mdev(ibqp
->device
);
1269 struct mlx4_ib_gid_entry
*ge
;
1271 ge
= kzalloc(sizeof *ge
, GFP_KERNEL
);
1276 if (mlx4_ib_add_mc(mdev
, mqp
, gid
)) {
1277 ge
->port
= mqp
->port
;
1281 mutex_lock(&mqp
->mutex
);
1282 list_add_tail(&ge
->list
, &mqp
->gid_list
);
1283 mutex_unlock(&mqp
->mutex
);
1288 static void mlx4_ib_delete_counters_table(struct mlx4_ib_dev
*ibdev
,
1289 struct mlx4_ib_counters
*ctr_table
)
1291 struct counter_index
*counter
, *tmp_count
;
1293 mutex_lock(&ctr_table
->mutex
);
1294 list_for_each_entry_safe(counter
, tmp_count
, &ctr_table
->counters_list
,
1296 if (counter
->allocated
)
1297 mlx4_counter_free(ibdev
->dev
, counter
->index
);
1298 list_del(&counter
->list
);
1301 mutex_unlock(&ctr_table
->mutex
);
1304 int mlx4_ib_add_mc(struct mlx4_ib_dev
*mdev
, struct mlx4_ib_qp
*mqp
,
1307 struct net_device
*ndev
;
1313 spin_lock_bh(&mdev
->iboe
.lock
);
1314 ndev
= mdev
->iboe
.netdevs
[mqp
->port
- 1];
1316 spin_unlock_bh(&mdev
->iboe
.lock
);
1326 struct mlx4_ib_steering
{
1327 struct list_head list
;
1328 struct mlx4_flow_reg_id reg_id
;
1332 #define LAST_ETH_FIELD vlan_tag
1333 #define LAST_IB_FIELD sl
1334 #define LAST_IPV4_FIELD dst_ip
1335 #define LAST_TCP_UDP_FIELD src_port
1337 /* Field is the last supported field */
1338 #define FIELDS_NOT_SUPPORTED(filter, field)\
1339 memchr_inv((void *)&filter.field +\
1340 sizeof(filter.field), 0,\
1342 offsetof(typeof(filter), field) -\
1343 sizeof(filter.field))
1345 static int parse_flow_attr(struct mlx4_dev
*dev
,
1347 union ib_flow_spec
*ib_spec
,
1348 struct _rule_hw
*mlx4_spec
)
1350 enum mlx4_net_trans_rule_id type
;
1352 switch (ib_spec
->type
) {
1353 case IB_FLOW_SPEC_ETH
:
1354 if (FIELDS_NOT_SUPPORTED(ib_spec
->eth
.mask
, LAST_ETH_FIELD
))
1357 type
= MLX4_NET_TRANS_RULE_ID_ETH
;
1358 memcpy(mlx4_spec
->eth
.dst_mac
, ib_spec
->eth
.val
.dst_mac
,
1360 memcpy(mlx4_spec
->eth
.dst_mac_msk
, ib_spec
->eth
.mask
.dst_mac
,
1362 mlx4_spec
->eth
.vlan_tag
= ib_spec
->eth
.val
.vlan_tag
;
1363 mlx4_spec
->eth
.vlan_tag_msk
= ib_spec
->eth
.mask
.vlan_tag
;
1365 case IB_FLOW_SPEC_IB
:
1366 if (FIELDS_NOT_SUPPORTED(ib_spec
->ib
.mask
, LAST_IB_FIELD
))
1369 type
= MLX4_NET_TRANS_RULE_ID_IB
;
1370 mlx4_spec
->ib
.l3_qpn
=
1371 cpu_to_be32(qp_num
);
1372 mlx4_spec
->ib
.qpn_mask
=
1373 cpu_to_be32(MLX4_IB_FLOW_QPN_MASK
);
1377 case IB_FLOW_SPEC_IPV4
:
1378 if (FIELDS_NOT_SUPPORTED(ib_spec
->ipv4
.mask
, LAST_IPV4_FIELD
))
1381 type
= MLX4_NET_TRANS_RULE_ID_IPV4
;
1382 mlx4_spec
->ipv4
.src_ip
= ib_spec
->ipv4
.val
.src_ip
;
1383 mlx4_spec
->ipv4
.src_ip_msk
= ib_spec
->ipv4
.mask
.src_ip
;
1384 mlx4_spec
->ipv4
.dst_ip
= ib_spec
->ipv4
.val
.dst_ip
;
1385 mlx4_spec
->ipv4
.dst_ip_msk
= ib_spec
->ipv4
.mask
.dst_ip
;
1388 case IB_FLOW_SPEC_TCP
:
1389 case IB_FLOW_SPEC_UDP
:
1390 if (FIELDS_NOT_SUPPORTED(ib_spec
->tcp_udp
.mask
, LAST_TCP_UDP_FIELD
))
1393 type
= ib_spec
->type
== IB_FLOW_SPEC_TCP
?
1394 MLX4_NET_TRANS_RULE_ID_TCP
:
1395 MLX4_NET_TRANS_RULE_ID_UDP
;
1396 mlx4_spec
->tcp_udp
.dst_port
= ib_spec
->tcp_udp
.val
.dst_port
;
1397 mlx4_spec
->tcp_udp
.dst_port_msk
= ib_spec
->tcp_udp
.mask
.dst_port
;
1398 mlx4_spec
->tcp_udp
.src_port
= ib_spec
->tcp_udp
.val
.src_port
;
1399 mlx4_spec
->tcp_udp
.src_port_msk
= ib_spec
->tcp_udp
.mask
.src_port
;
1405 if (mlx4_map_sw_to_hw_steering_id(dev
, type
) < 0 ||
1406 mlx4_hw_rule_sz(dev
, type
) < 0)
1408 mlx4_spec
->id
= cpu_to_be16(mlx4_map_sw_to_hw_steering_id(dev
, type
));
1409 mlx4_spec
->size
= mlx4_hw_rule_sz(dev
, type
) >> 2;
1410 return mlx4_hw_rule_sz(dev
, type
);
1413 struct default_rules
{
1414 __u32 mandatory_fields
[IB_FLOW_SPEC_SUPPORT_LAYERS
];
1415 __u32 mandatory_not_fields
[IB_FLOW_SPEC_SUPPORT_LAYERS
];
1416 __u32 rules_create_list
[IB_FLOW_SPEC_SUPPORT_LAYERS
];
1419 static const struct default_rules default_table
[] = {
1421 .mandatory_fields
= {IB_FLOW_SPEC_IPV4
},
1422 .mandatory_not_fields
= {IB_FLOW_SPEC_ETH
},
1423 .rules_create_list
= {IB_FLOW_SPEC_IB
},
1424 .link_layer
= IB_LINK_LAYER_INFINIBAND
1428 static int __mlx4_ib_default_rules_match(struct ib_qp
*qp
,
1429 struct ib_flow_attr
*flow_attr
)
1433 const struct default_rules
*pdefault_rules
= default_table
;
1434 u8 link_layer
= rdma_port_get_link_layer(qp
->device
, flow_attr
->port
);
1436 for (i
= 0; i
< ARRAY_SIZE(default_table
); i
++, pdefault_rules
++) {
1437 __u32 field_types
[IB_FLOW_SPEC_SUPPORT_LAYERS
];
1438 memset(&field_types
, 0, sizeof(field_types
));
1440 if (link_layer
!= pdefault_rules
->link_layer
)
1443 ib_flow
= flow_attr
+ 1;
1444 /* we assume the specs are sorted */
1445 for (j
= 0, k
= 0; k
< IB_FLOW_SPEC_SUPPORT_LAYERS
&&
1446 j
< flow_attr
->num_of_specs
; k
++) {
1447 union ib_flow_spec
*current_flow
=
1448 (union ib_flow_spec
*)ib_flow
;
1450 /* same layer but different type */
1451 if (((current_flow
->type
& IB_FLOW_SPEC_LAYER_MASK
) ==
1452 (pdefault_rules
->mandatory_fields
[k
] &
1453 IB_FLOW_SPEC_LAYER_MASK
)) &&
1454 (current_flow
->type
!=
1455 pdefault_rules
->mandatory_fields
[k
]))
1458 /* same layer, try match next one */
1459 if (current_flow
->type
==
1460 pdefault_rules
->mandatory_fields
[k
]) {
1463 ((union ib_flow_spec
*)ib_flow
)->size
;
1467 ib_flow
= flow_attr
+ 1;
1468 for (j
= 0; j
< flow_attr
->num_of_specs
;
1469 j
++, ib_flow
+= ((union ib_flow_spec
*)ib_flow
)->size
)
1470 for (k
= 0; k
< IB_FLOW_SPEC_SUPPORT_LAYERS
; k
++)
1471 /* same layer and same type */
1472 if (((union ib_flow_spec
*)ib_flow
)->type
==
1473 pdefault_rules
->mandatory_not_fields
[k
])
1482 static int __mlx4_ib_create_default_rules(
1483 struct mlx4_ib_dev
*mdev
,
1485 const struct default_rules
*pdefault_rules
,
1486 struct _rule_hw
*mlx4_spec
) {
1490 for (i
= 0; i
< ARRAY_SIZE(pdefault_rules
->rules_create_list
); i
++) {
1491 union ib_flow_spec ib_spec
= {};
1494 switch (pdefault_rules
->rules_create_list
[i
]) {
1498 case IB_FLOW_SPEC_IB
:
1499 ib_spec
.type
= IB_FLOW_SPEC_IB
;
1500 ib_spec
.size
= sizeof(struct ib_flow_spec_ib
);
1507 /* We must put empty rule, qpn is being ignored */
1508 ret
= parse_flow_attr(mdev
->dev
, 0, &ib_spec
,
1511 pr_info("invalid parsing\n");
1515 mlx4_spec
= (void *)mlx4_spec
+ ret
;
1521 static int __mlx4_ib_create_flow(struct ib_qp
*qp
, struct ib_flow_attr
*flow_attr
,
1523 enum mlx4_net_trans_promisc_mode flow_type
,
1529 struct mlx4_ib_dev
*mdev
= to_mdev(qp
->device
);
1530 struct mlx4_cmd_mailbox
*mailbox
;
1531 struct mlx4_net_trans_rule_hw_ctrl
*ctrl
;
1534 if (flow_attr
->priority
> MLX4_IB_FLOW_MAX_PRIO
) {
1535 pr_err("Invalid priority value %d\n", flow_attr
->priority
);
1539 if (mlx4_map_sw_to_hw_steering_mode(mdev
->dev
, flow_type
) < 0)
1542 mailbox
= mlx4_alloc_cmd_mailbox(mdev
->dev
);
1543 if (IS_ERR(mailbox
))
1544 return PTR_ERR(mailbox
);
1545 ctrl
= mailbox
->buf
;
1547 ctrl
->prio
= cpu_to_be16(domain
| flow_attr
->priority
);
1548 ctrl
->type
= mlx4_map_sw_to_hw_steering_mode(mdev
->dev
, flow_type
);
1549 ctrl
->port
= flow_attr
->port
;
1550 ctrl
->qpn
= cpu_to_be32(qp
->qp_num
);
1552 ib_flow
= flow_attr
+ 1;
1553 size
+= sizeof(struct mlx4_net_trans_rule_hw_ctrl
);
1554 /* Add default flows */
1555 default_flow
= __mlx4_ib_default_rules_match(qp
, flow_attr
);
1556 if (default_flow
>= 0) {
1557 ret
= __mlx4_ib_create_default_rules(
1558 mdev
, qp
, default_table
+ default_flow
,
1559 mailbox
->buf
+ size
);
1561 mlx4_free_cmd_mailbox(mdev
->dev
, mailbox
);
1566 for (i
= 0; i
< flow_attr
->num_of_specs
; i
++) {
1567 ret
= parse_flow_attr(mdev
->dev
, qp
->qp_num
, ib_flow
,
1568 mailbox
->buf
+ size
);
1570 mlx4_free_cmd_mailbox(mdev
->dev
, mailbox
);
1573 ib_flow
+= ((union ib_flow_spec
*) ib_flow
)->size
;
1577 if (mlx4_is_master(mdev
->dev
) && flow_type
== MLX4_FS_REGULAR
&&
1578 flow_attr
->num_of_specs
== 1) {
1579 struct _rule_hw
*rule_header
= (struct _rule_hw
*)(ctrl
+ 1);
1580 enum ib_flow_spec_type header_spec
=
1581 ((union ib_flow_spec
*)(flow_attr
+ 1))->type
;
1583 if (header_spec
== IB_FLOW_SPEC_ETH
)
1584 mlx4_handle_eth_header_mcast_prio(ctrl
, rule_header
);
1587 ret
= mlx4_cmd_imm(mdev
->dev
, mailbox
->dma
, reg_id
, size
>> 2, 0,
1588 MLX4_QP_FLOW_STEERING_ATTACH
, MLX4_CMD_TIME_CLASS_A
,
1591 pr_err("mcg table is full. Fail to register network rule.\n");
1592 else if (ret
== -ENXIO
)
1593 pr_err("Device managed flow steering is disabled. Fail to register network rule.\n");
1595 pr_err("Invalid argument. Fail to register network rule.\n");
1597 mlx4_free_cmd_mailbox(mdev
->dev
, mailbox
);
1601 static int __mlx4_ib_destroy_flow(struct mlx4_dev
*dev
, u64 reg_id
)
1604 err
= mlx4_cmd(dev
, reg_id
, 0, 0,
1605 MLX4_QP_FLOW_STEERING_DETACH
, MLX4_CMD_TIME_CLASS_A
,
1608 pr_err("Fail to detach network rule. registration id = 0x%llx\n",
1613 static int mlx4_ib_tunnel_steer_add(struct ib_qp
*qp
, struct ib_flow_attr
*flow_attr
,
1617 union ib_flow_spec
*ib_spec
;
1618 struct mlx4_dev
*dev
= to_mdev(qp
->device
)->dev
;
1621 if (dev
->caps
.tunnel_offload_mode
!= MLX4_TUNNEL_OFFLOAD_MODE_VXLAN
||
1622 dev
->caps
.dmfs_high_steer_mode
== MLX4_STEERING_DMFS_A0_STATIC
)
1623 return 0; /* do nothing */
1625 ib_flow
= flow_attr
+ 1;
1626 ib_spec
= (union ib_flow_spec
*)ib_flow
;
1628 if (ib_spec
->type
!= IB_FLOW_SPEC_ETH
|| flow_attr
->num_of_specs
!= 1)
1629 return 0; /* do nothing */
1631 err
= mlx4_tunnel_steer_add(to_mdev(qp
->device
)->dev
, ib_spec
->eth
.val
.dst_mac
,
1632 flow_attr
->port
, qp
->qp_num
,
1633 MLX4_DOMAIN_UVERBS
| (flow_attr
->priority
& 0xff),
1638 static int mlx4_ib_add_dont_trap_rule(struct mlx4_dev
*dev
,
1639 struct ib_flow_attr
*flow_attr
,
1640 enum mlx4_net_trans_promisc_mode
*type
)
1644 if (!(dev
->caps
.flags2
& MLX4_DEV_CAP_FLAG2_DMFS_UC_MC_SNIFFER
) ||
1645 (dev
->caps
.dmfs_high_steer_mode
== MLX4_STEERING_DMFS_A0_STATIC
) ||
1646 (flow_attr
->num_of_specs
> 1) || (flow_attr
->priority
!= 0)) {
1650 if (flow_attr
->num_of_specs
== 0) {
1651 type
[0] = MLX4_FS_MC_SNIFFER
;
1652 type
[1] = MLX4_FS_UC_SNIFFER
;
1654 union ib_flow_spec
*ib_spec
;
1656 ib_spec
= (union ib_flow_spec
*)(flow_attr
+ 1);
1657 if (ib_spec
->type
!= IB_FLOW_SPEC_ETH
)
1660 /* if all is zero than MC and UC */
1661 if (is_zero_ether_addr(ib_spec
->eth
.mask
.dst_mac
)) {
1662 type
[0] = MLX4_FS_MC_SNIFFER
;
1663 type
[1] = MLX4_FS_UC_SNIFFER
;
1665 u8 mac
[ETH_ALEN
] = {ib_spec
->eth
.mask
.dst_mac
[0] ^ 0x01,
1666 ib_spec
->eth
.mask
.dst_mac
[1],
1667 ib_spec
->eth
.mask
.dst_mac
[2],
1668 ib_spec
->eth
.mask
.dst_mac
[3],
1669 ib_spec
->eth
.mask
.dst_mac
[4],
1670 ib_spec
->eth
.mask
.dst_mac
[5]};
1672 /* Above xor was only on MC bit, non empty mask is valid
1673 * only if this bit is set and rest are zero.
1675 if (!is_zero_ether_addr(&mac
[0]))
1678 if (is_multicast_ether_addr(ib_spec
->eth
.val
.dst_mac
))
1679 type
[0] = MLX4_FS_MC_SNIFFER
;
1681 type
[0] = MLX4_FS_UC_SNIFFER
;
1688 static struct ib_flow
*mlx4_ib_create_flow(struct ib_qp
*qp
,
1689 struct ib_flow_attr
*flow_attr
,
1690 struct ib_udata
*udata
)
1692 int err
= 0, i
= 0, j
= 0;
1693 struct mlx4_ib_flow
*mflow
;
1694 enum mlx4_net_trans_promisc_mode type
[2];
1695 struct mlx4_dev
*dev
= (to_mdev(qp
->device
))->dev
;
1696 int is_bonded
= mlx4_is_bonded(dev
);
1698 if (flow_attr
->flags
& ~IB_FLOW_ATTR_FLAGS_DONT_TRAP
)
1699 return ERR_PTR(-EOPNOTSUPP
);
1701 if ((flow_attr
->flags
& IB_FLOW_ATTR_FLAGS_DONT_TRAP
) &&
1702 (flow_attr
->type
!= IB_FLOW_ATTR_NORMAL
))
1703 return ERR_PTR(-EOPNOTSUPP
);
1706 udata
->inlen
&& !ib_is_udata_cleared(udata
, 0, udata
->inlen
))
1707 return ERR_PTR(-EOPNOTSUPP
);
1709 memset(type
, 0, sizeof(type
));
1711 mflow
= kzalloc(sizeof(*mflow
), GFP_KERNEL
);
1717 switch (flow_attr
->type
) {
1718 case IB_FLOW_ATTR_NORMAL
:
1719 /* If dont trap flag (continue match) is set, under specific
1720 * condition traffic be replicated to given qp,
1721 * without stealing it
1723 if (unlikely(flow_attr
->flags
& IB_FLOW_ATTR_FLAGS_DONT_TRAP
)) {
1724 err
= mlx4_ib_add_dont_trap_rule(dev
,
1730 type
[0] = MLX4_FS_REGULAR
;
1734 case IB_FLOW_ATTR_ALL_DEFAULT
:
1735 type
[0] = MLX4_FS_ALL_DEFAULT
;
1738 case IB_FLOW_ATTR_MC_DEFAULT
:
1739 type
[0] = MLX4_FS_MC_DEFAULT
;
1742 case IB_FLOW_ATTR_SNIFFER
:
1743 type
[0] = MLX4_FS_MIRROR_RX_PORT
;
1744 type
[1] = MLX4_FS_MIRROR_SX_PORT
;
1752 while (i
< ARRAY_SIZE(type
) && type
[i
]) {
1753 err
= __mlx4_ib_create_flow(qp
, flow_attr
, MLX4_DOMAIN_UVERBS
,
1754 type
[i
], &mflow
->reg_id
[i
].id
);
1756 goto err_create_flow
;
1758 /* Application always sees one port so the mirror rule
1759 * must be on port #2
1761 flow_attr
->port
= 2;
1762 err
= __mlx4_ib_create_flow(qp
, flow_attr
,
1763 MLX4_DOMAIN_UVERBS
, type
[j
],
1764 &mflow
->reg_id
[j
].mirror
);
1765 flow_attr
->port
= 1;
1767 goto err_create_flow
;
1774 if (i
< ARRAY_SIZE(type
) && flow_attr
->type
== IB_FLOW_ATTR_NORMAL
) {
1775 err
= mlx4_ib_tunnel_steer_add(qp
, flow_attr
,
1776 &mflow
->reg_id
[i
].id
);
1778 goto err_create_flow
;
1781 flow_attr
->port
= 2;
1782 err
= mlx4_ib_tunnel_steer_add(qp
, flow_attr
,
1783 &mflow
->reg_id
[j
].mirror
);
1784 flow_attr
->port
= 1;
1786 goto err_create_flow
;
1789 /* function to create mirror rule */
1793 return &mflow
->ibflow
;
1797 (void)__mlx4_ib_destroy_flow(to_mdev(qp
->device
)->dev
,
1798 mflow
->reg_id
[i
].id
);
1803 (void)__mlx4_ib_destroy_flow(to_mdev(qp
->device
)->dev
,
1804 mflow
->reg_id
[j
].mirror
);
1809 return ERR_PTR(err
);
1812 static int mlx4_ib_destroy_flow(struct ib_flow
*flow_id
)
1816 struct mlx4_ib_dev
*mdev
= to_mdev(flow_id
->qp
->device
);
1817 struct mlx4_ib_flow
*mflow
= to_mflow(flow_id
);
1819 while (i
< ARRAY_SIZE(mflow
->reg_id
) && mflow
->reg_id
[i
].id
) {
1820 err
= __mlx4_ib_destroy_flow(mdev
->dev
, mflow
->reg_id
[i
].id
);
1823 if (mflow
->reg_id
[i
].mirror
) {
1824 err
= __mlx4_ib_destroy_flow(mdev
->dev
,
1825 mflow
->reg_id
[i
].mirror
);
1836 static int mlx4_ib_mcg_attach(struct ib_qp
*ibqp
, union ib_gid
*gid
, u16 lid
)
1839 struct mlx4_ib_dev
*mdev
= to_mdev(ibqp
->device
);
1840 struct mlx4_dev
*dev
= mdev
->dev
;
1841 struct mlx4_ib_qp
*mqp
= to_mqp(ibqp
);
1842 struct mlx4_ib_steering
*ib_steering
= NULL
;
1843 enum mlx4_protocol prot
= MLX4_PROT_IB_IPV6
;
1844 struct mlx4_flow_reg_id reg_id
;
1846 if (mdev
->dev
->caps
.steering_mode
==
1847 MLX4_STEERING_MODE_DEVICE_MANAGED
) {
1848 ib_steering
= kmalloc(sizeof(*ib_steering
), GFP_KERNEL
);
1853 err
= mlx4_multicast_attach(mdev
->dev
, &mqp
->mqp
, gid
->raw
, mqp
->port
,
1855 MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK
),
1858 pr_err("multicast attach op failed, err %d\n", err
);
1863 if (mlx4_is_bonded(dev
)) {
1864 err
= mlx4_multicast_attach(mdev
->dev
, &mqp
->mqp
, gid
->raw
,
1865 (mqp
->port
== 1) ? 2 : 1,
1867 MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK
),
1868 prot
, ®_id
.mirror
);
1873 err
= add_gid_entry(ibqp
, gid
);
1878 memcpy(ib_steering
->gid
.raw
, gid
->raw
, 16);
1879 ib_steering
->reg_id
= reg_id
;
1880 mutex_lock(&mqp
->mutex
);
1881 list_add(&ib_steering
->list
, &mqp
->steering_rules
);
1882 mutex_unlock(&mqp
->mutex
);
1887 mlx4_multicast_detach(mdev
->dev
, &mqp
->mqp
, gid
->raw
,
1890 mlx4_multicast_detach(mdev
->dev
, &mqp
->mqp
, gid
->raw
,
1891 prot
, reg_id
.mirror
);
1898 static struct mlx4_ib_gid_entry
*find_gid_entry(struct mlx4_ib_qp
*qp
, u8
*raw
)
1900 struct mlx4_ib_gid_entry
*ge
;
1901 struct mlx4_ib_gid_entry
*tmp
;
1902 struct mlx4_ib_gid_entry
*ret
= NULL
;
1904 list_for_each_entry_safe(ge
, tmp
, &qp
->gid_list
, list
) {
1905 if (!memcmp(raw
, ge
->gid
.raw
, 16)) {
1914 static int mlx4_ib_mcg_detach(struct ib_qp
*ibqp
, union ib_gid
*gid
, u16 lid
)
1917 struct mlx4_ib_dev
*mdev
= to_mdev(ibqp
->device
);
1918 struct mlx4_dev
*dev
= mdev
->dev
;
1919 struct mlx4_ib_qp
*mqp
= to_mqp(ibqp
);
1920 struct net_device
*ndev
;
1921 struct mlx4_ib_gid_entry
*ge
;
1922 struct mlx4_flow_reg_id reg_id
= {0, 0};
1923 enum mlx4_protocol prot
= MLX4_PROT_IB_IPV6
;
1925 if (mdev
->dev
->caps
.steering_mode
==
1926 MLX4_STEERING_MODE_DEVICE_MANAGED
) {
1927 struct mlx4_ib_steering
*ib_steering
;
1929 mutex_lock(&mqp
->mutex
);
1930 list_for_each_entry(ib_steering
, &mqp
->steering_rules
, list
) {
1931 if (!memcmp(ib_steering
->gid
.raw
, gid
->raw
, 16)) {
1932 list_del(&ib_steering
->list
);
1936 mutex_unlock(&mqp
->mutex
);
1937 if (&ib_steering
->list
== &mqp
->steering_rules
) {
1938 pr_err("Couldn't find reg_id for mgid. Steering rule is left attached\n");
1941 reg_id
= ib_steering
->reg_id
;
1945 err
= mlx4_multicast_detach(mdev
->dev
, &mqp
->mqp
, gid
->raw
,
1950 if (mlx4_is_bonded(dev
)) {
1951 err
= mlx4_multicast_detach(mdev
->dev
, &mqp
->mqp
, gid
->raw
,
1952 prot
, reg_id
.mirror
);
1957 mutex_lock(&mqp
->mutex
);
1958 ge
= find_gid_entry(mqp
, gid
->raw
);
1960 spin_lock_bh(&mdev
->iboe
.lock
);
1961 ndev
= ge
->added
? mdev
->iboe
.netdevs
[ge
->port
- 1] : NULL
;
1963 spin_unlock_bh(&mdev
->iboe
.lock
);
1965 list_del(&ge
->list
);
1968 pr_warn("could not find mgid entry\n");
1970 mutex_unlock(&mqp
->mutex
);
1975 static int init_node_data(struct mlx4_ib_dev
*dev
)
1977 struct ib_smp
*in_mad
;
1978 struct ib_smp
*out_mad
;
1979 int mad_ifc_flags
= MLX4_MAD_IFC_IGNORE_KEYS
;
1982 in_mad
= kzalloc(sizeof *in_mad
, GFP_KERNEL
);
1983 out_mad
= kmalloc(sizeof *out_mad
, GFP_KERNEL
);
1984 if (!in_mad
|| !out_mad
)
1987 ib_init_query_mad(in_mad
);
1988 in_mad
->attr_id
= IB_SMP_ATTR_NODE_DESC
;
1989 if (mlx4_is_master(dev
->dev
))
1990 mad_ifc_flags
|= MLX4_MAD_IFC_NET_VIEW
;
1992 err
= mlx4_MAD_IFC(dev
, mad_ifc_flags
, 1, NULL
, NULL
, in_mad
, out_mad
);
1996 memcpy(dev
->ib_dev
.node_desc
, out_mad
->data
, IB_DEVICE_NODE_DESC_MAX
);
1998 in_mad
->attr_id
= IB_SMP_ATTR_NODE_INFO
;
2000 err
= mlx4_MAD_IFC(dev
, mad_ifc_flags
, 1, NULL
, NULL
, in_mad
, out_mad
);
2004 dev
->dev
->rev_id
= be32_to_cpup((__be32
*) (out_mad
->data
+ 32));
2005 memcpy(&dev
->ib_dev
.node_guid
, out_mad
->data
+ 12, 8);
2013 static ssize_t
hca_type_show(struct device
*device
,
2014 struct device_attribute
*attr
, char *buf
)
2016 struct mlx4_ib_dev
*dev
=
2017 rdma_device_to_drv_device(device
, struct mlx4_ib_dev
, ib_dev
);
2019 return sysfs_emit(buf
, "MT%d\n", dev
->dev
->persist
->pdev
->device
);
2021 static DEVICE_ATTR_RO(hca_type
);
2023 static ssize_t
hw_rev_show(struct device
*device
,
2024 struct device_attribute
*attr
, char *buf
)
2026 struct mlx4_ib_dev
*dev
=
2027 rdma_device_to_drv_device(device
, struct mlx4_ib_dev
, ib_dev
);
2029 return sysfs_emit(buf
, "%x\n", dev
->dev
->rev_id
);
2031 static DEVICE_ATTR_RO(hw_rev
);
2033 static ssize_t
board_id_show(struct device
*device
,
2034 struct device_attribute
*attr
, char *buf
)
2036 struct mlx4_ib_dev
*dev
=
2037 rdma_device_to_drv_device(device
, struct mlx4_ib_dev
, ib_dev
);
2039 return sysfs_emit(buf
, "%.*s\n", MLX4_BOARD_ID_LEN
, dev
->dev
->board_id
);
2041 static DEVICE_ATTR_RO(board_id
);
2043 static struct attribute
*mlx4_class_attributes
[] = {
2044 &dev_attr_hw_rev
.attr
,
2045 &dev_attr_hca_type
.attr
,
2046 &dev_attr_board_id
.attr
,
2050 static const struct attribute_group mlx4_attr_group
= {
2051 .attrs
= mlx4_class_attributes
,
2054 struct diag_counter
{
2059 #define DIAG_COUNTER(_name, _offset) \
2060 { .name = #_name, .offset = _offset }
2062 static const struct diag_counter diag_basic
[] = {
2063 DIAG_COUNTER(rq_num_lle
, 0x00),
2064 DIAG_COUNTER(sq_num_lle
, 0x04),
2065 DIAG_COUNTER(rq_num_lqpoe
, 0x08),
2066 DIAG_COUNTER(sq_num_lqpoe
, 0x0C),
2067 DIAG_COUNTER(rq_num_lpe
, 0x18),
2068 DIAG_COUNTER(sq_num_lpe
, 0x1C),
2069 DIAG_COUNTER(rq_num_wrfe
, 0x20),
2070 DIAG_COUNTER(sq_num_wrfe
, 0x24),
2071 DIAG_COUNTER(sq_num_mwbe
, 0x2C),
2072 DIAG_COUNTER(sq_num_bre
, 0x34),
2073 DIAG_COUNTER(sq_num_rire
, 0x44),
2074 DIAG_COUNTER(rq_num_rire
, 0x48),
2075 DIAG_COUNTER(sq_num_rae
, 0x4C),
2076 DIAG_COUNTER(rq_num_rae
, 0x50),
2077 DIAG_COUNTER(sq_num_roe
, 0x54),
2078 DIAG_COUNTER(sq_num_tree
, 0x5C),
2079 DIAG_COUNTER(sq_num_rree
, 0x64),
2080 DIAG_COUNTER(rq_num_rnr
, 0x68),
2081 DIAG_COUNTER(sq_num_rnr
, 0x6C),
2082 DIAG_COUNTER(rq_num_oos
, 0x100),
2083 DIAG_COUNTER(sq_num_oos
, 0x104),
2086 static const struct diag_counter diag_ext
[] = {
2087 DIAG_COUNTER(rq_num_dup
, 0x130),
2088 DIAG_COUNTER(sq_num_to
, 0x134),
2091 static const struct diag_counter diag_device_only
[] = {
2092 DIAG_COUNTER(num_cqovf
, 0x1A0),
2093 DIAG_COUNTER(rq_num_udsdprd
, 0x118),
2096 static struct rdma_hw_stats
*
2097 mlx4_ib_alloc_hw_device_stats(struct ib_device
*ibdev
)
2099 struct mlx4_ib_dev
*dev
= to_mdev(ibdev
);
2100 struct mlx4_ib_diag_counters
*diag
= dev
->diag_counters
;
2105 return rdma_alloc_hw_stats_struct(diag
[0].descs
, diag
[0].num_counters
,
2106 RDMA_HW_STATS_DEFAULT_LIFESPAN
);
2109 static struct rdma_hw_stats
*
2110 mlx4_ib_alloc_hw_port_stats(struct ib_device
*ibdev
, u32 port_num
)
2112 struct mlx4_ib_dev
*dev
= to_mdev(ibdev
);
2113 struct mlx4_ib_diag_counters
*diag
= dev
->diag_counters
;
2118 return rdma_alloc_hw_stats_struct(diag
[1].descs
, diag
[1].num_counters
,
2119 RDMA_HW_STATS_DEFAULT_LIFESPAN
);
2122 static int mlx4_ib_get_hw_stats(struct ib_device
*ibdev
,
2123 struct rdma_hw_stats
*stats
,
2124 u32 port
, int index
)
2126 struct mlx4_ib_dev
*dev
= to_mdev(ibdev
);
2127 struct mlx4_ib_diag_counters
*diag
= dev
->diag_counters
;
2128 u32 hw_value
[ARRAY_SIZE(diag_device_only
) +
2129 ARRAY_SIZE(diag_ext
) + ARRAY_SIZE(diag_basic
)] = {};
2133 ret
= mlx4_query_diag_counters(dev
->dev
,
2134 MLX4_OP_MOD_QUERY_TRANSPORT_CI_ERRORS
,
2135 diag
[!!port
].offset
, hw_value
,
2136 diag
[!!port
].num_counters
, port
);
2141 for (i
= 0; i
< diag
[!!port
].num_counters
; i
++)
2142 stats
->value
[i
] = hw_value
[i
];
2144 return diag
[!!port
].num_counters
;
2147 static int __mlx4_ib_alloc_diag_counters(struct mlx4_ib_dev
*ibdev
,
2148 struct rdma_stat_desc
**pdescs
,
2149 u32
**offset
, u32
*num
, bool port
)
2153 num_counters
= ARRAY_SIZE(diag_basic
);
2155 if (ibdev
->dev
->caps
.flags2
& MLX4_DEV_CAP_FLAG2_DIAG_PER_PORT
)
2156 num_counters
+= ARRAY_SIZE(diag_ext
);
2159 num_counters
+= ARRAY_SIZE(diag_device_only
);
2161 *pdescs
= kcalloc(num_counters
, sizeof(struct rdma_stat_desc
),
2166 *offset
= kcalloc(num_counters
, sizeof(**offset
), GFP_KERNEL
);
2170 *num
= num_counters
;
2179 static void mlx4_ib_fill_diag_counters(struct mlx4_ib_dev
*ibdev
,
2180 struct rdma_stat_desc
*descs
,
2181 u32
*offset
, bool port
)
2186 for (i
= 0, j
= 0; i
< ARRAY_SIZE(diag_basic
); i
++, j
++) {
2187 descs
[i
].name
= diag_basic
[i
].name
;
2188 offset
[i
] = diag_basic
[i
].offset
;
2191 if (ibdev
->dev
->caps
.flags2
& MLX4_DEV_CAP_FLAG2_DIAG_PER_PORT
) {
2192 for (i
= 0; i
< ARRAY_SIZE(diag_ext
); i
++, j
++) {
2193 descs
[j
].name
= diag_ext
[i
].name
;
2194 offset
[j
] = diag_ext
[i
].offset
;
2199 for (i
= 0; i
< ARRAY_SIZE(diag_device_only
); i
++, j
++) {
2200 descs
[j
].name
= diag_device_only
[i
].name
;
2201 offset
[j
] = diag_device_only
[i
].offset
;
2206 static const struct ib_device_ops mlx4_ib_hw_stats_ops
= {
2207 .alloc_hw_device_stats
= mlx4_ib_alloc_hw_device_stats
,
2208 .alloc_hw_port_stats
= mlx4_ib_alloc_hw_port_stats
,
2209 .get_hw_stats
= mlx4_ib_get_hw_stats
,
2212 static const struct ib_device_ops mlx4_ib_hw_stats_ops1
= {
2213 .alloc_hw_device_stats
= mlx4_ib_alloc_hw_device_stats
,
2214 .get_hw_stats
= mlx4_ib_get_hw_stats
,
2217 static int mlx4_ib_alloc_diag_counters(struct mlx4_ib_dev
*ibdev
)
2219 struct mlx4_ib_diag_counters
*diag
= ibdev
->diag_counters
;
2222 bool per_port
= !!(ibdev
->dev
->caps
.flags2
&
2223 MLX4_DEV_CAP_FLAG2_DIAG_PER_PORT
);
2225 if (mlx4_is_slave(ibdev
->dev
))
2228 for (i
= 0; i
< MLX4_DIAG_COUNTERS_TYPES
; i
++) {
2230 * i == 1 means we are building port counters, set a different
2231 * stats ops without port stats callback.
2233 if (i
&& !per_port
) {
2234 ib_set_device_ops(&ibdev
->ib_dev
,
2235 &mlx4_ib_hw_stats_ops1
);
2240 ret
= __mlx4_ib_alloc_diag_counters(ibdev
, &diag
[i
].descs
,
2242 &diag
[i
].num_counters
, i
);
2246 mlx4_ib_fill_diag_counters(ibdev
, diag
[i
].descs
,
2250 ib_set_device_ops(&ibdev
->ib_dev
, &mlx4_ib_hw_stats_ops
);
2256 kfree(diag
[i
- 1].descs
);
2257 kfree(diag
[i
- 1].offset
);
2263 static void mlx4_ib_diag_cleanup(struct mlx4_ib_dev
*ibdev
)
2267 for (i
= 0; i
< MLX4_DIAG_COUNTERS_TYPES
; i
++) {
2268 kfree(ibdev
->diag_counters
[i
].offset
);
2269 kfree(ibdev
->diag_counters
[i
].descs
);
2273 #define MLX4_IB_INVALID_MAC ((u64)-1)
2274 static void mlx4_ib_update_qps(struct mlx4_ib_dev
*ibdev
,
2275 struct net_device
*dev
,
2279 u64 release_mac
= MLX4_IB_INVALID_MAC
;
2280 struct mlx4_ib_qp
*qp
;
2282 new_smac
= ether_addr_to_u64(dev
->dev_addr
);
2283 atomic64_set(&ibdev
->iboe
.mac
[port
- 1], new_smac
);
2285 /* no need for update QP1 and mac registration in non-SRIOV */
2286 if (!mlx4_is_mfunc(ibdev
->dev
))
2289 mutex_lock(&ibdev
->qp1_proxy_lock
[port
- 1]);
2290 qp
= ibdev
->qp1_proxy
[port
- 1];
2294 struct mlx4_update_qp_params update_params
;
2296 mutex_lock(&qp
->mutex
);
2297 old_smac
= qp
->pri
.smac
;
2298 if (new_smac
== old_smac
)
2301 new_smac_index
= mlx4_register_mac(ibdev
->dev
, port
, new_smac
);
2303 if (new_smac_index
< 0)
2306 update_params
.smac_index
= new_smac_index
;
2307 if (mlx4_update_qp(ibdev
->dev
, qp
->mqp
.qpn
, MLX4_UPDATE_QP_SMAC
,
2309 release_mac
= new_smac
;
2312 /* if old port was zero, no mac was yet registered for this QP */
2313 if (qp
->pri
.smac_port
)
2314 release_mac
= old_smac
;
2315 qp
->pri
.smac
= new_smac
;
2316 qp
->pri
.smac_port
= port
;
2317 qp
->pri
.smac_index
= new_smac_index
;
2321 if (release_mac
!= MLX4_IB_INVALID_MAC
)
2322 mlx4_unregister_mac(ibdev
->dev
, port
, release_mac
);
2324 mutex_unlock(&qp
->mutex
);
2325 mutex_unlock(&ibdev
->qp1_proxy_lock
[port
- 1]);
2328 static void mlx4_ib_scan_netdev(struct mlx4_ib_dev
*ibdev
,
2329 struct net_device
*dev
,
2330 unsigned long event
)
2333 struct mlx4_ib_iboe
*iboe
= &ibdev
->iboe
;
2337 if (dev
->dev
.parent
!= ibdev
->ib_dev
.dev
.parent
)
2340 spin_lock_bh(&iboe
->lock
);
2342 iboe
->netdevs
[dev
->dev_port
] = event
!= NETDEV_UNREGISTER
? dev
: NULL
;
2344 if (event
== NETDEV_UP
|| event
== NETDEV_DOWN
) {
2345 enum ib_port_state port_state
;
2346 struct ib_event ibev
= { };
2348 if (ib_get_cached_port_state(&ibdev
->ib_dev
, dev
->dev_port
+ 1,
2352 if (event
== NETDEV_UP
&&
2353 (port_state
!= IB_PORT_ACTIVE
||
2354 iboe
->last_port_state
[dev
->dev_port
] != IB_PORT_DOWN
))
2356 if (event
== NETDEV_DOWN
&&
2357 (port_state
!= IB_PORT_DOWN
||
2358 iboe
->last_port_state
[dev
->dev_port
] != IB_PORT_ACTIVE
))
2360 iboe
->last_port_state
[dev
->dev_port
] = port_state
;
2362 ibev
.device
= &ibdev
->ib_dev
;
2363 ibev
.element
.port_num
= dev
->dev_port
+ 1;
2364 ibev
.event
= event
== NETDEV_UP
? IB_EVENT_PORT_ACTIVE
:
2366 ib_dispatch_event(&ibev
);
2370 spin_unlock_bh(&iboe
->lock
);
2372 if (event
== NETDEV_CHANGEADDR
|| event
== NETDEV_REGISTER
||
2373 event
== NETDEV_UP
|| event
== NETDEV_CHANGE
)
2374 mlx4_ib_update_qps(ibdev
, dev
, dev
->dev_port
+ 1);
2377 static int mlx4_ib_netdev_event(struct notifier_block
*this,
2378 unsigned long event
, void *ptr
)
2380 struct net_device
*dev
= netdev_notifier_info_to_dev(ptr
);
2381 struct mlx4_ib_dev
*ibdev
;
2383 if (!net_eq(dev_net(dev
), &init_net
))
2386 ibdev
= container_of(this, struct mlx4_ib_dev
, iboe
.nb
);
2387 mlx4_ib_scan_netdev(ibdev
, dev
, event
);
2392 static void init_pkeys(struct mlx4_ib_dev
*ibdev
)
2398 if (mlx4_is_master(ibdev
->dev
)) {
2399 for (slave
= 0; slave
<= ibdev
->dev
->persist
->num_vfs
;
2401 for (port
= 1; port
<= ibdev
->dev
->caps
.num_ports
; ++port
) {
2403 i
< ibdev
->dev
->phys_caps
.pkey_phys_table_len
[port
];
2405 ibdev
->pkeys
.virt2phys_pkey
[slave
][port
- 1][i
] =
2406 /* master has the identity virt2phys pkey mapping */
2407 (slave
== mlx4_master_func_num(ibdev
->dev
) || !i
) ? i
:
2408 ibdev
->dev
->phys_caps
.pkey_phys_table_len
[port
] - 1;
2409 mlx4_sync_pkey_table(ibdev
->dev
, slave
, port
, i
,
2410 ibdev
->pkeys
.virt2phys_pkey
[slave
][port
- 1][i
]);
2414 /* initialize pkey cache */
2415 for (port
= 1; port
<= ibdev
->dev
->caps
.num_ports
; ++port
) {
2417 i
< ibdev
->dev
->phys_caps
.pkey_phys_table_len
[port
];
2419 ibdev
->pkeys
.phys_pkey_cache
[port
-1][i
] =
2425 static void mlx4_ib_alloc_eqs(struct mlx4_dev
*dev
, struct mlx4_ib_dev
*ibdev
)
2427 int i
, j
, eq
= 0, total_eqs
= 0;
2429 ibdev
->eq_table
= kcalloc(dev
->caps
.num_comp_vectors
,
2430 sizeof(ibdev
->eq_table
[0]), GFP_KERNEL
);
2431 if (!ibdev
->eq_table
)
2434 for (i
= 1; i
<= dev
->caps
.num_ports
; i
++) {
2435 for (j
= 0; j
< mlx4_get_eqs_per_port(dev
, i
);
2437 if (i
> 1 && mlx4_is_eq_shared(dev
, total_eqs
))
2439 ibdev
->eq_table
[eq
] = total_eqs
;
2440 if (!mlx4_assign_eq(dev
, i
,
2441 &ibdev
->eq_table
[eq
]))
2444 ibdev
->eq_table
[eq
] = -1;
2448 for (i
= eq
; i
< dev
->caps
.num_comp_vectors
;
2449 ibdev
->eq_table
[i
++] = -1)
2452 /* Advertise the new number of EQs to clients */
2453 ibdev
->ib_dev
.num_comp_vectors
= eq
;
2456 static void mlx4_ib_free_eqs(struct mlx4_dev
*dev
, struct mlx4_ib_dev
*ibdev
)
2459 int total_eqs
= ibdev
->ib_dev
.num_comp_vectors
;
2461 /* no eqs were allocated */
2462 if (!ibdev
->eq_table
)
2465 /* Reset the advertised EQ number */
2466 ibdev
->ib_dev
.num_comp_vectors
= 0;
2468 for (i
= 0; i
< total_eqs
; i
++)
2469 mlx4_release_eq(dev
, ibdev
->eq_table
[i
]);
2471 kfree(ibdev
->eq_table
);
2472 ibdev
->eq_table
= NULL
;
2475 static int mlx4_port_immutable(struct ib_device
*ibdev
, u32 port_num
,
2476 struct ib_port_immutable
*immutable
)
2478 struct ib_port_attr attr
;
2479 struct mlx4_ib_dev
*mdev
= to_mdev(ibdev
);
2482 if (mlx4_ib_port_link_layer(ibdev
, port_num
) == IB_LINK_LAYER_INFINIBAND
) {
2483 immutable
->core_cap_flags
= RDMA_CORE_PORT_IBA_IB
;
2484 immutable
->max_mad_size
= IB_MGMT_MAD_SIZE
;
2486 if (mdev
->dev
->caps
.flags
& MLX4_DEV_CAP_FLAG_IBOE
)
2487 immutable
->core_cap_flags
= RDMA_CORE_PORT_IBA_ROCE
;
2488 if (mdev
->dev
->caps
.flags2
& MLX4_DEV_CAP_FLAG2_ROCE_V1_V2
)
2489 immutable
->core_cap_flags
= RDMA_CORE_PORT_IBA_ROCE
|
2490 RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP
;
2491 immutable
->core_cap_flags
|= RDMA_CORE_PORT_RAW_PACKET
;
2492 if (immutable
->core_cap_flags
& (RDMA_CORE_PORT_IBA_ROCE
|
2493 RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP
))
2494 immutable
->max_mad_size
= IB_MGMT_MAD_SIZE
;
2497 err
= ib_query_port(ibdev
, port_num
, &attr
);
2501 immutable
->pkey_tbl_len
= attr
.pkey_tbl_len
;
2502 immutable
->gid_tbl_len
= attr
.gid_tbl_len
;
2507 static void get_fw_ver_str(struct ib_device
*device
, char *str
)
2509 struct mlx4_ib_dev
*dev
=
2510 container_of(device
, struct mlx4_ib_dev
, ib_dev
);
2511 snprintf(str
, IB_FW_VERSION_NAME_MAX
, "%d.%d.%d",
2512 (int) (dev
->dev
->caps
.fw_ver
>> 32),
2513 (int) (dev
->dev
->caps
.fw_ver
>> 16) & 0xffff,
2514 (int) dev
->dev
->caps
.fw_ver
& 0xffff);
2517 static const struct ib_device_ops mlx4_ib_dev_ops
= {
2518 .owner
= THIS_MODULE
,
2519 .driver_id
= RDMA_DRIVER_MLX4
,
2520 .uverbs_abi_ver
= MLX4_IB_UVERBS_ABI_VERSION
,
2522 .add_gid
= mlx4_ib_add_gid
,
2523 .alloc_mr
= mlx4_ib_alloc_mr
,
2524 .alloc_pd
= mlx4_ib_alloc_pd
,
2525 .alloc_ucontext
= mlx4_ib_alloc_ucontext
,
2526 .attach_mcast
= mlx4_ib_mcg_attach
,
2527 .create_ah
= mlx4_ib_create_ah
,
2528 .create_cq
= mlx4_ib_create_cq
,
2529 .create_qp
= mlx4_ib_create_qp
,
2530 .create_srq
= mlx4_ib_create_srq
,
2531 .dealloc_pd
= mlx4_ib_dealloc_pd
,
2532 .dealloc_ucontext
= mlx4_ib_dealloc_ucontext
,
2533 .del_gid
= mlx4_ib_del_gid
,
2534 .dereg_mr
= mlx4_ib_dereg_mr
,
2535 .destroy_ah
= mlx4_ib_destroy_ah
,
2536 .destroy_cq
= mlx4_ib_destroy_cq
,
2537 .destroy_qp
= mlx4_ib_destroy_qp
,
2538 .destroy_srq
= mlx4_ib_destroy_srq
,
2539 .detach_mcast
= mlx4_ib_mcg_detach
,
2540 .device_group
= &mlx4_attr_group
,
2541 .disassociate_ucontext
= mlx4_ib_disassociate_ucontext
,
2542 .drain_rq
= mlx4_ib_drain_rq
,
2543 .drain_sq
= mlx4_ib_drain_sq
,
2544 .get_dev_fw_str
= get_fw_ver_str
,
2545 .get_dma_mr
= mlx4_ib_get_dma_mr
,
2546 .get_link_layer
= mlx4_ib_port_link_layer
,
2547 .get_netdev
= mlx4_ib_get_netdev
,
2548 .get_port_immutable
= mlx4_port_immutable
,
2549 .map_mr_sg
= mlx4_ib_map_mr_sg
,
2550 .mmap
= mlx4_ib_mmap
,
2551 .modify_cq
= mlx4_ib_modify_cq
,
2552 .modify_device
= mlx4_ib_modify_device
,
2553 .modify_port
= mlx4_ib_modify_port
,
2554 .modify_qp
= mlx4_ib_modify_qp
,
2555 .modify_srq
= mlx4_ib_modify_srq
,
2556 .poll_cq
= mlx4_ib_poll_cq
,
2557 .post_recv
= mlx4_ib_post_recv
,
2558 .post_send
= mlx4_ib_post_send
,
2559 .post_srq_recv
= mlx4_ib_post_srq_recv
,
2560 .process_mad
= mlx4_ib_process_mad
,
2561 .query_ah
= mlx4_ib_query_ah
,
2562 .query_device
= mlx4_ib_query_device
,
2563 .query_gid
= mlx4_ib_query_gid
,
2564 .query_pkey
= mlx4_ib_query_pkey
,
2565 .query_port
= mlx4_ib_query_port
,
2566 .query_qp
= mlx4_ib_query_qp
,
2567 .query_srq
= mlx4_ib_query_srq
,
2568 .reg_user_mr
= mlx4_ib_reg_user_mr
,
2569 .req_notify_cq
= mlx4_ib_arm_cq
,
2570 .rereg_user_mr
= mlx4_ib_rereg_user_mr
,
2571 .resize_cq
= mlx4_ib_resize_cq
,
2573 INIT_RDMA_OBJ_SIZE(ib_ah
, mlx4_ib_ah
, ibah
),
2574 INIT_RDMA_OBJ_SIZE(ib_cq
, mlx4_ib_cq
, ibcq
),
2575 INIT_RDMA_OBJ_SIZE(ib_pd
, mlx4_ib_pd
, ibpd
),
2576 INIT_RDMA_OBJ_SIZE(ib_qp
, mlx4_ib_qp
, ibqp
),
2577 INIT_RDMA_OBJ_SIZE(ib_srq
, mlx4_ib_srq
, ibsrq
),
2578 INIT_RDMA_OBJ_SIZE(ib_ucontext
, mlx4_ib_ucontext
, ibucontext
),
2581 static const struct ib_device_ops mlx4_ib_dev_wq_ops
= {
2582 .create_rwq_ind_table
= mlx4_ib_create_rwq_ind_table
,
2583 .create_wq
= mlx4_ib_create_wq
,
2584 .destroy_rwq_ind_table
= mlx4_ib_destroy_rwq_ind_table
,
2585 .destroy_wq
= mlx4_ib_destroy_wq
,
2586 .modify_wq
= mlx4_ib_modify_wq
,
2588 INIT_RDMA_OBJ_SIZE(ib_rwq_ind_table
, mlx4_ib_rwq_ind_table
,
2592 static const struct ib_device_ops mlx4_ib_dev_mw_ops
= {
2593 .alloc_mw
= mlx4_ib_alloc_mw
,
2594 .dealloc_mw
= mlx4_ib_dealloc_mw
,
2596 INIT_RDMA_OBJ_SIZE(ib_mw
, mlx4_ib_mw
, ibmw
),
2599 static const struct ib_device_ops mlx4_ib_dev_xrc_ops
= {
2600 .alloc_xrcd
= mlx4_ib_alloc_xrcd
,
2601 .dealloc_xrcd
= mlx4_ib_dealloc_xrcd
,
2603 INIT_RDMA_OBJ_SIZE(ib_xrcd
, mlx4_ib_xrcd
, ibxrcd
),
2606 static const struct ib_device_ops mlx4_ib_dev_fs_ops
= {
2607 .create_flow
= mlx4_ib_create_flow
,
2608 .destroy_flow
= mlx4_ib_destroy_flow
,
2611 static int mlx4_ib_probe(struct auxiliary_device
*adev
,
2612 const struct auxiliary_device_id
*id
)
2614 struct mlx4_adev
*madev
= container_of(adev
, struct mlx4_adev
, adev
);
2615 struct mlx4_dev
*dev
= madev
->mdev
;
2616 struct mlx4_ib_dev
*ibdev
;
2620 struct mlx4_ib_iboe
*iboe
;
2621 int ib_num_ports
= 0;
2622 int num_req_counters
;
2625 struct counter_index
*new_counter_index
;
2627 pr_info_once("%s", mlx4_ib_version
);
2630 mlx4_foreach_ib_transport_port(i
, dev
)
2633 /* No point in registering a device with no ports... */
2637 ibdev
= ib_alloc_device(mlx4_ib_dev
, ib_dev
);
2639 dev_err(&dev
->persist
->pdev
->dev
,
2640 "Device struct alloc failed\n");
2644 iboe
= &ibdev
->iboe
;
2646 err
= mlx4_pd_alloc(dev
, &ibdev
->priv_pdn
);
2650 err
= mlx4_uar_alloc(dev
, &ibdev
->priv_uar
);
2654 ibdev
->uar_map
= ioremap((phys_addr_t
) ibdev
->priv_uar
.pfn
<< PAGE_SHIFT
,
2656 if (!ibdev
->uar_map
) {
2660 MLX4_INIT_DOORBELL_LOCK(&ibdev
->uar_lock
);
2663 ibdev
->bond_next_port
= 0;
2665 ibdev
->ib_dev
.node_type
= RDMA_NODE_IB_CA
;
2666 ibdev
->ib_dev
.local_dma_lkey
= dev
->caps
.reserved_lkey
;
2667 ibdev
->num_ports
= num_ports
;
2668 ibdev
->ib_dev
.phys_port_cnt
= mlx4_is_bonded(dev
) ?
2669 1 : ibdev
->num_ports
;
2670 ibdev
->ib_dev
.num_comp_vectors
= dev
->caps
.num_comp_vectors
;
2671 ibdev
->ib_dev
.dev
.parent
= &dev
->persist
->pdev
->dev
;
2673 ib_set_device_ops(&ibdev
->ib_dev
, &mlx4_ib_dev_ops
);
2675 if ((dev
->caps
.flags2
& MLX4_DEV_CAP_FLAG2_RSS
) &&
2676 ((mlx4_ib_port_link_layer(&ibdev
->ib_dev
, 1) ==
2677 IB_LINK_LAYER_ETHERNET
) ||
2678 (mlx4_ib_port_link_layer(&ibdev
->ib_dev
, 2) ==
2679 IB_LINK_LAYER_ETHERNET
)))
2680 ib_set_device_ops(&ibdev
->ib_dev
, &mlx4_ib_dev_wq_ops
);
2682 if (dev
->caps
.flags
& MLX4_DEV_CAP_FLAG_MEM_WINDOW
||
2683 dev
->caps
.bmme_flags
& MLX4_BMME_FLAG_TYPE_2_WIN
)
2684 ib_set_device_ops(&ibdev
->ib_dev
, &mlx4_ib_dev_mw_ops
);
2686 if (dev
->caps
.flags
& MLX4_DEV_CAP_FLAG_XRC
) {
2687 ib_set_device_ops(&ibdev
->ib_dev
, &mlx4_ib_dev_xrc_ops
);
2690 if (check_flow_steering_support(dev
)) {
2691 ibdev
->steering_support
= MLX4_STEERING_MODE_DEVICE_MANAGED
;
2692 ib_set_device_ops(&ibdev
->ib_dev
, &mlx4_ib_dev_fs_ops
);
2695 if (!dev
->caps
.userspace_caps
)
2696 ibdev
->ib_dev
.ops
.uverbs_abi_ver
=
2697 MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION
;
2699 mlx4_ib_alloc_eqs(dev
, ibdev
);
2701 spin_lock_init(&iboe
->lock
);
2703 err
= init_node_data(ibdev
);
2706 mlx4_init_sl2vl_tbl(ibdev
);
2708 for (i
= 0; i
< ibdev
->num_ports
; ++i
) {
2709 mutex_init(&ibdev
->counters_table
[i
].mutex
);
2710 INIT_LIST_HEAD(&ibdev
->counters_table
[i
].counters_list
);
2711 iboe
->last_port_state
[i
] = IB_PORT_DOWN
;
2714 num_req_counters
= mlx4_is_bonded(dev
) ? 1 : ibdev
->num_ports
;
2715 for (i
= 0; i
< num_req_counters
; ++i
) {
2716 mutex_init(&ibdev
->qp1_proxy_lock
[i
]);
2718 if (mlx4_ib_port_link_layer(&ibdev
->ib_dev
, i
+ 1) ==
2719 IB_LINK_LAYER_ETHERNET
) {
2720 err
= mlx4_counter_alloc(ibdev
->dev
, &counter_index
,
2721 MLX4_RES_USAGE_DRIVER
);
2722 /* if failed to allocate a new counter, use default */
2725 mlx4_get_default_counter_index(dev
,
2729 } else { /* IB_LINK_LAYER_INFINIBAND use the default counter */
2730 counter_index
= mlx4_get_default_counter_index(dev
,
2733 new_counter_index
= kmalloc(sizeof(*new_counter_index
),
2735 if (!new_counter_index
) {
2738 mlx4_counter_free(ibdev
->dev
, counter_index
);
2741 new_counter_index
->index
= counter_index
;
2742 new_counter_index
->allocated
= allocated
;
2743 list_add_tail(&new_counter_index
->list
,
2744 &ibdev
->counters_table
[i
].counters_list
);
2745 ibdev
->counters_table
[i
].default_counter
= counter_index
;
2746 pr_info("counter index %d for port %d allocated %d\n",
2747 counter_index
, i
+ 1, allocated
);
2749 if (mlx4_is_bonded(dev
))
2750 for (i
= 1; i
< ibdev
->num_ports
; ++i
) {
2752 kmalloc(sizeof(struct counter_index
),
2754 if (!new_counter_index
) {
2758 new_counter_index
->index
= counter_index
;
2759 new_counter_index
->allocated
= 0;
2760 list_add_tail(&new_counter_index
->list
,
2761 &ibdev
->counters_table
[i
].counters_list
);
2762 ibdev
->counters_table
[i
].default_counter
=
2766 mlx4_foreach_port(i
, dev
, MLX4_PORT_TYPE_IB
)
2769 spin_lock_init(&ibdev
->sm_lock
);
2770 mutex_init(&ibdev
->cap_mask_mutex
);
2771 INIT_LIST_HEAD(&ibdev
->qp_list
);
2772 spin_lock_init(&ibdev
->reset_flow_resource_lock
);
2774 if (ibdev
->steering_support
== MLX4_STEERING_MODE_DEVICE_MANAGED
&&
2776 ibdev
->steer_qpn_count
= MLX4_IB_UC_MAX_NUM_QPS
;
2777 err
= mlx4_qp_reserve_range(dev
, ibdev
->steer_qpn_count
,
2778 MLX4_IB_UC_STEER_QPN_ALIGN
,
2779 &ibdev
->steer_qpn_base
, 0,
2780 MLX4_RES_USAGE_DRIVER
);
2784 ibdev
->ib_uc_qpns_bitmap
= bitmap_alloc(ibdev
->steer_qpn_count
,
2786 if (!ibdev
->ib_uc_qpns_bitmap
) {
2788 goto err_steer_qp_release
;
2791 if (dev
->caps
.flags2
& MLX4_DEV_CAP_FLAG2_DMFS_IPOIB
) {
2792 bitmap_zero(ibdev
->ib_uc_qpns_bitmap
,
2793 ibdev
->steer_qpn_count
);
2794 err
= mlx4_FLOW_STEERING_IB_UC_QP_RANGE(
2795 dev
, ibdev
->steer_qpn_base
,
2796 ibdev
->steer_qpn_base
+
2797 ibdev
->steer_qpn_count
- 1);
2799 goto err_steer_free_bitmap
;
2801 bitmap_fill(ibdev
->ib_uc_qpns_bitmap
,
2802 ibdev
->steer_qpn_count
);
2806 for (j
= 1; j
<= ibdev
->dev
->caps
.num_ports
; j
++)
2807 atomic64_set(&iboe
->mac
[j
- 1], ibdev
->dev
->caps
.def_mac
[j
]);
2809 err
= mlx4_ib_alloc_diag_counters(ibdev
);
2811 goto err_steer_free_bitmap
;
2813 err
= ib_register_device(&ibdev
->ib_dev
, "mlx4_%d",
2814 &dev
->persist
->pdev
->dev
);
2816 goto err_diag_counters
;
2818 err
= mlx4_ib_mad_init(ibdev
);
2822 err
= mlx4_ib_init_sriov(ibdev
);
2826 if (!iboe
->nb
.notifier_call
) {
2827 iboe
->nb
.notifier_call
= mlx4_ib_netdev_event
;
2828 err
= register_netdevice_notifier(&iboe
->nb
);
2830 iboe
->nb
.notifier_call
= NULL
;
2834 if (dev
->caps
.flags2
& MLX4_DEV_CAP_FLAG2_ROCE_V1_V2
) {
2835 err
= mlx4_config_roce_v2_port(dev
, ROCE_V2_UDP_DPORT
);
2840 ibdev
->ib_active
= true;
2841 mlx4_foreach_port(i
, dev
, MLX4_PORT_TYPE_IB
)
2842 devlink_port_type_ib_set(mlx4_get_devlink_port(dev
, i
),
2845 if (mlx4_is_mfunc(ibdev
->dev
))
2848 /* create paravirt contexts for any VFs which are active */
2849 if (mlx4_is_master(ibdev
->dev
)) {
2850 for (j
= 0; j
< MLX4_MFUNC_MAX
; j
++) {
2851 if (j
== mlx4_master_func_num(ibdev
->dev
))
2853 if (mlx4_is_slave_active(ibdev
->dev
, j
))
2854 do_slave_init(ibdev
, j
, 1);
2858 /* register mlx4 core notifier */
2859 ibdev
->mlx_nb
.notifier_call
= mlx4_ib_event
;
2860 err
= mlx4_register_event_notifier(dev
, &ibdev
->mlx_nb
);
2861 WARN(err
, "failed to register mlx4 event notifier (%d)", err
);
2863 auxiliary_set_drvdata(adev
, ibdev
);
2867 if (ibdev
->iboe
.nb
.notifier_call
) {
2868 if (unregister_netdevice_notifier(&ibdev
->iboe
.nb
))
2869 pr_warn("failure unregistering notifier\n");
2870 ibdev
->iboe
.nb
.notifier_call
= NULL
;
2872 flush_workqueue(wq
);
2874 mlx4_ib_close_sriov(ibdev
);
2877 mlx4_ib_mad_cleanup(ibdev
);
2880 ib_unregister_device(&ibdev
->ib_dev
);
2883 mlx4_ib_diag_cleanup(ibdev
);
2885 err_steer_free_bitmap
:
2886 bitmap_free(ibdev
->ib_uc_qpns_bitmap
);
2888 err_steer_qp_release
:
2889 mlx4_qp_release_range(dev
, ibdev
->steer_qpn_base
,
2890 ibdev
->steer_qpn_count
);
2892 for (i
= 0; i
< ibdev
->num_ports
; ++i
)
2893 mlx4_ib_delete_counters_table(ibdev
, &ibdev
->counters_table
[i
]);
2896 mlx4_ib_free_eqs(dev
, ibdev
);
2897 iounmap(ibdev
->uar_map
);
2900 mlx4_uar_free(dev
, &ibdev
->priv_uar
);
2903 mlx4_pd_free(dev
, ibdev
->priv_pdn
);
2906 ib_dealloc_device(&ibdev
->ib_dev
);
2911 int mlx4_ib_steer_qp_alloc(struct mlx4_ib_dev
*dev
, int count
, int *qpn
)
2915 WARN_ON(!dev
->ib_uc_qpns_bitmap
);
2917 offset
= bitmap_find_free_region(dev
->ib_uc_qpns_bitmap
,
2918 dev
->steer_qpn_count
,
2919 get_count_order(count
));
2923 *qpn
= dev
->steer_qpn_base
+ offset
;
2927 void mlx4_ib_steer_qp_free(struct mlx4_ib_dev
*dev
, u32 qpn
, int count
)
2930 dev
->steering_support
!= MLX4_STEERING_MODE_DEVICE_MANAGED
)
2933 if (WARN(qpn
< dev
->steer_qpn_base
, "qpn = %u, steer_qpn_base = %u\n",
2934 qpn
, dev
->steer_qpn_base
))
2935 /* not supposed to be here */
2938 bitmap_release_region(dev
->ib_uc_qpns_bitmap
,
2939 qpn
- dev
->steer_qpn_base
,
2940 get_count_order(count
));
2943 int mlx4_ib_steer_qp_reg(struct mlx4_ib_dev
*mdev
, struct mlx4_ib_qp
*mqp
,
2948 struct ib_flow_attr
*flow
;
2949 struct ib_flow_spec_ib
*ib_spec
;
2952 flow_size
= sizeof(struct ib_flow_attr
) +
2953 sizeof(struct ib_flow_spec_ib
);
2954 flow
= kzalloc(flow_size
, GFP_KERNEL
);
2957 flow
->port
= mqp
->port
;
2958 flow
->num_of_specs
= 1;
2959 flow
->size
= flow_size
;
2960 ib_spec
= (struct ib_flow_spec_ib
*)(flow
+ 1);
2961 ib_spec
->type
= IB_FLOW_SPEC_IB
;
2962 ib_spec
->size
= sizeof(struct ib_flow_spec_ib
);
2963 /* Add an empty rule for IB L2 */
2964 memset(&ib_spec
->mask
, 0, sizeof(ib_spec
->mask
));
2966 err
= __mlx4_ib_create_flow(&mqp
->ibqp
, flow
, MLX4_DOMAIN_NIC
,
2967 MLX4_FS_REGULAR
, &mqp
->reg_id
);
2972 return __mlx4_ib_destroy_flow(mdev
->dev
, mqp
->reg_id
);
2975 static void mlx4_ib_remove(struct auxiliary_device
*adev
)
2977 struct mlx4_adev
*madev
= container_of(adev
, struct mlx4_adev
, adev
);
2978 struct mlx4_dev
*dev
= madev
->mdev
;
2979 struct mlx4_ib_dev
*ibdev
= auxiliary_get_drvdata(adev
);
2983 mlx4_unregister_event_notifier(dev
, &ibdev
->mlx_nb
);
2985 mlx4_foreach_port(i
, dev
, MLX4_PORT_TYPE_IB
)
2986 devlink_port_type_clear(mlx4_get_devlink_port(dev
, i
));
2987 ibdev
->ib_active
= false;
2988 flush_workqueue(wq
);
2990 if (ibdev
->iboe
.nb
.notifier_call
) {
2991 if (unregister_netdevice_notifier(&ibdev
->iboe
.nb
))
2992 pr_warn("failure unregistering notifier\n");
2993 ibdev
->iboe
.nb
.notifier_call
= NULL
;
2996 mlx4_ib_close_sriov(ibdev
);
2997 mlx4_ib_mad_cleanup(ibdev
);
2998 ib_unregister_device(&ibdev
->ib_dev
);
2999 mlx4_ib_diag_cleanup(ibdev
);
3001 mlx4_qp_release_range(dev
, ibdev
->steer_qpn_base
,
3002 ibdev
->steer_qpn_count
);
3003 bitmap_free(ibdev
->ib_uc_qpns_bitmap
);
3005 iounmap(ibdev
->uar_map
);
3006 for (p
= 0; p
< ibdev
->num_ports
; ++p
)
3007 mlx4_ib_delete_counters_table(ibdev
, &ibdev
->counters_table
[p
]);
3009 mlx4_foreach_port(p
, dev
, MLX4_PORT_TYPE_IB
)
3010 mlx4_CLOSE_PORT(dev
, p
);
3012 mlx4_ib_free_eqs(dev
, ibdev
);
3014 mlx4_uar_free(dev
, &ibdev
->priv_uar
);
3015 mlx4_pd_free(dev
, ibdev
->priv_pdn
);
3016 ib_dealloc_device(&ibdev
->ib_dev
);
3019 static void do_slave_init(struct mlx4_ib_dev
*ibdev
, int slave
, int do_init
)
3021 struct mlx4_ib_demux_work
**dm
;
3022 struct mlx4_dev
*dev
= ibdev
->dev
;
3024 unsigned long flags
;
3025 struct mlx4_active_ports actv_ports
;
3027 unsigned int first_port
;
3029 if (!mlx4_is_master(dev
))
3032 actv_ports
= mlx4_get_active_ports(dev
, slave
);
3033 ports
= bitmap_weight(actv_ports
.ports
, dev
->caps
.num_ports
);
3034 first_port
= find_first_bit(actv_ports
.ports
, dev
->caps
.num_ports
);
3036 dm
= kcalloc(ports
, sizeof(*dm
), GFP_ATOMIC
);
3040 for (i
= 0; i
< ports
; i
++) {
3041 dm
[i
] = kmalloc(sizeof (struct mlx4_ib_demux_work
), GFP_ATOMIC
);
3047 INIT_WORK(&dm
[i
]->work
, mlx4_ib_tunnels_update_work
);
3048 dm
[i
]->port
= first_port
+ i
+ 1;
3049 dm
[i
]->slave
= slave
;
3050 dm
[i
]->do_init
= do_init
;
3053 /* initialize or tear down tunnel QPs for the slave */
3054 spin_lock_irqsave(&ibdev
->sriov
.going_down_lock
, flags
);
3055 if (!ibdev
->sriov
.is_going_down
) {
3056 for (i
= 0; i
< ports
; i
++)
3057 queue_work(ibdev
->sriov
.demux
[i
].ud_wq
, &dm
[i
]->work
);
3058 spin_unlock_irqrestore(&ibdev
->sriov
.going_down_lock
, flags
);
3060 spin_unlock_irqrestore(&ibdev
->sriov
.going_down_lock
, flags
);
3061 for (i
= 0; i
< ports
; i
++)
3069 static void mlx4_ib_handle_catas_error(struct mlx4_ib_dev
*ibdev
)
3071 struct mlx4_ib_qp
*mqp
;
3072 unsigned long flags_qp
;
3073 unsigned long flags_cq
;
3074 struct mlx4_ib_cq
*send_mcq
, *recv_mcq
;
3075 struct list_head cq_notify_list
;
3076 struct mlx4_cq
*mcq
;
3077 unsigned long flags
;
3079 pr_warn("mlx4_ib_handle_catas_error was started\n");
3080 INIT_LIST_HEAD(&cq_notify_list
);
3082 /* Go over qp list reside on that ibdev, sync with create/destroy qp.*/
3083 spin_lock_irqsave(&ibdev
->reset_flow_resource_lock
, flags
);
3085 list_for_each_entry(mqp
, &ibdev
->qp_list
, qps_list
) {
3086 spin_lock_irqsave(&mqp
->sq
.lock
, flags_qp
);
3087 if (mqp
->sq
.tail
!= mqp
->sq
.head
) {
3088 send_mcq
= to_mcq(mqp
->ibqp
.send_cq
);
3089 spin_lock_irqsave(&send_mcq
->lock
, flags_cq
);
3090 if (send_mcq
->mcq
.comp
&&
3091 mqp
->ibqp
.send_cq
->comp_handler
) {
3092 if (!send_mcq
->mcq
.reset_notify_added
) {
3093 send_mcq
->mcq
.reset_notify_added
= 1;
3094 list_add_tail(&send_mcq
->mcq
.reset_notify
,
3098 spin_unlock_irqrestore(&send_mcq
->lock
, flags_cq
);
3100 spin_unlock_irqrestore(&mqp
->sq
.lock
, flags_qp
);
3101 /* Now, handle the QP's receive queue */
3102 spin_lock_irqsave(&mqp
->rq
.lock
, flags_qp
);
3103 /* no handling is needed for SRQ */
3104 if (!mqp
->ibqp
.srq
) {
3105 if (mqp
->rq
.tail
!= mqp
->rq
.head
) {
3106 recv_mcq
= to_mcq(mqp
->ibqp
.recv_cq
);
3107 spin_lock_irqsave(&recv_mcq
->lock
, flags_cq
);
3108 if (recv_mcq
->mcq
.comp
&&
3109 mqp
->ibqp
.recv_cq
->comp_handler
) {
3110 if (!recv_mcq
->mcq
.reset_notify_added
) {
3111 recv_mcq
->mcq
.reset_notify_added
= 1;
3112 list_add_tail(&recv_mcq
->mcq
.reset_notify
,
3116 spin_unlock_irqrestore(&recv_mcq
->lock
,
3120 spin_unlock_irqrestore(&mqp
->rq
.lock
, flags_qp
);
3123 list_for_each_entry(mcq
, &cq_notify_list
, reset_notify
) {
3126 spin_unlock_irqrestore(&ibdev
->reset_flow_resource_lock
, flags
);
3127 pr_warn("mlx4_ib_handle_catas_error ended\n");
3130 static void handle_bonded_port_state_event(struct work_struct
*work
)
3132 struct ib_event_work
*ew
=
3133 container_of(work
, struct ib_event_work
, work
);
3134 struct mlx4_ib_dev
*ibdev
= ew
->ib_dev
;
3135 enum ib_port_state bonded_port_state
= IB_PORT_NOP
;
3137 struct ib_event ibev
;
3140 spin_lock_bh(&ibdev
->iboe
.lock
);
3141 for (i
= 0; i
< MLX4_MAX_PORTS
; ++i
) {
3142 struct net_device
*curr_netdev
= ibdev
->iboe
.netdevs
[i
];
3143 enum ib_port_state curr_port_state
;
3149 (netif_running(curr_netdev
) &&
3150 netif_carrier_ok(curr_netdev
)) ?
3151 IB_PORT_ACTIVE
: IB_PORT_DOWN
;
3153 bonded_port_state
= (bonded_port_state
!= IB_PORT_ACTIVE
) ?
3154 curr_port_state
: IB_PORT_ACTIVE
;
3156 spin_unlock_bh(&ibdev
->iboe
.lock
);
3158 ibev
.device
= &ibdev
->ib_dev
;
3159 ibev
.element
.port_num
= 1;
3160 ibev
.event
= (bonded_port_state
== IB_PORT_ACTIVE
) ?
3161 IB_EVENT_PORT_ACTIVE
: IB_EVENT_PORT_ERR
;
3163 ib_dispatch_event(&ibev
);
3166 void mlx4_ib_sl2vl_update(struct mlx4_ib_dev
*mdev
, int port
)
3171 err
= mlx4_ib_query_sl2vl(&mdev
->ib_dev
, port
, &sl2vl
);
3173 pr_err("Unable to get current sl to vl mapping for port %d. Using all zeroes (%d)\n",
3177 atomic64_set(&mdev
->sl2vl
[port
- 1], sl2vl
);
3180 static void ib_sl2vl_update_work(struct work_struct
*work
)
3182 struct ib_event_work
*ew
= container_of(work
, struct ib_event_work
, work
);
3183 struct mlx4_ib_dev
*mdev
= ew
->ib_dev
;
3184 int port
= ew
->port
;
3186 mlx4_ib_sl2vl_update(mdev
, port
);
3191 void mlx4_sched_ib_sl2vl_update_work(struct mlx4_ib_dev
*ibdev
,
3194 struct ib_event_work
*ew
;
3196 ew
= kmalloc(sizeof(*ew
), GFP_ATOMIC
);
3198 INIT_WORK(&ew
->work
, ib_sl2vl_update_work
);
3201 queue_work(wq
, &ew
->work
);
3205 static int mlx4_ib_event(struct notifier_block
*this, unsigned long event
,
3208 struct mlx4_ib_dev
*ibdev
=
3209 container_of(this, struct mlx4_ib_dev
, mlx_nb
);
3210 struct mlx4_dev
*dev
= ibdev
->dev
;
3211 struct ib_event ibev
;
3212 struct mlx4_eqe
*eqe
= NULL
;
3213 struct ib_event_work
*ew
;
3216 if (mlx4_is_bonded(dev
) &&
3217 ((event
== MLX4_DEV_EVENT_PORT_UP
) ||
3218 (event
== MLX4_DEV_EVENT_PORT_DOWN
))) {
3219 ew
= kmalloc(sizeof(*ew
), GFP_ATOMIC
);
3222 INIT_WORK(&ew
->work
, handle_bonded_port_state_event
);
3224 queue_work(wq
, &ew
->work
);
3229 case MLX4_DEV_EVENT_CATASTROPHIC_ERROR
:
3231 case MLX4_DEV_EVENT_PORT_MGMT_CHANGE
:
3232 eqe
= (struct mlx4_eqe
*)param
;
3240 case MLX4_DEV_EVENT_PORT_UP
:
3241 if (p
> ibdev
->num_ports
)
3243 if (!mlx4_is_slave(dev
) &&
3244 rdma_port_get_link_layer(&ibdev
->ib_dev
, p
) ==
3245 IB_LINK_LAYER_INFINIBAND
) {
3246 if (mlx4_is_master(dev
))
3247 mlx4_ib_invalidate_all_guid_record(ibdev
, p
);
3248 if (ibdev
->dev
->flags
& MLX4_FLAG_SECURE_HOST
&&
3249 !(ibdev
->dev
->caps
.flags2
& MLX4_DEV_CAP_FLAG2_SL_TO_VL_CHANGE_EVENT
))
3250 mlx4_sched_ib_sl2vl_update_work(ibdev
, p
);
3252 ibev
.event
= IB_EVENT_PORT_ACTIVE
;
3255 case MLX4_DEV_EVENT_PORT_DOWN
:
3256 if (p
> ibdev
->num_ports
)
3258 ibev
.event
= IB_EVENT_PORT_ERR
;
3261 case MLX4_DEV_EVENT_CATASTROPHIC_ERROR
:
3262 ibdev
->ib_active
= false;
3263 ibev
.event
= IB_EVENT_DEVICE_FATAL
;
3264 mlx4_ib_handle_catas_error(ibdev
);
3267 case MLX4_DEV_EVENT_PORT_MGMT_CHANGE
:
3268 ew
= kmalloc(sizeof *ew
, GFP_ATOMIC
);
3272 INIT_WORK(&ew
->work
, handle_port_mgmt_change_event
);
3273 memcpy(&ew
->ib_eqe
, eqe
, sizeof *eqe
);
3275 /* need to queue only for port owner, which uses GEN_EQE */
3276 if (mlx4_is_master(dev
))
3277 queue_work(wq
, &ew
->work
);
3279 handle_port_mgmt_change_event(&ew
->work
);
3282 case MLX4_DEV_EVENT_SLAVE_INIT
:
3283 /* here, p is the slave id */
3284 do_slave_init(ibdev
, p
, 1);
3285 if (mlx4_is_master(dev
)) {
3288 for (i
= 1; i
<= ibdev
->num_ports
; i
++) {
3289 if (rdma_port_get_link_layer(&ibdev
->ib_dev
, i
)
3290 == IB_LINK_LAYER_INFINIBAND
)
3291 mlx4_ib_slave_alias_guid_event(ibdev
,
3298 case MLX4_DEV_EVENT_SLAVE_SHUTDOWN
:
3299 if (mlx4_is_master(dev
)) {
3302 for (i
= 1; i
<= ibdev
->num_ports
; i
++) {
3303 if (rdma_port_get_link_layer(&ibdev
->ib_dev
, i
)
3304 == IB_LINK_LAYER_INFINIBAND
)
3305 mlx4_ib_slave_alias_guid_event(ibdev
,
3310 /* here, p is the slave id */
3311 do_slave_init(ibdev
, p
, 0);
3318 ibev
.device
= &ibdev
->ib_dev
;
3319 ibev
.element
.port_num
= mlx4_is_bonded(ibdev
->dev
) ? 1 : (u8
)p
;
3321 ib_dispatch_event(&ibev
);
3325 static const struct auxiliary_device_id mlx4_ib_id_table
[] = {
3326 { .name
= MLX4_ADEV_NAME
".ib" },
3330 MODULE_DEVICE_TABLE(auxiliary
, mlx4_ib_id_table
);
3332 static struct mlx4_adrv mlx4_ib_adrv
= {
3335 .probe
= mlx4_ib_probe
,
3336 .remove
= mlx4_ib_remove
,
3337 .id_table
= mlx4_ib_id_table
,
3339 .protocol
= MLX4_PROT_IB_IPV6
,
3340 .flags
= MLX4_INTFF_BONDING
3343 static int __init
mlx4_ib_init(void)
3347 wq
= alloc_ordered_workqueue("mlx4_ib", WQ_MEM_RECLAIM
);
3351 err
= mlx4_ib_qp_event_init();
3353 goto clean_qp_event
;
3355 err
= mlx4_ib_cm_init();
3359 err
= mlx4_ib_mcg_init();
3363 err
= mlx4_register_auxiliary_driver(&mlx4_ib_adrv
);
3370 mlx4_ib_mcg_destroy();
3373 mlx4_ib_cm_destroy();
3376 mlx4_ib_qp_event_cleanup();
3379 destroy_workqueue(wq
);
3383 static void __exit
mlx4_ib_cleanup(void)
3385 mlx4_unregister_auxiliary_driver(&mlx4_ib_adrv
);
3386 mlx4_ib_mcg_destroy();
3387 mlx4_ib_cm_destroy();
3388 mlx4_ib_qp_event_cleanup();
3389 destroy_workqueue(wq
);
3392 module_init(mlx4_ib_init
);
3393 module_exit(mlx4_ib_cleanup
);