2 * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <rdma/ib_mad.h>
34 #include <rdma/ib_smi.h>
36 #include <linux/mlx4/cmd.h>
37 #include <linux/gfp.h>
38 #include <rdma/ib_pma.h>
43 MLX4_IB_VENDOR_CLASS1
= 0x9,
44 MLX4_IB_VENDOR_CLASS2
= 0xa
47 int mlx4_MAD_IFC(struct mlx4_ib_dev
*dev
, int ignore_mkey
, int ignore_bkey
,
48 int port
, struct ib_wc
*in_wc
, struct ib_grh
*in_grh
,
49 void *in_mad
, void *response_mad
)
51 struct mlx4_cmd_mailbox
*inmailbox
, *outmailbox
;
54 u32 in_modifier
= port
;
57 inmailbox
= mlx4_alloc_cmd_mailbox(dev
->dev
);
58 if (IS_ERR(inmailbox
))
59 return PTR_ERR(inmailbox
);
60 inbox
= inmailbox
->buf
;
62 outmailbox
= mlx4_alloc_cmd_mailbox(dev
->dev
);
63 if (IS_ERR(outmailbox
)) {
64 mlx4_free_cmd_mailbox(dev
->dev
, inmailbox
);
65 return PTR_ERR(outmailbox
);
68 memcpy(inbox
, in_mad
, 256);
71 * Key check traps can't be generated unless we have in_wc to
72 * tell us where to send the trap.
74 if (ignore_mkey
|| !in_wc
)
76 if (ignore_bkey
|| !in_wc
)
92 memset(inbox
+ 256, 0, 256);
93 ext_info
= inbox
+ 256;
95 ext_info
->my_qpn
= cpu_to_be32(in_wc
->qp
->qp_num
);
96 ext_info
->rqpn
= cpu_to_be32(in_wc
->src_qp
);
97 ext_info
->sl
= in_wc
->sl
<< 4;
98 ext_info
->g_path
= in_wc
->dlid_path_bits
|
99 (in_wc
->wc_flags
& IB_WC_GRH
? 0x80 : 0);
100 ext_info
->pkey
= cpu_to_be16(in_wc
->pkey_index
);
103 memcpy(ext_info
->grh
, in_grh
, 40);
107 in_modifier
|= in_wc
->slid
<< 16;
110 err
= mlx4_cmd_box(dev
->dev
, inmailbox
->dma
, outmailbox
->dma
,
111 in_modifier
, op_modifier
,
112 MLX4_CMD_MAD_IFC
, MLX4_CMD_TIME_CLASS_C
);
115 memcpy(response_mad
, outmailbox
->buf
, 256);
117 mlx4_free_cmd_mailbox(dev
->dev
, inmailbox
);
118 mlx4_free_cmd_mailbox(dev
->dev
, outmailbox
);
123 static void update_sm_ah(struct mlx4_ib_dev
*dev
, u8 port_num
, u16 lid
, u8 sl
)
125 struct ib_ah
*new_ah
;
126 struct ib_ah_attr ah_attr
;
128 if (!dev
->send_agent
[port_num
- 1][0])
131 memset(&ah_attr
, 0, sizeof ah_attr
);
134 ah_attr
.port_num
= port_num
;
136 new_ah
= ib_create_ah(dev
->send_agent
[port_num
- 1][0]->qp
->pd
,
141 spin_lock(&dev
->sm_lock
);
142 if (dev
->sm_ah
[port_num
- 1])
143 ib_destroy_ah(dev
->sm_ah
[port_num
- 1]);
144 dev
->sm_ah
[port_num
- 1] = new_ah
;
145 spin_unlock(&dev
->sm_lock
);
149 * Snoop SM MADs for port info and P_Key table sets, so we can
150 * synthesize LID change and P_Key change events.
152 static void smp_snoop(struct ib_device
*ibdev
, u8 port_num
, struct ib_mad
*mad
,
155 struct ib_event event
;
157 if ((mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_SUBN_LID_ROUTED
||
158 mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE
) &&
159 mad
->mad_hdr
.method
== IB_MGMT_METHOD_SET
) {
160 if (mad
->mad_hdr
.attr_id
== IB_SMP_ATTR_PORT_INFO
) {
161 struct ib_port_info
*pinfo
=
162 (struct ib_port_info
*) ((struct ib_smp
*) mad
)->data
;
163 u16 lid
= be16_to_cpu(pinfo
->lid
);
165 update_sm_ah(to_mdev(ibdev
), port_num
,
166 be16_to_cpu(pinfo
->sm_lid
),
167 pinfo
->neighbormtu_mastersmsl
& 0xf);
169 event
.device
= ibdev
;
170 event
.element
.port_num
= port_num
;
172 if (pinfo
->clientrereg_resv_subnetto
& 0x80) {
173 event
.event
= IB_EVENT_CLIENT_REREGISTER
;
174 ib_dispatch_event(&event
);
177 if (prev_lid
!= lid
) {
178 event
.event
= IB_EVENT_LID_CHANGE
;
179 ib_dispatch_event(&event
);
183 if (mad
->mad_hdr
.attr_id
== IB_SMP_ATTR_PKEY_TABLE
) {
184 event
.device
= ibdev
;
185 event
.event
= IB_EVENT_PKEY_CHANGE
;
186 event
.element
.port_num
= port_num
;
187 ib_dispatch_event(&event
);
192 static void node_desc_override(struct ib_device
*dev
,
195 if ((mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_SUBN_LID_ROUTED
||
196 mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE
) &&
197 mad
->mad_hdr
.method
== IB_MGMT_METHOD_GET_RESP
&&
198 mad
->mad_hdr
.attr_id
== IB_SMP_ATTR_NODE_DESC
) {
199 spin_lock(&to_mdev(dev
)->sm_lock
);
200 memcpy(((struct ib_smp
*) mad
)->data
, dev
->node_desc
, 64);
201 spin_unlock(&to_mdev(dev
)->sm_lock
);
205 static void forward_trap(struct mlx4_ib_dev
*dev
, u8 port_num
, struct ib_mad
*mad
)
207 int qpn
= mad
->mad_hdr
.mgmt_class
!= IB_MGMT_CLASS_SUBN_LID_ROUTED
;
208 struct ib_mad_send_buf
*send_buf
;
209 struct ib_mad_agent
*agent
= dev
->send_agent
[port_num
- 1][qpn
];
213 send_buf
= ib_create_send_mad(agent
, qpn
, 0, 0, IB_MGMT_MAD_HDR
,
214 IB_MGMT_MAD_DATA
, GFP_ATOMIC
);
215 if (IS_ERR(send_buf
))
218 * We rely here on the fact that MLX QPs don't use the
219 * address handle after the send is posted (this is
220 * wrong following the IB spec strictly, but we know
221 * it's OK for our devices).
223 spin_lock(&dev
->sm_lock
);
224 memcpy(send_buf
->mad
, mad
, sizeof *mad
);
225 if ((send_buf
->ah
= dev
->sm_ah
[port_num
- 1]))
226 ret
= ib_post_send_mad(send_buf
, NULL
);
229 spin_unlock(&dev
->sm_lock
);
232 ib_free_send_mad(send_buf
);
236 static int ib_process_mad(struct ib_device
*ibdev
, int mad_flags
, u8 port_num
,
237 struct ib_wc
*in_wc
, struct ib_grh
*in_grh
,
238 struct ib_mad
*in_mad
, struct ib_mad
*out_mad
)
240 u16 slid
, prev_lid
= 0;
242 struct ib_port_attr pattr
;
244 slid
= in_wc
? in_wc
->slid
: be16_to_cpu(IB_LID_PERMISSIVE
);
246 if (in_mad
->mad_hdr
.method
== IB_MGMT_METHOD_TRAP
&& slid
== 0) {
247 forward_trap(to_mdev(ibdev
), port_num
, in_mad
);
248 return IB_MAD_RESULT_SUCCESS
| IB_MAD_RESULT_CONSUMED
;
251 if (in_mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_SUBN_LID_ROUTED
||
252 in_mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE
) {
253 if (in_mad
->mad_hdr
.method
!= IB_MGMT_METHOD_GET
&&
254 in_mad
->mad_hdr
.method
!= IB_MGMT_METHOD_SET
&&
255 in_mad
->mad_hdr
.method
!= IB_MGMT_METHOD_TRAP_REPRESS
)
256 return IB_MAD_RESULT_SUCCESS
;
259 * Don't process SMInfo queries or vendor-specific
260 * MADs -- the SMA can't handle them.
262 if (in_mad
->mad_hdr
.attr_id
== IB_SMP_ATTR_SM_INFO
||
263 ((in_mad
->mad_hdr
.attr_id
& IB_SMP_ATTR_VENDOR_MASK
) ==
264 IB_SMP_ATTR_VENDOR_MASK
))
265 return IB_MAD_RESULT_SUCCESS
;
266 } else if (in_mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_PERF_MGMT
||
267 in_mad
->mad_hdr
.mgmt_class
== MLX4_IB_VENDOR_CLASS1
||
268 in_mad
->mad_hdr
.mgmt_class
== MLX4_IB_VENDOR_CLASS2
||
269 in_mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_CONG_MGMT
) {
270 if (in_mad
->mad_hdr
.method
!= IB_MGMT_METHOD_GET
&&
271 in_mad
->mad_hdr
.method
!= IB_MGMT_METHOD_SET
)
272 return IB_MAD_RESULT_SUCCESS
;
274 return IB_MAD_RESULT_SUCCESS
;
276 if ((in_mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_SUBN_LID_ROUTED
||
277 in_mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE
) &&
278 in_mad
->mad_hdr
.method
== IB_MGMT_METHOD_SET
&&
279 in_mad
->mad_hdr
.attr_id
== IB_SMP_ATTR_PORT_INFO
&&
280 !ib_query_port(ibdev
, port_num
, &pattr
))
281 prev_lid
= pattr
.lid
;
283 err
= mlx4_MAD_IFC(to_mdev(ibdev
),
284 mad_flags
& IB_MAD_IGNORE_MKEY
,
285 mad_flags
& IB_MAD_IGNORE_BKEY
,
286 port_num
, in_wc
, in_grh
, in_mad
, out_mad
);
288 return IB_MAD_RESULT_FAILURE
;
290 if (!out_mad
->mad_hdr
.status
) {
291 smp_snoop(ibdev
, port_num
, in_mad
, prev_lid
);
292 node_desc_override(ibdev
, out_mad
);
295 /* set return bit in status of directed route responses */
296 if (in_mad
->mad_hdr
.mgmt_class
== IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE
)
297 out_mad
->mad_hdr
.status
|= cpu_to_be16(1 << 15);
299 if (in_mad
->mad_hdr
.method
== IB_MGMT_METHOD_TRAP_REPRESS
)
300 /* no response for trap repress */
301 return IB_MAD_RESULT_SUCCESS
| IB_MAD_RESULT_CONSUMED
;
303 return IB_MAD_RESULT_SUCCESS
| IB_MAD_RESULT_REPLY
;
306 static void edit_counter(struct mlx4_counter
*cnt
,
307 struct ib_pma_portcounters
*pma_cnt
)
309 pma_cnt
->port_xmit_data
= cpu_to_be32((be64_to_cpu(cnt
->tx_bytes
)>>2));
310 pma_cnt
->port_rcv_data
= cpu_to_be32((be64_to_cpu(cnt
->rx_bytes
)>>2));
311 pma_cnt
->port_xmit_packets
= cpu_to_be32(be64_to_cpu(cnt
->tx_frames
));
312 pma_cnt
->port_rcv_packets
= cpu_to_be32(be64_to_cpu(cnt
->rx_frames
));
315 static int iboe_process_mad(struct ib_device
*ibdev
, int mad_flags
, u8 port_num
,
316 struct ib_wc
*in_wc
, struct ib_grh
*in_grh
,
317 struct ib_mad
*in_mad
, struct ib_mad
*out_mad
)
319 struct mlx4_cmd_mailbox
*mailbox
;
320 struct mlx4_ib_dev
*dev
= to_mdev(ibdev
);
322 u32 inmod
= dev
->counters
[port_num
- 1] & 0xffff;
325 if (in_mad
->mad_hdr
.mgmt_class
!= IB_MGMT_CLASS_PERF_MGMT
)
328 mailbox
= mlx4_alloc_cmd_mailbox(dev
->dev
);
330 return IB_MAD_RESULT_FAILURE
;
332 err
= mlx4_cmd_box(dev
->dev
, 0, mailbox
->dma
, inmod
, 0,
333 MLX4_CMD_QUERY_IF_STAT
, MLX4_CMD_TIME_CLASS_C
);
335 err
= IB_MAD_RESULT_FAILURE
;
337 memset(out_mad
->data
, 0, sizeof out_mad
->data
);
338 mode
= ((struct mlx4_counter
*)mailbox
->buf
)->counter_mode
;
339 switch (mode
& 0xf) {
341 edit_counter(mailbox
->buf
,
342 (void *)(out_mad
->data
+ 40));
343 err
= IB_MAD_RESULT_SUCCESS
| IB_MAD_RESULT_REPLY
;
346 err
= IB_MAD_RESULT_FAILURE
;
350 mlx4_free_cmd_mailbox(dev
->dev
, mailbox
);
355 int mlx4_ib_process_mad(struct ib_device
*ibdev
, int mad_flags
, u8 port_num
,
356 struct ib_wc
*in_wc
, struct ib_grh
*in_grh
,
357 struct ib_mad
*in_mad
, struct ib_mad
*out_mad
)
359 switch (rdma_port_get_link_layer(ibdev
, port_num
)) {
360 case IB_LINK_LAYER_INFINIBAND
:
361 return ib_process_mad(ibdev
, mad_flags
, port_num
, in_wc
,
362 in_grh
, in_mad
, out_mad
);
363 case IB_LINK_LAYER_ETHERNET
:
364 return iboe_process_mad(ibdev
, mad_flags
, port_num
, in_wc
,
365 in_grh
, in_mad
, out_mad
);
371 static void send_handler(struct ib_mad_agent
*agent
,
372 struct ib_mad_send_wc
*mad_send_wc
)
374 ib_free_send_mad(mad_send_wc
->send_buf
);
377 int mlx4_ib_mad_init(struct mlx4_ib_dev
*dev
)
379 struct ib_mad_agent
*agent
;
382 enum rdma_link_layer ll
;
384 for (p
= 0; p
< dev
->num_ports
; ++p
) {
385 ll
= rdma_port_get_link_layer(&dev
->ib_dev
, p
+ 1);
386 for (q
= 0; q
<= 1; ++q
) {
387 if (ll
== IB_LINK_LAYER_INFINIBAND
) {
388 agent
= ib_register_mad_agent(&dev
->ib_dev
, p
+ 1,
389 q
? IB_QPT_GSI
: IB_QPT_SMI
,
390 NULL
, 0, send_handler
,
393 ret
= PTR_ERR(agent
);
396 dev
->send_agent
[p
][q
] = agent
;
398 dev
->send_agent
[p
][q
] = NULL
;
405 for (p
= 0; p
< dev
->num_ports
; ++p
)
406 for (q
= 0; q
<= 1; ++q
)
407 if (dev
->send_agent
[p
][q
])
408 ib_unregister_mad_agent(dev
->send_agent
[p
][q
]);
413 void mlx4_ib_mad_cleanup(struct mlx4_ib_dev
*dev
)
415 struct ib_mad_agent
*agent
;
418 for (p
= 0; p
< dev
->num_ports
; ++p
) {
419 for (q
= 0; q
<= 1; ++q
) {
420 agent
= dev
->send_agent
[p
][q
];
422 dev
->send_agent
[p
][q
] = NULL
;
423 ib_unregister_mad_agent(agent
);
428 ib_destroy_ah(dev
->sm_ah
[p
]);