4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
26 #include <libdevinfo.h>
27 #include <sys/types.h>
37 #include <net/if_types.h>
38 #include <net/if_dl.h>
40 #include <sys/ib/ib_types.h>
41 #include <sys/ibpart.h>
42 #include <libdllink.h>
45 #include <libdladm_impl.h>
48 * IP over IB administration API; see PSARC/2010/085
54 dladm_status_t
dladm_part_create(dladm_handle_t
, datalink_id_t
, ib_pkey_t
,
55 uint32_t, char *, datalink_id_t
*, dladm_arg_list_t
*);
56 static dladm_status_t
i_dladm_part_create(dladm_handle_t
,
58 static dladm_status_t
dladm_part_persist_conf(dladm_handle_t
, const char *,
60 static dladm_status_t
i_dladm_part_delete(dladm_handle_t
, datalink_id_t
);
61 dladm_status_t
dladm_part_delete(dladm_handle_t
, datalink_id_t
, int);
62 static int i_dladm_part_up(dladm_handle_t
, datalink_id_t
, void *);
63 dladm_status_t
dladm_part_up(dladm_handle_t
, datalink_id_t
, uint32_t);
66 * Convert a error status returned by the IP over IB kernel driver to a
70 dladm_ib_ioctl_err2status(int err
)
74 return (DLADM_STATUS_OK
);
75 case IBD_INVALID_PORT_INST
:
76 return (DLADM_STATUS_INVALID_PORT_INSTANCE
);
77 case IBD_PORT_IS_DOWN
:
78 return (DLADM_STATUS_PORT_IS_DOWN
);
79 case IBD_PKEY_NOT_PRESENT
:
80 return (DLADM_STATUS_PKEY_NOT_PRESENT
);
81 case IBD_PARTITION_EXISTS
:
82 return (DLADM_STATUS_PARTITION_EXISTS
);
83 case IBD_INVALID_PKEY
:
84 return (DLADM_STATUS_INVALID_PKEY
);
85 case IBD_NO_HW_RESOURCE
:
86 return (DLADM_STATUS_NO_IB_HW_RESOURCE
);
87 case IBD_INVALID_PKEY_TBL_SIZE
:
88 return (DLADM_STATUS_INVALID_PKEY_TBL_SIZE
);
90 return (DLADM_STATUS_FAILED
);
95 i_dladm_ib_ioctl(dladm_handle_t handle
, int ioccmd
, ibd_ioctl_t
*iocp
)
97 if (ioctl(dladm_dld_fd(handle
), ioccmd
, iocp
) == 0)
98 return (DLADM_STATUS_OK
);
100 if (iocp
->ioc_status
== 0)
101 return (dladm_errno2status(errno
));
103 return (dladm_ib_ioctl_err2status(iocp
->ioc_status
));
107 * Get the active configuration information for the partition given by
110 static dladm_status_t
111 i_dladm_part_info_active(dladm_handle_t handle
, datalink_id_t linkid
,
112 dladm_part_attr_t
*attrp
)
115 dladm_status_t status
= DLADM_STATUS_OK
;
117 bzero(&ioc
, sizeof (ioc
));
118 bzero(attrp
, sizeof (*attrp
));
120 * The ioc_linkid here will contain the data link id of the IB partition
123 ioc
.ibdioc
.ioc_linkid
= linkid
;
124 ioc
.ibdioc
.ioc_info_cmd
= IBD_INFO_CMD_IBPART
;
126 status
= i_dladm_ib_ioctl(handle
, IBD_INFO_IBPART
, (ibd_ioctl_t
*)&ioc
);
127 if (status
!= DLADM_STATUS_OK
)
131 * On return from the ioctl ioc_linkid field contains the IB port's
134 attrp
->dia_physlinkid
= ioc
.ibdioc
.ioc_linkid
;
135 attrp
->dia_partlinkid
= ioc
.ioc_partid
;
136 attrp
->dia_pkey
= ioc
.ioc_pkey
;
137 attrp
->dia_portnum
= ioc
.ibdioc
.ioc_portnum
;
138 attrp
->dia_hca_guid
= ioc
.ibdioc
.ioc_hcaguid
;
139 attrp
->dia_port_guid
= ioc
.ibdioc
.ioc_portguid
;
140 attrp
->dia_instance
= ioc
.ibdioc
.ioc_port_inst
;
143 * If the IP over IB driver reports that this partition was created
144 * forcibly, then set the force create flag.
146 if (ioc
.ioc_force_create
)
147 attrp
->dia_flags
|= DLADM_PART_FORCE_CREATE
;
154 * Get the configuration information about the IB partition 'linkid' from the
155 * persistent configuration.
157 static dladm_status_t
158 i_dladm_part_info_persist(dladm_handle_t handle
, datalink_id_t linkid
,
159 dladm_part_attr_t
*attrp
)
162 dladm_status_t status
;
163 char linkover
[MAXLINKNAMELEN
];
164 datalink_class_t
class;
165 boolean_t force
= B_FALSE
;
167 /* Get the IB partition's datalink ID */
168 if ((status
= dladm_datalink_id2info(handle
, linkid
, NULL
, &class,
169 NULL
, NULL
, 0)) != DLADM_STATUS_OK
)
172 bzero(attrp
, sizeof (*attrp
));
173 attrp
->dia_partlinkid
= linkid
;
174 if ((status
= dladm_getsnap_conf(handle
, linkid
, &conf
)) !=
179 * Get the name of the IB Phys link over which IB partition was
182 status
= dladm_get_conf_field(handle
, conf
, FLINKOVER
, linkover
,
184 if (status
!= DLADM_STATUS_OK
) {
185 attrp
->dia_physlinkid
= DATALINK_INVALID_LINKID
;
188 /* Get the IB Phys link's datalink ID */
189 if ((status
= dladm_name2info(handle
, linkover
,
190 &attrp
->dia_physlinkid
, NULL
, NULL
, NULL
)) !=
195 /* Get the IB partition's P_Key */
196 status
= dladm_get_conf_field(handle
, conf
, FPORTPKEY
,
197 &attrp
->dia_pkey
, sizeof (uint64_t));
198 if (status
!= DLADM_STATUS_OK
)
201 if (class != DATALINK_CLASS_PART
) {
202 status
= DLADM_STATUS_BADARG
;
207 * If the FFORCE field is set in the persistent configuration database
208 * set the force create flag in the partition attributes.
210 status
= dladm_get_conf_field(handle
, conf
, FFORCE
, &force
,
212 if (status
!= DLADM_STATUS_OK
) {
213 if (status
!= DLADM_STATUS_NOTFOUND
)
215 } else if (force
== B_TRUE
) {
216 attrp
->dia_flags
|= DLADM_PART_FORCE_CREATE
;
219 status
= DLADM_STATUS_OK
;
221 dladm_destroy_conf(handle
, conf
);
226 * Get the configuration information for the IB partition given by the datalink
227 * ID 'linkid'. Based on the 'flags' field the information is either from the
228 * active system (DLADM_OPT_ACTIVE) or from the persistent configuration
232 dladm_part_info(dladm_handle_t handle
, datalink_id_t linkid
,
233 dladm_part_attr_t
*attrp
, uint32_t flags
)
235 if (flags
== DLADM_OPT_ACTIVE
)
236 return (i_dladm_part_info_active(handle
, linkid
, attrp
));
237 else if (flags
== DLADM_OPT_PERSIST
)
238 return (i_dladm_part_info_persist(handle
, linkid
, attrp
));
240 return (DLADM_STATUS_BADARG
);
244 * Get the configuration information for the IB Phys link given by the datalink
249 dladm_ib_info(dladm_handle_t handle
, datalink_id_t linkid
,
250 dladm_ib_attr_t
*attrp
, uint32_t flags
)
254 dladm_phys_attr_t dpa
;
255 dladm_status_t status
= DLADM_STATUS_OK
;
258 * We need to get the device name of the IB Phys link to get the
259 * correct instance number of the IP over IB driver instance.
261 if (dladm_phys_info(handle
, linkid
, &dpa
, DLADM_OPT_ACTIVE
)
263 return (DLADM_STATUS_BADARG
);
266 * Get the instance number of the IP over IB driver instance which
267 * represents this IB Phys link.
269 if (dladm_parselink(dpa
.dp_dev
, NULL
, &instance
) != DLADM_STATUS_OK
)
270 return (DLADM_STATUS_FAILED
);
272 bzero(&ioc
, sizeof (ioc
));
274 * The ioc_linkid here will contain IB port linkid here. We make the
275 * first ioctl call to get the P_Key table size for this HCA port.
277 ioc
.ibdioc
.ioc_linkid
= linkid
;
278 ioc
.ibdioc
.ioc_info_cmd
= IBD_INFO_CMD_PKEYTBLSZ
;
279 ioc
.ioc_pkey_tbl_sz
= 0;
280 ioc
.ibdioc
.ioc_port_inst
= instance
;
282 status
= i_dladm_ib_ioctl(handle
, IBD_INFO_IBPART
, (ibd_ioctl_t
*)&ioc
);
283 if (status
!= DLADM_STATUS_OK
)
287 * Now allocate the memory for the P_Key table based on the table size
288 * return by the ioctl.
290 ioc
.ioc_pkeys
= calloc(sizeof (ib_pkey_t
), ioc
.ioc_pkey_tbl_sz
);
291 if (ioc
.ioc_pkeys
== NULL
) {
292 status
= dladm_errno2status(errno
);
297 * Call the ioctl again to get the P_Key table and other IB Phys link
300 ioc
.ibdioc
.ioc_linkid
= linkid
;
301 ioc
.ibdioc
.ioc_port_inst
= instance
;
302 ioc
.ibdioc
.ioc_info_cmd
= IBD_INFO_CMD_IBPORT
;
304 status
= i_dladm_ib_ioctl(handle
, IBD_INFO_IBPART
, (ibd_ioctl_t
*)&ioc
);
305 if (status
!= DLADM_STATUS_OK
)
308 attrp
->dia_physlinkid
= ioc
.ibdioc
.ioc_linkid
;
309 attrp
->dia_portnum
= ioc
.ibdioc
.ioc_portnum
;
310 attrp
->dia_port_pkey_tbl_sz
= ioc
.ioc_pkey_tbl_sz
;
311 attrp
->dia_port_pkeys
= ioc
.ioc_pkeys
;
312 attrp
->dia_hca_guid
= ioc
.ibdioc
.ioc_hcaguid
;
313 attrp
->dia_port_guid
= ioc
.ibdioc
.ioc_portguid
;
314 attrp
->dia_instance
= ioc
.ibdioc
.ioc_port_inst
;
322 * Free the memory allocated for the IB HCA port's P_Key table by
323 * dladm_ib_info library call.
326 dladm_free_ib_info(dladm_ib_attr_t
*attr
)
329 free(attr
->dia_port_pkeys
);
333 * Call into the IP over IB driver to create a partition object.
335 static dladm_status_t
336 i_dladm_part_create(dladm_handle_t handle
, dladm_part_attr_t
*pattr
)
340 bzero(&ioc
, sizeof (ioc
));
342 /* IB Physical datalink ID */
343 ioc
.ibdioc
.ioc_linkid
= pattr
->dia_physlinkid
;
344 /* IB Partition datalink ID */
345 ioc
.ioc_partid
= pattr
->dia_partlinkid
;
346 ioc
.ioc_pkey
= pattr
->dia_pkey
;
347 ioc
.ibdioc
.ioc_port_inst
= pattr
->dia_instance
;
348 ioc
.ioc_force_create
= ((pattr
->dia_flags
& DLADM_OPT_FORCE
)
351 return (i_dladm_ib_ioctl(handle
, IBD_CREATE_IBPART
, &ioc
.ibdioc
));
355 * Create an entry in the dladm persistent configuration database for the
356 * partition specified by pattr.
359 dladm_part_persist_conf(dladm_handle_t handle
, const char *pname
,
360 dladm_part_attr_t
*pattr
)
364 dladm_status_t status
;
365 char linkover
[MAXLINKNAMELEN
];
368 status
= dladm_create_conf(handle
, pname
, pattr
->dia_partlinkid
,
369 DATALINK_CLASS_PART
, DL_IB
, &conf
);
370 if (status
!= DLADM_STATUS_OK
)
374 * Get the name of the IB Phys link over which this partition was
377 status
= dladm_datalink_id2info(handle
, pattr
->dia_physlinkid
,
378 NULL
, NULL
, NULL
, linkover
, sizeof (linkover
));
379 if (status
!= DLADM_STATUS_OK
)
382 /* Store IB Phys link name (linkover) */
383 status
= dladm_set_conf_field(handle
, conf
, FLINKOVER
, DLADM_TYPE_STR
,
385 if (status
!= DLADM_STATUS_OK
)
388 u64
= pattr
->dia_pkey
;
390 /* Store the IB Partitions P_Key */
391 status
= dladm_set_conf_field(handle
, conf
, FPORTPKEY
,
392 DLADM_TYPE_UINT64
, &u64
);
393 if (status
!= DLADM_STATUS_OK
)
396 if (pattr
->dia_flags
& DLADM_OPT_FORCE
) {
397 boolean_t force
= B_TRUE
;
398 /* Store the force create flag. */
399 status
= dladm_set_conf_field(handle
, conf
, FFORCE
,
400 DLADM_TYPE_BOOLEAN
, &force
);
401 if (status
!= DLADM_STATUS_OK
)
405 status
= dladm_write_conf(handle
, conf
);
406 if (status
!= DLADM_STATUS_OK
)
409 dladm_destroy_conf(handle
, conf
);
415 * Create a new IB Partition datalink of name 'pname' over the IB Physical link
416 * given in 'physlinkid' with the P_key 'pkey' and return the datalink ID in
417 * 'partlinkid'. If the 'force' option is set in the 'flags' argument, the
418 * partition will be created even if the P_Key 'pkey' does not exist or if the
419 * HCA port represented by the IB Phys link is down. If the 'temporary' flag is
420 * set, then the configuration information is not added to the persistent
424 dladm_part_create(dladm_handle_t handle
, datalink_id_t physlinkid
,
425 ib_pkey_t pkey
, uint32_t flags
, char *pname
, datalink_id_t
*partlinkid
,
426 dladm_arg_list_t
*proplist
)
429 dladm_status_t status
;
431 boolean_t part_created
= B_FALSE
;
432 boolean_t conf_set
= B_FALSE
;
433 dladm_phys_attr_t dpa
;
434 dladm_part_attr_t pattr
;
436 pattr
.dia_pkey
= pkey
;
437 pattr
.dia_physlinkid
= physlinkid
; /* IB Phys link's datalink id */
438 pattr
.dia_flags
= flags
;
440 flags
&= ~DLADM_OPT_FORCE
;
443 * Check whether the PKEY is valid. If not, return immediately
444 * Only full members are allowed as per the IPoIB specification
446 if (pattr
.dia_pkey
<= IB_PKEY_INVALID_FULL
)
447 return (DLADM_STATUS_INVALID_PKEY
);
450 * Get the media type of the Phys link datalink ID provided and
451 * make sure that it is Infiniband media DL_IB)
453 if ((status
= dladm_datalink_id2info(handle
, pattr
.dia_physlinkid
, NULL
,
454 NULL
, &media
, NULL
, 0)) != DLADM_STATUS_OK
)
458 return (dladm_errno2status(ENOTSUP
));
461 * Get the instance number of the IP over IB driver instance which the
462 * IB Phys link 'physlinkid' over which we will be creating our IB
465 if ((status
= dladm_phys_info(handle
, pattr
.dia_physlinkid
, &dpa
,
466 DLADM_OPT_ACTIVE
)) != DLADM_STATUS_OK
)
469 if (dladm_parselink(dpa
.dp_dev
, NULL
, (uint_t
*)&pattr
.dia_instance
) !=
471 return (DLADM_STATUS_FAILED
);
474 if ((status
= dladm_create_datalink_id(handle
, pname
,
475 DATALINK_CLASS_PART
, DL_IB
, flags
, &pattr
.dia_partlinkid
)) !=
480 * Create the IB partition object.
482 status
= i_dladm_part_create(handle
, &pattr
);
483 if (status
!= DLADM_STATUS_OK
)
486 part_created
= B_TRUE
;
489 * If the persist flag is set then write this partition information
490 * to the persistent configuration.
492 if (pattr
.dia_flags
& DLADM_OPT_PERSIST
) {
493 status
= dladm_part_persist_conf(handle
, pname
, &pattr
);
494 if (status
!= DLADM_STATUS_OK
)
500 * If the name-value pair list of properties were provided set those
501 * properties over the datalink.
503 if (proplist
!= NULL
) {
504 for (i
= 0; i
< proplist
->al_count
; i
++) {
505 dladm_arg_info_t
*aip
= &proplist
->al_info
[i
];
507 status
= dladm_set_linkprop(handle
,
508 pattr
.dia_partlinkid
, aip
->ai_name
, aip
->ai_val
,
509 aip
->ai_count
, pattr
.dia_flags
);
510 if (status
!= DLADM_STATUS_OK
)
515 if (status
!= DLADM_STATUS_OK
) {
517 (void) dladm_remove_conf(handle
, pattr
.dia_partlinkid
);
519 (void) i_dladm_part_delete(handle
,
520 pattr
.dia_partlinkid
);
521 (void) dladm_destroy_datalink_id(handle
, pattr
.dia_partlinkid
,
525 if (partlinkid
!= NULL
)
526 *partlinkid
= pattr
.dia_partlinkid
;
532 * Call into the IP over IB driver to delete the IB partition and free up all
533 * the resources allocated for it.
535 static dladm_status_t
536 i_dladm_part_delete(dladm_handle_t handle
, datalink_id_t partid
)
540 bzero(&ioc
, sizeof (ioc
));
541 ioc
.ioc_partid
= partid
;
542 return (i_dladm_ib_ioctl(handle
, IBD_DELETE_IBPART
, &ioc
.ibdioc
));
546 * Delete an IB partition if 'flags' contains the active flag. Update the
547 * persistent configuration if 'flags' contains the persist flag.
550 dladm_part_delete(dladm_handle_t handle
, datalink_id_t partid
, int flags
)
552 dladm_status_t status
= DLADM_STATUS_OK
;
553 datalink_class_t
class;
556 return (DLADM_STATUS_BADARG
);
559 * Make sure that the datalinkid provided is an IB partition class
562 if ((dladm_datalink_id2info(handle
, partid
, NULL
, &class, NULL
, NULL
, 0)
564 return (DLADM_STATUS_BADARG
);
566 if (class != DATALINK_CLASS_PART
)
567 return (DLADM_STATUS_BADARG
);
569 if ((flags
& DLADM_OPT_ACTIVE
) != 0) {
570 status
= i_dladm_part_delete(handle
, partid
);
571 if (status
== DLADM_STATUS_OK
) {
572 (void) dladm_set_linkprop(handle
, partid
, NULL
, NULL
, 0,
574 (void) dladm_destroy_datalink_id(handle
, partid
,
576 } else if (status
!= DLADM_STATUS_NOTFOUND
||
577 !(flags
& DLADM_OPT_PERSIST
)) {
582 if ((flags
& DLADM_OPT_PERSIST
) != 0) {
583 dladm_status_t db_status
;
584 db_status
= dladm_remove_conf(handle
, partid
);
587 * A partition could have been temporarily deleted in which
588 * case the delete of the active partition above would have
589 * failed. In that case, we update the status to be returned
590 * to that of the status returned for deleting the persistent
593 if (status
== DLADM_STATUS_NOTFOUND
)
596 (void) dladm_destroy_datalink_id(handle
, partid
,
604 * Call into the IP over IB driver to create the active instances of one or all
605 * IB partitions present in the persistent configuration.
609 i_dladm_part_up(dladm_handle_t handle
, datalink_id_t plinkid
, void *arg
)
612 datalink_id_t linkid
;
615 char linkover
[MAXLINKNAMELEN
];
616 dladm_status_t status
;
617 dladm_phys_attr_t dpa
;
618 dladm_part_attr_t pattr
;
621 * plinkid is the IB partition datalink's ID. Get an handle to the
622 * persistent configuration entry for this datalink ID. If this datalink
623 * ID is not present in the persistent configuration return.
625 if ((status
= dladm_getsnap_conf(handle
, plinkid
, &conf
)) !=
630 * Get the name of the IB Phys link over which this partition was
633 status
= dladm_get_conf_field(handle
, conf
, FLINKOVER
, linkover
,
635 if (status
!= DLADM_STATUS_OK
)
638 if ((status
= dladm_name2info(handle
, linkover
, &linkid
, NULL
, NULL
,
639 NULL
)) != DLADM_STATUS_OK
)
643 * Get the phys attribute of the IB Phys link to get the device name
644 * associated with the phys link. We need this to get the IP over IB
645 * driver instance number.
647 if (dladm_phys_info(handle
, linkid
, &dpa
, DLADM_OPT_ACTIVE
)
651 /* Get the IB partition's P_key */
652 status
= dladm_get_conf_field(handle
, conf
, FPORTPKEY
, &u64
,
654 if (status
!= DLADM_STATUS_OK
)
657 pkey
= (ib_pkey_t
)u64
;
660 * We always set the force flag during dladm_part_up because we want
661 * the partition creation to succeed even if the IB HCA port over which
662 * the partition is being created is still down. Since dladm_part_up
663 * is usually invoked during early boot sequence, it is possible under
664 * some IB subnet configurations for dladm_up_part to be called before
665 * the IB link negotiation is completed and port state is set to active
666 * and P_Key table is updated.
668 pattr
.dia_flags
= DLADM_OPT_FORCE
| DLADM_OPT_ACTIVE
|
670 /* IB Phys link's datalink ID. */
671 pattr
.dia_physlinkid
= linkid
;
672 /* IB Partition's datalink ID. */
673 pattr
.dia_partlinkid
= plinkid
;
674 pattr
.dia_pkey
= pkey
;
675 if (dladm_parselink(dpa
.dp_dev
, NULL
, (uint_t
*)&pattr
.dia_instance
) !=
677 return (DLADM_WALK_CONTINUE
);
679 /* Create the active IB Partition object. */
680 if (i_dladm_part_create(handle
, &pattr
) == DLADM_STATUS_OK
&&
681 dladm_up_datalink_id(handle
, plinkid
) != DLADM_STATUS_OK
)
682 (void) i_dladm_part_delete(handle
, linkid
);
685 dladm_destroy_conf(handle
, conf
);
686 return (DLADM_WALK_CONTINUE
);
690 * Bring up one or all IB partition(s) present in the persistent configuration
691 * database. If we need to bring up one IB Partition, its datalink ID is
692 * provided in 'linkid'.
696 dladm_part_up(dladm_handle_t handle
, datalink_id_t linkid
, uint32_t flags
)
698 dladm_status_t status
= DLADM_STATUS_OK
;
700 if (linkid
== DATALINK_ALL_LINKID
) {
701 (void) dladm_walk_datalink_id(i_dladm_part_up
, handle
,
702 &status
, DATALINK_CLASS_PART
, DATALINK_ANY_MEDIATYPE
,
704 return (DLADM_STATUS_OK
);
706 (void) i_dladm_part_up(handle
, linkid
, &status
);