usr/src/uts/common/io/ib/clients/rdsv3/ib.c

   1 /*
   2  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
   3  */
   4
   5 /*
   6  * This file contains code imported from the OFED rds source file ib.c
   7  * Oracle elects to have and use the contents of ib.c under and governed
   8  * by the OpenIB.org BSD license (see below for full license text). However,
   9  * the following notice accompanied the original version of this file:
  10  */
  11
  12 /*
  13  * Copyright (c) 2006 Oracle.  All rights reserved.
  14  *
  15  * This software is available to you under a choice of one of two
  16  * licenses.  You may choose to be licensed under the terms of the GNU
  17  * General Public License (GPL) Version 2, available from the file
  18  * COPYING in the main directory of this source tree, or the
  19  * OpenIB.org BSD license below:
  20  *
  21  *     Redistribution and use in source and binary forms, with or
  22  *     without modification, are permitted provided that the following
  23  *     conditions are met:
  24  *
  25  *      - Redistributions of source code must retain the above
  26  *        copyright notice, this list of conditions and the following
  27  *        disclaimer.
  28  *
  29  *      - Redistributions in binary form must reproduce the above
  30  *        copyright notice, this list of conditions and the following
  31  *        disclaimer in the documentation and/or other materials
  32  *        provided with the distribution.
  33  *
  34  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  35  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  36  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  37  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  38  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  39  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  40  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  41  * SOFTWARE.
  42  *
  43  */
  44 #include <sys/sysmacros.h>
  45 #include <sys/rds.h>
  46
  47 #include <sys/ib/ibtl/ibti.h>
  48 #include <sys/ib/clients/rdsv3/rdsv3.h>
  49 #include <sys/ib/clients/rdsv3/ib.h>
  50 #include <sys/ib/clients/rdsv3/rdsv3_debug.h>
  51
  52 unsigned int rdsv3_ib_retry_count = RDSV3_IB_DEFAULT_RETRY_COUNT;
  53
  54 struct list     rdsv3_ib_devices;
  55
  56 /* NOTE: if also grabbing ibdev lock, grab this first */
  57 kmutex_t ib_nodev_conns_lock;
  58 list_t ib_nodev_conns;
  59
  60 extern int rdsv3_ib_frag_constructor(void *buf, void *arg, int kmflags);
  61 extern void rdsv3_ib_frag_destructor(void *buf, void *arg);
  62
  63 void
  64 rdsv3_ib_add_one(ib_device_t *device)
  65 {
  66         struct rdsv3_ib_device *rds_ibdev;
  67         ibt_hca_attr_t *dev_attr;
  68         char name[64];
  69
  70         RDSV3_DPRINTF2("rdsv3_ib_add_one", "device: %p", device);
  71
  72         /* Only handle IB (no iWARP) devices */
  73         if (device->node_type != RDMA_NODE_IB_CA)
  74                 return;
  75
  76         dev_attr = (ibt_hca_attr_t *)kmem_alloc(sizeof (*dev_attr),
  77             KM_NOSLEEP);
  78         if (!dev_attr)
  79                 return;
  80
  81         if (ibt_query_hca(ib_get_ibt_hca_hdl(device), dev_attr)) {
  82                 RDSV3_DPRINTF2("rdsv3_ib_add_one",
  83                     "Query device failed for %s", device->name);
  84                 goto free_attr;
  85         }
  86
  87         /* We depend on Reserved Lkey */
  88         if (!(dev_attr->hca_flags2 & IBT_HCA2_RES_LKEY)) {
  89                 RDSV3_DPRINTF2("rdsv3_ib_add_one",
  90                     "Reserved Lkey support is required: %s",
  91                     device->name);
  92                 goto free_attr;
  93         }
  94
  95         rds_ibdev = kmem_zalloc(sizeof (*rds_ibdev), KM_NOSLEEP);
  96         if (!rds_ibdev)
  97                 goto free_attr;
  98
  99         rds_ibdev->ibt_hca_hdl = ib_get_ibt_hca_hdl(device);
 100         rds_ibdev->hca_attr =  *dev_attr;
 101
 102         rw_init(&rds_ibdev->rwlock, NULL, RW_DRIVER, NULL);
 103         mutex_init(&rds_ibdev->spinlock, NULL, MUTEX_DRIVER, NULL);
 104
 105         rds_ibdev->max_wrs = dev_attr->hca_max_chan_sz;
 106         rds_ibdev->max_sge = min(dev_attr->hca_max_sgl, RDSV3_IB_MAX_SGE);
 107
 108         rds_ibdev->max_initiator_depth = (uint_t)dev_attr->hca_max_rdma_in_qp;
 109         rds_ibdev->max_responder_resources =
 110             (uint_t)dev_attr->hca_max_rdma_in_qp;
 111
 112         rds_ibdev->dev = device;
 113         rds_ibdev->pd = ib_alloc_pd(device);
 114         if (IS_ERR(rds_ibdev->pd))
 115                 goto free_dev;
 116
 117         if (rdsv3_ib_create_mr_pool(rds_ibdev) != 0) {
 118                 goto free_dev;
 119         }
 120
 121         if (rdsv3_ib_create_inc_pool(rds_ibdev) != 0) {
 122                 rdsv3_ib_destroy_mr_pool(rds_ibdev);
 123                 goto free_dev;
 124         }
 125
 126         (void) snprintf(name, 64, "RDSV3_IB_FRAG_%llx",
 127             (longlong_t)htonll(dev_attr->hca_node_guid));
 128         rds_ibdev->ib_frag_slab = kmem_cache_create(name,
 129             sizeof (struct rdsv3_page_frag), 0, rdsv3_ib_frag_constructor,
 130             rdsv3_ib_frag_destructor, NULL, (void *)rds_ibdev, NULL, 0);
 131         if (rds_ibdev->ib_frag_slab == NULL) {
 132                 RDSV3_DPRINTF2("rdsv3_ib_add_one",
 133                     "kmem_cache_create for ib_frag_slab failed for device: %s",
 134                     device->name);
 135                 rdsv3_ib_destroy_mr_pool(rds_ibdev);
 136                 rdsv3_ib_destroy_inc_pool(rds_ibdev);
 137                 goto free_dev;
 138         }
 139
 140         rds_ibdev->aft_hcagp = rdsv3_af_grp_create(rds_ibdev->ibt_hca_hdl,
 141             (uint64_t)rds_ibdev->hca_attr.hca_node_guid);
 142         if (rds_ibdev->aft_hcagp == NULL) {
 143                 rdsv3_ib_destroy_mr_pool(rds_ibdev);
 144                 rdsv3_ib_destroy_inc_pool(rds_ibdev);
 145                 kmem_cache_destroy(rds_ibdev->ib_frag_slab);
 146                 goto free_dev;
 147         }
 148         rds_ibdev->fmr_soft_cq = rdsv3_af_thr_create(rdsv3_ib_drain_mrlist_fn,
 149             (void *)rds_ibdev->fmr_pool, SCQ_HCA_BIND_CPU,
 150             rds_ibdev->aft_hcagp);
 151         if (rds_ibdev->fmr_soft_cq == NULL) {
 152                 rdsv3_af_grp_destroy(rds_ibdev->aft_hcagp);
 153                 rdsv3_ib_destroy_mr_pool(rds_ibdev);
 154                 rdsv3_ib_destroy_inc_pool(rds_ibdev);
 155                 kmem_cache_destroy(rds_ibdev->ib_frag_slab);
 156                 goto free_dev;
 157         }
 158
 159         rds_ibdev->inc_soft_cq = rdsv3_af_thr_create(rdsv3_ib_drain_inclist,
 160             (void *)rds_ibdev->inc_pool, SCQ_HCA_BIND_CPU,
 161             rds_ibdev->aft_hcagp);
 162         if (rds_ibdev->inc_soft_cq == NULL) {
 163                 rdsv3_af_thr_destroy(rds_ibdev->fmr_soft_cq);
 164                 rdsv3_af_grp_destroy(rds_ibdev->aft_hcagp);
 165                 rdsv3_ib_destroy_mr_pool(rds_ibdev);
 166                 rdsv3_ib_destroy_inc_pool(rds_ibdev);
 167                 kmem_cache_destroy(rds_ibdev->ib_frag_slab);
 168                 goto free_dev;
 169         }
 170
 171         list_create(&rds_ibdev->ipaddr_list, sizeof (struct rdsv3_ib_ipaddr),
 172             offsetof(struct rdsv3_ib_ipaddr, list));
 173         list_create(&rds_ibdev->conn_list, sizeof (struct rdsv3_ib_connection),
 174             offsetof(struct rdsv3_ib_connection, ib_node));
 175
 176         list_insert_tail(&rdsv3_ib_devices, rds_ibdev);
 177
 178         ib_set_client_data(device, &rdsv3_ib_client, rds_ibdev);
 179
 180         RDSV3_DPRINTF2("rdsv3_ib_add_one", "Return: device: %p", device);
 181
 182         goto free_attr;
 183
 184 err_pd:
 185         (void) ib_dealloc_pd(rds_ibdev->pd);
 186 free_dev:
 187         mutex_destroy(&rds_ibdev->spinlock);
 188         rw_destroy(&rds_ibdev->rwlock);
 189         kmem_free(rds_ibdev, sizeof (*rds_ibdev));
 190 free_attr:
 191         kmem_free(dev_attr, sizeof (*dev_attr));
 192 }
 193
 194 void
 195 rdsv3_ib_remove_one(struct ib_device *device)
 196 {
 197         struct rdsv3_ib_device *rds_ibdev;
 198         struct rdsv3_ib_ipaddr *i_ipaddr, *i_next;
 199
 200         RDSV3_DPRINTF2("rdsv3_ib_remove_one", "device: %p", device);
 201
 202         rds_ibdev = ib_get_client_data(device, &rdsv3_ib_client);
 203         if (!rds_ibdev)
 204                 return;
 205
 206         RDSV3_FOR_EACH_LIST_NODE_SAFE(i_ipaddr, i_next, &rds_ibdev->ipaddr_list,
 207             list) {
 208                 list_remove_node(&i_ipaddr->list);
 209                 kmem_free(i_ipaddr, sizeof (*i_ipaddr));
 210         }
 211
 212         rdsv3_ib_destroy_conns(rds_ibdev);
 213
 214         if (rds_ibdev->fmr_soft_cq)
 215                 rdsv3_af_thr_destroy(rds_ibdev->fmr_soft_cq);
 216         if (rds_ibdev->inc_soft_cq)
 217                 rdsv3_af_thr_destroy(rds_ibdev->inc_soft_cq);
 218
 219         rdsv3_ib_destroy_mr_pool(rds_ibdev);
 220         rdsv3_ib_destroy_inc_pool(rds_ibdev);
 221
 222         kmem_cache_destroy(rds_ibdev->ib_frag_slab);
 223
 224         rdsv3_af_grp_destroy(rds_ibdev->aft_hcagp);
 225
 226 #if 0
 227         while (ib_dealloc_pd(rds_ibdev->pd)) {
 228 #ifndef __lock_lint
 229                 RDSV3_DPRINTF5("rdsv3_ib_remove_one",
 230                     "%s-%d Failed to dealloc pd %p",
 231                     __func__, __LINE__, rds_ibdev->pd);
 232 #endif
 233                 delay(drv_usectohz(1000));
 234         }
 235 #else
 236         if (ib_dealloc_pd(rds_ibdev->pd)) {
 237 #ifndef __lock_lint
 238                 RDSV3_DPRINTF2("rdsv3_ib_remove_one",
 239                     "Failed to dealloc pd %p\n", rds_ibdev->pd);
 240 #endif
 241         }
 242 #endif
 243
 244         list_destroy(&rds_ibdev->ipaddr_list);
 245         list_destroy(&rds_ibdev->conn_list);
 246         list_remove_node(&rds_ibdev->list);
 247         mutex_destroy(&rds_ibdev->spinlock);
 248         rw_destroy(&rds_ibdev->rwlock);
 249         kmem_free(rds_ibdev, sizeof (*rds_ibdev));
 250
 251         RDSV3_DPRINTF2("rdsv3_ib_remove_one", "Return: device: %p", device);
 252 }
 253
 254 #ifndef __lock_lint
 255 struct ib_client rdsv3_ib_client = {
 256         .name           = "rdsv3_ib",
 257         .add            = rdsv3_ib_add_one,
 258         .remove         = rdsv3_ib_remove_one,
 259         .clnt_hdl       = NULL,
 260         .state          = IB_CLNT_UNINITIALIZED
 261 };
 262 #else
 263 struct ib_client rdsv3_ib_client = {
 264         "rdsv3_ib",
 265         rdsv3_ib_add_one,
 266         rdsv3_ib_remove_one,
 267         NULL,
 268         NULL,
 269         IB_CLNT_UNINITIALIZED
 270 };
 271 #endif
 272
 273 static int
 274 rds_ib_conn_info_visitor(struct rdsv3_connection *conn,
 275     void *buffer)
 276 {
 277         struct rds_info_rdma_connection *iinfo = buffer;
 278         struct rdsv3_ib_connection *ic;
 279
 280         RDSV3_DPRINTF4("rds_ib_conn_info_visitor", "conn: %p buffer: %p",
 281             conn, buffer);
 282
 283         /* We will only ever look at IB transports */
 284         if (conn->c_trans != &rdsv3_ib_transport)
 285                 return (0);
 286
 287         iinfo->src_addr = conn->c_laddr;
 288         iinfo->dst_addr = conn->c_faddr;
 289
 290         (void) memset(&iinfo->src_gid, 0, sizeof (iinfo->src_gid));
 291         (void) memset(&iinfo->dst_gid, 0, sizeof (iinfo->dst_gid));
 292         if (rdsv3_conn_state(conn) == RDSV3_CONN_UP) {
 293                 struct rdsv3_ib_device *rds_ibdev;
 294                 struct rdma_dev_addr *dev_addr;
 295
 296                 ic = conn->c_transport_data;
 297                 dev_addr = &ic->i_cm_id->route.addr.dev_addr;
 298
 299                 ib_addr_get_sgid(dev_addr, (union ib_gid *)&iinfo->src_gid);
 300                 ib_addr_get_dgid(dev_addr, (union ib_gid *)&iinfo->dst_gid);
 301
 302                 rds_ibdev = ib_get_client_data(ic->i_cm_id->device,
 303                     &rdsv3_ib_client);
 304                 iinfo->max_send_wr = ic->i_send_ring.w_nr;
 305                 iinfo->max_recv_wr = ic->i_recv_ring.w_nr;
 306                 iinfo->max_send_sge = rds_ibdev->max_sge;
 307         }
 308
 309         RDSV3_DPRINTF4("rds_ib_conn_info_visitor", "conn: %p buffer: %p",
 310             conn, buffer);
 311         return (1);
 312 }
 313
 314 static void
 315 rds_ib_ic_info(struct rsock *sock, unsigned int len,
 316     struct rdsv3_info_iterator *iter,
 317     struct rdsv3_info_lengths *lens)
 318 {
 319         RDSV3_DPRINTF4("rds_ib_ic_info", "sk: %p iter: %p, lens: %p, len: %d",
 320             sock, iter, lens, len);
 321
 322         rdsv3_for_each_conn_info(sock, len, iter, lens,
 323             rds_ib_conn_info_visitor,
 324             sizeof (struct rds_info_rdma_connection));
 325 }
 326
 327 /*
 328  * Early RDS/IB was built to only bind to an address if there is an IPoIB
 329  * device with that address set.
 330  *
 331  * If it were me, I'd advocate for something more flexible.  Sending and
 332  * receiving should be device-agnostic.  Transports would try and maintain
 333  * connections between peers who have messages queued.  Userspace would be
 334  * allowed to influence which paths have priority.  We could call userspace
 335  * asserting this policy "routing".
 336  */
 337 static int
 338 rds_ib_laddr_check(uint32_be_t addr)
 339 {
 340         int ret;
 341         struct rdma_cm_id *cm_id;
 342         struct sockaddr_in sin;
 343
 344         RDSV3_DPRINTF4("rds_ib_laddr_check", "addr: %x", ntohl(addr));
 345
 346         /*
 347          * Create a CMA ID and try to bind it. This catches both
 348          * IB and iWARP capable NICs.
 349          */
 350         cm_id = rdma_create_id(NULL, NULL, RDMA_PS_TCP);
 351         if (!cm_id)
 352                 return (-EADDRNOTAVAIL);
 353
 354         (void) memset(&sin, 0, sizeof (sin));
 355         sin.sin_family = AF_INET;
 356         sin.sin_addr.s_addr = rdsv3_scaddr_to_ibaddr(addr);
 357
 358         /* rdma_bind_addr will only succeed for IB & iWARP devices */
 359         ret = rdma_bind_addr(cm_id, (struct sockaddr *)&sin);
 360         /*
 361          * due to this, we will claim to support iWARP devices unless we
 362          * check node_type.
 363          */
 364         if (ret || cm_id->device->node_type != RDMA_NODE_IB_CA)
 365                 ret = -EADDRNOTAVAIL;
 366
 367         RDSV3_DPRINTF5("rds_ib_laddr_check",
 368             "addr %u.%u.%u.%u ret %d node type %d",
 369             NIPQUAD(addr), ret,
 370             cm_id->device ? cm_id->device->node_type : -1);
 371
 372         rdma_destroy_id(cm_id);
 373
 374         return (ret);
 375 }
 376
 377 void
 378 rdsv3_ib_exit(void)
 379 {
 380         RDSV3_DPRINTF4("rds_ib_exit", "Enter");
 381
 382         rdsv3_info_deregister_func(RDS_INFO_IB_CONNECTIONS, rds_ib_ic_info);
 383         rdsv3_ib_destroy_nodev_conns();
 384         ib_unregister_client(&rdsv3_ib_client);
 385         rdsv3_ib_sysctl_exit();
 386         rdsv3_ib_recv_exit();
 387         rdsv3_trans_unregister(&rdsv3_ib_transport);
 388         kmem_free(rdsv3_ib_stats,
 389             nr_cpus * sizeof (struct rdsv3_ib_statistics));
 390         mutex_destroy(&ib_nodev_conns_lock);
 391         list_destroy(&ib_nodev_conns);
 392         list_destroy(&rdsv3_ib_devices);
 393
 394         RDSV3_DPRINTF4("rds_ib_exit", "Return");
 395 }
 396
 397 #ifndef __lock_lint
 398 struct rdsv3_transport rdsv3_ib_transport = {
 399         .laddr_check            = rds_ib_laddr_check,
 400         .xmit_complete          = rdsv3_ib_xmit_complete,
 401         .xmit                   = rdsv3_ib_xmit,
 402         .xmit_cong_map          = NULL,
 403         .xmit_rdma              = rdsv3_ib_xmit_rdma,
 404         .recv                   = rdsv3_ib_recv,
 405         .conn_alloc             = rdsv3_ib_conn_alloc,
 406         .conn_free              = rdsv3_ib_conn_free,
 407         .conn_connect           = rdsv3_ib_conn_connect,
 408         .conn_shutdown          = rdsv3_ib_conn_shutdown,
 409         .inc_copy_to_user       = rdsv3_ib_inc_copy_to_user,
 410         .inc_free               = rdsv3_ib_inc_free,
 411         .cm_initiate_connect    = rdsv3_ib_cm_initiate_connect,
 412         .cm_handle_connect      = rdsv3_ib_cm_handle_connect,
 413         .cm_connect_complete    = rdsv3_ib_cm_connect_complete,
 414         .stats_info_copy        = rdsv3_ib_stats_info_copy,
 415         .exit                   = rdsv3_ib_exit,
 416         .get_mr                 = rdsv3_ib_get_mr,
 417         .sync_mr                = rdsv3_ib_sync_mr,
 418         .free_mr                = rdsv3_ib_free_mr,
 419         .flush_mrs              = rdsv3_ib_flush_mrs,
 420         .t_name                 = "infiniband",
 421         .t_type                 = RDS_TRANS_IB
 422 };
 423 #else
 424 struct rdsv3_transport rdsv3_ib_transport;
 425 #endif
 426
 427 int
 428 rdsv3_ib_init(void)
 429 {
 430         int ret;
 431
 432         RDSV3_DPRINTF4("rds_ib_init", "Enter");
 433
 434         list_create(&rdsv3_ib_devices, sizeof (struct rdsv3_ib_device),
 435             offsetof(struct rdsv3_ib_device, list));
 436         list_create(&ib_nodev_conns, sizeof (struct rdsv3_ib_connection),
 437             offsetof(struct rdsv3_ib_connection, ib_node));
 438         mutex_init(&ib_nodev_conns_lock, NULL, MUTEX_DRIVER, NULL);
 439
 440         /* allocate space for ib statistics */
 441         ASSERT(rdsv3_ib_stats == NULL);
 442         rdsv3_ib_stats = kmem_zalloc(nr_cpus *
 443             sizeof (struct rdsv3_ib_statistics), KM_SLEEP);
 444
 445         rdsv3_ib_client.dip = rdsv3_dev_info;
 446         ret = ib_register_client(&rdsv3_ib_client);
 447         if (ret)
 448                 goto out;
 449
 450         ret = rdsv3_ib_sysctl_init();
 451         if (ret)
 452                 goto out_ibreg;
 453
 454         ret = rdsv3_ib_recv_init();
 455         if (ret)
 456                 goto out_sysctl;
 457
 458         ret = rdsv3_trans_register(&rdsv3_ib_transport);
 459         if (ret)
 460                 goto out_recv;
 461
 462         rdsv3_info_register_func(RDS_INFO_IB_CONNECTIONS, rds_ib_ic_info);
 463
 464         RDSV3_DPRINTF4("rds_ib_init", "Return");
 465
 466         return (0);
 467
 468 out_recv:
 469         rdsv3_ib_recv_exit();
 470 out_sysctl:
 471         rdsv3_ib_sysctl_exit();
 472 out_ibreg:
 473         ib_unregister_client(&rdsv3_ib_client);
 474 out:
 475         kmem_free(rdsv3_ib_stats,
 476             nr_cpus * sizeof (struct rdsv3_ib_statistics));
 477         mutex_destroy(&ib_nodev_conns_lock);
 478         list_destroy(&ib_nodev_conns);
 479         list_destroy(&rdsv3_ib_devices);
 480         return (ret);
 481 }