drivers/infiniband/sw/rdmavt/vt.c

   1 /*
   2  * Copyright(c) 2016 Intel Corporation.
   3  *
   4  * This file is provided under a dual BSD/GPLv2 license.  When using or
   5  * redistributing this file, you may do so under either license.
   6  *
   7  * GPL LICENSE SUMMARY
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of version 2 of the GNU General Public License as
  11  * published by the Free Software Foundation.
  12  *
  13  * This program is distributed in the hope that it will be useful, but
  14  * WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * General Public License for more details.
  17  *
  18  * BSD LICENSE
  19  *
  20  * Redistribution and use in source and binary forms, with or without
  21  * modification, are permitted provided that the following conditions
  22  * are met:
  23  *
  24  *  - Redistributions of source code must retain the above copyright
  25  *    notice, this list of conditions and the following disclaimer.
  26  *  - Redistributions in binary form must reproduce the above copyright
  27  *    notice, this list of conditions and the following disclaimer in
  28  *    the documentation and/or other materials provided with the
  29  *    distribution.
  30  *  - Neither the name of Intel Corporation nor the names of its
  31  *    contributors may be used to endorse or promote products derived
  32  *    from this software without specific prior written permission.
  33  *
  34  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  35  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  36  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  37  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  38  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  39  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  40  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  41  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  42  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  43  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  44  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  45  *
  46  */
  47
  48 #include <linux/module.h>
  49 #include <linux/kernel.h>
  50 #include <linux/dma-mapping.h>
  51 #include "vt.h"
  52 #include "trace.h"
  53
  54 #define RVT_UVERBS_ABI_VERSION 2
  55
  56 MODULE_LICENSE("Dual BSD/GPL");
  57 MODULE_DESCRIPTION("RDMA Verbs Transport Library");
  58
  59 static int rvt_init(void)
  60 {
  61         /*
  62          * rdmavt does not need to do anything special when it starts up. All it
  63          * needs to do is sit and wait until a driver attempts registration.
  64          */
  65         return 0;
  66 }
  67 module_init(rvt_init);
  68
  69 static void rvt_cleanup(void)
  70 {
  71         /*
  72          * Nothing to do at exit time either. The module won't be able to be
  73          * removed until all drivers are gone which means all the dev structs
  74          * are gone so there is really nothing to do.
  75          */
  76 }
  77 module_exit(rvt_cleanup);
  78
  79 /**
  80  * rvt_alloc_device - allocate rdi
  81  * @size: how big of a structure to allocate
  82  * @nports: number of ports to allocate array slots for
  83  *
  84  * Use IB core device alloc to allocate space for the rdi which is assumed to be
  85  * inside of the ib_device. Any extra space that drivers require should be
  86  * included in size.
  87  *
  88  * We also allocate a port array based on the number of ports.
  89  *
  90  * Return: pointer to allocated rdi
  91  */
  92 struct rvt_dev_info *rvt_alloc_device(size_t size, int nports)
  93 {
  94         struct rvt_dev_info *rdi = ERR_PTR(-ENOMEM);
  95
  96         rdi = (struct rvt_dev_info *)ib_alloc_device(size);
  97         if (!rdi)
  98                 return rdi;
  99
 100         rdi->ports = kcalloc(nports,
 101                              sizeof(struct rvt_ibport **),
 102                              GFP_KERNEL);
 103         if (!rdi->ports)
 104                 ib_dealloc_device(&rdi->ibdev);
 105
 106         return rdi;
 107 }
 108 EXPORT_SYMBOL(rvt_alloc_device);
 109
 110 /**
 111  * rvt_dealloc_device - deallocate rdi
 112  * @rdi: structure to free
 113  *
 114  * Free a structure allocated with rvt_alloc_device()
 115  */
 116 void rvt_dealloc_device(struct rvt_dev_info *rdi)
 117 {
 118         kfree(rdi->ports);
 119         ib_dealloc_device(&rdi->ibdev);
 120 }
 121 EXPORT_SYMBOL(rvt_dealloc_device);
 122
 123 static int rvt_query_device(struct ib_device *ibdev,
 124                             struct ib_device_attr *props,
 125                             struct ib_udata *uhw)
 126 {
 127         struct rvt_dev_info *rdi = ib_to_rvt(ibdev);
 128
 129         if (uhw->inlen || uhw->outlen)
 130                 return -EINVAL;
 131         /*
 132          * Return rvt_dev_info.dparms.props contents
 133          */
 134         *props = rdi->dparms.props;
 135         return 0;
 136 }
 137
 138 static int rvt_modify_device(struct ib_device *device,
 139                              int device_modify_mask,
 140                              struct ib_device_modify *device_modify)
 141 {
 142         /*
 143          * There is currently no need to supply this based on qib and hfi1.
 144          * Future drivers may need to implement this though.
 145          */
 146
 147         return -EOPNOTSUPP;
 148 }
 149
 150 /**
 151  * rvt_query_port: Passes the query port call to the driver
 152  * @ibdev: Verbs IB dev
 153  * @port_num: port number, 1 based from ib core
 154  * @props: structure to hold returned properties
 155  *
 156  * Return: 0 on success
 157  */
 158 static int rvt_query_port(struct ib_device *ibdev, u8 port_num,
 159                           struct ib_port_attr *props)
 160 {
 161         struct rvt_dev_info *rdi = ib_to_rvt(ibdev);
 162         struct rvt_ibport *rvp;
 163         int port_index = ibport_num_to_idx(ibdev, port_num);
 164
 165         if (port_index < 0)
 166                 return -EINVAL;
 167
 168         rvp = rdi->ports[port_index];
 169         /* props being zeroed by the caller, avoid zeroing it here */
 170         props->sm_lid = rvp->sm_lid;
 171         props->sm_sl = rvp->sm_sl;
 172         props->port_cap_flags = rvp->port_cap_flags;
 173         props->max_msg_sz = 0x80000000;
 174         props->pkey_tbl_len = rvt_get_npkeys(rdi);
 175         props->bad_pkey_cntr = rvp->pkey_violations;
 176         props->qkey_viol_cntr = rvp->qkey_violations;
 177         props->subnet_timeout = rvp->subnet_timeout;
 178         props->init_type_reply = 0;
 179
 180         /* Populate the remaining ib_port_attr elements */
 181         return rdi->driver_f.query_port_state(rdi, port_num, props);
 182 }
 183
 184 /**
 185  * rvt_modify_port
 186  * @ibdev: Verbs IB dev
 187  * @port_num: Port number, 1 based from ib core
 188  * @port_modify_mask: How to change the port
 189  * @props: Structure to fill in
 190  *
 191  * Return: 0 on success
 192  */
 193 static int rvt_modify_port(struct ib_device *ibdev, u8 port_num,
 194                            int port_modify_mask, struct ib_port_modify *props)
 195 {
 196         struct rvt_dev_info *rdi = ib_to_rvt(ibdev);
 197         struct rvt_ibport *rvp;
 198         int ret = 0;
 199         int port_index = ibport_num_to_idx(ibdev, port_num);
 200
 201         if (port_index < 0)
 202                 return -EINVAL;
 203
 204         rvp = rdi->ports[port_index];
 205         if (port_modify_mask & IB_PORT_OPA_MASK_CHG) {
 206                 rvp->port_cap3_flags |= props->set_port_cap_mask;
 207                 rvp->port_cap3_flags &= ~props->clr_port_cap_mask;
 208         } else {
 209                 rvp->port_cap_flags |= props->set_port_cap_mask;
 210                 rvp->port_cap_flags &= ~props->clr_port_cap_mask;
 211         }
 212
 213         if (props->set_port_cap_mask || props->clr_port_cap_mask)
 214                 rdi->driver_f.cap_mask_chg(rdi, port_num);
 215         if (port_modify_mask & IB_PORT_SHUTDOWN)
 216                 ret = rdi->driver_f.shut_down_port(rdi, port_num);
 217         if (port_modify_mask & IB_PORT_RESET_QKEY_CNTR)
 218                 rvp->qkey_violations = 0;
 219
 220         return ret;
 221 }
 222
 223 /**
 224  * rvt_query_pkey - Return a pkey from the table at a given index
 225  * @ibdev: Verbs IB dev
 226  * @port_num: Port number, 1 based from ib core
 227  * @index: Index into pkey table
 228  * @pkey: returned pkey from the port pkey table
 229  *
 230  * Return: 0 on failure pkey otherwise
 231  */
 232 static int rvt_query_pkey(struct ib_device *ibdev, u8 port_num, u16 index,
 233                           u16 *pkey)
 234 {
 235         /*
 236          * Driver will be responsible for keeping rvt_dev_info.pkey_table up to
 237          * date. This function will just return that value. There is no need to
 238          * lock, if a stale value is read and sent to the user so be it there is
 239          * no way to protect against that anyway.
 240          */
 241         struct rvt_dev_info *rdi = ib_to_rvt(ibdev);
 242         int port_index;
 243
 244         port_index = ibport_num_to_idx(ibdev, port_num);
 245         if (port_index < 0)
 246                 return -EINVAL;
 247
 248         if (index >= rvt_get_npkeys(rdi))
 249                 return -EINVAL;
 250
 251         *pkey = rvt_get_pkey(rdi, port_index, index);
 252         return 0;
 253 }
 254
 255 /**
 256  * rvt_query_gid - Return a gid from the table
 257  * @ibdev: Verbs IB dev
 258  * @port_num: Port number, 1 based from ib core
 259  * @guid_index: Index in table
 260  * @gid: Gid to return
 261  *
 262  * Return: 0 on success
 263  */
 264 static int rvt_query_gid(struct ib_device *ibdev, u8 port_num,
 265                          int guid_index, union ib_gid *gid)
 266 {
 267         struct rvt_dev_info *rdi;
 268         struct rvt_ibport *rvp;
 269         int port_index;
 270
 271         /*
 272          * Driver is responsible for updating the guid table. Which will be used
 273          * to craft the return value. This will work similar to how query_pkey()
 274          * is being done.
 275          */
 276         port_index = ibport_num_to_idx(ibdev, port_num);
 277         if (port_index < 0)
 278                 return -EINVAL;
 279
 280         rdi = ib_to_rvt(ibdev);
 281         rvp = rdi->ports[port_index];
 282
 283         gid->global.subnet_prefix = rvp->gid_prefix;
 284
 285         return rdi->driver_f.get_guid_be(rdi, rvp, guid_index,
 286                                          &gid->global.interface_id);
 287 }
 288
 289 struct rvt_ucontext {
 290         struct ib_ucontext ibucontext;
 291 };
 292
 293 static inline struct rvt_ucontext *to_iucontext(struct ib_ucontext
 294                                                 *ibucontext)
 295 {
 296         return container_of(ibucontext, struct rvt_ucontext, ibucontext);
 297 }
 298
 299 /**
 300  * rvt_alloc_ucontext - Allocate a user context
 301  * @ibdev: Verbs IB dev
 302  * @udata: User data allocated
 303  */
 304 static struct ib_ucontext *rvt_alloc_ucontext(struct ib_device *ibdev,
 305                                               struct ib_udata *udata)
 306 {
 307         struct rvt_ucontext *context;
 308
 309         context = kmalloc(sizeof(*context), GFP_KERNEL);
 310         if (!context)
 311                 return ERR_PTR(-ENOMEM);
 312         return &context->ibucontext;
 313 }
 314
 315 /**
 316  *rvt_dealloc_ucontext - Free a user context
 317  *@context - Free this
 318  */
 319 static int rvt_dealloc_ucontext(struct ib_ucontext *context)
 320 {
 321         kfree(to_iucontext(context));
 322         return 0;
 323 }
 324
 325 static int rvt_get_port_immutable(struct ib_device *ibdev, u8 port_num,
 326                                   struct ib_port_immutable *immutable)
 327 {
 328         struct rvt_dev_info *rdi = ib_to_rvt(ibdev);
 329         struct ib_port_attr attr;
 330         int err, port_index;
 331
 332         port_index = ibport_num_to_idx(ibdev, port_num);
 333         if (port_index < 0)
 334                 return -EINVAL;
 335
 336         immutable->core_cap_flags = rdi->dparms.core_cap_flags;
 337
 338         err = ib_query_port(ibdev, port_num, &attr);
 339         if (err)
 340                 return err;
 341
 342         immutable->pkey_tbl_len = attr.pkey_tbl_len;
 343         immutable->gid_tbl_len = attr.gid_tbl_len;
 344         immutable->max_mad_size = rdi->dparms.max_mad_size;
 345
 346         return 0;
 347 }
 348
 349 enum {
 350         MISC,
 351         QUERY_DEVICE,
 352         MODIFY_DEVICE,
 353         QUERY_PORT,
 354         MODIFY_PORT,
 355         QUERY_PKEY,
 356         QUERY_GID,
 357         ALLOC_UCONTEXT,
 358         DEALLOC_UCONTEXT,
 359         GET_PORT_IMMUTABLE,
 360         CREATE_QP,
 361         MODIFY_QP,
 362         DESTROY_QP,
 363         QUERY_QP,
 364         POST_SEND,
 365         POST_RECV,
 366         POST_SRQ_RECV,
 367         CREATE_AH,
 368         DESTROY_AH,
 369         MODIFY_AH,
 370         QUERY_AH,
 371         CREATE_SRQ,
 372         MODIFY_SRQ,
 373         DESTROY_SRQ,
 374         QUERY_SRQ,
 375         ATTACH_MCAST,
 376         DETACH_MCAST,
 377         GET_DMA_MR,
 378         REG_USER_MR,
 379         DEREG_MR,
 380         ALLOC_MR,
 381         MAP_MR_SG,
 382         ALLOC_FMR,
 383         MAP_PHYS_FMR,
 384         UNMAP_FMR,
 385         DEALLOC_FMR,
 386         MMAP,
 387         CREATE_CQ,
 388         DESTROY_CQ,
 389         POLL_CQ,
 390         REQ_NOTFIY_CQ,
 391         RESIZE_CQ,
 392         ALLOC_PD,
 393         DEALLOC_PD,
 394         _VERB_IDX_MAX /* Must always be last! */
 395 };
 396
 397 static inline int check_driver_override(struct rvt_dev_info *rdi,
 398                                         size_t offset, void *func)
 399 {
 400         if (!*(void **)((void *)&rdi->ibdev + offset)) {
 401                 *(void **)((void *)&rdi->ibdev + offset) = func;
 402                 return 0;
 403         }
 404
 405         return 1;
 406 }
 407
 408 static noinline int check_support(struct rvt_dev_info *rdi, int verb)
 409 {
 410         switch (verb) {
 411         case MISC:
 412                 /*
 413                  * These functions are not part of verbs specifically but are
 414                  * required for rdmavt to function.
 415                  */
 416                 if ((!rdi->driver_f.port_callback) ||
 417                     (!rdi->driver_f.get_pci_dev))
 418                         return -EINVAL;
 419                 break;
 420
 421         case QUERY_DEVICE:
 422                 check_driver_override(rdi, offsetof(struct ib_device,
 423                                                     query_device),
 424                                                     rvt_query_device);
 425                 break;
 426
 427         case MODIFY_DEVICE:
 428                 /*
 429                  * rdmavt does not support modify device currently drivers must
 430                  * provide.
 431                  */
 432                 if (!check_driver_override(rdi, offsetof(struct ib_device,
 433                                                          modify_device),
 434                                            rvt_modify_device))
 435                         return -EOPNOTSUPP;
 436                 break;
 437
 438         case QUERY_PORT:
 439                 if (!check_driver_override(rdi, offsetof(struct ib_device,
 440                                                          query_port),
 441                                            rvt_query_port))
 442                         if (!rdi->driver_f.query_port_state)
 443                                 return -EINVAL;
 444                 break;
 445
 446         case MODIFY_PORT:
 447                 if (!check_driver_override(rdi, offsetof(struct ib_device,
 448                                                          modify_port),
 449                                            rvt_modify_port))
 450                         if (!rdi->driver_f.cap_mask_chg ||
 451                             !rdi->driver_f.shut_down_port)
 452                                 return -EINVAL;
 453                 break;
 454
 455         case QUERY_PKEY:
 456                 check_driver_override(rdi, offsetof(struct ib_device,
 457                                                     query_pkey),
 458                                       rvt_query_pkey);
 459                 break;
 460
 461         case QUERY_GID:
 462                 if (!check_driver_override(rdi, offsetof(struct ib_device,
 463                                                          query_gid),
 464                                            rvt_query_gid))
 465                         if (!rdi->driver_f.get_guid_be)
 466                                 return -EINVAL;
 467                 break;
 468
 469         case ALLOC_UCONTEXT:
 470                 check_driver_override(rdi, offsetof(struct ib_device,
 471                                                     alloc_ucontext),
 472                                       rvt_alloc_ucontext);
 473                 break;
 474
 475         case DEALLOC_UCONTEXT:
 476                 check_driver_override(rdi, offsetof(struct ib_device,
 477                                                     dealloc_ucontext),
 478                                       rvt_dealloc_ucontext);
 479                 break;
 480
 481         case GET_PORT_IMMUTABLE:
 482                 check_driver_override(rdi, offsetof(struct ib_device,
 483                                                     get_port_immutable),
 484                                       rvt_get_port_immutable);
 485                 break;
 486
 487         case CREATE_QP:
 488                 if (!check_driver_override(rdi, offsetof(struct ib_device,
 489                                                          create_qp),
 490                                            rvt_create_qp))
 491                         if (!rdi->driver_f.qp_priv_alloc ||
 492                             !rdi->driver_f.qp_priv_free ||
 493                             !rdi->driver_f.notify_qp_reset ||
 494                             !rdi->driver_f.flush_qp_waiters ||
 495                             !rdi->driver_f.stop_send_queue ||
 496                             !rdi->driver_f.quiesce_qp)
 497                                 return -EINVAL;
 498                 break;
 499
 500         case MODIFY_QP:
 501                 if (!check_driver_override(rdi, offsetof(struct ib_device,
 502                                                          modify_qp),
 503                                            rvt_modify_qp))
 504                         if (!rdi->driver_f.notify_qp_reset ||
 505                             !rdi->driver_f.schedule_send ||
 506                             !rdi->driver_f.get_pmtu_from_attr ||
 507                             !rdi->driver_f.flush_qp_waiters ||
 508                             !rdi->driver_f.stop_send_queue ||
 509                             !rdi->driver_f.quiesce_qp ||
 510                             !rdi->driver_f.notify_error_qp ||
 511                             !rdi->driver_f.mtu_from_qp ||
 512                             !rdi->driver_f.mtu_to_path_mtu)
 513                                 return -EINVAL;
 514                 break;
 515
 516         case DESTROY_QP:
 517                 if (!check_driver_override(rdi, offsetof(struct ib_device,
 518                                                          destroy_qp),
 519                                            rvt_destroy_qp))
 520                         if (!rdi->driver_f.qp_priv_free ||
 521                             !rdi->driver_f.notify_qp_reset ||
 522                             !rdi->driver_f.flush_qp_waiters ||
 523                             !rdi->driver_f.stop_send_queue ||
 524                             !rdi->driver_f.quiesce_qp)
 525                                 return -EINVAL;
 526                 break;
 527
 528         case QUERY_QP:
 529                 check_driver_override(rdi, offsetof(struct ib_device,
 530                                                     query_qp),
 531                                                     rvt_query_qp);
 532                 break;
 533
 534         case POST_SEND:
 535                 if (!check_driver_override(rdi, offsetof(struct ib_device,
 536                                                          post_send),
 537                                            rvt_post_send))
 538                         if (!rdi->driver_f.schedule_send ||
 539                             !rdi->driver_f.do_send ||
 540                             !rdi->post_parms)
 541                                 return -EINVAL;
 542                 break;
 543
 544         case POST_RECV:
 545                 check_driver_override(rdi, offsetof(struct ib_device,
 546                                                     post_recv),
 547                                       rvt_post_recv);
 548                 break;
 549         case POST_SRQ_RECV:
 550                 check_driver_override(rdi, offsetof(struct ib_device,
 551                                                     post_srq_recv),
 552                                       rvt_post_srq_recv);
 553                 break;
 554
 555         case CREATE_AH:
 556                 check_driver_override(rdi, offsetof(struct ib_device,
 557                                                     create_ah),
 558                                       rvt_create_ah);
 559                 break;
 560
 561         case DESTROY_AH:
 562                 check_driver_override(rdi, offsetof(struct ib_device,
 563                                                     destroy_ah),
 564                                       rvt_destroy_ah);
 565                 break;
 566
 567         case MODIFY_AH:
 568                 check_driver_override(rdi, offsetof(struct ib_device,
 569                                                     modify_ah),
 570                                       rvt_modify_ah);
 571                 break;
 572
 573         case QUERY_AH:
 574                 check_driver_override(rdi, offsetof(struct ib_device,
 575                                                     query_ah),
 576                                       rvt_query_ah);
 577                 break;
 578
 579         case CREATE_SRQ:
 580                 check_driver_override(rdi, offsetof(struct ib_device,
 581                                                     create_srq),
 582                                       rvt_create_srq);
 583                 break;
 584
 585         case MODIFY_SRQ:
 586                 check_driver_override(rdi, offsetof(struct ib_device,
 587                                                     modify_srq),
 588                                       rvt_modify_srq);
 589                 break;
 590
 591         case DESTROY_SRQ:
 592                 check_driver_override(rdi, offsetof(struct ib_device,
 593                                                     destroy_srq),
 594                                       rvt_destroy_srq);
 595                 break;
 596
 597         case QUERY_SRQ:
 598                 check_driver_override(rdi, offsetof(struct ib_device,
 599                                                     query_srq),
 600                                       rvt_query_srq);
 601                 break;
 602
 603         case ATTACH_MCAST:
 604                 check_driver_override(rdi, offsetof(struct ib_device,
 605                                                     attach_mcast),
 606                                       rvt_attach_mcast);
 607                 break;
 608
 609         case DETACH_MCAST:
 610                 check_driver_override(rdi, offsetof(struct ib_device,
 611                                                     detach_mcast),
 612                                       rvt_detach_mcast);
 613                 break;
 614
 615         case GET_DMA_MR:
 616                 check_driver_override(rdi, offsetof(struct ib_device,
 617                                                     get_dma_mr),
 618                                       rvt_get_dma_mr);
 619                 break;
 620
 621         case REG_USER_MR:
 622                 check_driver_override(rdi, offsetof(struct ib_device,
 623                                                     reg_user_mr),
 624                                       rvt_reg_user_mr);
 625                 break;
 626
 627         case DEREG_MR:
 628                 check_driver_override(rdi, offsetof(struct ib_device,
 629                                                     dereg_mr),
 630                                       rvt_dereg_mr);
 631                 break;
 632
 633         case ALLOC_FMR:
 634                 check_driver_override(rdi, offsetof(struct ib_device,
 635                                                     alloc_fmr),
 636                                       rvt_alloc_fmr);
 637                 break;
 638
 639         case ALLOC_MR:
 640                 check_driver_override(rdi, offsetof(struct ib_device,
 641                                                     alloc_mr),
 642                                       rvt_alloc_mr);
 643                 break;
 644
 645         case MAP_MR_SG:
 646                 check_driver_override(rdi, offsetof(struct ib_device,
 647                                                     map_mr_sg),
 648                                       rvt_map_mr_sg);
 649                 break;
 650
 651         case MAP_PHYS_FMR:
 652                 check_driver_override(rdi, offsetof(struct ib_device,
 653                                                     map_phys_fmr),
 654                                       rvt_map_phys_fmr);
 655                 break;
 656
 657         case UNMAP_FMR:
 658                 check_driver_override(rdi, offsetof(struct ib_device,
 659                                                     unmap_fmr),
 660                                       rvt_unmap_fmr);
 661                 break;
 662
 663         case DEALLOC_FMR:
 664                 check_driver_override(rdi, offsetof(struct ib_device,
 665                                                     dealloc_fmr),
 666                                       rvt_dealloc_fmr);
 667                 break;
 668
 669         case MMAP:
 670                 check_driver_override(rdi, offsetof(struct ib_device,
 671                                                     mmap),
 672                                       rvt_mmap);
 673                 break;
 674
 675         case CREATE_CQ:
 676                 check_driver_override(rdi, offsetof(struct ib_device,
 677                                                     create_cq),
 678                                       rvt_create_cq);
 679                 break;
 680
 681         case DESTROY_CQ:
 682                 check_driver_override(rdi, offsetof(struct ib_device,
 683                                                     destroy_cq),
 684                                       rvt_destroy_cq);
 685                 break;
 686
 687         case POLL_CQ:
 688                 check_driver_override(rdi, offsetof(struct ib_device,
 689                                                     poll_cq),
 690                                       rvt_poll_cq);
 691                 break;
 692
 693         case REQ_NOTFIY_CQ:
 694                 check_driver_override(rdi, offsetof(struct ib_device,
 695                                                     req_notify_cq),
 696                                       rvt_req_notify_cq);
 697                 break;
 698
 699         case RESIZE_CQ:
 700                 check_driver_override(rdi, offsetof(struct ib_device,
 701                                                     resize_cq),
 702                                       rvt_resize_cq);
 703                 break;
 704
 705         case ALLOC_PD:
 706                 check_driver_override(rdi, offsetof(struct ib_device,
 707                                                     alloc_pd),
 708                                       rvt_alloc_pd);
 709                 break;
 710
 711         case DEALLOC_PD:
 712                 check_driver_override(rdi, offsetof(struct ib_device,
 713                                                     dealloc_pd),
 714                                       rvt_dealloc_pd);
 715                 break;
 716
 717         default:
 718                 return -EINVAL;
 719         }
 720
 721         return 0;
 722 }
 723
 724 /**
 725  * rvt_register_device - register a driver
 726  * @rdi: main dev structure for all of rdmavt operations
 727  *
 728  * It is up to drivers to allocate the rdi and fill in the appropriate
 729  * information.
 730  *
 731  * Return: 0 on success otherwise an errno.
 732  */
 733 int rvt_register_device(struct rvt_dev_info *rdi)
 734 {
 735         int ret = 0, i;
 736
 737         if (!rdi)
 738                 return -EINVAL;
 739
 740         /*
 741          * Check to ensure drivers have setup the required helpers for the verbs
 742          * they want rdmavt to handle
 743          */
 744         for (i = 0; i < _VERB_IDX_MAX; i++)
 745                 if (check_support(rdi, i)) {
 746                         pr_err("Driver support req not met at %d\n", i);
 747                         return -EINVAL;
 748                 }
 749
 750
 751         /* Once we get past here we can use rvt_pr macros and tracepoints */
 752         trace_rvt_dbg(rdi, "Driver attempting registration");
 753         rvt_mmap_init(rdi);
 754
 755         /* Queue Pairs */
 756         ret = rvt_driver_qp_init(rdi);
 757         if (ret) {
 758                 pr_err("Error in driver QP init.\n");
 759                 return -EINVAL;
 760         }
 761
 762         /* Address Handle */
 763         spin_lock_init(&rdi->n_ahs_lock);
 764         rdi->n_ahs_allocated = 0;
 765
 766         /* Shared Receive Queue */
 767         rvt_driver_srq_init(rdi);
 768
 769         /* Multicast */
 770         rvt_driver_mcast_init(rdi);
 771
 772         /* Mem Region */
 773         ret = rvt_driver_mr_init(rdi);
 774         if (ret) {
 775                 pr_err("Error in driver MR init.\n");
 776                 goto bail_no_mr;
 777         }
 778
 779         /* Completion queues */
 780         ret = rvt_driver_cq_init(rdi);
 781         if (ret) {
 782                 pr_err("Error in driver CQ init.\n");
 783                 goto bail_mr;
 784         }
 785
 786         /* DMA Operations */
 787         rdi->ibdev.dev.dma_ops = rdi->ibdev.dev.dma_ops ? : &dma_virt_ops;
 788
 789         /* Protection Domain */
 790         spin_lock_init(&rdi->n_pds_lock);
 791         rdi->n_pds_allocated = 0;
 792
 793         /*
 794          * There are some things which could be set by underlying drivers but
 795          * really should be up to rdmavt to set. For instance drivers can't know
 796          * exactly which functions rdmavt supports, nor do they know the ABI
 797          * version, so we do all of this sort of stuff here.
 798          */
 799         rdi->ibdev.uverbs_abi_ver = RVT_UVERBS_ABI_VERSION;
 800         rdi->ibdev.uverbs_cmd_mask =
 801                 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT)         |
 802                 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE)        |
 803                 (1ull << IB_USER_VERBS_CMD_QUERY_PORT)          |
 804                 (1ull << IB_USER_VERBS_CMD_ALLOC_PD)            |
 805                 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD)          |
 806                 (1ull << IB_USER_VERBS_CMD_CREATE_AH)           |
 807                 (1ull << IB_USER_VERBS_CMD_MODIFY_AH)           |
 808                 (1ull << IB_USER_VERBS_CMD_QUERY_AH)            |
 809                 (1ull << IB_USER_VERBS_CMD_DESTROY_AH)          |
 810                 (1ull << IB_USER_VERBS_CMD_REG_MR)              |
 811                 (1ull << IB_USER_VERBS_CMD_DEREG_MR)            |
 812                 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
 813                 (1ull << IB_USER_VERBS_CMD_CREATE_CQ)           |
 814                 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ)           |
 815                 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ)          |
 816                 (1ull << IB_USER_VERBS_CMD_POLL_CQ)             |
 817                 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ)       |
 818                 (1ull << IB_USER_VERBS_CMD_CREATE_QP)           |
 819                 (1ull << IB_USER_VERBS_CMD_QUERY_QP)            |
 820                 (1ull << IB_USER_VERBS_CMD_MODIFY_QP)           |
 821                 (1ull << IB_USER_VERBS_CMD_DESTROY_QP)          |
 822                 (1ull << IB_USER_VERBS_CMD_POST_SEND)           |
 823                 (1ull << IB_USER_VERBS_CMD_POST_RECV)           |
 824                 (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST)        |
 825                 (1ull << IB_USER_VERBS_CMD_DETACH_MCAST)        |
 826                 (1ull << IB_USER_VERBS_CMD_CREATE_SRQ)          |
 827                 (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ)          |
 828                 (1ull << IB_USER_VERBS_CMD_QUERY_SRQ)           |
 829                 (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ)         |
 830                 (1ull << IB_USER_VERBS_CMD_POST_SRQ_RECV);
 831         rdi->ibdev.node_type = RDMA_NODE_IB_CA;
 832         rdi->ibdev.num_comp_vectors = 1;
 833
 834         /* We are now good to announce we exist */
 835         ret =  ib_register_device(&rdi->ibdev, rdi->driver_f.port_callback);
 836         if (ret) {
 837                 rvt_pr_err(rdi, "Failed to register driver with ib core.\n");
 838                 goto bail_cq;
 839         }
 840
 841         rvt_create_mad_agents(rdi);
 842
 843         rvt_pr_info(rdi, "Registration with rdmavt done.\n");
 844         return ret;
 845
 846 bail_cq:
 847         rvt_cq_exit(rdi);
 848
 849 bail_mr:
 850         rvt_mr_exit(rdi);
 851
 852 bail_no_mr:
 853         rvt_qp_exit(rdi);
 854
 855         return ret;
 856 }
 857 EXPORT_SYMBOL(rvt_register_device);
 858
 859 /**
 860  * rvt_unregister_device - remove a driver
 861  * @rdi: rvt dev struct
 862  */
 863 void rvt_unregister_device(struct rvt_dev_info *rdi)
 864 {
 865         trace_rvt_dbg(rdi, "Driver is unregistering.");
 866         if (!rdi)
 867                 return;
 868
 869         rvt_free_mad_agents(rdi);
 870
 871         ib_unregister_device(&rdi->ibdev);
 872         rvt_cq_exit(rdi);
 873         rvt_mr_exit(rdi);
 874         rvt_qp_exit(rdi);
 875 }
 876 EXPORT_SYMBOL(rvt_unregister_device);
 877
 878 /**
 879  * rvt_init_port - init internal data for driver port
 880  * @rdi: rvt dev strut
 881  * @port: rvt port
 882  * @port_index: 0 based index of ports, different from IB core port num
 883  *
 884  * Keep track of a list of ports. No need to have a detach port.
 885  * They persist until the driver goes away.
 886  *
 887  * Return: always 0
 888  */
 889 int rvt_init_port(struct rvt_dev_info *rdi, struct rvt_ibport *port,
 890                   int port_index, u16 *pkey_table)
 891 {
 892
 893         rdi->ports[port_index] = port;
 894         rdi->ports[port_index]->pkey_table = pkey_table;
 895
 896         return 0;
 897 }
 898 EXPORT_SYMBOL(rvt_init_port);