src/net/infiniband.c

   1 /*
   2  * Copyright (C) 2007 Michael Brown <mbrown@fensystems.co.uk>.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public License as
   6  * published by the Free Software Foundation; either version 2 of the
   7  * License, or any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful, but
  10  * WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17  */
  18
  19 #include <stdint.h>
  20 #include <stdlib.h>
  21 #include <stdio.h>
  22 #include <string.h>
  23 #include <byteswap.h>
  24 #include <errno.h>
  25 #include <assert.h>
  26 #include <gpxe/list.h>
  27 #include <gpxe/if_arp.h>
  28 #include <gpxe/netdevice.h>
  29 #include <gpxe/iobuf.h>
  30 #include <gpxe/infiniband.h>
  31
  32 /** @file
  33  *
  34  * Infiniband protocol
  35  *
  36  */
  37
  38 /**
  39  * Create completion queue
  40  *
  41  * @v ibdev             Infiniband device
  42  * @v num_cqes          Number of completion queue entries
  43  * @ret cq              New completion queue
  44  */
  45 struct ib_completion_queue * ib_create_cq ( struct ib_device *ibdev,
  46                                             unsigned int num_cqes ) {
  47         struct ib_completion_queue *cq;
  48         int rc;
  49
  50         DBGC ( ibdev, "IBDEV %p creating completion queue\n", ibdev );
  51
  52         /* Allocate and initialise data structure */
  53         cq = zalloc ( sizeof ( *cq ) );
  54         if ( ! cq )
  55                 return NULL;
  56         cq->num_cqes = num_cqes;
  57         INIT_LIST_HEAD ( &cq->work_queues );
  58
  59         /* Perform device-specific initialisation and get CQN */
  60         if ( ( rc = ibdev->op->create_cq ( ibdev, cq ) ) != 0 ) {
  61                 DBGC ( ibdev, "IBDEV %p could not initialise completion "
  62                        "queue: %s\n", ibdev, strerror ( rc ) );
  63                 free ( cq );
  64                 return NULL;
  65         }
  66
  67         DBGC ( ibdev, "IBDEV %p created %d-entry completion queue %p (%p) "
  68                "with CQN %#lx\n", ibdev, num_cqes, cq, cq->dev_priv, cq->cqn );
  69         return cq;
  70 }
  71
  72 /**
  73  * Destroy completion queue
  74  *
  75  * @v ibdev             Infiniband device
  76  * @v cq                Completion queue
  77  */
  78 void ib_destroy_cq ( struct ib_device *ibdev,
  79                      struct ib_completion_queue *cq ) {
  80         DBGC ( ibdev, "IBDEV %p destroying completion queue %#lx\n",
  81                ibdev, cq->cqn );
  82         assert ( list_empty ( &cq->work_queues ) );
  83         ibdev->op->destroy_cq ( ibdev, cq );
  84         free ( cq );
  85 }
  86
  87 /**
  88  * Create queue pair
  89  *
  90  * @v ibdev             Infiniband device
  91  * @v num_send_wqes     Number of send work queue entries
  92  * @v send_cq           Send completion queue
  93  * @v num_recv_wqes     Number of receive work queue entries
  94  * @v recv_cq           Receive completion queue
  95  * @v qkey              Queue key
  96  * @ret qp              Queue pair
  97  */
  98 struct ib_queue_pair * ib_create_qp ( struct ib_device *ibdev,
  99                                       unsigned int num_send_wqes,
 100                                       struct ib_completion_queue *send_cq,
 101                                       unsigned int num_recv_wqes,
 102                                       struct ib_completion_queue *recv_cq,
 103                                       unsigned long qkey ) {
 104         struct ib_queue_pair *qp;
 105         size_t total_size;
 106         int rc;
 107
 108         DBGC ( ibdev, "IBDEV %p creating queue pair\n", ibdev );
 109
 110         /* Allocate and initialise data structure */
 111         total_size = ( sizeof ( *qp ) +
 112                        ( num_send_wqes * sizeof ( qp->send.iobufs[0] ) ) +
 113                        ( num_recv_wqes * sizeof ( qp->recv.iobufs[0] ) ) );
 114         qp = zalloc ( total_size );
 115         if ( ! qp )
 116                 return NULL;
 117         qp->qkey = qkey;
 118         qp->send.qp = qp;
 119         qp->send.is_send = 1;
 120         qp->send.cq = send_cq;
 121         list_add ( &qp->send.list, &send_cq->work_queues );
 122         qp->send.num_wqes = num_send_wqes;
 123         qp->send.iobufs = ( ( ( void * ) qp ) + sizeof ( *qp ) );
 124         qp->recv.qp = qp;
 125         qp->recv.cq = recv_cq;
 126         list_add ( &qp->recv.list, &recv_cq->work_queues );
 127         qp->recv.num_wqes = num_recv_wqes;
 128         qp->recv.iobufs = ( ( ( void * ) qp ) + sizeof ( *qp ) +
 129                             ( num_send_wqes * sizeof ( qp->send.iobufs[0] ) ));
 130
 131         /* Perform device-specific initialisation and get QPN */
 132         if ( ( rc = ibdev->op->create_qp ( ibdev, qp ) ) != 0 ) {
 133                 DBGC ( ibdev, "IBDEV %p could not initialise queue pair: "
 134                        "%s\n", ibdev, strerror ( rc ) );
 135                 free ( qp );
 136                 return NULL;
 137         }
 138
 139         DBGC ( ibdev, "IBDEV %p created queue pair %p (%p) with QPN %#lx\n",
 140                ibdev, qp, qp->dev_priv, qp->qpn );
 141         DBGC ( ibdev, "IBDEV %p QPN %#lx has %d send entries at [%p,%p)\n",
 142                ibdev, qp->qpn, num_send_wqes, qp->send.iobufs,
 143                qp->recv.iobufs );
 144         DBGC ( ibdev, "IBDEV %p QPN %#lx has %d receive entries at [%p,%p)\n",
 145                ibdev, qp->qpn, num_send_wqes, qp->recv.iobufs,
 146                ( ( ( void * ) qp ) + total_size ) );
 147         return qp;
 148 }
 149
 150 /**
 151  * Destroy queue pair
 152  *
 153  * @v ibdev             Infiniband device
 154  * @v qp                Queue pair
 155  */
 156 void ib_destroy_qp ( struct ib_device *ibdev,
 157                      struct ib_queue_pair *qp ) {
 158         DBGC ( ibdev, "IBDEV %p destroying queue pair %#lx\n",
 159                ibdev, qp->qpn );
 160         ibdev->op->destroy_qp ( ibdev, qp );
 161         list_del ( &qp->send.list );
 162         list_del ( &qp->recv.list );
 163         free ( qp );
 164 }
 165
 166 /**
 167  * Find work queue belonging to completion queue
 168  *
 169  * @v cq                Completion queue
 170  * @v qpn               Queue pair number
 171  * @v is_send           Find send work queue (rather than receive)
 172  * @ret wq              Work queue, or NULL if not found
 173  */
 174 struct ib_work_queue * ib_find_wq ( struct ib_completion_queue *cq,
 175                                     unsigned long qpn, int is_send ) {
 176         struct ib_work_queue *wq;
 177
 178         list_for_each_entry ( wq, &cq->work_queues, list ) {
 179                 if ( ( wq->qp->qpn == qpn ) && ( wq->is_send == is_send ) )
 180                         return wq;
 181         }
 182         return NULL;
 183 }
 184
 185 /**
 186  * Allocate Infiniband device
 187  *
 188  * @v priv_size         Size of private data area
 189  * @ret ibdev           Infiniband device, or NULL
 190  */
 191 struct ib_device * alloc_ibdev ( size_t priv_size ) {
 192         struct ib_device *ibdev;
 193         size_t total_len;
 194
 195         total_len = ( sizeof ( *ibdev ) + priv_size );
 196         ibdev = zalloc ( total_len );
 197         if ( ibdev ) {
 198                 ibdev->dev_priv = ( ( ( void * ) ibdev ) + sizeof ( *ibdev ) );
 199         }
 200         return ibdev;
 201 }
 202
 203 /**
 204  * Free Infiniband device
 205  *
 206  * @v ibdev             Infiniband device
 207  */
 208 void free_ibdev ( struct ib_device *ibdev ) {
 209         free ( ibdev );
 210 }