gpxe/src/net/infiniband/ib_packet.c

   1 /*
   2  * Copyright (C) 2008 Michael Brown <mbrown@fensystems.co.uk>.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public License as
   6  * published by the Free Software Foundation; either version 2 of the
   7  * License, or any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful, but
  10  * WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17  */
  18
  19 FILE_LICENCE ( GPL2_OR_LATER );
  20
  21 #include <stdint.h>
  22 #include <stdlib.h>
  23 #include <string.h>
  24 #include <errno.h>
  25 #include <byteswap.h>
  26 #include <gpxe/iobuf.h>
  27 #include <gpxe/infiniband.h>
  28 #include <gpxe/ib_packet.h>
  29
  30 /**
  31  * @file
  32  *
  33  * Infiniband Packet Formats
  34  *
  35  */
  36
  37 /**
  38  * Add IB headers
  39  *
  40  * @v ibdev             Infiniband device
  41  * @v iobuf             I/O buffer to contain headers
  42  * @v qp                Queue pair
  43  * @v payload_len       Payload length
  44  * @v av                Address vector
  45  */
  46 int ib_push ( struct ib_device *ibdev, struct io_buffer *iobuf,
  47               struct ib_queue_pair *qp, size_t payload_len,
  48               const struct ib_address_vector *av ) {
  49         struct ib_local_route_header *lrh;
  50         struct ib_global_route_header *grh;
  51         struct ib_base_transport_header *bth;
  52         struct ib_datagram_extended_transport_header *deth;
  53         size_t orig_iob_len = iob_len ( iobuf );
  54         size_t pad_len;
  55         size_t lrh_len;
  56         size_t grh_len;
  57         unsigned int vl;
  58         unsigned int lnh;
  59
  60         DBGC2 ( ibdev, "IBDEV %p TX %04x:%08lx => %04x:%08lx (key %08lx)\n",
  61                 ibdev, ibdev->lid, qp->ext_qpn, av->lid, av->qpn, av->qkey );
  62
  63         /* Calculate packet length */
  64         pad_len = ( (-payload_len) & 0x3 );
  65         payload_len += pad_len;
  66         payload_len += 4; /* ICRC */
  67
  68         /* Reserve space for headers */
  69         orig_iob_len = iob_len ( iobuf );
  70         deth = iob_push ( iobuf, sizeof ( *deth ) );
  71         bth = iob_push ( iobuf, sizeof ( *bth ) );
  72         grh_len = ( payload_len + iob_len ( iobuf ) - orig_iob_len );
  73         grh = ( av->gid_present ?
  74                 iob_push ( iobuf, sizeof ( *grh ) ) : NULL );
  75         lrh = iob_push ( iobuf, sizeof ( *lrh ) );
  76         lrh_len = ( payload_len + iob_len ( iobuf ) - orig_iob_len );
  77
  78         /* Construct LRH */
  79         vl = ( ( qp->ext_qpn == IB_QPN_SMI ) ? IB_VL_SMP : IB_VL_DEFAULT );
  80         lrh->vl__lver = ( vl << 4 );
  81         lnh = ( grh ? IB_LNH_GRH : IB_LNH_BTH );
  82         lrh->sl__lnh = ( ( av->sl << 4 ) | lnh );
  83         lrh->dlid = htons ( av->lid );
  84         lrh->length = htons ( lrh_len >> 2 );
  85         lrh->slid = htons ( ibdev->lid );
  86
  87         /* Construct GRH, if required */
  88         if ( grh ) {
  89                 grh->ipver__tclass__flowlabel =
  90                         htonl ( IB_GRH_IPVER_IPv6 << 28 );
  91                 grh->paylen = htons ( grh_len );
  92                 grh->nxthdr = IB_GRH_NXTHDR_IBA;
  93                 grh->hoplmt = 0;
  94                 memcpy ( &grh->sgid, &ibdev->gid, sizeof ( grh->sgid ) );
  95                 memcpy ( &grh->dgid, &av->gid, sizeof ( grh->dgid ) );
  96         }
  97
  98         /* Construct BTH */
  99         bth->opcode = BTH_OPCODE_UD_SEND;
 100         bth->se__m__padcnt__tver = ( pad_len << 4 );
 101         bth->pkey = htons ( ibdev->pkey );
 102         bth->dest_qp = htonl ( av->qpn );
 103         bth->ack__psn = htonl ( ( qp->send.psn++ ) & 0xffffffUL );
 104
 105         /* Construct DETH */
 106         deth->qkey = htonl ( av->qkey );
 107         deth->src_qp = htonl ( qp->ext_qpn );
 108
 109         DBGCP_HDA ( ibdev, 0, iobuf->data,
 110                     ( iob_len ( iobuf ) - orig_iob_len ) );
 111
 112         return 0;
 113 }
 114
 115 /**
 116  * Remove IB headers
 117  *
 118  * @v ibdev             Infiniband device
 119  * @v iobuf             I/O buffer containing headers
 120  * @v qp                Queue pair to fill in, or NULL
 121  * @v payload_len       Payload length to fill in, or NULL
 122  * @v av                Address vector to fill in
 123  */
 124 int ib_pull ( struct ib_device *ibdev, struct io_buffer *iobuf,
 125               struct ib_queue_pair **qp, size_t *payload_len,
 126               struct ib_address_vector *av ) {
 127         struct ib_local_route_header *lrh;
 128         struct ib_global_route_header *grh;
 129         struct ib_base_transport_header *bth;
 130         struct ib_datagram_extended_transport_header *deth;
 131         size_t orig_iob_len = iob_len ( iobuf );
 132         unsigned int lnh;
 133         size_t pad_len;
 134         unsigned long qpn;
 135         unsigned int lid;
 136
 137         /* Clear return values */
 138         if ( qp )
 139                 *qp = NULL;
 140         if ( payload_len )
 141                 *payload_len = 0;
 142         memset ( av, 0, sizeof ( *av ) );
 143
 144         /* Extract LRH */
 145         if ( iob_len ( iobuf ) < sizeof ( *lrh ) ) {
 146                 DBGC ( ibdev, "IBDEV %p RX too short (%zd bytes) for LRH\n",
 147                        ibdev, iob_len ( iobuf ) );
 148                 return -EINVAL;
 149         }
 150         lrh = iobuf->data;
 151         iob_pull ( iobuf, sizeof ( *lrh ) );
 152         av->lid = ntohs ( lrh->slid );
 153         av->sl = ( lrh->sl__lnh >> 4 );
 154         lnh = ( lrh->sl__lnh & 0x3 );
 155         lid = ntohs ( lrh->dlid );
 156
 157         /* Reject unsupported packets */
 158         if ( ! ( ( lnh == IB_LNH_BTH ) || ( lnh == IB_LNH_GRH ) ) ) {
 159                 DBGC ( ibdev, "IBDEV %p RX unsupported LNH %x\n",
 160                        ibdev, lnh );
 161                 return -ENOTSUP;
 162         }
 163
 164         /* Extract GRH, if present */
 165         if ( lnh == IB_LNH_GRH ) {
 166                 if ( iob_len ( iobuf ) < sizeof ( *grh ) ) {
 167                         DBGC ( ibdev, "IBDEV %p RX too short (%zd bytes) "
 168                                "for GRH\n", ibdev, iob_len ( iobuf ) );
 169                         return -EINVAL;
 170                 }
 171                 grh = iobuf->data;
 172                 iob_pull ( iobuf, sizeof ( *grh ) );
 173                 av->gid_present = 1;
 174                 memcpy ( &av->gid, &grh->sgid, sizeof ( av->gid ) );
 175         } else {
 176                 grh = NULL;
 177         }
 178
 179         /* Extract BTH */
 180         if ( iob_len ( iobuf ) < sizeof ( *bth ) ) {
 181                 DBGC ( ibdev, "IBDEV %p RX too short (%zd bytes) for BTH\n",
 182                        ibdev, iob_len ( iobuf ) );
 183                 return -EINVAL;
 184         }
 185         bth = iobuf->data;
 186         iob_pull ( iobuf, sizeof ( *bth ) );
 187         if ( bth->opcode != BTH_OPCODE_UD_SEND ) {
 188                 DBGC ( ibdev, "IBDEV %p unsupported BTH opcode %x\n",
 189                        ibdev, bth->opcode );
 190                 return -ENOTSUP;
 191         }
 192         qpn = ntohl ( bth->dest_qp );
 193
 194         /* Extract DETH */
 195         if ( iob_len ( iobuf ) < sizeof ( *deth ) ) {
 196                 DBGC ( ibdev, "IBDEV %p RX too short (%zd bytes) for DETH\n",
 197                        ibdev, iob_len ( iobuf ) );
 198                 return -EINVAL;
 199         }
 200         deth = iobuf->data;
 201         iob_pull ( iobuf, sizeof ( *deth ) );
 202         av->qpn = ntohl ( deth->src_qp );
 203         av->qkey = ntohl ( deth->qkey );
 204
 205         /* Calculate payload length, if applicable */
 206         if ( payload_len ) {
 207                 pad_len = ( ( bth->se__m__padcnt__tver >> 4 ) & 0x3 );
 208                 *payload_len = ( ( ntohs ( lrh->length ) << 2 )
 209                                  - ( orig_iob_len - iob_len ( iobuf ) )
 210                                  - pad_len - 4 /* ICRC */ );
 211         }
 212
 213         /* Determine destination QP, if applicable */
 214         if ( qp ) {
 215                 if ( IB_LID_MULTICAST ( lid ) && grh ) {
 216                         if ( ! ( *qp = ib_find_qp_mgid ( ibdev, &grh->dgid ))){
 217                                 DBGC ( ibdev, "IBDEV %p RX for unknown MGID "
 218                                        "%08x:%08x:%08x:%08x\n", ibdev,
 219                                        ntohl ( grh->dgid.u.dwords[0] ),
 220                                        ntohl ( grh->dgid.u.dwords[1] ),
 221                                        ntohl ( grh->dgid.u.dwords[2] ),
 222                                        ntohl ( grh->dgid.u.dwords[3] ) );
 223                                 return -ENODEV;
 224                         }
 225                 } else {
 226                         if ( ! ( *qp = ib_find_qp_qpn ( ibdev, qpn ) ) ) {
 227                                 DBGC ( ibdev, "IBDEV %p RX for nonexistent "
 228                                        "QPN %lx\n", ibdev, qpn );
 229                                 return -ENODEV;
 230                         }
 231                 }
 232                 assert ( *qp );
 233         }
 234
 235         DBGC2 ( ibdev, "IBDEV %p RX %04x:%08lx <= %04x:%08lx (key %08x)\n",
 236                 ibdev, lid, ( IB_LID_MULTICAST( lid ) ?
 237                               ( qp ? (*qp)->ext_qpn : -1UL ) : qpn ),
 238                 av->lid, av->qpn, ntohl ( deth->qkey ) );
 239         DBGCP_HDA ( ibdev, 0,
 240                     ( iobuf->data - ( orig_iob_len - iob_len ( iobuf ) ) ),
 241                     ( orig_iob_len - iob_len ( iobuf ) ) );
 242
 243         return 0;
 244 }