gpxe/src/net/infiniband/ib_pathrec.c

   1 /*
   2  * Copyright (C) 2009 Michael Brown <mbrown@fensystems.co.uk>.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public License as
   6  * published by the Free Software Foundation; either version 2 of the
   7  * License, or any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful, but
  10  * WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17  */
  18
  19 FILE_LICENCE ( GPL2_OR_LATER );
  20
  21 #include <stdint.h>
  22 #include <stdlib.h>
  23 #include <string.h>
  24 #include <byteswap.h>
  25 #include <errno.h>
  26 #include <gpxe/infiniband.h>
  27 #include <gpxe/ib_mi.h>
  28 #include <gpxe/ib_pathrec.h>
  29
  30 /** @file
  31  *
  32  * Infiniband path lookups
  33  *
  34  */
  35
  36 /**
  37  * Handle path transaction completion
  38  *
  39  * @v ibdev             Infiniband device
  40  * @v mi                Management interface
  41  * @v madx              Management transaction
  42  * @v rc                Status code
  43  * @v mad               Received MAD (or NULL on error)
  44  * @v av                Source address vector (or NULL on error)
  45  */
  46 static void ib_path_complete ( struct ib_device *ibdev,
  47                                struct ib_mad_interface *mi,
  48                                struct ib_mad_transaction *madx,
  49                                int rc, union ib_mad *mad,
  50                                struct ib_address_vector *av __unused ) {
  51         struct ib_path *path = ib_madx_get_ownerdata ( madx );
  52         struct ib_gid *dgid = &path->av.gid;
  53         struct ib_path_record *pathrec = &mad->sa.sa_data.path_record;
  54
  55         /* Report failures */
  56         if ( ( rc == 0 ) && ( mad->hdr.status != htons ( IB_MGMT_STATUS_OK ) ))
  57                 rc = -ENETUNREACH;
  58         if ( rc != 0 ) {
  59                 DBGC ( ibdev, "IBDEV %p path lookup for %08x:%08x:%08x:%08x "
  60                        "failed: %s\n", ibdev, htonl ( dgid->u.dwords[0] ),
  61                        htonl ( dgid->u.dwords[1] ),
  62                        htonl ( dgid->u.dwords[2] ),
  63                        htonl ( dgid->u.dwords[3] ), strerror ( rc ) );
  64                 goto out;
  65         }
  66
  67         /* Extract values from MAD */
  68         path->av.lid = ntohs ( pathrec->dlid );
  69         path->av.sl = ( pathrec->reserved__sl & 0x0f );
  70         path->av.rate = ( pathrec->rate_selector__rate & 0x3f );
  71         DBGC ( ibdev, "IBDEV %p path to %08x:%08x:%08x:%08x is %04x sl %d "
  72                "rate %d\n", ibdev, htonl ( dgid->u.dwords[0] ),
  73                htonl ( dgid->u.dwords[1] ), htonl ( dgid->u.dwords[2] ),
  74                htonl ( dgid->u.dwords[3] ), path->av.lid, path->av.sl,
  75                path->av.rate );
  76
  77  out:
  78         /* Destroy the completed transaction */
  79         ib_destroy_madx ( ibdev, mi, madx );
  80         path->madx = NULL;
  81
  82         /* Hand off to upper completion handler */
  83         path->op->complete ( ibdev, path, rc, &path->av );
  84 }
  85
  86 /** Path transaction completion operations */
  87 static struct ib_mad_transaction_operations ib_path_op = {
  88         .complete = ib_path_complete,
  89 };
  90
  91 /**
  92  * Create path
  93  *
  94  * @v ibdev             Infiniband device
  95  * @v av                Address vector to complete
  96  * @v op                Path operations
  97  * @ret path            Path
  98  */
  99 struct ib_path *
 100 ib_create_path ( struct ib_device *ibdev, struct ib_address_vector *av,
 101                  struct ib_path_operations *op ) {
 102         struct ib_path *path;
 103         union ib_mad mad;
 104         struct ib_mad_sa *sa = &mad.sa;
 105
 106         /* Allocate and initialise structure */
 107         path = zalloc ( sizeof ( *path ) );
 108         if ( ! path )
 109                 goto err_alloc_path;
 110         path->ibdev = ibdev;
 111         memcpy ( &path->av, av, sizeof ( path->av ) );
 112         path->op = op;
 113
 114         /* Construct path request */
 115         memset ( sa, 0, sizeof ( *sa ) );
 116         sa->mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM;
 117         sa->mad_hdr.class_version = IB_SA_CLASS_VERSION;
 118         sa->mad_hdr.method = IB_MGMT_METHOD_GET;
 119         sa->mad_hdr.attr_id = htons ( IB_SA_ATTR_PATH_REC );
 120         sa->sa_hdr.comp_mask[1] =
 121                 htonl ( IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID );
 122         memcpy ( &sa->sa_data.path_record.dgid, &path->av.gid,
 123                  sizeof ( sa->sa_data.path_record.dgid ) );
 124         memcpy ( &sa->sa_data.path_record.sgid, &ibdev->gid,
 125                  sizeof ( sa->sa_data.path_record.sgid ) );
 126
 127         /* Create management transaction */
 128         path->madx = ib_create_madx ( ibdev, ibdev->gsi, &mad, NULL,
 129                                       &ib_path_op );
 130         if ( ! path->madx )
 131                 goto err_create_madx;
 132         ib_madx_set_ownerdata ( path->madx, path );
 133
 134         return path;
 135
 136         ib_destroy_madx ( ibdev, ibdev->gsi, path->madx );
 137  err_create_madx:
 138         free ( path );
 139  err_alloc_path:
 140         return NULL;
 141 }
 142
 143 /**
 144  * Destroy path
 145  *
 146  * @v ibdev             Infiniband device
 147  * @v path              Path
 148  */
 149 void ib_destroy_path ( struct ib_device *ibdev, struct ib_path *path ) {
 150
 151         if ( path->madx )
 152                 ib_destroy_madx ( ibdev, ibdev->gsi, path->madx );
 153         free ( path );
 154 }
 155
 156 /** Number of path cache entries
 157  *
 158  * Must be a power of two.
 159  */
 160 #define IB_NUM_CACHED_PATHS 4
 161
 162 /** A cached path */
 163 struct ib_cached_path {
 164         /** Path */
 165         struct ib_path *path;
 166 };
 167
 168 /** Path cache */
 169 static struct ib_cached_path ib_path_cache[IB_NUM_CACHED_PATHS];
 170
 171 /** Oldest path cache entry index */
 172 static unsigned int ib_path_cache_idx;
 173
 174 /**
 175  * Find path cache entry
 176  *
 177  * @v ibdev             Infiniband device
 178  * @v dgid              Destination GID
 179  * @ret path            Path cache entry, or NULL
 180  */
 181 static struct ib_cached_path *
 182 ib_find_path_cache_entry ( struct ib_device *ibdev, struct ib_gid *dgid ) {
 183         struct ib_cached_path *cached;
 184         unsigned int i;
 185
 186         for ( i = 0 ; i < IB_NUM_CACHED_PATHS ; i++ ) {
 187                 cached = &ib_path_cache[i];
 188                 if ( ! cached->path )
 189                         continue;
 190                 if ( cached->path->ibdev != ibdev )
 191                         continue;
 192                 if ( memcmp ( &cached->path->av.gid, dgid,
 193                               sizeof ( cached->path->av.gid ) ) != 0 )
 194                         continue;
 195                 return cached;
 196         }
 197
 198         return NULL;
 199 }
 200
 201 /**
 202  * Handle cached path transaction completion
 203  *
 204  * @v ibdev             Infiniband device
 205  * @v path              Path
 206  * @v rc                Status code
 207  * @v av                Address vector, or NULL on error
 208  */
 209 static void ib_cached_path_complete ( struct ib_device *ibdev,
 210                                       struct ib_path *path, int rc,
 211                                       struct ib_address_vector *av __unused ) {
 212         struct ib_cached_path *cached = ib_path_get_ownerdata ( path );
 213
 214         /* If the transaction failed, erase the cache entry */
 215         if ( rc != 0 ) {
 216                 /* Destroy the old cache entry */
 217                 ib_destroy_path ( ibdev, path );
 218                 memset ( cached, 0, sizeof ( *cached ) );
 219                 return;
 220         }
 221
 222         /* Do not destroy the completed transaction; we still need to
 223          * refer to the resolved path.
 224          */
 225 }
 226
 227 /** Cached path transaction completion operations */
 228 static struct ib_path_operations ib_cached_path_op = {
 229         .complete = ib_cached_path_complete,
 230 };
 231
 232 /**
 233  * Resolve path
 234  *
 235  * @v ibdev             Infiniband device
 236  * @v av                Address vector to complete
 237  * @ret rc              Return status code
 238  *
 239  * This provides a non-transactional way to resolve a path, via a
 240  * cache similar to ARP.
 241  */
 242 int ib_resolve_path ( struct ib_device *ibdev, struct ib_address_vector *av ) {
 243         struct ib_gid *gid = &av->gid;
 244         struct ib_cached_path *cached;
 245         unsigned int cache_idx;
 246
 247         /* Sanity check */
 248         if ( ! av->gid_present ) {
 249                 DBGC ( ibdev, "IBDEV %p attempt to look up path "
 250                        "without GID\n", ibdev );
 251                 return -EINVAL;
 252         }
 253
 254         /* Look in cache for a matching entry */
 255         cached = ib_find_path_cache_entry ( ibdev, gid );
 256         if ( cached && cached->path->av.lid ) {
 257                 /* Populated entry found */
 258                 av->lid = cached->path->av.lid;
 259                 av->rate = cached->path->av.rate;
 260                 av->sl = cached->path->av.sl;
 261                 DBGC2 ( ibdev, "IBDEV %p cache hit for %08x:%08x:%08x:%08x\n",
 262                         ibdev, htonl ( gid->u.dwords[0] ),
 263                         htonl ( gid->u.dwords[1] ), htonl ( gid->u.dwords[2] ),
 264                         htonl ( gid->u.dwords[3] ) );
 265                 return 0;
 266         }
 267         DBGC ( ibdev, "IBDEV %p cache miss for %08x:%08x:%08x:%08x%s\n",
 268                ibdev, htonl ( gid->u.dwords[0] ), htonl ( gid->u.dwords[1] ),
 269                htonl ( gid->u.dwords[2] ), htonl ( gid->u.dwords[3] ),
 270                ( cached ? " (in progress)" : "" ) );
 271
 272         /* If lookup is already in progress, do nothing */
 273         if ( cached )
 274                 return -ENOENT;
 275
 276         /* Locate a new cache entry to use */
 277         cache_idx = ( (ib_path_cache_idx++) % IB_NUM_CACHED_PATHS );
 278         cached = &ib_path_cache[cache_idx];
 279
 280         /* Destroy the old cache entry */
 281         if ( cached->path )
 282                 ib_destroy_path ( ibdev, cached->path );
 283         memset ( cached, 0, sizeof ( *cached ) );
 284
 285         /* Create new path */
 286         cached->path = ib_create_path ( ibdev, av, &ib_cached_path_op );
 287         if ( ! cached->path ) {
 288                 DBGC ( ibdev, "IBDEV %p could not create path\n",
 289                        ibdev );
 290                 return -ENOMEM;
 291         }
 292         ib_path_set_ownerdata ( cached->path, cached );
 293
 294         /* Not found yet */
 295         return -ENOENT;
 296 }