net/sunrpc/xprtrdma/fmr_ops.c

   1 /*
   2  * Copyright (c) 2015 Oracle.  All rights reserved.
   3  * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
   4  */
   5
   6 /* Lightweight memory registration using Fast Memory Regions (FMR).
   7  * Referred to sometimes as MTHCAFMR mode.
   8  *
   9  * FMR uses synchronous memory registration and deregistration.
  10  * FMR registration is known to be fast, but FMR deregistration
  11  * can take tens of usecs to complete.
  12  */
  13
  14 /* Normal operation
  15  *
  16  * A Memory Region is prepared for RDMA READ or WRITE using the
  17  * ib_map_phys_fmr verb (fmr_op_map). When the RDMA operation is
  18  * finished, the Memory Region is unmapped using the ib_unmap_fmr
  19  * verb (fmr_op_unmap).
  20  */
  21
  22 /* Transport recovery
  23  *
  24  * After a transport reconnect, fmr_op_map re-uses the MR already
  25  * allocated for the RPC, but generates a fresh rkey then maps the
  26  * MR again. This process is synchronous.
  27  */
  28
  29 #include "xprt_rdma.h"
  30
  31 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
  32 # define RPCDBG_FACILITY        RPCDBG_TRANS
  33 #endif
  34
  35 /* Maximum scatter/gather per FMR */
  36 #define RPCRDMA_MAX_FMR_SGES    (64)
  37
  38 static struct workqueue_struct *fmr_recovery_wq;
  39
  40 #define FMR_RECOVERY_WQ_FLAGS           (WQ_UNBOUND)
  41
  42 int
  43 fmr_alloc_recovery_wq(void)
  44 {
  45         fmr_recovery_wq = alloc_workqueue("fmr_recovery", WQ_UNBOUND, 0);
  46         return !fmr_recovery_wq ? -ENOMEM : 0;
  47 }
  48
  49 void
  50 fmr_destroy_recovery_wq(void)
  51 {
  52         struct workqueue_struct *wq;
  53
  54         if (!fmr_recovery_wq)
  55                 return;
  56
  57         wq = fmr_recovery_wq;
  58         fmr_recovery_wq = NULL;
  59         destroy_workqueue(wq);
  60 }
  61
  62 static int
  63 __fmr_unmap(struct rpcrdma_mw *mw)
  64 {
  65         LIST_HEAD(l);
  66
  67         list_add(&mw->fmr.fmr->list, &l);
  68         return ib_unmap_fmr(&l);
  69 }
  70
  71 /* Deferred reset of a single FMR. Generate a fresh rkey by
  72  * replacing the MR. There's no recovery if this fails.
  73  */
  74 static void
  75 __fmr_recovery_worker(struct work_struct *work)
  76 {
  77         struct rpcrdma_mw *mw = container_of(work, struct rpcrdma_mw,
  78                                             mw_work);
  79         struct rpcrdma_xprt *r_xprt = mw->mw_xprt;
  80
  81         __fmr_unmap(mw);
  82         rpcrdma_put_mw(r_xprt, mw);
  83         return;
  84 }
  85
  86 /* A broken MR was discovered in a context that can't sleep.
  87  * Defer recovery to the recovery worker.
  88  */
  89 static void
  90 __fmr_queue_recovery(struct rpcrdma_mw *mw)
  91 {
  92         INIT_WORK(&mw->mw_work, __fmr_recovery_worker);
  93         queue_work(fmr_recovery_wq, &mw->mw_work);
  94 }
  95
  96 static int
  97 fmr_op_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep,
  98             struct rpcrdma_create_data_internal *cdata)
  99 {
 100         rpcrdma_set_max_header_sizes(ia, cdata, max_t(unsigned int, 1,
 101                                                       RPCRDMA_MAX_DATA_SEGS /
 102                                                       RPCRDMA_MAX_FMR_SGES));
 103         return 0;
 104 }
 105
 106 /* FMR mode conveys up to 64 pages of payload per chunk segment.
 107  */
 108 static size_t
 109 fmr_op_maxpages(struct rpcrdma_xprt *r_xprt)
 110 {
 111         return min_t(unsigned int, RPCRDMA_MAX_DATA_SEGS,
 112                      RPCRDMA_MAX_HDR_SEGS * RPCRDMA_MAX_FMR_SGES);
 113 }
 114
 115 static int
 116 fmr_op_init(struct rpcrdma_xprt *r_xprt)
 117 {
 118         struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
 119         int mr_access_flags = IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ;
 120         struct ib_fmr_attr fmr_attr = {
 121                 .max_pages      = RPCRDMA_MAX_FMR_SGES,
 122                 .max_maps       = 1,
 123                 .page_shift     = PAGE_SHIFT
 124         };
 125         struct ib_pd *pd = r_xprt->rx_ia.ri_pd;
 126         struct rpcrdma_mw *r;
 127         int i, rc;
 128
 129         spin_lock_init(&buf->rb_mwlock);
 130         INIT_LIST_HEAD(&buf->rb_mws);
 131         INIT_LIST_HEAD(&buf->rb_all);
 132
 133         i = max_t(int, RPCRDMA_MAX_DATA_SEGS / RPCRDMA_MAX_FMR_SGES, 1);
 134         i += 2;                         /* head + tail */
 135         i *= buf->rb_max_requests;      /* one set for each RPC slot */
 136         dprintk("RPC:       %s: initalizing %d FMRs\n", __func__, i);
 137
 138         rc = -ENOMEM;
 139         while (i--) {
 140                 r = kzalloc(sizeof(*r), GFP_KERNEL);
 141                 if (!r)
 142                         goto out;
 143
 144                 r->fmr.physaddrs = kmalloc(RPCRDMA_MAX_FMR_SGES *
 145                                            sizeof(u64), GFP_KERNEL);
 146                 if (!r->fmr.physaddrs)
 147                         goto out_free;
 148
 149                 r->fmr.fmr = ib_alloc_fmr(pd, mr_access_flags, &fmr_attr);
 150                 if (IS_ERR(r->fmr.fmr))
 151                         goto out_fmr_err;
 152
 153                 r->mw_xprt = r_xprt;
 154                 list_add(&r->mw_list, &buf->rb_mws);
 155                 list_add(&r->mw_all, &buf->rb_all);
 156         }
 157         return 0;
 158
 159 out_fmr_err:
 160         rc = PTR_ERR(r->fmr.fmr);
 161         dprintk("RPC:       %s: ib_alloc_fmr status %i\n", __func__, rc);
 162         kfree(r->fmr.physaddrs);
 163 out_free:
 164         kfree(r);
 165 out:
 166         return rc;
 167 }
 168
 169 /* Use the ib_map_phys_fmr() verb to register a memory region
 170  * for remote access via RDMA READ or RDMA WRITE.
 171  */
 172 static int
 173 fmr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg,
 174            int nsegs, bool writing)
 175 {
 176         struct rpcrdma_ia *ia = &r_xprt->rx_ia;
 177         struct ib_device *device = ia->ri_device;
 178         enum dma_data_direction direction = rpcrdma_data_dir(writing);
 179         struct rpcrdma_mr_seg *seg1 = seg;
 180         int len, pageoff, i, rc;
 181         struct rpcrdma_mw *mw;
 182
 183         mw = seg1->rl_mw;
 184         seg1->rl_mw = NULL;
 185         if (!mw) {
 186                 mw = rpcrdma_get_mw(r_xprt);
 187                 if (!mw)
 188                         return -ENOMEM;
 189         } else {
 190                 /* this is a retransmit; generate a fresh rkey */
 191                 rc = __fmr_unmap(mw);
 192                 if (rc)
 193                         return rc;
 194         }
 195
 196         pageoff = offset_in_page(seg1->mr_offset);
 197         seg1->mr_offset -= pageoff;     /* start of page */
 198         seg1->mr_len += pageoff;
 199         len = -pageoff;
 200         if (nsegs > RPCRDMA_MAX_FMR_SGES)
 201                 nsegs = RPCRDMA_MAX_FMR_SGES;
 202         for (i = 0; i < nsegs;) {
 203                 rpcrdma_map_one(device, seg, direction);
 204                 mw->fmr.physaddrs[i] = seg->mr_dma;
 205                 len += seg->mr_len;
 206                 ++seg;
 207                 ++i;
 208                 /* Check for holes */
 209                 if ((i < nsegs && offset_in_page(seg->mr_offset)) ||
 210                     offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len))
 211                         break;
 212         }
 213
 214         rc = ib_map_phys_fmr(mw->fmr.fmr, mw->fmr.physaddrs,
 215                              i, seg1->mr_dma);
 216         if (rc)
 217                 goto out_maperr;
 218
 219         seg1->rl_mw = mw;
 220         seg1->mr_rkey = mw->fmr.fmr->rkey;
 221         seg1->mr_base = seg1->mr_dma + pageoff;
 222         seg1->mr_nsegs = i;
 223         seg1->mr_len = len;
 224         return i;
 225
 226 out_maperr:
 227         dprintk("RPC:       %s: ib_map_phys_fmr %u@0x%llx+%i (%d) status %i\n",
 228                 __func__, len, (unsigned long long)seg1->mr_dma,
 229                 pageoff, i, rc);
 230         while (i--)
 231                 rpcrdma_unmap_one(device, --seg);
 232         return rc;
 233 }
 234
 235 static void
 236 __fmr_dma_unmap(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg)
 237 {
 238         struct ib_device *device = r_xprt->rx_ia.ri_device;
 239         int nsegs = seg->mr_nsegs;
 240
 241         while (nsegs--)
 242                 rpcrdma_unmap_one(device, seg++);
 243 }
 244
 245 /* Invalidate all memory regions that were registered for "req".
 246  *
 247  * Sleeps until it is safe for the host CPU to access the
 248  * previously mapped memory regions.
 249  */
 250 static void
 251 fmr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)
 252 {
 253         struct rpcrdma_mr_seg *seg;
 254         unsigned int i, nchunks;
 255         struct rpcrdma_mw *mw;
 256         LIST_HEAD(unmap_list);
 257         int rc;
 258
 259         dprintk("RPC:       %s: req %p\n", __func__, req);
 260
 261         /* ORDER: Invalidate all of the req's MRs first
 262          *
 263          * ib_unmap_fmr() is slow, so use a single call instead
 264          * of one call per mapped MR.
 265          */
 266         for (i = 0, nchunks = req->rl_nchunks; nchunks; nchunks--) {
 267                 seg = &req->rl_segments[i];
 268                 mw = seg->rl_mw;
 269
 270                 list_add(&mw->fmr.fmr->list, &unmap_list);
 271
 272                 i += seg->mr_nsegs;
 273         }
 274         rc = ib_unmap_fmr(&unmap_list);
 275         if (rc)
 276                 pr_warn("%s: ib_unmap_fmr failed (%i)\n", __func__, rc);
 277
 278         /* ORDER: Now DMA unmap all of the req's MRs, and return
 279          * them to the free MW list.
 280          */
 281         for (i = 0, nchunks = req->rl_nchunks; nchunks; nchunks--) {
 282                 seg = &req->rl_segments[i];
 283
 284                 __fmr_dma_unmap(r_xprt, seg);
 285                 rpcrdma_put_mw(r_xprt, seg->rl_mw);
 286
 287                 i += seg->mr_nsegs;
 288                 seg->mr_nsegs = 0;
 289                 seg->rl_mw = NULL;
 290         }
 291
 292         req->rl_nchunks = 0;
 293 }
 294
 295 /* Use a slow, safe mechanism to invalidate all memory regions
 296  * that were registered for "req".
 297  *
 298  * In the asynchronous case, DMA unmapping occurs first here
 299  * because the rpcrdma_mr_seg is released immediately after this
 300  * call. It's contents won't be available in __fmr_dma_unmap later.
 301  * FIXME.
 302  */
 303 static void
 304 fmr_op_unmap_safe(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req,
 305                   bool sync)
 306 {
 307         struct rpcrdma_mr_seg *seg;
 308         struct rpcrdma_mw *mw;
 309         unsigned int i;
 310
 311         for (i = 0; req->rl_nchunks; req->rl_nchunks--) {
 312                 seg = &req->rl_segments[i];
 313                 mw = seg->rl_mw;
 314
 315                 if (sync) {
 316                         /* ORDER */
 317                         __fmr_unmap(mw);
 318                         __fmr_dma_unmap(r_xprt, seg);
 319                         rpcrdma_put_mw(r_xprt, mw);
 320                 } else {
 321                         __fmr_dma_unmap(r_xprt, seg);
 322                         __fmr_queue_recovery(mw);
 323                 }
 324
 325                 i += seg->mr_nsegs;
 326                 seg->mr_nsegs = 0;
 327                 seg->rl_mw = NULL;
 328         }
 329 }
 330
 331 static void
 332 fmr_op_destroy(struct rpcrdma_buffer *buf)
 333 {
 334         struct rpcrdma_mw *r;
 335         int rc;
 336
 337         while (!list_empty(&buf->rb_all)) {
 338                 r = list_entry(buf->rb_all.next, struct rpcrdma_mw, mw_all);
 339                 list_del(&r->mw_all);
 340                 kfree(r->fmr.physaddrs);
 341
 342                 rc = ib_dealloc_fmr(r->fmr.fmr);
 343                 if (rc)
 344                         dprintk("RPC:       %s: ib_dealloc_fmr failed %i\n",
 345                                 __func__, rc);
 346
 347                 kfree(r);
 348         }
 349 }
 350
 351 const struct rpcrdma_memreg_ops rpcrdma_fmr_memreg_ops = {
 352         .ro_map                         = fmr_op_map,
 353         .ro_unmap_sync                  = fmr_op_unmap_sync,
 354         .ro_unmap_safe                  = fmr_op_unmap_safe,
 355         .ro_open                        = fmr_op_open,
 356         .ro_maxpages                    = fmr_op_maxpages,
 357         .ro_init                        = fmr_op_init,
 358         .ro_destroy                     = fmr_op_destroy,
 359         .ro_displayname                 = "fmr",
 360 };