include/trace/events/rpcrdma.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  * Copyright (c) 2017, 2018 Oracle.  All rights reserved.
   4  *
   5  * Trace point definitions for the "rpcrdma" subsystem.
   6  */
   7 #undef TRACE_SYSTEM
   8 #define TRACE_SYSTEM rpcrdma
   9
  10 #if !defined(_TRACE_RPCRDMA_H) || defined(TRACE_HEADER_MULTI_READ)
  11 #define _TRACE_RPCRDMA_H
  12
  13 #include <linux/scatterlist.h>
  14 #include <linux/sunrpc/rpc_rdma_cid.h>
  15 #include <linux/tracepoint.h>
  16 #include <rdma/ib_cm.h>
  17
  18 #include <trace/misc/rdma.h>
  19 #include <trace/misc/sunrpc.h>
  20
  21 /**
  22  ** Event classes
  23  **/
  24
  25 DECLARE_EVENT_CLASS(rpcrdma_simple_cid_class,
  26         TP_PROTO(
  27                 const struct rpc_rdma_cid *cid
  28         ),
  29
  30         TP_ARGS(cid),
  31
  32         TP_STRUCT__entry(
  33                 __field(u32, cq_id)
  34                 __field(int, completion_id)
  35         ),
  36
  37         TP_fast_assign(
  38                 __entry->cq_id = cid->ci_queue_id;
  39                 __entry->completion_id = cid->ci_completion_id;
  40         ),
  41
  42         TP_printk("cq.id=%d cid=%d",
  43                 __entry->cq_id, __entry->completion_id
  44         )
  45 );
  46
  47 #define DEFINE_SIMPLE_CID_EVENT(name)                                   \
  48                 DEFINE_EVENT(rpcrdma_simple_cid_class, name,            \
  49                                 TP_PROTO(                               \
  50                                         const struct rpc_rdma_cid *cid  \
  51                                 ),                                      \
  52                                 TP_ARGS(cid)                            \
  53                 )
  54
  55 DECLARE_EVENT_CLASS(rpcrdma_completion_class,
  56         TP_PROTO(
  57                 const struct ib_wc *wc,
  58                 const struct rpc_rdma_cid *cid
  59         ),
  60
  61         TP_ARGS(wc, cid),
  62
  63         TP_STRUCT__entry(
  64                 __field(u32, cq_id)
  65                 __field(int, completion_id)
  66                 __field(unsigned long, status)
  67                 __field(unsigned int, vendor_err)
  68         ),
  69
  70         TP_fast_assign(
  71                 __entry->cq_id = cid->ci_queue_id;
  72                 __entry->completion_id = cid->ci_completion_id;
  73                 __entry->status = wc->status;
  74                 if (wc->status)
  75                         __entry->vendor_err = wc->vendor_err;
  76                 else
  77                         __entry->vendor_err = 0;
  78         ),
  79
  80         TP_printk("cq.id=%u cid=%d status=%s (%lu/0x%x)",
  81                 __entry->cq_id, __entry->completion_id,
  82                 rdma_show_wc_status(__entry->status),
  83                 __entry->status, __entry->vendor_err
  84         )
  85 );
  86
  87 #define DEFINE_COMPLETION_EVENT(name)                                   \
  88                 DEFINE_EVENT(rpcrdma_completion_class, name,            \
  89                                 TP_PROTO(                               \
  90                                         const struct ib_wc *wc,         \
  91                                         const struct rpc_rdma_cid *cid  \
  92                                 ),                                      \
  93                                 TP_ARGS(wc, cid))
  94
  95 DECLARE_EVENT_CLASS(rpcrdma_send_flush_class,
  96         TP_PROTO(
  97                 const struct ib_wc *wc,
  98                 const struct rpc_rdma_cid *cid
  99         ),
 100
 101         TP_ARGS(wc, cid),
 102
 103         TP_STRUCT__entry(
 104                 __field(u32, cq_id)
 105                 __field(int, completion_id)
 106                 __field(unsigned long, status)
 107                 __field(unsigned int, vendor_err)
 108         ),
 109
 110         TP_fast_assign(
 111                 __entry->cq_id = cid->ci_queue_id;
 112                 __entry->completion_id = cid->ci_completion_id;
 113                 __entry->status = wc->status;
 114                 __entry->vendor_err = wc->vendor_err;
 115         ),
 116
 117         TP_printk("cq.id=%u cid=%d status=%s (%lu/0x%x)",
 118                 __entry->cq_id, __entry->completion_id,
 119                 rdma_show_wc_status(__entry->status),
 120                 __entry->status, __entry->vendor_err
 121         )
 122 );
 123
 124 #define DEFINE_SEND_FLUSH_EVENT(name)                                   \
 125                 DEFINE_EVENT(rpcrdma_send_flush_class, name,            \
 126                                 TP_PROTO(                               \
 127                                         const struct ib_wc *wc,         \
 128                                         const struct rpc_rdma_cid *cid  \
 129                                 ),                                      \
 130                                 TP_ARGS(wc, cid))
 131
 132 DECLARE_EVENT_CLASS(rpcrdma_mr_completion_class,
 133         TP_PROTO(
 134                 const struct ib_wc *wc,
 135                 const struct rpc_rdma_cid *cid
 136         ),
 137
 138         TP_ARGS(wc, cid),
 139
 140         TP_STRUCT__entry(
 141                 __field(u32, cq_id)
 142                 __field(int, completion_id)
 143                 __field(unsigned long, status)
 144                 __field(unsigned int, vendor_err)
 145         ),
 146
 147         TP_fast_assign(
 148                 __entry->cq_id = cid->ci_queue_id;
 149                 __entry->completion_id = cid->ci_completion_id;
 150                 __entry->status = wc->status;
 151                 if (wc->status)
 152                         __entry->vendor_err = wc->vendor_err;
 153                 else
 154                         __entry->vendor_err = 0;
 155         ),
 156
 157         TP_printk("cq.id=%u mr.id=%d status=%s (%lu/0x%x)",
 158                 __entry->cq_id, __entry->completion_id,
 159                 rdma_show_wc_status(__entry->status),
 160                 __entry->status, __entry->vendor_err
 161         )
 162 );
 163
 164 #define DEFINE_MR_COMPLETION_EVENT(name)                                \
 165                 DEFINE_EVENT(rpcrdma_mr_completion_class, name,         \
 166                                 TP_PROTO(                               \
 167                                         const struct ib_wc *wc,         \
 168                                         const struct rpc_rdma_cid *cid  \
 169                                 ),                                      \
 170                                 TP_ARGS(wc, cid))
 171
 172 DECLARE_EVENT_CLASS(rpcrdma_receive_completion_class,
 173         TP_PROTO(
 174                 const struct ib_wc *wc,
 175                 const struct rpc_rdma_cid *cid
 176         ),
 177
 178         TP_ARGS(wc, cid),
 179
 180         TP_STRUCT__entry(
 181                 __field(u32, cq_id)
 182                 __field(int, completion_id)
 183                 __field(u32, received)
 184                 __field(unsigned long, status)
 185                 __field(unsigned int, vendor_err)
 186         ),
 187
 188         TP_fast_assign(
 189                 __entry->cq_id = cid->ci_queue_id;
 190                 __entry->completion_id = cid->ci_completion_id;
 191                 __entry->status = wc->status;
 192                 if (wc->status) {
 193                         __entry->received = 0;
 194                         __entry->vendor_err = wc->vendor_err;
 195                 } else {
 196                         __entry->received = wc->byte_len;
 197                         __entry->vendor_err = 0;
 198                 }
 199         ),
 200
 201         TP_printk("cq.id=%u cid=%d status=%s (%lu/0x%x) received=%u",
 202                 __entry->cq_id, __entry->completion_id,
 203                 rdma_show_wc_status(__entry->status),
 204                 __entry->status, __entry->vendor_err,
 205                 __entry->received
 206         )
 207 );
 208
 209 #define DEFINE_RECEIVE_COMPLETION_EVENT(name)                           \
 210                 DEFINE_EVENT(rpcrdma_receive_completion_class, name,    \
 211                                 TP_PROTO(                               \
 212                                         const struct ib_wc *wc,         \
 213                                         const struct rpc_rdma_cid *cid  \
 214                                 ),                                      \
 215                                 TP_ARGS(wc, cid))
 216
 217 DECLARE_EVENT_CLASS(rpcrdma_receive_success_class,
 218         TP_PROTO(
 219                 const struct ib_wc *wc,
 220                 const struct rpc_rdma_cid *cid
 221         ),
 222
 223         TP_ARGS(wc, cid),
 224
 225         TP_STRUCT__entry(
 226                 __field(u32, cq_id)
 227                 __field(int, completion_id)
 228                 __field(u32, received)
 229         ),
 230
 231         TP_fast_assign(
 232                 __entry->cq_id = cid->ci_queue_id;
 233                 __entry->completion_id = cid->ci_completion_id;
 234                 __entry->received = wc->byte_len;
 235         ),
 236
 237         TP_printk("cq.id=%u cid=%d received=%u",
 238                 __entry->cq_id, __entry->completion_id,
 239                 __entry->received
 240         )
 241 );
 242
 243 #define DEFINE_RECEIVE_SUCCESS_EVENT(name)                              \
 244                 DEFINE_EVENT(rpcrdma_receive_success_class, name,       \
 245                                 TP_PROTO(                               \
 246                                         const struct ib_wc *wc,         \
 247                                         const struct rpc_rdma_cid *cid  \
 248                                 ),                                      \
 249                                 TP_ARGS(wc, cid))
 250
 251 DECLARE_EVENT_CLASS(rpcrdma_receive_flush_class,
 252         TP_PROTO(
 253                 const struct ib_wc *wc,
 254                 const struct rpc_rdma_cid *cid
 255         ),
 256
 257         TP_ARGS(wc, cid),
 258
 259         TP_STRUCT__entry(
 260                 __field(u32, cq_id)
 261                 __field(int, completion_id)
 262                 __field(unsigned long, status)
 263                 __field(unsigned int, vendor_err)
 264         ),
 265
 266         TP_fast_assign(
 267                 __entry->cq_id = cid->ci_queue_id;
 268                 __entry->completion_id = cid->ci_completion_id;
 269                 __entry->status = wc->status;
 270                 __entry->vendor_err = wc->vendor_err;
 271         ),
 272
 273         TP_printk("cq.id=%u cid=%d status=%s (%lu/0x%x)",
 274                 __entry->cq_id, __entry->completion_id,
 275                 rdma_show_wc_status(__entry->status),
 276                 __entry->status, __entry->vendor_err
 277         )
 278 );
 279
 280 #define DEFINE_RECEIVE_FLUSH_EVENT(name)                                \
 281                 DEFINE_EVENT(rpcrdma_receive_flush_class, name,         \
 282                                 TP_PROTO(                               \
 283                                         const struct ib_wc *wc,         \
 284                                         const struct rpc_rdma_cid *cid  \
 285                                 ),                                      \
 286                                 TP_ARGS(wc, cid))
 287
 288 DECLARE_EVENT_CLASS(xprtrdma_reply_class,
 289         TP_PROTO(
 290                 const struct rpcrdma_rep *rep
 291         ),
 292
 293         TP_ARGS(rep),
 294
 295         TP_STRUCT__entry(
 296                 __field(u32, xid)
 297                 __field(u32, version)
 298                 __field(u32, proc)
 299                 __string(addr, rpcrdma_addrstr(rep->rr_rxprt))
 300                 __string(port, rpcrdma_portstr(rep->rr_rxprt))
 301         ),
 302
 303         TP_fast_assign(
 304                 __entry->xid = be32_to_cpu(rep->rr_xid);
 305                 __entry->version = be32_to_cpu(rep->rr_vers);
 306                 __entry->proc = be32_to_cpu(rep->rr_proc);
 307                 __assign_str(addr);
 308                 __assign_str(port);
 309         ),
 310
 311         TP_printk("peer=[%s]:%s xid=0x%08x version=%u proc=%u",
 312                 __get_str(addr), __get_str(port),
 313                 __entry->xid, __entry->version, __entry->proc
 314         )
 315 );
 316
 317 #define DEFINE_REPLY_EVENT(name)                                        \
 318                 DEFINE_EVENT(xprtrdma_reply_class,                      \
 319                                 xprtrdma_reply_##name##_err,            \
 320                                 TP_PROTO(                               \
 321                                         const struct rpcrdma_rep *rep   \
 322                                 ),                                      \
 323                                 TP_ARGS(rep))
 324
 325 DECLARE_EVENT_CLASS(xprtrdma_rxprt,
 326         TP_PROTO(
 327                 const struct rpcrdma_xprt *r_xprt
 328         ),
 329
 330         TP_ARGS(r_xprt),
 331
 332         TP_STRUCT__entry(
 333                 __string(addr, rpcrdma_addrstr(r_xprt))
 334                 __string(port, rpcrdma_portstr(r_xprt))
 335         ),
 336
 337         TP_fast_assign(
 338                 __assign_str(addr);
 339                 __assign_str(port);
 340         ),
 341
 342         TP_printk("peer=[%s]:%s",
 343                 __get_str(addr), __get_str(port)
 344         )
 345 );
 346
 347 #define DEFINE_RXPRT_EVENT(name)                                        \
 348                 DEFINE_EVENT(xprtrdma_rxprt, name,                      \
 349                                 TP_PROTO(                               \
 350                                         const struct rpcrdma_xprt *r_xprt \
 351                                 ),                                      \
 352                                 TP_ARGS(r_xprt))
 353
 354 DECLARE_EVENT_CLASS(xprtrdma_connect_class,
 355         TP_PROTO(
 356                 const struct rpcrdma_xprt *r_xprt,
 357                 int rc
 358         ),
 359
 360         TP_ARGS(r_xprt, rc),
 361
 362         TP_STRUCT__entry(
 363                 __field(int, rc)
 364                 __field(int, connect_status)
 365                 __string(addr, rpcrdma_addrstr(r_xprt))
 366                 __string(port, rpcrdma_portstr(r_xprt))
 367         ),
 368
 369         TP_fast_assign(
 370                 __entry->rc = rc;
 371                 __entry->connect_status = r_xprt->rx_ep->re_connect_status;
 372                 __assign_str(addr);
 373                 __assign_str(port);
 374         ),
 375
 376         TP_printk("peer=[%s]:%s rc=%d connection status=%d",
 377                 __get_str(addr), __get_str(port),
 378                 __entry->rc, __entry->connect_status
 379         )
 380 );
 381
 382 #define DEFINE_CONN_EVENT(name)                                         \
 383                 DEFINE_EVENT(xprtrdma_connect_class, xprtrdma_##name,   \
 384                                 TP_PROTO(                               \
 385                                         const struct rpcrdma_xprt *r_xprt, \
 386                                         int rc                          \
 387                                 ),                                      \
 388                                 TP_ARGS(r_xprt, rc))
 389
 390 DECLARE_EVENT_CLASS(xprtrdma_rdch_event,
 391         TP_PROTO(
 392                 const struct rpc_task *task,
 393                 unsigned int pos,
 394                 struct rpcrdma_mr *mr,
 395                 int nsegs
 396         ),
 397
 398         TP_ARGS(task, pos, mr, nsegs),
 399
 400         TP_STRUCT__entry(
 401                 __field(unsigned int, task_id)
 402                 __field(unsigned int, client_id)
 403                 __field(unsigned int, pos)
 404                 __field(int, nents)
 405                 __field(u32, handle)
 406                 __field(u32, length)
 407                 __field(u64, offset)
 408                 __field(int, nsegs)
 409         ),
 410
 411         TP_fast_assign(
 412                 __entry->task_id = task->tk_pid;
 413                 __entry->client_id = task->tk_client->cl_clid;
 414                 __entry->pos = pos;
 415                 __entry->nents = mr->mr_nents;
 416                 __entry->handle = mr->mr_handle;
 417                 __entry->length = mr->mr_length;
 418                 __entry->offset = mr->mr_offset;
 419                 __entry->nsegs = nsegs;
 420         ),
 421
 422         TP_printk(SUNRPC_TRACE_TASK_SPECIFIER
 423                   " pos=%u %u@0x%016llx:0x%08x (%s)",
 424                 __entry->task_id, __entry->client_id,
 425                 __entry->pos, __entry->length,
 426                 (unsigned long long)__entry->offset, __entry->handle,
 427                 __entry->nents < __entry->nsegs ? "more" : "last"
 428         )
 429 );
 430
 431 #define DEFINE_RDCH_EVENT(name)                                         \
 432                 DEFINE_EVENT(xprtrdma_rdch_event, xprtrdma_chunk_##name,\
 433                                 TP_PROTO(                               \
 434                                         const struct rpc_task *task,    \
 435                                         unsigned int pos,               \
 436                                         struct rpcrdma_mr *mr,          \
 437                                         int nsegs                       \
 438                                 ),                                      \
 439                                 TP_ARGS(task, pos, mr, nsegs))
 440
 441 DECLARE_EVENT_CLASS(xprtrdma_wrch_event,
 442         TP_PROTO(
 443                 const struct rpc_task *task,
 444                 struct rpcrdma_mr *mr,
 445                 int nsegs
 446         ),
 447
 448         TP_ARGS(task, mr, nsegs),
 449
 450         TP_STRUCT__entry(
 451                 __field(unsigned int, task_id)
 452                 __field(unsigned int, client_id)
 453                 __field(int, nents)
 454                 __field(u32, handle)
 455                 __field(u32, length)
 456                 __field(u64, offset)
 457                 __field(int, nsegs)
 458         ),
 459
 460         TP_fast_assign(
 461                 __entry->task_id = task->tk_pid;
 462                 __entry->client_id = task->tk_client->cl_clid;
 463                 __entry->nents = mr->mr_nents;
 464                 __entry->handle = mr->mr_handle;
 465                 __entry->length = mr->mr_length;
 466                 __entry->offset = mr->mr_offset;
 467                 __entry->nsegs = nsegs;
 468         ),
 469
 470         TP_printk(SUNRPC_TRACE_TASK_SPECIFIER
 471                   " %u@0x%016llx:0x%08x (%s)",
 472                 __entry->task_id, __entry->client_id,
 473                 __entry->length, (unsigned long long)__entry->offset,
 474                 __entry->handle,
 475                 __entry->nents < __entry->nsegs ? "more" : "last"
 476         )
 477 );
 478
 479 #define DEFINE_WRCH_EVENT(name)                                         \
 480                 DEFINE_EVENT(xprtrdma_wrch_event, xprtrdma_chunk_##name,\
 481                                 TP_PROTO(                               \
 482                                         const struct rpc_task *task,    \
 483                                         struct rpcrdma_mr *mr,          \
 484                                         int nsegs                       \
 485                                 ),                                      \
 486                                 TP_ARGS(task, mr, nsegs))
 487
 488 TRACE_DEFINE_ENUM(DMA_BIDIRECTIONAL);
 489 TRACE_DEFINE_ENUM(DMA_TO_DEVICE);
 490 TRACE_DEFINE_ENUM(DMA_FROM_DEVICE);
 491 TRACE_DEFINE_ENUM(DMA_NONE);
 492
 493 #define xprtrdma_show_direction(x)                                      \
 494                 __print_symbolic(x,                                     \
 495                                 { DMA_BIDIRECTIONAL, "BIDIR" },         \
 496                                 { DMA_TO_DEVICE, "TO_DEVICE" },         \
 497                                 { DMA_FROM_DEVICE, "FROM_DEVICE" },     \
 498                                 { DMA_NONE, "NONE" })
 499
 500 DECLARE_EVENT_CLASS(xprtrdma_mr_class,
 501         TP_PROTO(
 502                 const struct rpcrdma_mr *mr
 503         ),
 504
 505         TP_ARGS(mr),
 506
 507         TP_STRUCT__entry(
 508                 __field(unsigned int, task_id)
 509                 __field(unsigned int, client_id)
 510                 __field(u32, mr_id)
 511                 __field(int, nents)
 512                 __field(u32, handle)
 513                 __field(u32, length)
 514                 __field(u64, offset)
 515                 __field(u32, dir)
 516         ),
 517
 518         TP_fast_assign(
 519                 const struct rpcrdma_req *req = mr->mr_req;
 520
 521                 if (req) {
 522                         const struct rpc_task *task = req->rl_slot.rq_task;
 523
 524                         __entry->task_id = task->tk_pid;
 525                         __entry->client_id = task->tk_client->cl_clid;
 526                 } else {
 527                         __entry->task_id = 0;
 528                         __entry->client_id = -1;
 529                 }
 530                 __entry->mr_id  = mr->mr_ibmr->res.id;
 531                 __entry->nents  = mr->mr_nents;
 532                 __entry->handle = mr->mr_handle;
 533                 __entry->length = mr->mr_length;
 534                 __entry->offset = mr->mr_offset;
 535                 __entry->dir    = mr->mr_dir;
 536         ),
 537
 538         TP_printk(SUNRPC_TRACE_TASK_SPECIFIER
 539                   " mr.id=%u nents=%d %u@0x%016llx:0x%08x (%s)",
 540                 __entry->task_id, __entry->client_id,
 541                 __entry->mr_id, __entry->nents, __entry->length,
 542                 (unsigned long long)__entry->offset, __entry->handle,
 543                 xprtrdma_show_direction(__entry->dir)
 544         )
 545 );
 546
 547 #define DEFINE_MR_EVENT(name)                                           \
 548                 DEFINE_EVENT(xprtrdma_mr_class,                         \
 549                                 xprtrdma_mr_##name,                     \
 550                                 TP_PROTO(                               \
 551                                         const struct rpcrdma_mr *mr     \
 552                                 ),                                      \
 553                                 TP_ARGS(mr))
 554
 555 DECLARE_EVENT_CLASS(xprtrdma_anonymous_mr_class,
 556         TP_PROTO(
 557                 const struct rpcrdma_mr *mr
 558         ),
 559
 560         TP_ARGS(mr),
 561
 562         TP_STRUCT__entry(
 563                 __field(u32, mr_id)
 564                 __field(int, nents)
 565                 __field(u32, handle)
 566                 __field(u32, length)
 567                 __field(u64, offset)
 568                 __field(u32, dir)
 569         ),
 570
 571         TP_fast_assign(
 572                 __entry->mr_id  = mr->mr_ibmr->res.id;
 573                 __entry->nents  = mr->mr_nents;
 574                 __entry->handle = mr->mr_handle;
 575                 __entry->length = mr->mr_length;
 576                 __entry->offset = mr->mr_offset;
 577                 __entry->dir    = mr->mr_dir;
 578         ),
 579
 580         TP_printk("mr.id=%u nents=%d %u@0x%016llx:0x%08x (%s)",
 581                 __entry->mr_id, __entry->nents, __entry->length,
 582                 (unsigned long long)__entry->offset, __entry->handle,
 583                 xprtrdma_show_direction(__entry->dir)
 584         )
 585 );
 586
 587 #define DEFINE_ANON_MR_EVENT(name)                                      \
 588                 DEFINE_EVENT(xprtrdma_anonymous_mr_class,               \
 589                                 xprtrdma_mr_##name,                     \
 590                                 TP_PROTO(                               \
 591                                         const struct rpcrdma_mr *mr     \
 592                                 ),                                      \
 593                                 TP_ARGS(mr))
 594
 595 DECLARE_EVENT_CLASS(xprtrdma_callback_class,
 596         TP_PROTO(
 597                 const struct rpcrdma_xprt *r_xprt,
 598                 const struct rpc_rqst *rqst
 599         ),
 600
 601         TP_ARGS(r_xprt, rqst),
 602
 603         TP_STRUCT__entry(
 604                 __field(u32, xid)
 605                 __string(addr, rpcrdma_addrstr(r_xprt))
 606                 __string(port, rpcrdma_portstr(r_xprt))
 607         ),
 608
 609         TP_fast_assign(
 610                 __entry->xid = be32_to_cpu(rqst->rq_xid);
 611                 __assign_str(addr);
 612                 __assign_str(port);
 613         ),
 614
 615         TP_printk("peer=[%s]:%s xid=0x%08x",
 616                 __get_str(addr), __get_str(port), __entry->xid
 617         )
 618 );
 619
 620 #define DEFINE_CALLBACK_EVENT(name)                                     \
 621                 DEFINE_EVENT(xprtrdma_callback_class,                   \
 622                                 xprtrdma_cb_##name,                     \
 623                                 TP_PROTO(                               \
 624                                         const struct rpcrdma_xprt *r_xprt, \
 625                                         const struct rpc_rqst *rqst     \
 626                                 ),                                      \
 627                                 TP_ARGS(r_xprt, rqst))
 628
 629 /**
 630  ** Connection events
 631  **/
 632
 633 TRACE_EVENT(xprtrdma_inline_thresh,
 634         TP_PROTO(
 635                 const struct rpcrdma_ep *ep
 636         ),
 637
 638         TP_ARGS(ep),
 639
 640         TP_STRUCT__entry(
 641                 __field(unsigned int, inline_send)
 642                 __field(unsigned int, inline_recv)
 643                 __field(unsigned int, max_send)
 644                 __field(unsigned int, max_recv)
 645                 __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6))
 646                 __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6))
 647         ),
 648
 649         TP_fast_assign(
 650                 const struct rdma_cm_id *id = ep->re_id;
 651
 652                 __entry->inline_send = ep->re_inline_send;
 653                 __entry->inline_recv = ep->re_inline_recv;
 654                 __entry->max_send = ep->re_max_inline_send;
 655                 __entry->max_recv = ep->re_max_inline_recv;
 656                 memcpy(__entry->srcaddr, &id->route.addr.src_addr,
 657                        sizeof(struct sockaddr_in6));
 658                 memcpy(__entry->dstaddr, &id->route.addr.dst_addr,
 659                        sizeof(struct sockaddr_in6));
 660         ),
 661
 662         TP_printk("%pISpc -> %pISpc neg send/recv=%u/%u, calc send/recv=%u/%u",
 663                 __entry->srcaddr, __entry->dstaddr,
 664                 __entry->inline_send, __entry->inline_recv,
 665                 __entry->max_send, __entry->max_recv
 666         )
 667 );
 668
 669 DEFINE_CONN_EVENT(connect);
 670 DEFINE_CONN_EVENT(disconnect);
 671
 672 TRACE_EVENT(xprtrdma_device_removal,
 673         TP_PROTO(
 674                 const struct rdma_cm_id *id
 675         ),
 676
 677         TP_ARGS(id),
 678
 679         TP_STRUCT__entry(
 680                 __string(name, id->device->name)
 681                 __array(unsigned char, addr, sizeof(struct sockaddr_in6))
 682         ),
 683
 684         TP_fast_assign(
 685                 __assign_str(name);
 686                 memcpy(__entry->addr, &id->route.addr.dst_addr,
 687                        sizeof(struct sockaddr_in6));
 688         ),
 689
 690         TP_printk("device %s to be removed, disconnecting %pISpc\n",
 691                 __get_str(name), __entry->addr
 692         )
 693 );
 694
 695 DEFINE_RXPRT_EVENT(xprtrdma_op_inject_dsc);
 696
 697 TRACE_EVENT(xprtrdma_op_connect,
 698         TP_PROTO(
 699                 const struct rpcrdma_xprt *r_xprt,
 700                 unsigned long delay
 701         ),
 702
 703         TP_ARGS(r_xprt, delay),
 704
 705         TP_STRUCT__entry(
 706                 __field(unsigned long, delay)
 707                 __string(addr, rpcrdma_addrstr(r_xprt))
 708                 __string(port, rpcrdma_portstr(r_xprt))
 709         ),
 710
 711         TP_fast_assign(
 712                 __entry->delay = delay;
 713                 __assign_str(addr);
 714                 __assign_str(port);
 715         ),
 716
 717         TP_printk("peer=[%s]:%s delay=%lu",
 718                 __get_str(addr), __get_str(port), __entry->delay
 719         )
 720 );
 721
 722
 723 TRACE_EVENT(xprtrdma_op_set_cto,
 724         TP_PROTO(
 725                 const struct rpcrdma_xprt *r_xprt,
 726                 unsigned long connect,
 727                 unsigned long reconnect
 728         ),
 729
 730         TP_ARGS(r_xprt, connect, reconnect),
 731
 732         TP_STRUCT__entry(
 733                 __field(unsigned long, connect)
 734                 __field(unsigned long, reconnect)
 735                 __string(addr, rpcrdma_addrstr(r_xprt))
 736                 __string(port, rpcrdma_portstr(r_xprt))
 737         ),
 738
 739         TP_fast_assign(
 740                 __entry->connect = connect;
 741                 __entry->reconnect = reconnect;
 742                 __assign_str(addr);
 743                 __assign_str(port);
 744         ),
 745
 746         TP_printk("peer=[%s]:%s connect=%lu reconnect=%lu",
 747                 __get_str(addr), __get_str(port),
 748                 __entry->connect / HZ, __entry->reconnect / HZ
 749         )
 750 );
 751
 752 /**
 753  ** Call events
 754  **/
 755
 756 TRACE_EVENT(xprtrdma_createmrs,
 757         TP_PROTO(
 758                 const struct rpcrdma_xprt *r_xprt,
 759                 unsigned int count
 760         ),
 761
 762         TP_ARGS(r_xprt, count),
 763
 764         TP_STRUCT__entry(
 765                 __string(addr, rpcrdma_addrstr(r_xprt))
 766                 __string(port, rpcrdma_portstr(r_xprt))
 767                 __field(unsigned int, count)
 768         ),
 769
 770         TP_fast_assign(
 771                 __entry->count = count;
 772                 __assign_str(addr);
 773                 __assign_str(port);
 774         ),
 775
 776         TP_printk("peer=[%s]:%s created %u MRs",
 777                 __get_str(addr), __get_str(port), __entry->count
 778         )
 779 );
 780
 781 TRACE_EVENT(xprtrdma_nomrs_err,
 782         TP_PROTO(
 783                 const struct rpcrdma_xprt *r_xprt,
 784                 const struct rpcrdma_req *req
 785         ),
 786
 787         TP_ARGS(r_xprt, req),
 788
 789         TP_STRUCT__entry(
 790                 __field(unsigned int, task_id)
 791                 __field(unsigned int, client_id)
 792                 __string(addr, rpcrdma_addrstr(r_xprt))
 793                 __string(port, rpcrdma_portstr(r_xprt))
 794         ),
 795
 796         TP_fast_assign(
 797                 const struct rpc_rqst *rqst = &req->rl_slot;
 798
 799                 __entry->task_id = rqst->rq_task->tk_pid;
 800                 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
 801                 __assign_str(addr);
 802                 __assign_str(port);
 803         ),
 804
 805         TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " peer=[%s]:%s",
 806                 __entry->task_id, __entry->client_id,
 807                 __get_str(addr), __get_str(port)
 808         )
 809 );
 810
 811 DEFINE_RDCH_EVENT(read);
 812 DEFINE_WRCH_EVENT(write);
 813 DEFINE_WRCH_EVENT(reply);
 814 DEFINE_WRCH_EVENT(wp);
 815
 816 TRACE_DEFINE_ENUM(rpcrdma_noch);
 817 TRACE_DEFINE_ENUM(rpcrdma_noch_pullup);
 818 TRACE_DEFINE_ENUM(rpcrdma_noch_mapped);
 819 TRACE_DEFINE_ENUM(rpcrdma_readch);
 820 TRACE_DEFINE_ENUM(rpcrdma_areadch);
 821 TRACE_DEFINE_ENUM(rpcrdma_writech);
 822 TRACE_DEFINE_ENUM(rpcrdma_replych);
 823
 824 #define xprtrdma_show_chunktype(x)                                      \
 825                 __print_symbolic(x,                                     \
 826                                 { rpcrdma_noch, "inline" },             \
 827                                 { rpcrdma_noch_pullup, "pullup" },      \
 828                                 { rpcrdma_noch_mapped, "mapped" },      \
 829                                 { rpcrdma_readch, "read list" },        \
 830                                 { rpcrdma_areadch, "*read list" },      \
 831                                 { rpcrdma_writech, "write list" },      \
 832                                 { rpcrdma_replych, "reply chunk" })
 833
 834 TRACE_EVENT(xprtrdma_marshal,
 835         TP_PROTO(
 836                 const struct rpcrdma_req *req,
 837                 unsigned int rtype,
 838                 unsigned int wtype
 839         ),
 840
 841         TP_ARGS(req, rtype, wtype),
 842
 843         TP_STRUCT__entry(
 844                 __field(unsigned int, task_id)
 845                 __field(unsigned int, client_id)
 846                 __field(u32, xid)
 847                 __field(unsigned int, hdrlen)
 848                 __field(unsigned int, headlen)
 849                 __field(unsigned int, pagelen)
 850                 __field(unsigned int, taillen)
 851                 __field(unsigned int, rtype)
 852                 __field(unsigned int, wtype)
 853         ),
 854
 855         TP_fast_assign(
 856                 const struct rpc_rqst *rqst = &req->rl_slot;
 857
 858                 __entry->task_id = rqst->rq_task->tk_pid;
 859                 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
 860                 __entry->xid = be32_to_cpu(rqst->rq_xid);
 861                 __entry->hdrlen = req->rl_hdrbuf.len;
 862                 __entry->headlen = rqst->rq_snd_buf.head[0].iov_len;
 863                 __entry->pagelen = rqst->rq_snd_buf.page_len;
 864                 __entry->taillen = rqst->rq_snd_buf.tail[0].iov_len;
 865                 __entry->rtype = rtype;
 866                 __entry->wtype = wtype;
 867         ),
 868
 869         TP_printk(SUNRPC_TRACE_TASK_SPECIFIER
 870                   " xid=0x%08x hdr=%u xdr=%u/%u/%u %s/%s",
 871                 __entry->task_id, __entry->client_id, __entry->xid,
 872                 __entry->hdrlen,
 873                 __entry->headlen, __entry->pagelen, __entry->taillen,
 874                 xprtrdma_show_chunktype(__entry->rtype),
 875                 xprtrdma_show_chunktype(__entry->wtype)
 876         )
 877 );
 878
 879 TRACE_EVENT(xprtrdma_marshal_failed,
 880         TP_PROTO(const struct rpc_rqst *rqst,
 881                  int ret
 882         ),
 883
 884         TP_ARGS(rqst, ret),
 885
 886         TP_STRUCT__entry(
 887                 __field(unsigned int, task_id)
 888                 __field(unsigned int, client_id)
 889                 __field(u32, xid)
 890                 __field(int, ret)
 891         ),
 892
 893         TP_fast_assign(
 894                 __entry->task_id = rqst->rq_task->tk_pid;
 895                 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
 896                 __entry->xid = be32_to_cpu(rqst->rq_xid);
 897                 __entry->ret = ret;
 898         ),
 899
 900         TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " xid=0x%08x ret=%d",
 901                 __entry->task_id, __entry->client_id, __entry->xid,
 902                 __entry->ret
 903         )
 904 );
 905
 906 TRACE_EVENT(xprtrdma_prepsend_failed,
 907         TP_PROTO(const struct rpc_rqst *rqst,
 908                  int ret
 909         ),
 910
 911         TP_ARGS(rqst, ret),
 912
 913         TP_STRUCT__entry(
 914                 __field(unsigned int, task_id)
 915                 __field(unsigned int, client_id)
 916                 __field(u32, xid)
 917                 __field(int, ret)
 918         ),
 919
 920         TP_fast_assign(
 921                 __entry->task_id = rqst->rq_task->tk_pid;
 922                 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
 923                 __entry->xid = be32_to_cpu(rqst->rq_xid);
 924                 __entry->ret = ret;
 925         ),
 926
 927         TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " xid=0x%08x ret=%d",
 928                 __entry->task_id, __entry->client_id, __entry->xid,
 929                 __entry->ret
 930         )
 931 );
 932
 933 TRACE_EVENT(xprtrdma_post_send,
 934         TP_PROTO(
 935                 const struct rpcrdma_req *req
 936         ),
 937
 938         TP_ARGS(req),
 939
 940         TP_STRUCT__entry(
 941                 __field(u32, cq_id)
 942                 __field(int, completion_id)
 943                 __field(unsigned int, task_id)
 944                 __field(unsigned int, client_id)
 945                 __field(int, num_sge)
 946                 __field(int, signaled)
 947         ),
 948
 949         TP_fast_assign(
 950                 const struct rpc_rqst *rqst = &req->rl_slot;
 951                 const struct rpcrdma_sendctx *sc = req->rl_sendctx;
 952
 953                 __entry->cq_id = sc->sc_cid.ci_queue_id;
 954                 __entry->completion_id = sc->sc_cid.ci_completion_id;
 955                 __entry->task_id = rqst->rq_task->tk_pid;
 956                 __entry->client_id = rqst->rq_task->tk_client ?
 957                                      rqst->rq_task->tk_client->cl_clid : -1;
 958                 __entry->num_sge = req->rl_wr.num_sge;
 959                 __entry->signaled = req->rl_wr.send_flags & IB_SEND_SIGNALED;
 960         ),
 961
 962         TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " cq.id=%u cid=%d (%d SGE%s) %s",
 963                 __entry->task_id, __entry->client_id,
 964                 __entry->cq_id, __entry->completion_id,
 965                 __entry->num_sge, (__entry->num_sge == 1 ? "" : "s"),
 966                 (__entry->signaled ? "signaled" : "")
 967         )
 968 );
 969
 970 TRACE_EVENT(xprtrdma_post_send_err,
 971         TP_PROTO(
 972                 const struct rpcrdma_xprt *r_xprt,
 973                 const struct rpcrdma_req *req,
 974                 int rc
 975         ),
 976
 977         TP_ARGS(r_xprt, req, rc),
 978
 979         TP_STRUCT__entry(
 980                 __field(u32, cq_id)
 981                 __field(unsigned int, task_id)
 982                 __field(unsigned int, client_id)
 983                 __field(int, rc)
 984         ),
 985
 986         TP_fast_assign(
 987                 const struct rpc_rqst *rqst = &req->rl_slot;
 988                 const struct rpcrdma_ep *ep = r_xprt->rx_ep;
 989
 990                 __entry->cq_id = ep ? ep->re_attr.recv_cq->res.id : 0;
 991                 __entry->task_id = rqst->rq_task->tk_pid;
 992                 __entry->client_id = rqst->rq_task->tk_client ?
 993                                      rqst->rq_task->tk_client->cl_clid : -1;
 994                 __entry->rc = rc;
 995         ),
 996
 997         TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " cq.id=%u rc=%d",
 998                 __entry->task_id, __entry->client_id,
 999                 __entry->cq_id, __entry->rc
1000         )
1001 );
1002
1003 DEFINE_SIMPLE_CID_EVENT(xprtrdma_post_recv);
1004
1005 TRACE_EVENT(xprtrdma_post_recvs,
1006         TP_PROTO(
1007                 const struct rpcrdma_xprt *r_xprt,
1008                 unsigned int count
1009         ),
1010
1011         TP_ARGS(r_xprt, count),
1012
1013         TP_STRUCT__entry(
1014                 __field(u32, cq_id)
1015                 __field(unsigned int, count)
1016                 __field(int, posted)
1017                 __string(addr, rpcrdma_addrstr(r_xprt))
1018                 __string(port, rpcrdma_portstr(r_xprt))
1019         ),
1020
1021         TP_fast_assign(
1022                 const struct rpcrdma_ep *ep = r_xprt->rx_ep;
1023
1024                 __entry->cq_id = ep->re_attr.recv_cq->res.id;
1025                 __entry->count = count;
1026                 __entry->posted = ep->re_receive_count;
1027                 __assign_str(addr);
1028                 __assign_str(port);
1029         ),
1030
1031         TP_printk("peer=[%s]:%s cq.id=%d %u new recvs, %d active",
1032                 __get_str(addr), __get_str(port), __entry->cq_id,
1033                 __entry->count, __entry->posted
1034         )
1035 );
1036
1037 TRACE_EVENT(xprtrdma_post_recvs_err,
1038         TP_PROTO(
1039                 const struct rpcrdma_xprt *r_xprt,
1040                 int status
1041         ),
1042
1043         TP_ARGS(r_xprt, status),
1044
1045         TP_STRUCT__entry(
1046                 __field(u32, cq_id)
1047                 __field(int, status)
1048                 __string(addr, rpcrdma_addrstr(r_xprt))
1049                 __string(port, rpcrdma_portstr(r_xprt))
1050         ),
1051
1052         TP_fast_assign(
1053                 const struct rpcrdma_ep *ep = r_xprt->rx_ep;
1054
1055                 __entry->cq_id = ep->re_attr.recv_cq->res.id;
1056                 __entry->status = status;
1057                 __assign_str(addr);
1058                 __assign_str(port);
1059         ),
1060
1061         TP_printk("peer=[%s]:%s cq.id=%d rc=%d",
1062                 __get_str(addr), __get_str(port), __entry->cq_id,
1063                 __entry->status
1064         )
1065 );
1066
1067 TRACE_EVENT(xprtrdma_post_linv_err,
1068         TP_PROTO(
1069                 const struct rpcrdma_req *req,
1070                 int status
1071         ),
1072
1073         TP_ARGS(req, status),
1074
1075         TP_STRUCT__entry(
1076                 __field(unsigned int, task_id)
1077                 __field(unsigned int, client_id)
1078                 __field(int, status)
1079         ),
1080
1081         TP_fast_assign(
1082                 const struct rpc_task *task = req->rl_slot.rq_task;
1083
1084                 __entry->task_id = task->tk_pid;
1085                 __entry->client_id = task->tk_client->cl_clid;
1086                 __entry->status = status;
1087         ),
1088
1089         TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " status=%d",
1090                 __entry->task_id, __entry->client_id, __entry->status
1091         )
1092 );
1093
1094 /**
1095  ** Completion events
1096  **/
1097
1098 DEFINE_RECEIVE_COMPLETION_EVENT(xprtrdma_wc_receive);
1099
1100 DEFINE_COMPLETION_EVENT(xprtrdma_wc_send);
1101 DEFINE_MR_COMPLETION_EVENT(xprtrdma_wc_fastreg);
1102 DEFINE_MR_COMPLETION_EVENT(xprtrdma_wc_li);
1103 DEFINE_MR_COMPLETION_EVENT(xprtrdma_wc_li_wake);
1104 DEFINE_MR_COMPLETION_EVENT(xprtrdma_wc_li_done);
1105
1106 TRACE_EVENT(xprtrdma_frwr_alloc,
1107         TP_PROTO(
1108                 const struct rpcrdma_mr *mr,
1109                 int rc
1110         ),
1111
1112         TP_ARGS(mr, rc),
1113
1114         TP_STRUCT__entry(
1115                 __field(u32, mr_id)
1116                 __field(int, rc)
1117         ),
1118
1119         TP_fast_assign(
1120                 __entry->mr_id = mr->mr_ibmr->res.id;
1121                 __entry->rc = rc;
1122         ),
1123
1124         TP_printk("mr.id=%u: rc=%d",
1125                 __entry->mr_id, __entry->rc
1126         )
1127 );
1128
1129 TRACE_EVENT(xprtrdma_frwr_dereg,
1130         TP_PROTO(
1131                 const struct rpcrdma_mr *mr,
1132                 int rc
1133         ),
1134
1135         TP_ARGS(mr, rc),
1136
1137         TP_STRUCT__entry(
1138                 __field(u32, mr_id)
1139                 __field(int, nents)
1140                 __field(u32, handle)
1141                 __field(u32, length)
1142                 __field(u64, offset)
1143                 __field(u32, dir)
1144                 __field(int, rc)
1145         ),
1146
1147         TP_fast_assign(
1148                 __entry->mr_id  = mr->mr_ibmr->res.id;
1149                 __entry->nents  = mr->mr_nents;
1150                 __entry->handle = mr->mr_handle;
1151                 __entry->length = mr->mr_length;
1152                 __entry->offset = mr->mr_offset;
1153                 __entry->dir    = mr->mr_dir;
1154                 __entry->rc     = rc;
1155         ),
1156
1157         TP_printk("mr.id=%u nents=%d %u@0x%016llx:0x%08x (%s): rc=%d",
1158                 __entry->mr_id, __entry->nents, __entry->length,
1159                 (unsigned long long)__entry->offset, __entry->handle,
1160                 xprtrdma_show_direction(__entry->dir),
1161                 __entry->rc
1162         )
1163 );
1164
1165 TRACE_EVENT(xprtrdma_frwr_sgerr,
1166         TP_PROTO(
1167                 const struct rpcrdma_mr *mr,
1168                 int sg_nents
1169         ),
1170
1171         TP_ARGS(mr, sg_nents),
1172
1173         TP_STRUCT__entry(
1174                 __field(u32, mr_id)
1175                 __field(u64, addr)
1176                 __field(u32, dir)
1177                 __field(int, nents)
1178         ),
1179
1180         TP_fast_assign(
1181                 __entry->mr_id = mr->mr_ibmr->res.id;
1182                 __entry->addr = mr->mr_sg->dma_address;
1183                 __entry->dir = mr->mr_dir;
1184                 __entry->nents = sg_nents;
1185         ),
1186
1187         TP_printk("mr.id=%u DMA addr=0x%llx (%s) sg_nents=%d",
1188                 __entry->mr_id, __entry->addr,
1189                 xprtrdma_show_direction(__entry->dir),
1190                 __entry->nents
1191         )
1192 );
1193
1194 TRACE_EVENT(xprtrdma_frwr_maperr,
1195         TP_PROTO(
1196                 const struct rpcrdma_mr *mr,
1197                 int num_mapped
1198         ),
1199
1200         TP_ARGS(mr, num_mapped),
1201
1202         TP_STRUCT__entry(
1203                 __field(u32, mr_id)
1204                 __field(u64, addr)
1205                 __field(u32, dir)
1206                 __field(int, num_mapped)
1207                 __field(int, nents)
1208         ),
1209
1210         TP_fast_assign(
1211                 __entry->mr_id = mr->mr_ibmr->res.id;
1212                 __entry->addr = mr->mr_sg->dma_address;
1213                 __entry->dir = mr->mr_dir;
1214                 __entry->num_mapped = num_mapped;
1215                 __entry->nents = mr->mr_nents;
1216         ),
1217
1218         TP_printk("mr.id=%u DMA addr=0x%llx (%s) nents=%d of %d",
1219                 __entry->mr_id, __entry->addr,
1220                 xprtrdma_show_direction(__entry->dir),
1221                 __entry->num_mapped, __entry->nents
1222         )
1223 );
1224
1225 DEFINE_MR_EVENT(fastreg);
1226 DEFINE_MR_EVENT(localinv);
1227 DEFINE_MR_EVENT(reminv);
1228 DEFINE_MR_EVENT(map);
1229
1230 DEFINE_ANON_MR_EVENT(unmap);
1231
1232 TRACE_EVENT(xprtrdma_dma_maperr,
1233         TP_PROTO(
1234                 u64 addr
1235         ),
1236
1237         TP_ARGS(addr),
1238
1239         TP_STRUCT__entry(
1240                 __field(u64, addr)
1241         ),
1242
1243         TP_fast_assign(
1244                 __entry->addr = addr;
1245         ),
1246
1247         TP_printk("dma addr=0x%llx\n", __entry->addr)
1248 );
1249
1250 /**
1251  ** Reply events
1252  **/
1253
1254 TRACE_EVENT(xprtrdma_reply,
1255         TP_PROTO(
1256                 const struct rpc_task *task,
1257                 const struct rpcrdma_rep *rep,
1258                 unsigned int credits
1259         ),
1260
1261         TP_ARGS(task, rep, credits),
1262
1263         TP_STRUCT__entry(
1264                 __field(unsigned int, task_id)
1265                 __field(unsigned int, client_id)
1266                 __field(u32, xid)
1267                 __field(unsigned int, credits)
1268         ),
1269
1270         TP_fast_assign(
1271                 __entry->task_id = task->tk_pid;
1272                 __entry->client_id = task->tk_client->cl_clid;
1273                 __entry->xid = be32_to_cpu(rep->rr_xid);
1274                 __entry->credits = credits;
1275         ),
1276
1277         TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " xid=0x%08x credits=%u",
1278                 __entry->task_id, __entry->client_id, __entry->xid,
1279                 __entry->credits
1280         )
1281 );
1282
1283 DEFINE_REPLY_EVENT(vers);
1284 DEFINE_REPLY_EVENT(rqst);
1285 DEFINE_REPLY_EVENT(short);
1286 DEFINE_REPLY_EVENT(hdr);
1287
1288 TRACE_EVENT(xprtrdma_err_vers,
1289         TP_PROTO(
1290                 const struct rpc_rqst *rqst,
1291                 __be32 *min,
1292                 __be32 *max
1293         ),
1294
1295         TP_ARGS(rqst, min, max),
1296
1297         TP_STRUCT__entry(
1298                 __field(unsigned int, task_id)
1299                 __field(unsigned int, client_id)
1300                 __field(u32, xid)
1301                 __field(u32, min)
1302                 __field(u32, max)
1303         ),
1304
1305         TP_fast_assign(
1306                 __entry->task_id = rqst->rq_task->tk_pid;
1307                 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
1308                 __entry->xid = be32_to_cpu(rqst->rq_xid);
1309                 __entry->min = be32_to_cpup(min);
1310                 __entry->max = be32_to_cpup(max);
1311         ),
1312
1313         TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " xid=0x%08x versions=[%u, %u]",
1314                 __entry->task_id, __entry->client_id, __entry->xid,
1315                 __entry->min, __entry->max
1316         )
1317 );
1318
1319 TRACE_EVENT(xprtrdma_err_chunk,
1320         TP_PROTO(
1321                 const struct rpc_rqst *rqst
1322         ),
1323
1324         TP_ARGS(rqst),
1325
1326         TP_STRUCT__entry(
1327                 __field(unsigned int, task_id)
1328                 __field(unsigned int, client_id)
1329                 __field(u32, xid)
1330         ),
1331
1332         TP_fast_assign(
1333                 __entry->task_id = rqst->rq_task->tk_pid;
1334                 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
1335                 __entry->xid = be32_to_cpu(rqst->rq_xid);
1336         ),
1337
1338         TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " xid=0x%08x",
1339                 __entry->task_id, __entry->client_id, __entry->xid
1340         )
1341 );
1342
1343 TRACE_EVENT(xprtrdma_err_unrecognized,
1344         TP_PROTO(
1345                 const struct rpc_rqst *rqst,
1346                 __be32 *procedure
1347         ),
1348
1349         TP_ARGS(rqst, procedure),
1350
1351         TP_STRUCT__entry(
1352                 __field(unsigned int, task_id)
1353                 __field(unsigned int, client_id)
1354                 __field(u32, xid)
1355                 __field(u32, procedure)
1356         ),
1357
1358         TP_fast_assign(
1359                 __entry->task_id = rqst->rq_task->tk_pid;
1360                 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
1361                 __entry->procedure = be32_to_cpup(procedure);
1362         ),
1363
1364         TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " xid=0x%08x procedure=%u",
1365                 __entry->task_id, __entry->client_id, __entry->xid,
1366                 __entry->procedure
1367         )
1368 );
1369
1370 TRACE_EVENT(xprtrdma_fixup,
1371         TP_PROTO(
1372                 const struct rpc_rqst *rqst,
1373                 unsigned long fixup
1374         ),
1375
1376         TP_ARGS(rqst, fixup),
1377
1378         TP_STRUCT__entry(
1379                 __field(unsigned int, task_id)
1380                 __field(unsigned int, client_id)
1381                 __field(unsigned long, fixup)
1382                 __field(size_t, headlen)
1383                 __field(unsigned int, pagelen)
1384                 __field(size_t, taillen)
1385         ),
1386
1387         TP_fast_assign(
1388                 __entry->task_id = rqst->rq_task->tk_pid;
1389                 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
1390                 __entry->fixup = fixup;
1391                 __entry->headlen = rqst->rq_rcv_buf.head[0].iov_len;
1392                 __entry->pagelen = rqst->rq_rcv_buf.page_len;
1393                 __entry->taillen = rqst->rq_rcv_buf.tail[0].iov_len;
1394         ),
1395
1396         TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " fixup=%lu xdr=%zu/%u/%zu",
1397                 __entry->task_id, __entry->client_id, __entry->fixup,
1398                 __entry->headlen, __entry->pagelen, __entry->taillen
1399         )
1400 );
1401
1402 TRACE_EVENT(xprtrdma_decode_seg,
1403         TP_PROTO(
1404                 u32 handle,
1405                 u32 length,
1406                 u64 offset
1407         ),
1408
1409         TP_ARGS(handle, length, offset),
1410
1411         TP_STRUCT__entry(
1412                 __field(u32, handle)
1413                 __field(u32, length)
1414                 __field(u64, offset)
1415         ),
1416
1417         TP_fast_assign(
1418                 __entry->handle = handle;
1419                 __entry->length = length;
1420                 __entry->offset = offset;
1421         ),
1422
1423         TP_printk("%u@0x%016llx:0x%08x",
1424                 __entry->length, (unsigned long long)__entry->offset,
1425                 __entry->handle
1426         )
1427 );
1428
1429 TRACE_EVENT(xprtrdma_mrs_zap,
1430         TP_PROTO(
1431                 const struct rpc_task *task
1432         ),
1433
1434         TP_ARGS(task),
1435
1436         TP_STRUCT__entry(
1437                 __field(unsigned int, task_id)
1438                 __field(unsigned int, client_id)
1439         ),
1440
1441         TP_fast_assign(
1442                 __entry->task_id = task->tk_pid;
1443                 __entry->client_id = task->tk_client->cl_clid;
1444         ),
1445
1446         TP_printk(SUNRPC_TRACE_TASK_SPECIFIER,
1447                 __entry->task_id, __entry->client_id
1448         )
1449 );
1450
1451 /**
1452  ** Callback events
1453  **/
1454
1455 TRACE_EVENT(xprtrdma_cb_setup,
1456         TP_PROTO(
1457                 const struct rpcrdma_xprt *r_xprt,
1458                 unsigned int reqs
1459         ),
1460
1461         TP_ARGS(r_xprt, reqs),
1462
1463         TP_STRUCT__entry(
1464                 __field(unsigned int, reqs)
1465                 __string(addr, rpcrdma_addrstr(r_xprt))
1466                 __string(port, rpcrdma_portstr(r_xprt))
1467         ),
1468
1469         TP_fast_assign(
1470                 __entry->reqs = reqs;
1471                 __assign_str(addr);
1472                 __assign_str(port);
1473         ),
1474
1475         TP_printk("peer=[%s]:%s %u reqs",
1476                 __get_str(addr), __get_str(port), __entry->reqs
1477         )
1478 );
1479
1480 DEFINE_CALLBACK_EVENT(call);
1481 DEFINE_CALLBACK_EVENT(reply);
1482
1483 /**
1484  ** Server-side RPC/RDMA events
1485  **/
1486
1487 DECLARE_EVENT_CLASS(svcrdma_accept_class,
1488         TP_PROTO(
1489                 const struct svcxprt_rdma *rdma,
1490                 long status
1491         ),
1492
1493         TP_ARGS(rdma, status),
1494
1495         TP_STRUCT__entry(
1496                 __field(long, status)
1497                 __string(addr, rdma->sc_xprt.xpt_remotebuf)
1498         ),
1499
1500         TP_fast_assign(
1501                 __entry->status = status;
1502                 __assign_str(addr);
1503         ),
1504
1505         TP_printk("addr=%s status=%ld",
1506                 __get_str(addr), __entry->status
1507         )
1508 );
1509
1510 #define DEFINE_ACCEPT_EVENT(name) \
1511                 DEFINE_EVENT(svcrdma_accept_class, svcrdma_##name##_err, \
1512                                 TP_PROTO( \
1513                                         const struct svcxprt_rdma *rdma, \
1514                                         long status \
1515                                 ), \
1516                                 TP_ARGS(rdma, status))
1517
1518 DEFINE_ACCEPT_EVENT(pd);
1519 DEFINE_ACCEPT_EVENT(qp);
1520 DEFINE_ACCEPT_EVENT(fabric);
1521 DEFINE_ACCEPT_EVENT(initdepth);
1522 DEFINE_ACCEPT_EVENT(accept);
1523
1524 TRACE_DEFINE_ENUM(RDMA_MSG);
1525 TRACE_DEFINE_ENUM(RDMA_NOMSG);
1526 TRACE_DEFINE_ENUM(RDMA_MSGP);
1527 TRACE_DEFINE_ENUM(RDMA_DONE);
1528 TRACE_DEFINE_ENUM(RDMA_ERROR);
1529
1530 #define show_rpcrdma_proc(x)                                            \
1531                 __print_symbolic(x,                                     \
1532                                 { RDMA_MSG, "RDMA_MSG" },               \
1533                                 { RDMA_NOMSG, "RDMA_NOMSG" },           \
1534                                 { RDMA_MSGP, "RDMA_MSGP" },             \
1535                                 { RDMA_DONE, "RDMA_DONE" },             \
1536                                 { RDMA_ERROR, "RDMA_ERROR" })
1537
1538 TRACE_EVENT(svcrdma_decode_rqst,
1539         TP_PROTO(
1540                 const struct svc_rdma_recv_ctxt *ctxt,
1541                 __be32 *p,
1542                 unsigned int hdrlen
1543         ),
1544
1545         TP_ARGS(ctxt, p, hdrlen),
1546
1547         TP_STRUCT__entry(
1548                 __field(u32, cq_id)
1549                 __field(int, completion_id)
1550                 __field(u32, xid)
1551                 __field(u32, vers)
1552                 __field(u32, proc)
1553                 __field(u32, credits)
1554                 __field(unsigned int, hdrlen)
1555         ),
1556
1557         TP_fast_assign(
1558                 __entry->cq_id = ctxt->rc_cid.ci_queue_id;
1559                 __entry->completion_id = ctxt->rc_cid.ci_completion_id;
1560                 __entry->xid = be32_to_cpup(p++);
1561                 __entry->vers = be32_to_cpup(p++);
1562                 __entry->credits = be32_to_cpup(p++);
1563                 __entry->proc = be32_to_cpup(p);
1564                 __entry->hdrlen = hdrlen;
1565         ),
1566
1567         TP_printk("cq.id=%u cid=%d xid=0x%08x vers=%u credits=%u proc=%s hdrlen=%u",
1568                 __entry->cq_id, __entry->completion_id,
1569                 __entry->xid, __entry->vers, __entry->credits,
1570                 show_rpcrdma_proc(__entry->proc), __entry->hdrlen)
1571 );
1572
1573 TRACE_EVENT(svcrdma_decode_short_err,
1574         TP_PROTO(
1575                 const struct svc_rdma_recv_ctxt *ctxt,
1576                 unsigned int hdrlen
1577         ),
1578
1579         TP_ARGS(ctxt, hdrlen),
1580
1581         TP_STRUCT__entry(
1582                 __field(u32, cq_id)
1583                 __field(int, completion_id)
1584                 __field(unsigned int, hdrlen)
1585         ),
1586
1587         TP_fast_assign(
1588                 __entry->cq_id = ctxt->rc_cid.ci_queue_id;
1589                 __entry->completion_id = ctxt->rc_cid.ci_completion_id;
1590                 __entry->hdrlen = hdrlen;
1591         ),
1592
1593         TP_printk("cq.id=%u cid=%d hdrlen=%u",
1594                 __entry->cq_id, __entry->completion_id,
1595                 __entry->hdrlen)
1596 );
1597
1598 DECLARE_EVENT_CLASS(svcrdma_badreq_event,
1599         TP_PROTO(
1600                 const struct svc_rdma_recv_ctxt *ctxt,
1601                 __be32 *p
1602         ),
1603
1604         TP_ARGS(ctxt, p),
1605
1606         TP_STRUCT__entry(
1607                 __field(u32, cq_id)
1608                 __field(int, completion_id)
1609                 __field(u32, xid)
1610                 __field(u32, vers)
1611                 __field(u32, proc)
1612                 __field(u32, credits)
1613         ),
1614
1615         TP_fast_assign(
1616                 __entry->cq_id = ctxt->rc_cid.ci_queue_id;
1617                 __entry->completion_id = ctxt->rc_cid.ci_completion_id;
1618                 __entry->xid = be32_to_cpup(p++);
1619                 __entry->vers = be32_to_cpup(p++);
1620                 __entry->credits = be32_to_cpup(p++);
1621                 __entry->proc = be32_to_cpup(p);
1622         ),
1623
1624         TP_printk("cq.id=%u cid=%d xid=0x%08x vers=%u credits=%u proc=%u",
1625                 __entry->cq_id, __entry->completion_id,
1626                 __entry->xid, __entry->vers, __entry->credits, __entry->proc)
1627 );
1628
1629 #define DEFINE_BADREQ_EVENT(name)                                       \
1630                 DEFINE_EVENT(svcrdma_badreq_event,                      \
1631                              svcrdma_decode_##name##_err,               \
1632                                 TP_PROTO(                               \
1633                                         const struct svc_rdma_recv_ctxt *ctxt,  \
1634                                         __be32 *p                       \
1635                                 ),                                      \
1636                                 TP_ARGS(ctxt, p))
1637
1638 DEFINE_BADREQ_EVENT(badvers);
1639 DEFINE_BADREQ_EVENT(drop);
1640 DEFINE_BADREQ_EVENT(badproc);
1641 DEFINE_BADREQ_EVENT(parse);
1642
1643 TRACE_EVENT(svcrdma_encode_wseg,
1644         TP_PROTO(
1645                 const struct svc_rdma_send_ctxt *ctxt,
1646                 u32 segno,
1647                 u32 handle,
1648                 u32 length,
1649                 u64 offset
1650         ),
1651
1652         TP_ARGS(ctxt, segno, handle, length, offset),
1653
1654         TP_STRUCT__entry(
1655                 __field(u32, cq_id)
1656                 __field(int, completion_id)
1657                 __field(u32, segno)
1658                 __field(u32, handle)
1659                 __field(u32, length)
1660                 __field(u64, offset)
1661         ),
1662
1663         TP_fast_assign(
1664                 __entry->cq_id = ctxt->sc_cid.ci_queue_id;
1665                 __entry->completion_id = ctxt->sc_cid.ci_completion_id;
1666                 __entry->segno = segno;
1667                 __entry->handle = handle;
1668                 __entry->length = length;
1669                 __entry->offset = offset;
1670         ),
1671
1672         TP_printk("cq.id=%u cid=%d segno=%u %u@0x%016llx:0x%08x",
1673                 __entry->cq_id, __entry->completion_id,
1674                 __entry->segno, __entry->length,
1675                 (unsigned long long)__entry->offset, __entry->handle
1676         )
1677 );
1678
1679 TRACE_EVENT(svcrdma_decode_rseg,
1680         TP_PROTO(
1681                 const struct rpc_rdma_cid *cid,
1682                 const struct svc_rdma_chunk *chunk,
1683                 const struct svc_rdma_segment *segment
1684         ),
1685
1686         TP_ARGS(cid, chunk, segment),
1687
1688         TP_STRUCT__entry(
1689                 __field(u32, cq_id)
1690                 __field(int, completion_id)
1691                 __field(u32, segno)
1692                 __field(u32, position)
1693                 __field(u32, handle)
1694                 __field(u32, length)
1695                 __field(u64, offset)
1696         ),
1697
1698         TP_fast_assign(
1699                 __entry->cq_id = cid->ci_queue_id;
1700                 __entry->completion_id = cid->ci_completion_id;
1701                 __entry->segno = chunk->ch_segcount;
1702                 __entry->position = chunk->ch_position;
1703                 __entry->handle = segment->rs_handle;
1704                 __entry->length = segment->rs_length;
1705                 __entry->offset = segment->rs_offset;
1706         ),
1707
1708         TP_printk("cq.id=%u cid=%d segno=%u position=%u %u@0x%016llx:0x%08x",
1709                 __entry->cq_id, __entry->completion_id,
1710                 __entry->segno, __entry->position, __entry->length,
1711                 (unsigned long long)__entry->offset, __entry->handle
1712         )
1713 );
1714
1715 TRACE_EVENT(svcrdma_decode_wseg,
1716         TP_PROTO(
1717                 const struct rpc_rdma_cid *cid,
1718                 const struct svc_rdma_chunk *chunk,
1719                 u32 segno
1720         ),
1721
1722         TP_ARGS(cid, chunk, segno),
1723
1724         TP_STRUCT__entry(
1725                 __field(u32, cq_id)
1726                 __field(int, completion_id)
1727                 __field(u32, segno)
1728                 __field(u32, handle)
1729                 __field(u32, length)
1730                 __field(u64, offset)
1731         ),
1732
1733         TP_fast_assign(
1734                 const struct svc_rdma_segment *segment =
1735                         &chunk->ch_segments[segno];
1736
1737                 __entry->cq_id = cid->ci_queue_id;
1738                 __entry->completion_id = cid->ci_completion_id;
1739                 __entry->segno = segno;
1740                 __entry->handle = segment->rs_handle;
1741                 __entry->length = segment->rs_length;
1742                 __entry->offset = segment->rs_offset;
1743         ),
1744
1745         TP_printk("cq.id=%u cid=%d segno=%u %u@0x%016llx:0x%08x",
1746                 __entry->cq_id, __entry->completion_id,
1747                 __entry->segno, __entry->length,
1748                 (unsigned long long)__entry->offset, __entry->handle
1749         )
1750 );
1751
1752 DECLARE_EVENT_CLASS(svcrdma_error_event,
1753         TP_PROTO(
1754                 __be32 xid
1755         ),
1756
1757         TP_ARGS(xid),
1758
1759         TP_STRUCT__entry(
1760                 __field(u32, xid)
1761         ),
1762
1763         TP_fast_assign(
1764                 __entry->xid = be32_to_cpu(xid);
1765         ),
1766
1767         TP_printk("xid=0x%08x",
1768                 __entry->xid
1769         )
1770 );
1771
1772 #define DEFINE_ERROR_EVENT(name)                                        \
1773                 DEFINE_EVENT(svcrdma_error_event, svcrdma_err_##name,   \
1774                                 TP_PROTO(                               \
1775                                         __be32 xid                      \
1776                                 ),                                      \
1777                                 TP_ARGS(xid))
1778
1779 DEFINE_ERROR_EVENT(vers);
1780 DEFINE_ERROR_EVENT(chunk);
1781
1782 /**
1783  ** Server-side RDMA API events
1784  **/
1785
1786 DECLARE_EVENT_CLASS(svcrdma_dma_map_class,
1787         TP_PROTO(
1788                 const struct rpc_rdma_cid *cid,
1789                 u64 dma_addr,
1790                 u32 length
1791         ),
1792
1793         TP_ARGS(cid, dma_addr, length),
1794
1795         TP_STRUCT__entry(
1796                 __field(u32, cq_id)
1797                 __field(int, completion_id)
1798                 __field(u64, dma_addr)
1799                 __field(u32, length)
1800         ),
1801
1802         TP_fast_assign(
1803                 __entry->cq_id = cid->ci_queue_id;
1804                 __entry->completion_id = cid->ci_completion_id;
1805                 __entry->dma_addr = dma_addr;
1806                 __entry->length = length;
1807         ),
1808
1809         TP_printk("cq.id=%u cid=%d dma_addr=%llu length=%u",
1810                 __entry->cq_id, __entry->completion_id,
1811                 __entry->dma_addr, __entry->length
1812         )
1813 );
1814
1815 #define DEFINE_SVC_DMA_EVENT(name)                                      \
1816                 DEFINE_EVENT(svcrdma_dma_map_class, svcrdma_##name,     \
1817                                 TP_PROTO(                               \
1818                                         const struct rpc_rdma_cid *cid, \
1819                                         u64 dma_addr,                   \
1820                                         u32 length                      \
1821                                 ),                                      \
1822                                 TP_ARGS(cid, dma_addr, length)          \
1823                 )
1824
1825 DEFINE_SVC_DMA_EVENT(dma_map_page);
1826 DEFINE_SVC_DMA_EVENT(dma_map_err);
1827 DEFINE_SVC_DMA_EVENT(dma_unmap_page);
1828
1829 TRACE_EVENT(svcrdma_dma_map_rw_err,
1830         TP_PROTO(
1831                 const struct svcxprt_rdma *rdma,
1832                 u64 offset,
1833                 u32 handle,
1834                 unsigned int nents,
1835                 int status
1836         ),
1837
1838         TP_ARGS(rdma, offset, handle, nents, status),
1839
1840         TP_STRUCT__entry(
1841                 __field(u32, cq_id)
1842                 __field(u32, handle)
1843                 __field(u64, offset)
1844                 __field(unsigned int, nents)
1845                 __field(int, status)
1846         ),
1847
1848         TP_fast_assign(
1849                 __entry->cq_id = rdma->sc_sq_cq->res.id;
1850                 __entry->handle = handle;
1851                 __entry->offset = offset;
1852                 __entry->nents = nents;
1853                 __entry->status = status;
1854         ),
1855
1856         TP_printk("cq.id=%u 0x%016llx:0x%08x nents=%u status=%d",
1857                 __entry->cq_id, (unsigned long long)__entry->offset,
1858                 __entry->handle, __entry->nents, __entry->status
1859         )
1860 );
1861
1862 TRACE_EVENT(svcrdma_rwctx_empty,
1863         TP_PROTO(
1864                 const struct svcxprt_rdma *rdma,
1865                 unsigned int num_sges
1866         ),
1867
1868         TP_ARGS(rdma, num_sges),
1869
1870         TP_STRUCT__entry(
1871                 __field(u32, cq_id)
1872                 __field(unsigned int, num_sges)
1873         ),
1874
1875         TP_fast_assign(
1876                 __entry->cq_id = rdma->sc_sq_cq->res.id;
1877                 __entry->num_sges = num_sges;
1878         ),
1879
1880         TP_printk("cq.id=%u num_sges=%d",
1881                 __entry->cq_id, __entry->num_sges
1882         )
1883 );
1884
1885 TRACE_EVENT(svcrdma_page_overrun_err,
1886         TP_PROTO(
1887                 const struct rpc_rdma_cid *cid,
1888                 unsigned int pageno
1889         ),
1890
1891         TP_ARGS(cid, pageno),
1892
1893         TP_STRUCT__entry(
1894                 __field(u32, cq_id)
1895                 __field(int, completion_id)
1896                 __field(unsigned int, pageno)
1897         ),
1898
1899         TP_fast_assign(
1900                 __entry->cq_id = cid->ci_queue_id;
1901                 __entry->completion_id = cid->ci_completion_id;
1902                 __entry->pageno = pageno;
1903         ),
1904
1905         TP_printk("cq.id=%u cid=%d pageno=%u",
1906                 __entry->cq_id, __entry->completion_id,
1907                 __entry->pageno
1908         )
1909 );
1910
1911 TRACE_EVENT(svcrdma_small_wrch_err,
1912         TP_PROTO(
1913                 const struct rpc_rdma_cid *cid,
1914                 unsigned int remaining,
1915                 unsigned int seg_no,
1916                 unsigned int num_segs
1917         ),
1918
1919         TP_ARGS(cid, remaining, seg_no, num_segs),
1920
1921         TP_STRUCT__entry(
1922                 __field(u32, cq_id)
1923                 __field(int, completion_id)
1924                 __field(unsigned int, remaining)
1925                 __field(unsigned int, seg_no)
1926                 __field(unsigned int, num_segs)
1927         ),
1928
1929         TP_fast_assign(
1930                 __entry->cq_id = cid->ci_queue_id;
1931                 __entry->completion_id = cid->ci_completion_id;
1932                 __entry->remaining = remaining;
1933                 __entry->seg_no = seg_no;
1934                 __entry->num_segs = num_segs;
1935         ),
1936
1937         TP_printk("cq.id=%u cid=%d remaining=%u seg_no=%u num_segs=%u",
1938                 __entry->cq_id, __entry->completion_id,
1939                 __entry->remaining, __entry->seg_no, __entry->num_segs
1940         )
1941 );
1942
1943 TRACE_EVENT(svcrdma_send_pullup,
1944         TP_PROTO(
1945                 const struct svc_rdma_send_ctxt *ctxt,
1946                 unsigned int msglen
1947         ),
1948
1949         TP_ARGS(ctxt, msglen),
1950
1951         TP_STRUCT__entry(
1952                 __field(u32, cq_id)
1953                 __field(int, completion_id)
1954                 __field(unsigned int, hdrlen)
1955                 __field(unsigned int, msglen)
1956         ),
1957
1958         TP_fast_assign(
1959                 __entry->cq_id = ctxt->sc_cid.ci_queue_id;
1960                 __entry->completion_id = ctxt->sc_cid.ci_completion_id;
1961                 __entry->hdrlen = ctxt->sc_hdrbuf.len,
1962                 __entry->msglen = msglen;
1963         ),
1964
1965         TP_printk("cq.id=%u cid=%d hdr=%u msg=%u (total %u)",
1966                 __entry->cq_id, __entry->completion_id,
1967                 __entry->hdrlen, __entry->msglen,
1968                 __entry->hdrlen + __entry->msglen)
1969 );
1970
1971 TRACE_EVENT(svcrdma_send_err,
1972         TP_PROTO(
1973                 const struct svc_rqst *rqst,
1974                 int status
1975         ),
1976
1977         TP_ARGS(rqst, status),
1978
1979         TP_STRUCT__entry(
1980                 __field(int, status)
1981                 __field(u32, xid)
1982                 __string(addr, rqst->rq_xprt->xpt_remotebuf)
1983         ),
1984
1985         TP_fast_assign(
1986                 __entry->status = status;
1987                 __entry->xid = __be32_to_cpu(rqst->rq_xid);
1988                 __assign_str(addr);
1989         ),
1990
1991         TP_printk("addr=%s xid=0x%08x status=%d", __get_str(addr),
1992                 __entry->xid, __entry->status
1993         )
1994 );
1995
1996 TRACE_EVENT(svcrdma_post_send,
1997         TP_PROTO(
1998                 const struct svc_rdma_send_ctxt *ctxt
1999         ),
2000
2001         TP_ARGS(ctxt),
2002
2003         TP_STRUCT__entry(
2004                 __field(u32, cq_id)
2005                 __field(int, completion_id)
2006                 __field(unsigned int, num_sge)
2007                 __field(u32, inv_rkey)
2008         ),
2009
2010         TP_fast_assign(
2011                 const struct ib_send_wr *wr = &ctxt->sc_send_wr;
2012
2013                 __entry->cq_id = ctxt->sc_cid.ci_queue_id;
2014                 __entry->completion_id = ctxt->sc_cid.ci_completion_id;
2015                 __entry->num_sge = wr->num_sge;
2016                 __entry->inv_rkey = (wr->opcode == IB_WR_SEND_WITH_INV) ?
2017                                         wr->ex.invalidate_rkey : 0;
2018         ),
2019
2020         TP_printk("cq.id=%u cid=%d num_sge=%u inv_rkey=0x%08x",
2021                 __entry->cq_id, __entry->completion_id,
2022                 __entry->num_sge, __entry->inv_rkey
2023         )
2024 );
2025
2026 DEFINE_SIMPLE_CID_EVENT(svcrdma_wc_send);
2027 DEFINE_SEND_FLUSH_EVENT(svcrdma_wc_send_flush);
2028 DEFINE_SEND_FLUSH_EVENT(svcrdma_wc_send_err);
2029
2030 DEFINE_SIMPLE_CID_EVENT(svcrdma_post_recv);
2031
2032 DEFINE_RECEIVE_SUCCESS_EVENT(svcrdma_wc_recv);
2033 DEFINE_RECEIVE_FLUSH_EVENT(svcrdma_wc_recv_flush);
2034 DEFINE_RECEIVE_FLUSH_EVENT(svcrdma_wc_recv_err);
2035
2036 TRACE_EVENT(svcrdma_rq_post_err,
2037         TP_PROTO(
2038                 const struct svcxprt_rdma *rdma,
2039                 int status
2040         ),
2041
2042         TP_ARGS(rdma, status),
2043
2044         TP_STRUCT__entry(
2045                 __field(int, status)
2046                 __string(addr, rdma->sc_xprt.xpt_remotebuf)
2047         ),
2048
2049         TP_fast_assign(
2050                 __entry->status = status;
2051                 __assign_str(addr);
2052         ),
2053
2054         TP_printk("addr=%s status=%d",
2055                 __get_str(addr), __entry->status
2056         )
2057 );
2058
2059 DECLARE_EVENT_CLASS(svcrdma_post_chunk_class,
2060         TP_PROTO(
2061                 const struct rpc_rdma_cid *cid,
2062                 int sqecount
2063         ),
2064
2065         TP_ARGS(cid, sqecount),
2066
2067         TP_STRUCT__entry(
2068                 __field(u32, cq_id)
2069                 __field(int, completion_id)
2070                 __field(int, sqecount)
2071         ),
2072
2073         TP_fast_assign(
2074                 __entry->cq_id = cid->ci_queue_id;
2075                 __entry->completion_id = cid->ci_completion_id;
2076                 __entry->sqecount = sqecount;
2077         ),
2078
2079         TP_printk("cq.id=%u cid=%d sqecount=%d",
2080                 __entry->cq_id, __entry->completion_id,
2081                 __entry->sqecount
2082         )
2083 );
2084
2085 #define DEFINE_POST_CHUNK_EVENT(name)                                   \
2086                 DEFINE_EVENT(svcrdma_post_chunk_class,                  \
2087                                 svcrdma_post_##name##_chunk,            \
2088                                 TP_PROTO(                               \
2089                                         const struct rpc_rdma_cid *cid, \
2090                                         int sqecount                    \
2091                                 ),                                      \
2092                                 TP_ARGS(cid, sqecount))
2093
2094 DEFINE_POST_CHUNK_EVENT(read);
2095 DEFINE_POST_CHUNK_EVENT(write);
2096 DEFINE_POST_CHUNK_EVENT(reply);
2097
2098 DEFINE_EVENT(svcrdma_post_chunk_class, svcrdma_cc_release,
2099         TP_PROTO(
2100                 const struct rpc_rdma_cid *cid,
2101                 int sqecount
2102         ),
2103         TP_ARGS(cid, sqecount)
2104 );
2105
2106 TRACE_EVENT(svcrdma_wc_read,
2107         TP_PROTO(
2108                 const struct ib_wc *wc,
2109                 const struct rpc_rdma_cid *cid,
2110                 unsigned int totalbytes,
2111                 const ktime_t posttime
2112         ),
2113
2114         TP_ARGS(wc, cid, totalbytes, posttime),
2115
2116         TP_STRUCT__entry(
2117                 __field(u32, cq_id)
2118                 __field(int, completion_id)
2119                 __field(s64, read_latency)
2120                 __field(unsigned int, totalbytes)
2121         ),
2122
2123         TP_fast_assign(
2124                 __entry->cq_id = cid->ci_queue_id;
2125                 __entry->completion_id = cid->ci_completion_id;
2126                 __entry->totalbytes = totalbytes;
2127                 __entry->read_latency = ktime_us_delta(ktime_get(), posttime);
2128         ),
2129
2130         TP_printk("cq.id=%u cid=%d totalbytes=%u latency-us=%lld",
2131                 __entry->cq_id, __entry->completion_id,
2132                 __entry->totalbytes, __entry->read_latency
2133         )
2134 );
2135
2136 DEFINE_SEND_FLUSH_EVENT(svcrdma_wc_read_flush);
2137 DEFINE_SEND_FLUSH_EVENT(svcrdma_wc_read_err);
2138 DEFINE_SIMPLE_CID_EVENT(svcrdma_read_finished);
2139
2140 DEFINE_SIMPLE_CID_EVENT(svcrdma_wc_write);
2141 DEFINE_SEND_FLUSH_EVENT(svcrdma_wc_write_flush);
2142 DEFINE_SEND_FLUSH_EVENT(svcrdma_wc_write_err);
2143
2144 DEFINE_SIMPLE_CID_EVENT(svcrdma_wc_reply);
2145 DEFINE_SEND_FLUSH_EVENT(svcrdma_wc_reply_flush);
2146 DEFINE_SEND_FLUSH_EVENT(svcrdma_wc_reply_err);
2147
2148 TRACE_EVENT(svcrdma_qp_error,
2149         TP_PROTO(
2150                 const struct ib_event *event,
2151                 const struct sockaddr *sap
2152         ),
2153
2154         TP_ARGS(event, sap),
2155
2156         TP_STRUCT__entry(
2157                 __field(unsigned int, event)
2158                 __string(device, event->device->name)
2159                 __array(__u8, addr, INET6_ADDRSTRLEN + 10)
2160         ),
2161
2162         TP_fast_assign(
2163                 __entry->event = event->event;
2164                 __assign_str(device);
2165                 snprintf(__entry->addr, sizeof(__entry->addr) - 1,
2166                          "%pISpc", sap);
2167         ),
2168
2169         TP_printk("addr=%s dev=%s event=%s (%u)",
2170                 __entry->addr, __get_str(device),
2171                 rdma_show_ib_event(__entry->event), __entry->event
2172         )
2173 );
2174
2175 TRACE_EVENT(svcrdma_device_removal,
2176         TP_PROTO(
2177                 const struct rdma_cm_id *id
2178         ),
2179
2180         TP_ARGS(id),
2181
2182         TP_STRUCT__entry(
2183                 __string(name, id->device->name)
2184                 __array(unsigned char, addr, sizeof(struct sockaddr_in6))
2185         ),
2186
2187         TP_fast_assign(
2188                 __assign_str(name);
2189                 memcpy(__entry->addr, &id->route.addr.dst_addr,
2190                        sizeof(struct sockaddr_in6));
2191         ),
2192
2193         TP_printk("device %s to be removed, disconnecting %pISpc\n",
2194                 __get_str(name), __entry->addr
2195         )
2196 );
2197
2198 DECLARE_EVENT_CLASS(svcrdma_sendqueue_class,
2199         TP_PROTO(
2200                 const struct svcxprt_rdma *rdma,
2201                 const struct rpc_rdma_cid *cid
2202         ),
2203
2204         TP_ARGS(rdma, cid),
2205
2206         TP_STRUCT__entry(
2207                 __field(u32, cq_id)
2208                 __field(int, completion_id)
2209                 __field(int, avail)
2210                 __field(int, depth)
2211         ),
2212
2213         TP_fast_assign(
2214                 __entry->cq_id = cid->ci_queue_id;
2215                 __entry->completion_id = cid->ci_completion_id;
2216                 __entry->avail = atomic_read(&rdma->sc_sq_avail);
2217                 __entry->depth = rdma->sc_sq_depth;
2218         ),
2219
2220         TP_printk("cq.id=%u cid=%d sc_sq_avail=%d/%d",
2221                 __entry->cq_id, __entry->completion_id,
2222                 __entry->avail, __entry->depth
2223         )
2224 );
2225
2226 #define DEFINE_SQ_EVENT(name)                                           \
2227                 DEFINE_EVENT(svcrdma_sendqueue_class, name,             \
2228                         TP_PROTO(                                       \
2229                                 const struct svcxprt_rdma *rdma,        \
2230                                 const struct rpc_rdma_cid *cid          \
2231                         ),                                              \
2232                         TP_ARGS(rdma, cid)                              \
2233                 )
2234
2235 DEFINE_SQ_EVENT(svcrdma_sq_full);
2236 DEFINE_SQ_EVENT(svcrdma_sq_retry);
2237
2238 TRACE_EVENT(svcrdma_sq_post_err,
2239         TP_PROTO(
2240                 const struct svcxprt_rdma *rdma,
2241                 const struct rpc_rdma_cid *cid,
2242                 int status
2243         ),
2244
2245         TP_ARGS(rdma, cid, status),
2246
2247         TP_STRUCT__entry(
2248                 __field(u32, cq_id)
2249                 __field(int, completion_id)
2250                 __field(int, avail)
2251                 __field(int, depth)
2252                 __field(int, status)
2253         ),
2254
2255         TP_fast_assign(
2256                 __entry->cq_id = cid->ci_queue_id;
2257                 __entry->completion_id = cid->ci_completion_id;
2258                 __entry->avail = atomic_read(&rdma->sc_sq_avail);
2259                 __entry->depth = rdma->sc_sq_depth;
2260                 __entry->status = status;
2261         ),
2262
2263         TP_printk("cq.id=%u cid=%d sc_sq_avail=%d/%d status=%d",
2264                 __entry->cq_id, __entry->completion_id,
2265                 __entry->avail, __entry->depth, __entry->status
2266         )
2267 );
2268
2269 DECLARE_EVENT_CLASS(rpcrdma_client_device_class,
2270         TP_PROTO(
2271                 const struct ib_device *device
2272         ),
2273
2274         TP_ARGS(device),
2275
2276         TP_STRUCT__entry(
2277                 __string(name, device->name)
2278         ),
2279
2280         TP_fast_assign(
2281                 __assign_str(name);
2282         ),
2283
2284         TP_printk("device=%s",
2285                 __get_str(name)
2286         )
2287 );
2288
2289 #define DEFINE_CLIENT_DEVICE_EVENT(name)                                \
2290         DEFINE_EVENT(rpcrdma_client_device_class, name,                 \
2291                 TP_PROTO(                                               \
2292                         const struct ib_device *device                  \
2293                 ),                                                      \
2294                 TP_ARGS(device)                                         \
2295         )
2296
2297 DEFINE_CLIENT_DEVICE_EVENT(rpcrdma_client_completion);
2298 DEFINE_CLIENT_DEVICE_EVENT(rpcrdma_client_add_one);
2299 DEFINE_CLIENT_DEVICE_EVENT(rpcrdma_client_remove_one);
2300 DEFINE_CLIENT_DEVICE_EVENT(rpcrdma_client_wait_on);
2301 DEFINE_CLIENT_DEVICE_EVENT(rpcrdma_client_remove_one_done);
2302
2303 DECLARE_EVENT_CLASS(rpcrdma_client_register_class,
2304         TP_PROTO(
2305                 const struct ib_device *device,
2306                 const struct rpcrdma_notification *rn
2307         ),
2308
2309         TP_ARGS(device, rn),
2310
2311         TP_STRUCT__entry(
2312                 __string(name, device->name)
2313                 __field(void *, callback)
2314                 __field(u32, index)
2315         ),
2316
2317         TP_fast_assign(
2318                 __assign_str(name);
2319                 __entry->callback = rn->rn_done;
2320                 __entry->index = rn->rn_index;
2321         ),
2322
2323         TP_printk("device=%s index=%u done callback=%pS\n",
2324                 __get_str(name), __entry->index, __entry->callback
2325         )
2326 );
2327
2328 #define DEFINE_CLIENT_REGISTER_EVENT(name)                              \
2329         DEFINE_EVENT(rpcrdma_client_register_class, name,               \
2330         TP_PROTO(                                                       \
2331                 const struct ib_device *device,                         \
2332                 const struct rpcrdma_notification *rn                   \
2333         ),                                                              \
2334         TP_ARGS(device, rn))
2335
2336 DEFINE_CLIENT_REGISTER_EVENT(rpcrdma_client_register);
2337 DEFINE_CLIENT_REGISTER_EVENT(rpcrdma_client_unregister);
2338
2339 #endif /* _TRACE_RPCRDMA_H */
2340
2341 #include <trace/define_trace.h>