drivers/infiniband/core/counters.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright (c) 2019 Mellanox Technologies. All rights reserved.
   4  */
   5 #include <rdma/ib_verbs.h>
   6 #include <rdma/rdma_counter.h>
   7
   8 #include "core_priv.h"
   9 #include "restrack.h"
  10
  11 #define ALL_AUTO_MODE_MASKS (RDMA_COUNTER_MASK_QP_TYPE | RDMA_COUNTER_MASK_PID)
  12
  13 static int __counter_set_mode(struct rdma_counter_mode *curr,
  14                               enum rdma_nl_counter_mode new_mode,
  15                               enum rdma_nl_counter_mask new_mask)
  16 {
  17         if ((new_mode == RDMA_COUNTER_MODE_AUTO) &&
  18             ((new_mask & (~ALL_AUTO_MODE_MASKS)) ||
  19              (curr->mode != RDMA_COUNTER_MODE_NONE)))
  20                 return -EINVAL;
  21
  22         curr->mode = new_mode;
  23         curr->mask = new_mask;
  24         return 0;
  25 }
  26
  27 /**
  28  * rdma_counter_set_auto_mode() - Turn on/off per-port auto mode
  29  *
  30  * When @on is true, the @mask must be set; When @on is false, it goes
  31  * into manual mode if there's any counter, so that the user is able to
  32  * manually access them.
  33  */
  34 int rdma_counter_set_auto_mode(struct ib_device *dev, u8 port,
  35                                bool on, enum rdma_nl_counter_mask mask)
  36 {
  37         struct rdma_port_counter *port_counter;
  38         int ret;
  39
  40         port_counter = &dev->port_data[port].port_counter;
  41         if (!port_counter->hstats)
  42                 return -EOPNOTSUPP;
  43
  44         mutex_lock(&port_counter->lock);
  45         if (on) {
  46                 ret = __counter_set_mode(&port_counter->mode,
  47                                          RDMA_COUNTER_MODE_AUTO, mask);
  48         } else {
  49                 if (port_counter->mode.mode != RDMA_COUNTER_MODE_AUTO) {
  50                         ret = -EINVAL;
  51                         goto out;
  52                 }
  53
  54                 if (port_counter->num_counters)
  55                         ret = __counter_set_mode(&port_counter->mode,
  56                                                  RDMA_COUNTER_MODE_MANUAL, 0);
  57                 else
  58                         ret = __counter_set_mode(&port_counter->mode,
  59                                                  RDMA_COUNTER_MODE_NONE, 0);
  60         }
  61
  62 out:
  63         mutex_unlock(&port_counter->lock);
  64         return ret;
  65 }
  66
  67 static void auto_mode_init_counter(struct rdma_counter *counter,
  68                                    const struct ib_qp *qp,
  69                                    enum rdma_nl_counter_mask new_mask)
  70 {
  71         struct auto_mode_param *param = &counter->mode.param;
  72
  73         counter->mode.mode = RDMA_COUNTER_MODE_AUTO;
  74         counter->mode.mask = new_mask;
  75
  76         if (new_mask & RDMA_COUNTER_MASK_QP_TYPE)
  77                 param->qp_type = qp->qp_type;
  78 }
  79
  80 static int __rdma_counter_bind_qp(struct rdma_counter *counter,
  81                                   struct ib_qp *qp)
  82 {
  83         int ret;
  84
  85         if (qp->counter)
  86                 return -EINVAL;
  87
  88         if (!qp->device->ops.counter_bind_qp)
  89                 return -EOPNOTSUPP;
  90
  91         mutex_lock(&counter->lock);
  92         ret = qp->device->ops.counter_bind_qp(counter, qp);
  93         mutex_unlock(&counter->lock);
  94
  95         return ret;
  96 }
  97
  98 static struct rdma_counter *alloc_and_bind(struct ib_device *dev, u8 port,
  99                                            struct ib_qp *qp,
 100                                            enum rdma_nl_counter_mode mode)
 101 {
 102         struct rdma_port_counter *port_counter;
 103         struct rdma_counter *counter;
 104         int ret;
 105
 106         if (!dev->ops.counter_dealloc || !dev->ops.counter_alloc_stats)
 107                 return NULL;
 108
 109         counter = kzalloc(sizeof(*counter), GFP_KERNEL);
 110         if (!counter)
 111                 return NULL;
 112
 113         counter->device    = dev;
 114         counter->port      = port;
 115
 116         rdma_restrack_new(&counter->res, RDMA_RESTRACK_COUNTER);
 117         counter->stats = dev->ops.counter_alloc_stats(counter);
 118         if (!counter->stats)
 119                 goto err_stats;
 120
 121         port_counter = &dev->port_data[port].port_counter;
 122         mutex_lock(&port_counter->lock);
 123         switch (mode) {
 124         case RDMA_COUNTER_MODE_MANUAL:
 125                 ret = __counter_set_mode(&port_counter->mode,
 126                                          RDMA_COUNTER_MODE_MANUAL, 0);
 127                 if (ret) {
 128                         mutex_unlock(&port_counter->lock);
 129                         goto err_mode;
 130                 }
 131                 break;
 132         case RDMA_COUNTER_MODE_AUTO:
 133                 auto_mode_init_counter(counter, qp, port_counter->mode.mask);
 134                 break;
 135         default:
 136                 ret = -EOPNOTSUPP;
 137                 mutex_unlock(&port_counter->lock);
 138                 goto err_mode;
 139         }
 140
 141         port_counter->num_counters++;
 142         mutex_unlock(&port_counter->lock);
 143
 144         counter->mode.mode = mode;
 145         kref_init(&counter->kref);
 146         mutex_init(&counter->lock);
 147
 148         ret = __rdma_counter_bind_qp(counter, qp);
 149         if (ret)
 150                 goto err_mode;
 151
 152         rdma_restrack_parent_name(&counter->res, &qp->res);
 153         rdma_restrack_add(&counter->res);
 154         return counter;
 155
 156 err_mode:
 157         kfree(counter->stats);
 158 err_stats:
 159         rdma_restrack_put(&counter->res);
 160         kfree(counter);
 161         return NULL;
 162 }
 163
 164 static void rdma_counter_free(struct rdma_counter *counter)
 165 {
 166         struct rdma_port_counter *port_counter;
 167
 168         port_counter = &counter->device->port_data[counter->port].port_counter;
 169         mutex_lock(&port_counter->lock);
 170         port_counter->num_counters--;
 171         if (!port_counter->num_counters &&
 172             (port_counter->mode.mode == RDMA_COUNTER_MODE_MANUAL))
 173                 __counter_set_mode(&port_counter->mode, RDMA_COUNTER_MODE_NONE,
 174                                    0);
 175
 176         mutex_unlock(&port_counter->lock);
 177
 178         rdma_restrack_del(&counter->res);
 179         kfree(counter->stats);
 180         kfree(counter);
 181 }
 182
 183 static bool auto_mode_match(struct ib_qp *qp, struct rdma_counter *counter,
 184                             enum rdma_nl_counter_mask auto_mask)
 185 {
 186         struct auto_mode_param *param = &counter->mode.param;
 187         bool match = true;
 188
 189         if (auto_mask & RDMA_COUNTER_MASK_QP_TYPE)
 190                 match &= (param->qp_type == qp->qp_type);
 191
 192         if (auto_mask & RDMA_COUNTER_MASK_PID)
 193                 match &= (task_pid_nr(counter->res.task) ==
 194                           task_pid_nr(qp->res.task));
 195
 196         return match;
 197 }
 198
 199 static int __rdma_counter_unbind_qp(struct ib_qp *qp)
 200 {
 201         struct rdma_counter *counter = qp->counter;
 202         int ret;
 203
 204         if (!qp->device->ops.counter_unbind_qp)
 205                 return -EOPNOTSUPP;
 206
 207         mutex_lock(&counter->lock);
 208         ret = qp->device->ops.counter_unbind_qp(qp);
 209         mutex_unlock(&counter->lock);
 210
 211         return ret;
 212 }
 213
 214 static void counter_history_stat_update(struct rdma_counter *counter)
 215 {
 216         struct ib_device *dev = counter->device;
 217         struct rdma_port_counter *port_counter;
 218         int i;
 219
 220         port_counter = &dev->port_data[counter->port].port_counter;
 221         if (!port_counter->hstats)
 222                 return;
 223
 224         rdma_counter_query_stats(counter);
 225
 226         for (i = 0; i < counter->stats->num_counters; i++)
 227                 port_counter->hstats->value[i] += counter->stats->value[i];
 228 }
 229
 230 /**
 231  * rdma_get_counter_auto_mode - Find the counter that @qp should be bound
 232  *     with in auto mode
 233  *
 234  * Return: The counter (with ref-count increased) if found
 235  */
 236 static struct rdma_counter *rdma_get_counter_auto_mode(struct ib_qp *qp,
 237                                                        u8 port)
 238 {
 239         struct rdma_port_counter *port_counter;
 240         struct rdma_counter *counter = NULL;
 241         struct ib_device *dev = qp->device;
 242         struct rdma_restrack_entry *res;
 243         struct rdma_restrack_root *rt;
 244         unsigned long id = 0;
 245
 246         port_counter = &dev->port_data[port].port_counter;
 247         rt = &dev->res[RDMA_RESTRACK_COUNTER];
 248         xa_lock(&rt->xa);
 249         xa_for_each(&rt->xa, id, res) {
 250                 counter = container_of(res, struct rdma_counter, res);
 251                 if ((counter->device != qp->device) || (counter->port != port))
 252                         goto next;
 253
 254                 if (auto_mode_match(qp, counter, port_counter->mode.mask))
 255                         break;
 256 next:
 257                 counter = NULL;
 258         }
 259
 260         if (counter && !kref_get_unless_zero(&counter->kref))
 261                 counter = NULL;
 262
 263         xa_unlock(&rt->xa);
 264         return counter;
 265 }
 266
 267 static void counter_release(struct kref *kref)
 268 {
 269         struct rdma_counter *counter;
 270
 271         counter = container_of(kref, struct rdma_counter, kref);
 272         counter_history_stat_update(counter);
 273         counter->device->ops.counter_dealloc(counter);
 274         rdma_counter_free(counter);
 275 }
 276
 277 /**
 278  * rdma_counter_bind_qp_auto - Check and bind the QP to a counter base on
 279  *   the auto-mode rule
 280  */
 281 int rdma_counter_bind_qp_auto(struct ib_qp *qp, u8 port)
 282 {
 283         struct rdma_port_counter *port_counter;
 284         struct ib_device *dev = qp->device;
 285         struct rdma_counter *counter;
 286         int ret;
 287
 288         if (!rdma_restrack_is_tracked(&qp->res) || rdma_is_kernel_res(&qp->res))
 289                 return 0;
 290
 291         if (!rdma_is_port_valid(dev, port))
 292                 return -EINVAL;
 293
 294         port_counter = &dev->port_data[port].port_counter;
 295         if (port_counter->mode.mode != RDMA_COUNTER_MODE_AUTO)
 296                 return 0;
 297
 298         counter = rdma_get_counter_auto_mode(qp, port);
 299         if (counter) {
 300                 ret = __rdma_counter_bind_qp(counter, qp);
 301                 if (ret) {
 302                         kref_put(&counter->kref, counter_release);
 303                         return ret;
 304                 }
 305         } else {
 306                 counter = alloc_and_bind(dev, port, qp, RDMA_COUNTER_MODE_AUTO);
 307                 if (!counter)
 308                         return -ENOMEM;
 309         }
 310
 311         return 0;
 312 }
 313
 314 /**
 315  * rdma_counter_unbind_qp - Unbind a qp from a counter
 316  * @force:
 317  *   true - Decrease the counter ref-count anyway (e.g., qp destroy)
 318  */
 319 int rdma_counter_unbind_qp(struct ib_qp *qp, bool force)
 320 {
 321         struct rdma_counter *counter = qp->counter;
 322         int ret;
 323
 324         if (!counter)
 325                 return -EINVAL;
 326
 327         ret = __rdma_counter_unbind_qp(qp);
 328         if (ret && !force)
 329                 return ret;
 330
 331         kref_put(&counter->kref, counter_release);
 332         return 0;
 333 }
 334
 335 int rdma_counter_query_stats(struct rdma_counter *counter)
 336 {
 337         struct ib_device *dev = counter->device;
 338         int ret;
 339
 340         if (!dev->ops.counter_update_stats)
 341                 return -EINVAL;
 342
 343         mutex_lock(&counter->lock);
 344         ret = dev->ops.counter_update_stats(counter);
 345         mutex_unlock(&counter->lock);
 346
 347         return ret;
 348 }
 349
 350 static u64 get_running_counters_hwstat_sum(struct ib_device *dev,
 351                                            u8 port, u32 index)
 352 {
 353         struct rdma_restrack_entry *res;
 354         struct rdma_restrack_root *rt;
 355         struct rdma_counter *counter;
 356         unsigned long id = 0;
 357         u64 sum = 0;
 358
 359         rt = &dev->res[RDMA_RESTRACK_COUNTER];
 360         xa_lock(&rt->xa);
 361         xa_for_each(&rt->xa, id, res) {
 362                 if (!rdma_restrack_get(res))
 363                         continue;
 364
 365                 xa_unlock(&rt->xa);
 366
 367                 counter = container_of(res, struct rdma_counter, res);
 368                 if ((counter->device != dev) || (counter->port != port) ||
 369                     rdma_counter_query_stats(counter))
 370                         goto next;
 371
 372                 sum += counter->stats->value[index];
 373
 374 next:
 375                 xa_lock(&rt->xa);
 376                 rdma_restrack_put(res);
 377         }
 378
 379         xa_unlock(&rt->xa);
 380         return sum;
 381 }
 382
 383 /**
 384  * rdma_counter_get_hwstat_value() - Get the sum value of all counters on a
 385  *   specific port, including the running ones and history data
 386  */
 387 u64 rdma_counter_get_hwstat_value(struct ib_device *dev, u8 port, u32 index)
 388 {
 389         struct rdma_port_counter *port_counter;
 390         u64 sum;
 391
 392         port_counter = &dev->port_data[port].port_counter;
 393         if (!port_counter->hstats)
 394                 return 0;
 395
 396         sum = get_running_counters_hwstat_sum(dev, port, index);
 397         sum += port_counter->hstats->value[index];
 398
 399         return sum;
 400 }
 401
 402 static struct ib_qp *rdma_counter_get_qp(struct ib_device *dev, u32 qp_num)
 403 {
 404         struct rdma_restrack_entry *res = NULL;
 405         struct ib_qp *qp = NULL;
 406
 407         res = rdma_restrack_get_byid(dev, RDMA_RESTRACK_QP, qp_num);
 408         if (IS_ERR(res))
 409                 return NULL;
 410
 411         qp = container_of(res, struct ib_qp, res);
 412         if (qp->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW))
 413                 goto err;
 414
 415         return qp;
 416
 417 err:
 418         rdma_restrack_put(res);
 419         return NULL;
 420 }
 421
 422 static struct rdma_counter *rdma_get_counter_by_id(struct ib_device *dev,
 423                                                    u32 counter_id)
 424 {
 425         struct rdma_restrack_entry *res;
 426         struct rdma_counter *counter;
 427
 428         res = rdma_restrack_get_byid(dev, RDMA_RESTRACK_COUNTER, counter_id);
 429         if (IS_ERR(res))
 430                 return NULL;
 431
 432         counter = container_of(res, struct rdma_counter, res);
 433         kref_get(&counter->kref);
 434         rdma_restrack_put(res);
 435
 436         return counter;
 437 }
 438
 439 /**
 440  * rdma_counter_bind_qpn() - Bind QP @qp_num to counter @counter_id
 441  */
 442 int rdma_counter_bind_qpn(struct ib_device *dev, u8 port,
 443                           u32 qp_num, u32 counter_id)
 444 {
 445         struct rdma_port_counter *port_counter;
 446         struct rdma_counter *counter;
 447         struct ib_qp *qp;
 448         int ret;
 449
 450         port_counter = &dev->port_data[port].port_counter;
 451         if (port_counter->mode.mode == RDMA_COUNTER_MODE_AUTO)
 452                 return -EINVAL;
 453
 454         qp = rdma_counter_get_qp(dev, qp_num);
 455         if (!qp)
 456                 return -ENOENT;
 457
 458         counter = rdma_get_counter_by_id(dev, counter_id);
 459         if (!counter) {
 460                 ret = -ENOENT;
 461                 goto err;
 462         }
 463
 464         if (rdma_is_kernel_res(&counter->res) != rdma_is_kernel_res(&qp->res)) {
 465                 ret = -EINVAL;
 466                 goto err_task;
 467         }
 468
 469         if ((counter->device != qp->device) || (counter->port != qp->port)) {
 470                 ret = -EINVAL;
 471                 goto err_task;
 472         }
 473
 474         ret = __rdma_counter_bind_qp(counter, qp);
 475         if (ret)
 476                 goto err_task;
 477
 478         rdma_restrack_put(&qp->res);
 479         return 0;
 480
 481 err_task:
 482         kref_put(&counter->kref, counter_release);
 483 err:
 484         rdma_restrack_put(&qp->res);
 485         return ret;
 486 }
 487
 488 /**
 489  * rdma_counter_bind_qpn_alloc() - Alloc a counter and bind QP @qp_num to it
 490  *   The id of new counter is returned in @counter_id
 491  */
 492 int rdma_counter_bind_qpn_alloc(struct ib_device *dev, u8 port,
 493                                 u32 qp_num, u32 *counter_id)
 494 {
 495         struct rdma_port_counter *port_counter;
 496         struct rdma_counter *counter;
 497         struct ib_qp *qp;
 498         int ret;
 499
 500         if (!rdma_is_port_valid(dev, port))
 501                 return -EINVAL;
 502
 503         port_counter = &dev->port_data[port].port_counter;
 504         if (!port_counter->hstats)
 505                 return -EOPNOTSUPP;
 506
 507         if (port_counter->mode.mode == RDMA_COUNTER_MODE_AUTO)
 508                 return -EINVAL;
 509
 510         qp = rdma_counter_get_qp(dev, qp_num);
 511         if (!qp)
 512                 return -ENOENT;
 513
 514         if (rdma_is_port_valid(dev, qp->port) && (qp->port != port)) {
 515                 ret = -EINVAL;
 516                 goto err;
 517         }
 518
 519         counter = alloc_and_bind(dev, port, qp, RDMA_COUNTER_MODE_MANUAL);
 520         if (!counter) {
 521                 ret = -ENOMEM;
 522                 goto err;
 523         }
 524
 525         if (counter_id)
 526                 *counter_id = counter->id;
 527
 528         rdma_restrack_put(&qp->res);
 529         return 0;
 530
 531 err:
 532         rdma_restrack_put(&qp->res);
 533         return ret;
 534 }
 535
 536 /**
 537  * rdma_counter_unbind_qpn() - Unbind QP @qp_num from a counter
 538  */
 539 int rdma_counter_unbind_qpn(struct ib_device *dev, u8 port,
 540                             u32 qp_num, u32 counter_id)
 541 {
 542         struct rdma_port_counter *port_counter;
 543         struct ib_qp *qp;
 544         int ret;
 545
 546         if (!rdma_is_port_valid(dev, port))
 547                 return -EINVAL;
 548
 549         qp = rdma_counter_get_qp(dev, qp_num);
 550         if (!qp)
 551                 return -ENOENT;
 552
 553         if (rdma_is_port_valid(dev, qp->port) && (qp->port != port)) {
 554                 ret = -EINVAL;
 555                 goto out;
 556         }
 557
 558         port_counter = &dev->port_data[port].port_counter;
 559         if (!qp->counter || qp->counter->id != counter_id ||
 560             port_counter->mode.mode != RDMA_COUNTER_MODE_MANUAL) {
 561                 ret = -EINVAL;
 562                 goto out;
 563         }
 564
 565         ret = rdma_counter_unbind_qp(qp, false);
 566
 567 out:
 568         rdma_restrack_put(&qp->res);
 569         return ret;
 570 }
 571
 572 int rdma_counter_get_mode(struct ib_device *dev, u8 port,
 573                           enum rdma_nl_counter_mode *mode,
 574                           enum rdma_nl_counter_mask *mask)
 575 {
 576         struct rdma_port_counter *port_counter;
 577
 578         port_counter = &dev->port_data[port].port_counter;
 579         *mode = port_counter->mode.mode;
 580         *mask = port_counter->mode.mask;
 581
 582         return 0;
 583 }
 584
 585 void rdma_counter_init(struct ib_device *dev)
 586 {
 587         struct rdma_port_counter *port_counter;
 588         u32 port, i;
 589
 590         if (!dev->port_data)
 591                 return;
 592
 593         rdma_for_each_port(dev, port) {
 594                 port_counter = &dev->port_data[port].port_counter;
 595                 port_counter->mode.mode = RDMA_COUNTER_MODE_NONE;
 596                 mutex_init(&port_counter->lock);
 597
 598                 if (!dev->ops.alloc_hw_stats)
 599                         continue;
 600
 601                 port_counter->hstats = dev->ops.alloc_hw_stats(dev, port);
 602                 if (!port_counter->hstats)
 603                         goto fail;
 604         }
 605
 606         return;
 607
 608 fail:
 609         for (i = port; i >= rdma_start_port(dev); i--) {
 610                 port_counter = &dev->port_data[port].port_counter;
 611                 kfree(port_counter->hstats);
 612                 port_counter->hstats = NULL;
 613                 mutex_destroy(&port_counter->lock);
 614         }
 615 }
 616
 617 void rdma_counter_release(struct ib_device *dev)
 618 {
 619         struct rdma_port_counter *port_counter;
 620         u32 port;
 621
 622         rdma_for_each_port(dev, port) {
 623                 port_counter = &dev->port_data[port].port_counter;
 624                 kfree(port_counter->hstats);
 625                 mutex_destroy(&port_counter->lock);
 626         }
 627 }