drivers/net/ethernet/mellanox/mlx5/core/transobj.c

   1 /*
   2  * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <linux/mlx5/driver.h>
  34 #include "mlx5_core.h"
  35 #include <linux/mlx5/transobj.h>
  36
  37 int mlx5_core_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn)
  38 {
  39         u32 in[MLX5_ST_SZ_DW(alloc_transport_domain_in)]   = {0};
  40         u32 out[MLX5_ST_SZ_DW(alloc_transport_domain_out)] = {0};
  41         int err;
  42
  43         MLX5_SET(alloc_transport_domain_in, in, opcode,
  44                  MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN);
  45
  46         err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
  47         if (!err)
  48                 *tdn = MLX5_GET(alloc_transport_domain_out, out,
  49                                 transport_domain);
  50
  51         return err;
  52 }
  53 EXPORT_SYMBOL(mlx5_core_alloc_transport_domain);
  54
  55 void mlx5_core_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn)
  56 {
  57         u32 in[MLX5_ST_SZ_DW(dealloc_transport_domain_in)]   = {0};
  58         u32 out[MLX5_ST_SZ_DW(dealloc_transport_domain_out)] = {0};
  59
  60         MLX5_SET(dealloc_transport_domain_in, in, opcode,
  61                  MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
  62         MLX5_SET(dealloc_transport_domain_in, in, transport_domain, tdn);
  63         mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
  64 }
  65 EXPORT_SYMBOL(mlx5_core_dealloc_transport_domain);
  66
  67 int mlx5_core_create_rq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *rqn)
  68 {
  69         u32 out[MLX5_ST_SZ_DW(create_rq_out)] = {0};
  70         int err;
  71
  72         MLX5_SET(create_rq_in, in, opcode, MLX5_CMD_OP_CREATE_RQ);
  73         err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
  74         if (!err)
  75                 *rqn = MLX5_GET(create_rq_out, out, rqn);
  76
  77         return err;
  78 }
  79 EXPORT_SYMBOL(mlx5_core_create_rq);
  80
  81 int mlx5_core_modify_rq(struct mlx5_core_dev *dev, u32 rqn, u32 *in, int inlen)
  82 {
  83         u32 out[MLX5_ST_SZ_DW(modify_rq_out)];
  84
  85         MLX5_SET(modify_rq_in, in, rqn, rqn);
  86         MLX5_SET(modify_rq_in, in, opcode, MLX5_CMD_OP_MODIFY_RQ);
  87
  88         memset(out, 0, sizeof(out));
  89         return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
  90 }
  91 EXPORT_SYMBOL(mlx5_core_modify_rq);
  92
  93 void mlx5_core_destroy_rq(struct mlx5_core_dev *dev, u32 rqn)
  94 {
  95         u32 in[MLX5_ST_SZ_DW(destroy_rq_in)]   = {0};
  96         u32 out[MLX5_ST_SZ_DW(destroy_rq_out)] = {0};
  97
  98         MLX5_SET(destroy_rq_in, in, opcode, MLX5_CMD_OP_DESTROY_RQ);
  99         MLX5_SET(destroy_rq_in, in, rqn, rqn);
 100         mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 101 }
 102 EXPORT_SYMBOL(mlx5_core_destroy_rq);
 103
 104 int mlx5_core_query_rq(struct mlx5_core_dev *dev, u32 rqn, u32 *out)
 105 {
 106         u32 in[MLX5_ST_SZ_DW(query_rq_in)] = {0};
 107         int outlen = MLX5_ST_SZ_BYTES(query_rq_out);
 108
 109         MLX5_SET(query_rq_in, in, opcode, MLX5_CMD_OP_QUERY_RQ);
 110         MLX5_SET(query_rq_in, in, rqn, rqn);
 111
 112         return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
 113 }
 114 EXPORT_SYMBOL(mlx5_core_query_rq);
 115
 116 int mlx5_core_create_sq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *sqn)
 117 {
 118         u32 out[MLX5_ST_SZ_DW(create_sq_out)] = {0};
 119         int err;
 120
 121         MLX5_SET(create_sq_in, in, opcode, MLX5_CMD_OP_CREATE_SQ);
 122         err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 123         if (!err)
 124                 *sqn = MLX5_GET(create_sq_out, out, sqn);
 125
 126         return err;
 127 }
 128
 129 int mlx5_core_modify_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *in, int inlen)
 130 {
 131         u32 out[MLX5_ST_SZ_DW(modify_sq_out)] = {0};
 132
 133         MLX5_SET(modify_sq_in, in, sqn, sqn);
 134         MLX5_SET(modify_sq_in, in, opcode, MLX5_CMD_OP_MODIFY_SQ);
 135         return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 136 }
 137 EXPORT_SYMBOL(mlx5_core_modify_sq);
 138
 139 void mlx5_core_destroy_sq(struct mlx5_core_dev *dev, u32 sqn)
 140 {
 141         u32 in[MLX5_ST_SZ_DW(destroy_sq_in)]   = {0};
 142         u32 out[MLX5_ST_SZ_DW(destroy_sq_out)] = {0};
 143
 144         MLX5_SET(destroy_sq_in, in, opcode, MLX5_CMD_OP_DESTROY_SQ);
 145         MLX5_SET(destroy_sq_in, in, sqn, sqn);
 146         mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 147 }
 148
 149 int mlx5_core_query_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *out)
 150 {
 151         u32 in[MLX5_ST_SZ_DW(query_sq_in)] = {0};
 152         int outlen = MLX5_ST_SZ_BYTES(query_sq_out);
 153
 154         MLX5_SET(query_sq_in, in, opcode, MLX5_CMD_OP_QUERY_SQ);
 155         MLX5_SET(query_sq_in, in, sqn, sqn);
 156         return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
 157 }
 158 EXPORT_SYMBOL(mlx5_core_query_sq);
 159
 160 int mlx5_core_create_tir(struct mlx5_core_dev *dev, u32 *in, int inlen,
 161                          u32 *tirn)
 162 {
 163         u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {0};
 164         int err;
 165
 166         MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);
 167
 168         memset(out, 0, sizeof(out));
 169         err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 170         if (!err)
 171                 *tirn = MLX5_GET(create_tir_out, out, tirn);
 172
 173         return err;
 174 }
 175 EXPORT_SYMBOL(mlx5_core_create_tir);
 176
 177 int mlx5_core_modify_tir(struct mlx5_core_dev *dev, u32 tirn, u32 *in,
 178                          int inlen)
 179 {
 180         u32 out[MLX5_ST_SZ_DW(modify_tir_out)] = {0};
 181
 182         MLX5_SET(modify_tir_in, in, tirn, tirn);
 183         MLX5_SET(modify_tir_in, in, opcode, MLX5_CMD_OP_MODIFY_TIR);
 184         return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 185 }
 186
 187 void mlx5_core_destroy_tir(struct mlx5_core_dev *dev, u32 tirn)
 188 {
 189         u32 in[MLX5_ST_SZ_DW(destroy_tir_in)]   = {0};
 190         u32 out[MLX5_ST_SZ_DW(destroy_tir_out)] = {0};
 191
 192         MLX5_SET(destroy_tir_in, in, opcode, MLX5_CMD_OP_DESTROY_TIR);
 193         MLX5_SET(destroy_tir_in, in, tirn, tirn);
 194         mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 195 }
 196 EXPORT_SYMBOL(mlx5_core_destroy_tir);
 197
 198 int mlx5_core_create_tis(struct mlx5_core_dev *dev, u32 *in, int inlen,
 199                          u32 *tisn)
 200 {
 201         u32 out[MLX5_ST_SZ_DW(create_tis_out)] = {0};
 202         int err;
 203
 204         MLX5_SET(create_tis_in, in, opcode, MLX5_CMD_OP_CREATE_TIS);
 205         err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 206         if (!err)
 207                 *tisn = MLX5_GET(create_tis_out, out, tisn);
 208
 209         return err;
 210 }
 211 EXPORT_SYMBOL(mlx5_core_create_tis);
 212
 213 int mlx5_core_modify_tis(struct mlx5_core_dev *dev, u32 tisn, u32 *in,
 214                          int inlen)
 215 {
 216         u32 out[MLX5_ST_SZ_DW(modify_tis_out)] = {0};
 217
 218         MLX5_SET(modify_tis_in, in, tisn, tisn);
 219         MLX5_SET(modify_tis_in, in, opcode, MLX5_CMD_OP_MODIFY_TIS);
 220
 221         return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 222 }
 223 EXPORT_SYMBOL(mlx5_core_modify_tis);
 224
 225 void mlx5_core_destroy_tis(struct mlx5_core_dev *dev, u32 tisn)
 226 {
 227         u32 in[MLX5_ST_SZ_DW(destroy_tis_in)]   = {0};
 228         u32 out[MLX5_ST_SZ_DW(destroy_tis_out)] = {0};
 229
 230         MLX5_SET(destroy_tis_in, in, opcode, MLX5_CMD_OP_DESTROY_TIS);
 231         MLX5_SET(destroy_tis_in, in, tisn, tisn);
 232         mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 233 }
 234 EXPORT_SYMBOL(mlx5_core_destroy_tis);
 235
 236 int mlx5_core_create_rmp(struct mlx5_core_dev *dev, u32 *in, int inlen,
 237                          u32 *rmpn)
 238 {
 239         u32 out[MLX5_ST_SZ_DW(create_rmp_out)] = {0};
 240         int err;
 241
 242         MLX5_SET(create_rmp_in, in, opcode, MLX5_CMD_OP_CREATE_RMP);
 243         err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 244         if (!err)
 245                 *rmpn = MLX5_GET(create_rmp_out, out, rmpn);
 246
 247         return err;
 248 }
 249
 250 int mlx5_core_modify_rmp(struct mlx5_core_dev *dev, u32 *in, int inlen)
 251 {
 252         u32 out[MLX5_ST_SZ_DW(modify_rmp_out)] = {0};
 253
 254         MLX5_SET(modify_rmp_in, in, opcode, MLX5_CMD_OP_MODIFY_RMP);
 255         return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 256 }
 257
 258 int mlx5_core_destroy_rmp(struct mlx5_core_dev *dev, u32 rmpn)
 259 {
 260         u32 in[MLX5_ST_SZ_DW(destroy_rmp_in)]   = {0};
 261         u32 out[MLX5_ST_SZ_DW(destroy_rmp_out)] = {0};
 262
 263         MLX5_SET(destroy_rmp_in, in, opcode, MLX5_CMD_OP_DESTROY_RMP);
 264         MLX5_SET(destroy_rmp_in, in, rmpn, rmpn);
 265         return mlx5_cmd_exec(dev, in, sizeof(in), out,
 266                                           sizeof(out));
 267 }
 268
 269 int mlx5_core_query_rmp(struct mlx5_core_dev *dev, u32 rmpn, u32 *out)
 270 {
 271         u32 in[MLX5_ST_SZ_DW(query_rmp_in)] = {0};
 272         int outlen = MLX5_ST_SZ_BYTES(query_rmp_out);
 273
 274         MLX5_SET(query_rmp_in, in, opcode, MLX5_CMD_OP_QUERY_RMP);
 275         MLX5_SET(query_rmp_in, in, rmpn,   rmpn);
 276         return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
 277 }
 278
 279 int mlx5_core_arm_rmp(struct mlx5_core_dev *dev, u32 rmpn, u16 lwm)
 280 {
 281         void *in;
 282         void *rmpc;
 283         void *wq;
 284         void *bitmask;
 285         int  err;
 286
 287         in = kvzalloc(MLX5_ST_SZ_BYTES(modify_rmp_in), GFP_KERNEL);
 288         if (!in)
 289                 return -ENOMEM;
 290
 291         rmpc    = MLX5_ADDR_OF(modify_rmp_in,   in,   ctx);
 292         bitmask = MLX5_ADDR_OF(modify_rmp_in,   in,   bitmask);
 293         wq      = MLX5_ADDR_OF(rmpc,            rmpc, wq);
 294
 295         MLX5_SET(modify_rmp_in, in,      rmp_state, MLX5_RMPC_STATE_RDY);
 296         MLX5_SET(modify_rmp_in, in,      rmpn,      rmpn);
 297         MLX5_SET(wq,            wq,      lwm,       lwm);
 298         MLX5_SET(rmp_bitmask,   bitmask, lwm,       1);
 299         MLX5_SET(rmpc,          rmpc,    state,     MLX5_RMPC_STATE_RDY);
 300
 301         err =  mlx5_core_modify_rmp(dev, in, MLX5_ST_SZ_BYTES(modify_rmp_in));
 302
 303         kvfree(in);
 304
 305         return err;
 306 }
 307
 308 int mlx5_core_create_xsrq(struct mlx5_core_dev *dev, u32 *in, int inlen,
 309                           u32 *xsrqn)
 310 {
 311         u32 out[MLX5_ST_SZ_DW(create_xrc_srq_out)] = {0};
 312         int err;
 313
 314         MLX5_SET(create_xrc_srq_in, in, opcode,     MLX5_CMD_OP_CREATE_XRC_SRQ);
 315         err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 316         if (!err)
 317                 *xsrqn = MLX5_GET(create_xrc_srq_out, out, xrc_srqn);
 318
 319         return err;
 320 }
 321
 322 int mlx5_core_destroy_xsrq(struct mlx5_core_dev *dev, u32 xsrqn)
 323 {
 324         u32 in[MLX5_ST_SZ_DW(destroy_xrc_srq_in)]   = {0};
 325         u32 out[MLX5_ST_SZ_DW(destroy_xrc_srq_out)] = {0};
 326
 327         MLX5_SET(destroy_xrc_srq_in, in, opcode,   MLX5_CMD_OP_DESTROY_XRC_SRQ);
 328         MLX5_SET(destroy_xrc_srq_in, in, xrc_srqn, xsrqn);
 329         return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 330 }
 331
 332 int mlx5_core_query_xsrq(struct mlx5_core_dev *dev, u32 xsrqn, u32 *out)
 333 {
 334         u32 in[MLX5_ST_SZ_DW(query_xrc_srq_in)] = {0};
 335         void *srqc;
 336         void *xrc_srqc;
 337         int err;
 338
 339         MLX5_SET(query_xrc_srq_in, in, opcode,   MLX5_CMD_OP_QUERY_XRC_SRQ);
 340         MLX5_SET(query_xrc_srq_in, in, xrc_srqn, xsrqn);
 341         err = mlx5_cmd_exec(dev, in, sizeof(in), out,
 342                             MLX5_ST_SZ_BYTES(query_xrc_srq_out));
 343         if (!err) {
 344                 xrc_srqc = MLX5_ADDR_OF(query_xrc_srq_out, out,
 345                                         xrc_srq_context_entry);
 346                 srqc = MLX5_ADDR_OF(query_srq_out, out, srq_context_entry);
 347                 memcpy(srqc, xrc_srqc, MLX5_ST_SZ_BYTES(srqc));
 348         }
 349
 350         return err;
 351 }
 352
 353 int mlx5_core_arm_xsrq(struct mlx5_core_dev *dev, u32 xsrqn, u16 lwm)
 354 {
 355         u32 in[MLX5_ST_SZ_DW(arm_xrc_srq_in)]   = {0};
 356         u32 out[MLX5_ST_SZ_DW(arm_xrc_srq_out)] = {0};
 357
 358         MLX5_SET(arm_xrc_srq_in, in, opcode,   MLX5_CMD_OP_ARM_XRC_SRQ);
 359         MLX5_SET(arm_xrc_srq_in, in, xrc_srqn, xsrqn);
 360         MLX5_SET(arm_xrc_srq_in, in, lwm,      lwm);
 361         MLX5_SET(arm_xrc_srq_in, in, op_mod,
 362                  MLX5_ARM_XRC_SRQ_IN_OP_MOD_XRC_SRQ);
 363         return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 364 }
 365
 366 int mlx5_core_create_rqt(struct mlx5_core_dev *dev, u32 *in, int inlen,
 367                          u32 *rqtn)
 368 {
 369         u32 out[MLX5_ST_SZ_DW(create_rqt_out)] = {0};
 370         int err;
 371
 372         MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT);
 373         err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 374         if (!err)
 375                 *rqtn = MLX5_GET(create_rqt_out, out, rqtn);
 376
 377         return err;
 378 }
 379 EXPORT_SYMBOL(mlx5_core_create_rqt);
 380
 381 int mlx5_core_modify_rqt(struct mlx5_core_dev *dev, u32 rqtn, u32 *in,
 382                          int inlen)
 383 {
 384         u32 out[MLX5_ST_SZ_DW(modify_rqt_out)] = {0};
 385
 386         MLX5_SET(modify_rqt_in, in, rqtn, rqtn);
 387         MLX5_SET(modify_rqt_in, in, opcode, MLX5_CMD_OP_MODIFY_RQT);
 388         return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 389 }
 390
 391 void mlx5_core_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn)
 392 {
 393         u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)]   = {0};
 394         u32 out[MLX5_ST_SZ_DW(destroy_rqt_out)] = {0};
 395
 396         MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT);
 397         MLX5_SET(destroy_rqt_in, in, rqtn, rqtn);
 398         mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 399 }
 400 EXPORT_SYMBOL(mlx5_core_destroy_rqt);
 401
 402 static int mlx5_hairpin_create_rq(struct mlx5_core_dev *mdev,
 403                                   struct mlx5_hairpin_params *params, u32 *rqn)
 404 {
 405         u32 in[MLX5_ST_SZ_DW(create_rq_in)] = {0};
 406         void *rqc, *wq;
 407
 408         rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
 409         wq  = MLX5_ADDR_OF(rqc, rqc, wq);
 410
 411         MLX5_SET(rqc, rqc, hairpin, 1);
 412         MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
 413         MLX5_SET(rqc, rqc, counter_set_id, params->q_counter);
 414
 415         MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size);
 416         MLX5_SET(wq, wq, log_hairpin_num_packets, params->log_num_packets);
 417
 418         return mlx5_core_create_rq(mdev, in, MLX5_ST_SZ_BYTES(create_rq_in), rqn);
 419 }
 420
 421 static int mlx5_hairpin_create_sq(struct mlx5_core_dev *mdev,
 422                                   struct mlx5_hairpin_params *params, u32 *sqn)
 423 {
 424         u32 in[MLX5_ST_SZ_DW(create_sq_in)] = {0};
 425         void *sqc, *wq;
 426
 427         sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
 428         wq  = MLX5_ADDR_OF(sqc, sqc, wq);
 429
 430         MLX5_SET(sqc, sqc, hairpin, 1);
 431         MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
 432
 433         MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size);
 434         MLX5_SET(wq, wq, log_hairpin_num_packets, params->log_num_packets);
 435
 436         return mlx5_core_create_sq(mdev, in, MLX5_ST_SZ_BYTES(create_sq_in), sqn);
 437 }
 438
 439 static int mlx5_hairpin_create_queues(struct mlx5_hairpin *hp,
 440                                       struct mlx5_hairpin_params *params)
 441 {
 442         int i, j, err;
 443
 444         for (i = 0; i < hp->num_channels; i++) {
 445                 err = mlx5_hairpin_create_rq(hp->func_mdev, params, &hp->rqn[i]);
 446                 if (err)
 447                         goto out_err_rq;
 448         }
 449
 450         for (i = 0; i < hp->num_channels; i++) {
 451                 err = mlx5_hairpin_create_sq(hp->peer_mdev, params, &hp->sqn[i]);
 452                 if (err)
 453                         goto out_err_sq;
 454         }
 455
 456         return 0;
 457
 458 out_err_sq:
 459         for (j = 0; j < i; j++)
 460                 mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[j]);
 461         i = hp->num_channels;
 462 out_err_rq:
 463         for (j = 0; j < i; j++)
 464                 mlx5_core_destroy_rq(hp->func_mdev, hp->rqn[j]);
 465         return err;
 466 }
 467
 468 static void mlx5_hairpin_destroy_queues(struct mlx5_hairpin *hp)
 469 {
 470         int i;
 471
 472         for (i = 0; i < hp->num_channels; i++) {
 473                 mlx5_core_destroy_rq(hp->func_mdev, hp->rqn[i]);
 474                 mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[i]);
 475         }
 476 }
 477
 478 static int mlx5_hairpin_modify_rq(struct mlx5_core_dev *func_mdev, u32 rqn,
 479                                   int curr_state, int next_state,
 480                                   u16 peer_vhca, u32 peer_sq)
 481 {
 482         u32 in[MLX5_ST_SZ_DW(modify_rq_in)] = {0};
 483         void *rqc;
 484
 485         rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
 486
 487         if (next_state == MLX5_RQC_STATE_RDY) {
 488                 MLX5_SET(rqc, rqc, hairpin_peer_sq, peer_sq);
 489                 MLX5_SET(rqc, rqc, hairpin_peer_vhca, peer_vhca);
 490         }
 491
 492         MLX5_SET(modify_rq_in, in, rq_state, curr_state);
 493         MLX5_SET(rqc, rqc, state, next_state);
 494
 495         return mlx5_core_modify_rq(func_mdev, rqn,
 496                                    in, MLX5_ST_SZ_BYTES(modify_rq_in));
 497 }
 498
 499 static int mlx5_hairpin_modify_sq(struct mlx5_core_dev *peer_mdev, u32 sqn,
 500                                   int curr_state, int next_state,
 501                                   u16 peer_vhca, u32 peer_rq)
 502 {
 503         u32 in[MLX5_ST_SZ_DW(modify_sq_in)] = {0};
 504         void *sqc;
 505
 506         sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
 507
 508         if (next_state == MLX5_RQC_STATE_RDY) {
 509                 MLX5_SET(sqc, sqc, hairpin_peer_rq, peer_rq);
 510                 MLX5_SET(sqc, sqc, hairpin_peer_vhca, peer_vhca);
 511         }
 512
 513         MLX5_SET(modify_sq_in, in, sq_state, curr_state);
 514         MLX5_SET(sqc, sqc, state, next_state);
 515
 516         return mlx5_core_modify_sq(peer_mdev, sqn,
 517                                    in, MLX5_ST_SZ_BYTES(modify_sq_in));
 518 }
 519
 520 static int mlx5_hairpin_pair_queues(struct mlx5_hairpin *hp)
 521 {
 522         int i, j, err;
 523
 524         /* set peer SQs */
 525         for (i = 0; i < hp->num_channels; i++) {
 526                 err = mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[i],
 527                                              MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY,
 528                                              MLX5_CAP_GEN(hp->func_mdev, vhca_id), hp->rqn[i]);
 529                 if (err)
 530                         goto err_modify_sq;
 531         }
 532
 533         /* set func RQs */
 534         for (i = 0; i < hp->num_channels; i++) {
 535                 err = mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[i],
 536                                              MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY,
 537                                              MLX5_CAP_GEN(hp->peer_mdev, vhca_id), hp->sqn[i]);
 538                 if (err)
 539                         goto err_modify_rq;
 540         }
 541
 542         return 0;
 543
 544 err_modify_rq:
 545         for (j = 0; j < i; j++)
 546                 mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[j], MLX5_RQC_STATE_RDY,
 547                                        MLX5_RQC_STATE_RST, 0, 0);
 548         i = hp->num_channels;
 549 err_modify_sq:
 550         for (j = 0; j < i; j++)
 551                 mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[j], MLX5_SQC_STATE_RDY,
 552                                        MLX5_SQC_STATE_RST, 0, 0);
 553         return err;
 554 }
 555
 556 static void mlx5_hairpin_unpair_queues(struct mlx5_hairpin *hp)
 557 {
 558         int i;
 559
 560         /* unset func RQs */
 561         for (i = 0; i < hp->num_channels; i++)
 562                 mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[i], MLX5_RQC_STATE_RDY,
 563                                        MLX5_RQC_STATE_RST, 0, 0);
 564
 565         /* unset peer SQs */
 566         for (i = 0; i < hp->num_channels; i++)
 567                 mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[i], MLX5_SQC_STATE_RDY,
 568                                        MLX5_SQC_STATE_RST, 0, 0);
 569 }
 570
 571 struct mlx5_hairpin *
 572 mlx5_core_hairpin_create(struct mlx5_core_dev *func_mdev,
 573                          struct mlx5_core_dev *peer_mdev,
 574                          struct mlx5_hairpin_params *params)
 575 {
 576         struct mlx5_hairpin *hp;
 577         int size, err;
 578
 579         size = sizeof(*hp) + params->num_channels * 2 * sizeof(u32);
 580         hp = kzalloc(size, GFP_KERNEL);
 581         if (!hp)
 582                 return ERR_PTR(-ENOMEM);
 583
 584         hp->func_mdev = func_mdev;
 585         hp->peer_mdev = peer_mdev;
 586         hp->num_channels = params->num_channels;
 587
 588         hp->rqn = (void *)hp + sizeof(*hp);
 589         hp->sqn = hp->rqn + params->num_channels;
 590
 591         /* alloc and pair func --> peer hairpin */
 592         err = mlx5_hairpin_create_queues(hp, params);
 593         if (err)
 594                 goto err_create_queues;
 595
 596         err = mlx5_hairpin_pair_queues(hp);
 597         if (err)
 598                 goto err_pair_queues;
 599
 600         return hp;
 601
 602 err_pair_queues:
 603         mlx5_hairpin_destroy_queues(hp);
 604 err_create_queues:
 605         kfree(hp);
 606         return ERR_PTR(err);
 607 }
 608
 609 void mlx5_core_hairpin_destroy(struct mlx5_hairpin *hp)
 610 {
 611         mlx5_hairpin_unpair_queues(hp);
 612         mlx5_hairpin_destroy_queues(hp);
 613         kfree(hp);
 614 }