drivers/net/ethernet/qlogic/qed/qed_ooo.c

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
   2 /* QLogic qed NIC Driver
   3  * Copyright (c) 2015-2017  QLogic Corporation
   4  * Copyright (c) 2019-2020 Marvell International Ltd.
   5  */
   6
   7 #include <linux/types.h>
   8 #include <linux/dma-mapping.h>
   9 #include <linux/kernel.h>
  10 #include <linux/list.h>
  11 #include <linux/pci.h>
  12 #include <linux/slab.h>
  13 #include <linux/string.h>
  14 #include "qed.h"
  15 #include "qed_iscsi.h"
  16 #include "qed_ll2.h"
  17 #include "qed_ooo.h"
  18 #include "qed_cxt.h"
  19 #include "qed_nvmetcp.h"
  20 static struct qed_ooo_archipelago
  21 *qed_ooo_seek_archipelago(struct qed_hwfn *p_hwfn,
  22                           struct qed_ooo_info
  23                           *p_ooo_info,
  24                           u32 cid)
  25 {
  26         u32 idx = (cid & 0xffff) - p_ooo_info->cid_base;
  27         struct qed_ooo_archipelago *p_archipelago;
  28
  29         if (unlikely(idx >= p_ooo_info->max_num_archipelagos))
  30                 return NULL;
  31
  32         p_archipelago = &p_ooo_info->p_archipelagos_mem[idx];
  33
  34         if (unlikely(list_empty(&p_archipelago->isles_list)))
  35                 return NULL;
  36
  37         return p_archipelago;
  38 }
  39
  40 static struct qed_ooo_isle *qed_ooo_seek_isle(struct qed_hwfn *p_hwfn,
  41                                               struct qed_ooo_info *p_ooo_info,
  42                                               u32 cid, u8 isle)
  43 {
  44         struct qed_ooo_archipelago *p_archipelago = NULL;
  45         struct qed_ooo_isle *p_isle = NULL;
  46         u8 the_num_of_isle = 1;
  47
  48         p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid);
  49         if (unlikely(!p_archipelago)) {
  50                 DP_NOTICE(p_hwfn,
  51                           "Connection %d is not found in OOO list\n", cid);
  52                 return NULL;
  53         }
  54
  55         list_for_each_entry(p_isle, &p_archipelago->isles_list, list_entry) {
  56                 if (the_num_of_isle == isle)
  57                         return p_isle;
  58                 the_num_of_isle++;
  59         }
  60
  61         return NULL;
  62 }
  63
  64 void qed_ooo_save_history_entry(struct qed_hwfn *p_hwfn,
  65                                 struct qed_ooo_info *p_ooo_info,
  66                                 struct ooo_opaque *p_cqe)
  67 {
  68         struct qed_ooo_history *p_history = &p_ooo_info->ooo_history;
  69
  70         if (p_history->head_idx == p_history->num_of_cqes)
  71                 p_history->head_idx = 0;
  72         p_history->p_cqes[p_history->head_idx] = *p_cqe;
  73         p_history->head_idx++;
  74 }
  75
  76 int qed_ooo_alloc(struct qed_hwfn *p_hwfn)
  77 {
  78         u16 max_num_archipelagos = 0, cid_base;
  79         struct qed_ooo_info *p_ooo_info;
  80         enum protocol_type proto;
  81         u16 max_num_isles = 0;
  82         u32 i;
  83
  84         switch (p_hwfn->hw_info.personality) {
  85         case QED_PCI_ISCSI:
  86         case QED_PCI_NVMETCP:
  87                 proto = PROTOCOLID_TCP_ULP;
  88                 break;
  89         case QED_PCI_ETH_RDMA:
  90         case QED_PCI_ETH_IWARP:
  91                 proto = PROTOCOLID_IWARP;
  92                 break;
  93         default:
  94                 DP_NOTICE(p_hwfn,
  95                           "Failed to allocate qed_ooo_info: unknown personality\n");
  96                 return -EINVAL;
  97         }
  98
  99         max_num_archipelagos = (u16)qed_cxt_get_proto_cid_count(p_hwfn, proto,
 100                                                                 NULL);
 101         max_num_isles = QED_MAX_NUM_ISLES + max_num_archipelagos;
 102         cid_base = (u16)qed_cxt_get_proto_cid_start(p_hwfn, proto);
 103
 104         if (!max_num_archipelagos) {
 105                 DP_NOTICE(p_hwfn,
 106                           "Failed to allocate qed_ooo_info: unknown amount of connections\n");
 107                 return -EINVAL;
 108         }
 109
 110         p_ooo_info = kzalloc(sizeof(*p_ooo_info), GFP_KERNEL);
 111         if (!p_ooo_info)
 112                 return -ENOMEM;
 113
 114         p_ooo_info->cid_base = cid_base;
 115         p_ooo_info->max_num_archipelagos = max_num_archipelagos;
 116
 117         INIT_LIST_HEAD(&p_ooo_info->free_buffers_list);
 118         INIT_LIST_HEAD(&p_ooo_info->ready_buffers_list);
 119         INIT_LIST_HEAD(&p_ooo_info->free_isles_list);
 120
 121         p_ooo_info->p_isles_mem = kcalloc(max_num_isles,
 122                                           sizeof(struct qed_ooo_isle),
 123                                           GFP_KERNEL);
 124         if (!p_ooo_info->p_isles_mem)
 125                 goto no_isles_mem;
 126
 127         for (i = 0; i < max_num_isles; i++) {
 128                 INIT_LIST_HEAD(&p_ooo_info->p_isles_mem[i].buffers_list);
 129                 list_add_tail(&p_ooo_info->p_isles_mem[i].list_entry,
 130                               &p_ooo_info->free_isles_list);
 131         }
 132
 133         p_ooo_info->p_archipelagos_mem =
 134                                 kcalloc(max_num_archipelagos,
 135                                         sizeof(struct qed_ooo_archipelago),
 136                                         GFP_KERNEL);
 137         if (!p_ooo_info->p_archipelagos_mem)
 138                 goto no_archipelagos_mem;
 139
 140         for (i = 0; i < max_num_archipelagos; i++)
 141                 INIT_LIST_HEAD(&p_ooo_info->p_archipelagos_mem[i].isles_list);
 142
 143         p_ooo_info->ooo_history.p_cqes =
 144                                 kcalloc(QED_MAX_NUM_OOO_HISTORY_ENTRIES,
 145                                         sizeof(struct ooo_opaque),
 146                                         GFP_KERNEL);
 147         if (!p_ooo_info->ooo_history.p_cqes)
 148                 goto no_history_mem;
 149
 150         p_ooo_info->ooo_history.num_of_cqes = QED_MAX_NUM_OOO_HISTORY_ENTRIES;
 151
 152         p_hwfn->p_ooo_info = p_ooo_info;
 153         return 0;
 154
 155 no_history_mem:
 156         kfree(p_ooo_info->p_archipelagos_mem);
 157 no_archipelagos_mem:
 158         kfree(p_ooo_info->p_isles_mem);
 159 no_isles_mem:
 160         kfree(p_ooo_info);
 161         return -ENOMEM;
 162 }
 163
 164 void qed_ooo_release_connection_isles(struct qed_hwfn *p_hwfn,
 165                                       struct qed_ooo_info *p_ooo_info, u32 cid)
 166 {
 167         struct qed_ooo_archipelago *p_archipelago;
 168         struct qed_ooo_buffer *p_buffer;
 169         struct qed_ooo_isle *p_isle;
 170
 171         p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid);
 172         if (!p_archipelago)
 173                 return;
 174
 175         while (!list_empty(&p_archipelago->isles_list)) {
 176                 p_isle = list_first_entry(&p_archipelago->isles_list,
 177                                           struct qed_ooo_isle, list_entry);
 178
 179                 list_del(&p_isle->list_entry);
 180
 181                 while (!list_empty(&p_isle->buffers_list)) {
 182                         p_buffer = list_first_entry(&p_isle->buffers_list,
 183                                                     struct qed_ooo_buffer,
 184                                                     list_entry);
 185
 186                         if (!p_buffer)
 187                                 break;
 188
 189                         list_move_tail(&p_buffer->list_entry,
 190                                        &p_ooo_info->free_buffers_list);
 191                 }
 192                 list_add_tail(&p_isle->list_entry,
 193                               &p_ooo_info->free_isles_list);
 194         }
 195 }
 196
 197 void qed_ooo_release_all_isles(struct qed_hwfn *p_hwfn,
 198                                struct qed_ooo_info *p_ooo_info)
 199 {
 200         struct qed_ooo_archipelago *p_archipelago;
 201         struct qed_ooo_buffer *p_buffer;
 202         struct qed_ooo_isle *p_isle;
 203         u32 i;
 204
 205         for (i = 0; i < p_ooo_info->max_num_archipelagos; i++) {
 206                 p_archipelago = &(p_ooo_info->p_archipelagos_mem[i]);
 207
 208                 while (!list_empty(&p_archipelago->isles_list)) {
 209                         p_isle = list_first_entry(&p_archipelago->isles_list,
 210                                                   struct qed_ooo_isle,
 211                                                   list_entry);
 212
 213                         list_del(&p_isle->list_entry);
 214
 215                         while (!list_empty(&p_isle->buffers_list)) {
 216                                 p_buffer =
 217                                     list_first_entry(&p_isle->buffers_list,
 218                                                      struct qed_ooo_buffer,
 219                                                      list_entry);
 220
 221                                 if (!p_buffer)
 222                                         break;
 223
 224                                 list_move_tail(&p_buffer->list_entry,
 225                                                &p_ooo_info->free_buffers_list);
 226                         }
 227                         list_add_tail(&p_isle->list_entry,
 228                                       &p_ooo_info->free_isles_list);
 229                 }
 230         }
 231         if (!list_empty(&p_ooo_info->ready_buffers_list))
 232                 list_splice_tail_init(&p_ooo_info->ready_buffers_list,
 233                                       &p_ooo_info->free_buffers_list);
 234 }
 235
 236 void qed_ooo_setup(struct qed_hwfn *p_hwfn)
 237 {
 238         qed_ooo_release_all_isles(p_hwfn, p_hwfn->p_ooo_info);
 239         memset(p_hwfn->p_ooo_info->ooo_history.p_cqes, 0,
 240                p_hwfn->p_ooo_info->ooo_history.num_of_cqes *
 241                sizeof(struct ooo_opaque));
 242         p_hwfn->p_ooo_info->ooo_history.head_idx = 0;
 243 }
 244
 245 void qed_ooo_free(struct qed_hwfn *p_hwfn)
 246 {
 247         struct qed_ooo_info *p_ooo_info  = p_hwfn->p_ooo_info;
 248         struct qed_ooo_buffer *p_buffer;
 249
 250         if (!p_ooo_info)
 251                 return;
 252
 253         qed_ooo_release_all_isles(p_hwfn, p_ooo_info);
 254         while (!list_empty(&p_ooo_info->free_buffers_list)) {
 255                 p_buffer = list_first_entry(&p_ooo_info->free_buffers_list,
 256                                             struct qed_ooo_buffer, list_entry);
 257
 258                 if (!p_buffer)
 259                         break;
 260
 261                 list_del(&p_buffer->list_entry);
 262                 dma_free_coherent(&p_hwfn->cdev->pdev->dev,
 263                                   p_buffer->rx_buffer_size,
 264                                   p_buffer->rx_buffer_virt_addr,
 265                                   p_buffer->rx_buffer_phys_addr);
 266                 kfree(p_buffer);
 267         }
 268
 269         kfree(p_ooo_info->p_isles_mem);
 270         kfree(p_ooo_info->p_archipelagos_mem);
 271         kfree(p_ooo_info->ooo_history.p_cqes);
 272         kfree(p_ooo_info);
 273         p_hwfn->p_ooo_info = NULL;
 274 }
 275
 276 void qed_ooo_put_free_buffer(struct qed_hwfn *p_hwfn,
 277                              struct qed_ooo_info *p_ooo_info,
 278                              struct qed_ooo_buffer *p_buffer)
 279 {
 280         list_add_tail(&p_buffer->list_entry, &p_ooo_info->free_buffers_list);
 281 }
 282
 283 struct qed_ooo_buffer *qed_ooo_get_free_buffer(struct qed_hwfn *p_hwfn,
 284                                                struct qed_ooo_info *p_ooo_info)
 285 {
 286         struct qed_ooo_buffer *p_buffer = NULL;
 287
 288         if (!list_empty(&p_ooo_info->free_buffers_list)) {
 289                 p_buffer = list_first_entry(&p_ooo_info->free_buffers_list,
 290                                             struct qed_ooo_buffer, list_entry);
 291
 292                 list_del(&p_buffer->list_entry);
 293         }
 294
 295         return p_buffer;
 296 }
 297
 298 void qed_ooo_put_ready_buffer(struct qed_hwfn *p_hwfn,
 299                               struct qed_ooo_info *p_ooo_info,
 300                               struct qed_ooo_buffer *p_buffer, u8 on_tail)
 301 {
 302         if (on_tail)
 303                 list_add_tail(&p_buffer->list_entry,
 304                               &p_ooo_info->ready_buffers_list);
 305         else
 306                 list_add(&p_buffer->list_entry,
 307                          &p_ooo_info->ready_buffers_list);
 308 }
 309
 310 struct qed_ooo_buffer *qed_ooo_get_ready_buffer(struct qed_hwfn *p_hwfn,
 311                                                 struct qed_ooo_info *p_ooo_info)
 312 {
 313         struct qed_ooo_buffer *p_buffer = NULL;
 314
 315         if (!list_empty(&p_ooo_info->ready_buffers_list)) {
 316                 p_buffer = list_first_entry(&p_ooo_info->ready_buffers_list,
 317                                             struct qed_ooo_buffer, list_entry);
 318
 319                 list_del(&p_buffer->list_entry);
 320         }
 321
 322         return p_buffer;
 323 }
 324
 325 void qed_ooo_delete_isles(struct qed_hwfn *p_hwfn,
 326                           struct qed_ooo_info *p_ooo_info,
 327                           u32 cid, u8 drop_isle, u8 drop_size)
 328 {
 329         struct qed_ooo_isle *p_isle = NULL;
 330         u8 isle_idx;
 331
 332         for (isle_idx = 0; isle_idx < drop_size; isle_idx++) {
 333                 p_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, drop_isle);
 334                 if (!p_isle) {
 335                         DP_NOTICE(p_hwfn,
 336                                   "Isle %d is not found(cid %d)\n",
 337                                   drop_isle, cid);
 338                         return;
 339                 }
 340                 if (list_empty(&p_isle->buffers_list))
 341                         DP_NOTICE(p_hwfn,
 342                                   "Isle %d is empty(cid %d)\n", drop_isle, cid);
 343                 else
 344                         list_splice_tail_init(&p_isle->buffers_list,
 345                                               &p_ooo_info->free_buffers_list);
 346
 347                 list_del(&p_isle->list_entry);
 348                 p_ooo_info->cur_isles_number--;
 349                 list_add(&p_isle->list_entry, &p_ooo_info->free_isles_list);
 350         }
 351 }
 352
 353 void qed_ooo_add_new_isle(struct qed_hwfn *p_hwfn,
 354                           struct qed_ooo_info *p_ooo_info,
 355                           u32 cid, u8 ooo_isle,
 356                           struct qed_ooo_buffer *p_buffer)
 357 {
 358         struct qed_ooo_archipelago *p_archipelago = NULL;
 359         struct qed_ooo_isle *p_prev_isle = NULL;
 360         struct qed_ooo_isle *p_isle = NULL;
 361
 362         if (ooo_isle > 1) {
 363                 p_prev_isle = qed_ooo_seek_isle(p_hwfn,
 364                                                 p_ooo_info, cid, ooo_isle - 1);
 365                 if (unlikely(!p_prev_isle)) {
 366                         DP_NOTICE(p_hwfn,
 367                                   "Isle %d is not found(cid %d)\n",
 368                                   ooo_isle - 1, cid);
 369                         return;
 370                 }
 371         }
 372         p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid);
 373         if (unlikely(!p_archipelago && ooo_isle != 1)) {
 374                 DP_NOTICE(p_hwfn,
 375                           "Connection %d is not found in OOO list\n", cid);
 376                 return;
 377         }
 378
 379         if (!list_empty(&p_ooo_info->free_isles_list)) {
 380                 p_isle = list_first_entry(&p_ooo_info->free_isles_list,
 381                                           struct qed_ooo_isle, list_entry);
 382
 383                 list_del(&p_isle->list_entry);
 384                 if (unlikely(!list_empty(&p_isle->buffers_list))) {
 385                         DP_NOTICE(p_hwfn, "Free isle is not empty\n");
 386                         INIT_LIST_HEAD(&p_isle->buffers_list);
 387                 }
 388         } else {
 389                 DP_NOTICE(p_hwfn, "No more free isles\n");
 390                 return;
 391         }
 392
 393         if (!p_archipelago) {
 394                 u32 idx = (cid & 0xffff) - p_ooo_info->cid_base;
 395
 396                 p_archipelago = &p_ooo_info->p_archipelagos_mem[idx];
 397         }
 398
 399         list_add(&p_buffer->list_entry, &p_isle->buffers_list);
 400         p_ooo_info->cur_isles_number++;
 401         p_ooo_info->gen_isles_number++;
 402
 403         if (p_ooo_info->cur_isles_number > p_ooo_info->max_isles_number)
 404                 p_ooo_info->max_isles_number = p_ooo_info->cur_isles_number;
 405
 406         if (!p_prev_isle)
 407                 list_add(&p_isle->list_entry, &p_archipelago->isles_list);
 408         else
 409                 list_add(&p_isle->list_entry, &p_prev_isle->list_entry);
 410 }
 411
 412 void qed_ooo_add_new_buffer(struct qed_hwfn *p_hwfn,
 413                             struct qed_ooo_info *p_ooo_info,
 414                             u32 cid,
 415                             u8 ooo_isle,
 416                             struct qed_ooo_buffer *p_buffer, u8 buffer_side)
 417 {
 418         struct qed_ooo_isle *p_isle = NULL;
 419
 420         p_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, ooo_isle);
 421         if (unlikely(!p_isle)) {
 422                 DP_NOTICE(p_hwfn,
 423                           "Isle %d is not found(cid %d)\n", ooo_isle, cid);
 424                 return;
 425         }
 426
 427         if (unlikely(buffer_side == QED_OOO_LEFT_BUF))
 428                 list_add(&p_buffer->list_entry, &p_isle->buffers_list);
 429         else
 430                 list_add_tail(&p_buffer->list_entry, &p_isle->buffers_list);
 431 }
 432
 433 void qed_ooo_join_isles(struct qed_hwfn *p_hwfn,
 434                         struct qed_ooo_info *p_ooo_info, u32 cid, u8 left_isle)
 435 {
 436         struct qed_ooo_isle *p_right_isle = NULL;
 437         struct qed_ooo_isle *p_left_isle = NULL;
 438
 439         p_right_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid,
 440                                          left_isle + 1);
 441         if (unlikely(!p_right_isle)) {
 442                 DP_NOTICE(p_hwfn,
 443                           "Right isle %d is not found(cid %d)\n",
 444                           left_isle + 1, cid);
 445                 return;
 446         }
 447
 448         list_del(&p_right_isle->list_entry);
 449         p_ooo_info->cur_isles_number--;
 450         if (left_isle) {
 451                 p_left_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid,
 452                                                 left_isle);
 453                 if (unlikely(!p_left_isle)) {
 454                         DP_NOTICE(p_hwfn,
 455                                   "Left isle %d is not found(cid %d)\n",
 456                                   left_isle, cid);
 457                         return;
 458                 }
 459                 list_splice_tail_init(&p_right_isle->buffers_list,
 460                                       &p_left_isle->buffers_list);
 461         } else {
 462                 list_splice_tail_init(&p_right_isle->buffers_list,
 463                                       &p_ooo_info->ready_buffers_list);
 464         }
 465         list_add_tail(&p_right_isle->list_entry, &p_ooo_info->free_isles_list);
 466 }