drivers/net/ethernet/qlogic/qed/qed_ooo.c

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
   2 /* QLogic qed NIC Driver
   3  * Copyright (c) 2015-2017  QLogic Corporation
   4  * Copyright (c) 2019-2020 Marvell International Ltd.
   5  */
   6
   7 #include <linux/types.h>
   8 #include <linux/dma-mapping.h>
   9 #include <linux/kernel.h>
  10 #include <linux/list.h>
  11 #include <linux/pci.h>
  12 #include <linux/slab.h>
  13 #include <linux/string.h>
  14 #include "qed.h"
  15 #include "qed_iscsi.h"
  16 #include "qed_ll2.h"
  17 #include "qed_ooo.h"
  18 #include "qed_cxt.h"
  19
  20 static struct qed_ooo_archipelago
  21 *qed_ooo_seek_archipelago(struct qed_hwfn *p_hwfn,
  22                           struct qed_ooo_info
  23                           *p_ooo_info,
  24                           u32 cid)
  25 {
  26         u32 idx = (cid & 0xffff) - p_ooo_info->cid_base;
  27         struct qed_ooo_archipelago *p_archipelago;
  28
  29         if (idx >= p_ooo_info->max_num_archipelagos)
  30                 return NULL;
  31
  32         p_archipelago = &p_ooo_info->p_archipelagos_mem[idx];
  33
  34         if (list_empty(&p_archipelago->isles_list))
  35                 return NULL;
  36
  37         return p_archipelago;
  38 }
  39
  40 static struct qed_ooo_isle *qed_ooo_seek_isle(struct qed_hwfn *p_hwfn,
  41                                               struct qed_ooo_info *p_ooo_info,
  42                                               u32 cid, u8 isle)
  43 {
  44         struct qed_ooo_archipelago *p_archipelago = NULL;
  45         struct qed_ooo_isle *p_isle = NULL;
  46         u8 the_num_of_isle = 1;
  47
  48         p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid);
  49         if (!p_archipelago) {
  50                 DP_NOTICE(p_hwfn,
  51                           "Connection %d is not found in OOO list\n", cid);
  52                 return NULL;
  53         }
  54
  55         list_for_each_entry(p_isle, &p_archipelago->isles_list, list_entry) {
  56                 if (the_num_of_isle == isle)
  57                         return p_isle;
  58                 the_num_of_isle++;
  59         }
  60
  61         return NULL;
  62 }
  63
  64 void qed_ooo_save_history_entry(struct qed_hwfn *p_hwfn,
  65                                 struct qed_ooo_info *p_ooo_info,
  66                                 struct ooo_opaque *p_cqe)
  67 {
  68         struct qed_ooo_history *p_history = &p_ooo_info->ooo_history;
  69
  70         if (p_history->head_idx == p_history->num_of_cqes)
  71                 p_history->head_idx = 0;
  72         p_history->p_cqes[p_history->head_idx] = *p_cqe;
  73         p_history->head_idx++;
  74 }
  75
  76 int qed_ooo_alloc(struct qed_hwfn *p_hwfn)
  77 {
  78         u16 max_num_archipelagos = 0, cid_base;
  79         struct qed_ooo_info *p_ooo_info;
  80         enum protocol_type proto;
  81         u16 max_num_isles = 0;
  82         u32 i;
  83
  84         switch (p_hwfn->hw_info.personality) {
  85         case QED_PCI_ISCSI:
  86                 proto = PROTOCOLID_ISCSI;
  87                 break;
  88         case QED_PCI_ETH_RDMA:
  89         case QED_PCI_ETH_IWARP:
  90                 proto = PROTOCOLID_IWARP;
  91                 break;
  92         default:
  93                 DP_NOTICE(p_hwfn,
  94                           "Failed to allocate qed_ooo_info: unknown personality\n");
  95                 return -EINVAL;
  96         }
  97
  98         max_num_archipelagos = (u16)qed_cxt_get_proto_cid_count(p_hwfn, proto,
  99                                                                 NULL);
 100         max_num_isles = QED_MAX_NUM_ISLES + max_num_archipelagos;
 101         cid_base = (u16)qed_cxt_get_proto_cid_start(p_hwfn, proto);
 102
 103         if (!max_num_archipelagos) {
 104                 DP_NOTICE(p_hwfn,
 105                           "Failed to allocate qed_ooo_info: unknown amount of connections\n");
 106                 return -EINVAL;
 107         }
 108
 109         p_ooo_info = kzalloc(sizeof(*p_ooo_info), GFP_KERNEL);
 110         if (!p_ooo_info)
 111                 return -ENOMEM;
 112
 113         p_ooo_info->cid_base = cid_base;
 114         p_ooo_info->max_num_archipelagos = max_num_archipelagos;
 115
 116         INIT_LIST_HEAD(&p_ooo_info->free_buffers_list);
 117         INIT_LIST_HEAD(&p_ooo_info->ready_buffers_list);
 118         INIT_LIST_HEAD(&p_ooo_info->free_isles_list);
 119
 120         p_ooo_info->p_isles_mem = kcalloc(max_num_isles,
 121                                           sizeof(struct qed_ooo_isle),
 122                                           GFP_KERNEL);
 123         if (!p_ooo_info->p_isles_mem)
 124                 goto no_isles_mem;
 125
 126         for (i = 0; i < max_num_isles; i++) {
 127                 INIT_LIST_HEAD(&p_ooo_info->p_isles_mem[i].buffers_list);
 128                 list_add_tail(&p_ooo_info->p_isles_mem[i].list_entry,
 129                               &p_ooo_info->free_isles_list);
 130         }
 131
 132         p_ooo_info->p_archipelagos_mem =
 133                                 kcalloc(max_num_archipelagos,
 134                                         sizeof(struct qed_ooo_archipelago),
 135                                         GFP_KERNEL);
 136         if (!p_ooo_info->p_archipelagos_mem)
 137                 goto no_archipelagos_mem;
 138
 139         for (i = 0; i < max_num_archipelagos; i++)
 140                 INIT_LIST_HEAD(&p_ooo_info->p_archipelagos_mem[i].isles_list);
 141
 142         p_ooo_info->ooo_history.p_cqes =
 143                                 kcalloc(QED_MAX_NUM_OOO_HISTORY_ENTRIES,
 144                                         sizeof(struct ooo_opaque),
 145                                         GFP_KERNEL);
 146         if (!p_ooo_info->ooo_history.p_cqes)
 147                 goto no_history_mem;
 148
 149         p_ooo_info->ooo_history.num_of_cqes = QED_MAX_NUM_OOO_HISTORY_ENTRIES;
 150
 151         p_hwfn->p_ooo_info = p_ooo_info;
 152         return 0;
 153
 154 no_history_mem:
 155         kfree(p_ooo_info->p_archipelagos_mem);
 156 no_archipelagos_mem:
 157         kfree(p_ooo_info->p_isles_mem);
 158 no_isles_mem:
 159         kfree(p_ooo_info);
 160         return -ENOMEM;
 161 }
 162
 163 void qed_ooo_release_connection_isles(struct qed_hwfn *p_hwfn,
 164                                       struct qed_ooo_info *p_ooo_info, u32 cid)
 165 {
 166         struct qed_ooo_archipelago *p_archipelago;
 167         struct qed_ooo_buffer *p_buffer;
 168         struct qed_ooo_isle *p_isle;
 169
 170         p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid);
 171         if (!p_archipelago)
 172                 return;
 173
 174         while (!list_empty(&p_archipelago->isles_list)) {
 175                 p_isle = list_first_entry(&p_archipelago->isles_list,
 176                                           struct qed_ooo_isle, list_entry);
 177
 178                 list_del(&p_isle->list_entry);
 179
 180                 while (!list_empty(&p_isle->buffers_list)) {
 181                         p_buffer = list_first_entry(&p_isle->buffers_list,
 182                                                     struct qed_ooo_buffer,
 183                                                     list_entry);
 184
 185                         if (!p_buffer)
 186                                 break;
 187
 188                         list_move_tail(&p_buffer->list_entry,
 189                                        &p_ooo_info->free_buffers_list);
 190                 }
 191                 list_add_tail(&p_isle->list_entry,
 192                               &p_ooo_info->free_isles_list);
 193         }
 194 }
 195
 196 void qed_ooo_release_all_isles(struct qed_hwfn *p_hwfn,
 197                                struct qed_ooo_info *p_ooo_info)
 198 {
 199         struct qed_ooo_archipelago *p_archipelago;
 200         struct qed_ooo_buffer *p_buffer;
 201         struct qed_ooo_isle *p_isle;
 202         u32 i;
 203
 204         for (i = 0; i < p_ooo_info->max_num_archipelagos; i++) {
 205                 p_archipelago = &(p_ooo_info->p_archipelagos_mem[i]);
 206
 207                 while (!list_empty(&p_archipelago->isles_list)) {
 208                         p_isle = list_first_entry(&p_archipelago->isles_list,
 209                                                   struct qed_ooo_isle,
 210                                                   list_entry);
 211
 212                         list_del(&p_isle->list_entry);
 213
 214                         while (!list_empty(&p_isle->buffers_list)) {
 215                                 p_buffer =
 216                                     list_first_entry(&p_isle->buffers_list,
 217                                                      struct qed_ooo_buffer,
 218                                                      list_entry);
 219
 220                                 if (!p_buffer)
 221                                         break;
 222
 223                                 list_move_tail(&p_buffer->list_entry,
 224                                                &p_ooo_info->free_buffers_list);
 225                         }
 226                         list_add_tail(&p_isle->list_entry,
 227                                       &p_ooo_info->free_isles_list);
 228                 }
 229         }
 230         if (!list_empty(&p_ooo_info->ready_buffers_list))
 231                 list_splice_tail_init(&p_ooo_info->ready_buffers_list,
 232                                       &p_ooo_info->free_buffers_list);
 233 }
 234
 235 void qed_ooo_setup(struct qed_hwfn *p_hwfn)
 236 {
 237         qed_ooo_release_all_isles(p_hwfn, p_hwfn->p_ooo_info);
 238         memset(p_hwfn->p_ooo_info->ooo_history.p_cqes, 0,
 239                p_hwfn->p_ooo_info->ooo_history.num_of_cqes *
 240                sizeof(struct ooo_opaque));
 241         p_hwfn->p_ooo_info->ooo_history.head_idx = 0;
 242 }
 243
 244 void qed_ooo_free(struct qed_hwfn *p_hwfn)
 245 {
 246         struct qed_ooo_info *p_ooo_info  = p_hwfn->p_ooo_info;
 247         struct qed_ooo_buffer *p_buffer;
 248
 249         if (!p_ooo_info)
 250                 return;
 251
 252         qed_ooo_release_all_isles(p_hwfn, p_ooo_info);
 253         while (!list_empty(&p_ooo_info->free_buffers_list)) {
 254                 p_buffer = list_first_entry(&p_ooo_info->free_buffers_list,
 255                                             struct qed_ooo_buffer, list_entry);
 256
 257                 if (!p_buffer)
 258                         break;
 259
 260                 list_del(&p_buffer->list_entry);
 261                 dma_free_coherent(&p_hwfn->cdev->pdev->dev,
 262                                   p_buffer->rx_buffer_size,
 263                                   p_buffer->rx_buffer_virt_addr,
 264                                   p_buffer->rx_buffer_phys_addr);
 265                 kfree(p_buffer);
 266         }
 267
 268         kfree(p_ooo_info->p_isles_mem);
 269         kfree(p_ooo_info->p_archipelagos_mem);
 270         kfree(p_ooo_info->ooo_history.p_cqes);
 271         kfree(p_ooo_info);
 272         p_hwfn->p_ooo_info = NULL;
 273 }
 274
 275 void qed_ooo_put_free_buffer(struct qed_hwfn *p_hwfn,
 276                              struct qed_ooo_info *p_ooo_info,
 277                              struct qed_ooo_buffer *p_buffer)
 278 {
 279         list_add_tail(&p_buffer->list_entry, &p_ooo_info->free_buffers_list);
 280 }
 281
 282 struct qed_ooo_buffer *qed_ooo_get_free_buffer(struct qed_hwfn *p_hwfn,
 283                                                struct qed_ooo_info *p_ooo_info)
 284 {
 285         struct qed_ooo_buffer *p_buffer = NULL;
 286
 287         if (!list_empty(&p_ooo_info->free_buffers_list)) {
 288                 p_buffer = list_first_entry(&p_ooo_info->free_buffers_list,
 289                                             struct qed_ooo_buffer, list_entry);
 290
 291                 list_del(&p_buffer->list_entry);
 292         }
 293
 294         return p_buffer;
 295 }
 296
 297 void qed_ooo_put_ready_buffer(struct qed_hwfn *p_hwfn,
 298                               struct qed_ooo_info *p_ooo_info,
 299                               struct qed_ooo_buffer *p_buffer, u8 on_tail)
 300 {
 301         if (on_tail)
 302                 list_add_tail(&p_buffer->list_entry,
 303                               &p_ooo_info->ready_buffers_list);
 304         else
 305                 list_add(&p_buffer->list_entry,
 306                          &p_ooo_info->ready_buffers_list);
 307 }
 308
 309 struct qed_ooo_buffer *qed_ooo_get_ready_buffer(struct qed_hwfn *p_hwfn,
 310                                                 struct qed_ooo_info *p_ooo_info)
 311 {
 312         struct qed_ooo_buffer *p_buffer = NULL;
 313
 314         if (!list_empty(&p_ooo_info->ready_buffers_list)) {
 315                 p_buffer = list_first_entry(&p_ooo_info->ready_buffers_list,
 316                                             struct qed_ooo_buffer, list_entry);
 317
 318                 list_del(&p_buffer->list_entry);
 319         }
 320
 321         return p_buffer;
 322 }
 323
 324 void qed_ooo_delete_isles(struct qed_hwfn *p_hwfn,
 325                           struct qed_ooo_info *p_ooo_info,
 326                           u32 cid, u8 drop_isle, u8 drop_size)
 327 {
 328         struct qed_ooo_isle *p_isle = NULL;
 329         u8 isle_idx;
 330
 331         for (isle_idx = 0; isle_idx < drop_size; isle_idx++) {
 332                 p_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, drop_isle);
 333                 if (!p_isle) {
 334                         DP_NOTICE(p_hwfn,
 335                                   "Isle %d is not found(cid %d)\n",
 336                                   drop_isle, cid);
 337                         return;
 338                 }
 339                 if (list_empty(&p_isle->buffers_list))
 340                         DP_NOTICE(p_hwfn,
 341                                   "Isle %d is empty(cid %d)\n", drop_isle, cid);
 342                 else
 343                         list_splice_tail_init(&p_isle->buffers_list,
 344                                               &p_ooo_info->free_buffers_list);
 345
 346                 list_del(&p_isle->list_entry);
 347                 p_ooo_info->cur_isles_number--;
 348                 list_add(&p_isle->list_entry, &p_ooo_info->free_isles_list);
 349         }
 350 }
 351
 352 void qed_ooo_add_new_isle(struct qed_hwfn *p_hwfn,
 353                           struct qed_ooo_info *p_ooo_info,
 354                           u32 cid, u8 ooo_isle,
 355                           struct qed_ooo_buffer *p_buffer)
 356 {
 357         struct qed_ooo_archipelago *p_archipelago = NULL;
 358         struct qed_ooo_isle *p_prev_isle = NULL;
 359         struct qed_ooo_isle *p_isle = NULL;
 360
 361         if (ooo_isle > 1) {
 362                 p_prev_isle = qed_ooo_seek_isle(p_hwfn,
 363                                                 p_ooo_info, cid, ooo_isle - 1);
 364                 if (!p_prev_isle) {
 365                         DP_NOTICE(p_hwfn,
 366                                   "Isle %d is not found(cid %d)\n",
 367                                   ooo_isle - 1, cid);
 368                         return;
 369                 }
 370         }
 371         p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid);
 372         if (!p_archipelago && (ooo_isle != 1)) {
 373                 DP_NOTICE(p_hwfn,
 374                           "Connection %d is not found in OOO list\n", cid);
 375                 return;
 376         }
 377
 378         if (!list_empty(&p_ooo_info->free_isles_list)) {
 379                 p_isle = list_first_entry(&p_ooo_info->free_isles_list,
 380                                           struct qed_ooo_isle, list_entry);
 381
 382                 list_del(&p_isle->list_entry);
 383                 if (!list_empty(&p_isle->buffers_list)) {
 384                         DP_NOTICE(p_hwfn, "Free isle is not empty\n");
 385                         INIT_LIST_HEAD(&p_isle->buffers_list);
 386                 }
 387         } else {
 388                 DP_NOTICE(p_hwfn, "No more free isles\n");
 389                 return;
 390         }
 391
 392         if (!p_archipelago) {
 393                 u32 idx = (cid & 0xffff) - p_ooo_info->cid_base;
 394
 395                 p_archipelago = &p_ooo_info->p_archipelagos_mem[idx];
 396         }
 397
 398         list_add(&p_buffer->list_entry, &p_isle->buffers_list);
 399         p_ooo_info->cur_isles_number++;
 400         p_ooo_info->gen_isles_number++;
 401
 402         if (p_ooo_info->cur_isles_number > p_ooo_info->max_isles_number)
 403                 p_ooo_info->max_isles_number = p_ooo_info->cur_isles_number;
 404
 405         if (!p_prev_isle)
 406                 list_add(&p_isle->list_entry, &p_archipelago->isles_list);
 407         else
 408                 list_add(&p_isle->list_entry, &p_prev_isle->list_entry);
 409 }
 410
 411 void qed_ooo_add_new_buffer(struct qed_hwfn *p_hwfn,
 412                             struct qed_ooo_info *p_ooo_info,
 413                             u32 cid,
 414                             u8 ooo_isle,
 415                             struct qed_ooo_buffer *p_buffer, u8 buffer_side)
 416 {
 417         struct qed_ooo_isle *p_isle = NULL;
 418
 419         p_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, ooo_isle);
 420         if (!p_isle) {
 421                 DP_NOTICE(p_hwfn,
 422                           "Isle %d is not found(cid %d)\n", ooo_isle, cid);
 423                 return;
 424         }
 425
 426         if (buffer_side == QED_OOO_LEFT_BUF)
 427                 list_add(&p_buffer->list_entry, &p_isle->buffers_list);
 428         else
 429                 list_add_tail(&p_buffer->list_entry, &p_isle->buffers_list);
 430 }
 431
 432 void qed_ooo_join_isles(struct qed_hwfn *p_hwfn,
 433                         struct qed_ooo_info *p_ooo_info, u32 cid, u8 left_isle)
 434 {
 435         struct qed_ooo_isle *p_right_isle = NULL;
 436         struct qed_ooo_isle *p_left_isle = NULL;
 437
 438         p_right_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid,
 439                                          left_isle + 1);
 440         if (!p_right_isle) {
 441                 DP_NOTICE(p_hwfn,
 442                           "Right isle %d is not found(cid %d)\n",
 443                           left_isle + 1, cid);
 444                 return;
 445         }
 446
 447         list_del(&p_right_isle->list_entry);
 448         p_ooo_info->cur_isles_number--;
 449         if (left_isle) {
 450                 p_left_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid,
 451                                                 left_isle);
 452                 if (!p_left_isle) {
 453                         DP_NOTICE(p_hwfn,
 454                                   "Left isle %d is not found(cid %d)\n",
 455                                   left_isle, cid);
 456                         return;
 457                 }
 458                 list_splice_tail_init(&p_right_isle->buffers_list,
 459                                       &p_left_isle->buffers_list);
 460         } else {
 461                 list_splice_tail_init(&p_right_isle->buffers_list,
 462                                       &p_ooo_info->ready_buffers_list);
 463         }
 464         list_add_tail(&p_right_isle->list_entry, &p_ooo_info->free_isles_list);
 465 }