drivers/net/ethernet/intel/ice/ice_main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2018, Intel Corporation. */
   3
   4 /* Intel(R) Ethernet Connection E800 Series Linux Driver */
   5
   6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   7
   8 #include "ice.h"
   9
  10 #define DRV_VERSION     "ice-0.7.0-k"
  11 #define DRV_SUMMARY     "Intel(R) Ethernet Connection E800 Series Linux Driver"
  12 const char ice_drv_ver[] = DRV_VERSION;
  13 static const char ice_driver_string[] = DRV_SUMMARY;
  14 static const char ice_copyright[] = "Copyright (c) 2018, Intel Corporation.";
  15
  16 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
  17 MODULE_DESCRIPTION(DRV_SUMMARY);
  18 MODULE_LICENSE("GPL");
  19 MODULE_VERSION(DRV_VERSION);
  20
  21 static int debug = -1;
  22 module_param(debug, int, 0644);
  23 #ifndef CONFIG_DYNAMIC_DEBUG
  24 MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all), hw debug_mask (0x8XXXXXXX)");
  25 #else
  26 MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all)");
  27 #endif /* !CONFIG_DYNAMIC_DEBUG */
  28
  29 static struct workqueue_struct *ice_wq;
  30 static const struct net_device_ops ice_netdev_ops;
  31
  32 static void ice_pf_dis_all_vsi(struct ice_pf *pf);
  33 static void ice_rebuild(struct ice_pf *pf);
  34 static int ice_vsi_release(struct ice_vsi *vsi);
  35 static void ice_update_vsi_stats(struct ice_vsi *vsi);
  36 static void ice_update_pf_stats(struct ice_pf *pf);
  37
  38 /**
  39  * ice_get_free_slot - get the next non-NULL location index in array
  40  * @array: array to search
  41  * @size: size of the array
  42  * @curr: last known occupied index to be used as a search hint
  43  *
  44  * void * is being used to keep the functionality generic. This lets us use this
  45  * function on any array of pointers.
  46  */
  47 static int ice_get_free_slot(void *array, int size, int curr)
  48 {
  49         int **tmp_array = (int **)array;
  50         int next;
  51
  52         if (curr < (size - 1) && !tmp_array[curr + 1]) {
  53                 next = curr + 1;
  54         } else {
  55                 int i = 0;
  56
  57                 while ((i < size) && (tmp_array[i]))
  58                         i++;
  59                 if (i == size)
  60                         next = ICE_NO_VSI;
  61                 else
  62                         next = i;
  63         }
  64         return next;
  65 }
  66
  67 /**
  68  * ice_search_res - Search the tracker for a block of resources
  69  * @res: pointer to the resource
  70  * @needed: size of the block needed
  71  * @id: identifier to track owner
  72  * Returns the base item index of the block, or -ENOMEM for error
  73  */
  74 static int ice_search_res(struct ice_res_tracker *res, u16 needed, u16 id)
  75 {
  76         int start = res->search_hint;
  77         int end = start;
  78
  79         id |= ICE_RES_VALID_BIT;
  80
  81         do {
  82                 /* skip already allocated entries */
  83                 if (res->list[end++] & ICE_RES_VALID_BIT) {
  84                         start = end;
  85                         if ((start + needed) > res->num_entries)
  86                                 break;
  87                 }
  88
  89                 if (end == (start + needed)) {
  90                         int i = start;
  91
  92                         /* there was enough, so assign it to the requestor */
  93                         while (i != end)
  94                                 res->list[i++] = id;
  95
  96                         if (end == res->num_entries)
  97                                 end = 0;
  98
  99                         res->search_hint = end;
 100                         return start;
 101                 }
 102         } while (1);
 103
 104         return -ENOMEM;
 105 }
 106
 107 /**
 108  * ice_get_res - get a block of resources
 109  * @pf: board private structure
 110  * @res: pointer to the resource
 111  * @needed: size of the block needed
 112  * @id: identifier to track owner
 113  *
 114  * Returns the base item index of the block, or -ENOMEM for error
 115  * The search_hint trick and lack of advanced fit-finding only works
 116  * because we're highly likely to have all the same sized requests.
 117  * Linear search time and any fragmentation should be minimal.
 118  */
 119 static int
 120 ice_get_res(struct ice_pf *pf, struct ice_res_tracker *res, u16 needed, u16 id)
 121 {
 122         int ret;
 123
 124         if (!res || !pf)
 125                 return -EINVAL;
 126
 127         if (!needed || needed > res->num_entries || id >= ICE_RES_VALID_BIT) {
 128                 dev_err(&pf->pdev->dev,
 129                         "param err: needed=%d, num_entries = %d id=0x%04x\n",
 130                         needed, res->num_entries, id);
 131                 return -EINVAL;
 132         }
 133
 134         /* search based on search_hint */
 135         ret = ice_search_res(res, needed, id);
 136
 137         if (ret < 0) {
 138                 /* previous search failed. Reset search hint and try again */
 139                 res->search_hint = 0;
 140                 ret = ice_search_res(res, needed, id);
 141         }
 142
 143         return ret;
 144 }
 145
 146 /**
 147  * ice_free_res - free a block of resources
 148  * @res: pointer to the resource
 149  * @index: starting index previously returned by ice_get_res
 150  * @id: identifier to track owner
 151  * Returns number of resources freed
 152  */
 153 static int ice_free_res(struct ice_res_tracker *res, u16 index, u16 id)
 154 {
 155         int count = 0;
 156         int i;
 157
 158         if (!res || index >= res->num_entries)
 159                 return -EINVAL;
 160
 161         id |= ICE_RES_VALID_BIT;
 162         for (i = index; i < res->num_entries && res->list[i] == id; i++) {
 163                 res->list[i] = 0;
 164                 count++;
 165         }
 166
 167         return count;
 168 }
 169
 170 /**
 171  * ice_add_mac_to_list - Add a mac address filter entry to the list
 172  * @vsi: the VSI to be forwarded to
 173  * @add_list: pointer to the list which contains MAC filter entries
 174  * @macaddr: the MAC address to be added.
 175  *
 176  * Adds mac address filter entry to the temp list
 177  *
 178  * Returns 0 on success or ENOMEM on failure.
 179  */
 180 static int ice_add_mac_to_list(struct ice_vsi *vsi, struct list_head *add_list,
 181                                const u8 *macaddr)
 182 {
 183         struct ice_fltr_list_entry *tmp;
 184         struct ice_pf *pf = vsi->back;
 185
 186         tmp = devm_kzalloc(&pf->pdev->dev, sizeof(*tmp), GFP_ATOMIC);
 187         if (!tmp)
 188                 return -ENOMEM;
 189
 190         tmp->fltr_info.flag = ICE_FLTR_TX;
 191         tmp->fltr_info.src = vsi->vsi_num;
 192         tmp->fltr_info.lkup_type = ICE_SW_LKUP_MAC;
 193         tmp->fltr_info.fltr_act = ICE_FWD_TO_VSI;
 194         tmp->fltr_info.fwd_id.vsi_id = vsi->vsi_num;
 195         ether_addr_copy(tmp->fltr_info.l_data.mac.mac_addr, macaddr);
 196
 197         INIT_LIST_HEAD(&tmp->list_entry);
 198         list_add(&tmp->list_entry, add_list);
 199
 200         return 0;
 201 }
 202
 203 /**
 204  * ice_add_mac_to_sync_list - creates list of mac addresses to be synced
 205  * @netdev: the net device on which the sync is happening
 206  * @addr: mac address to sync
 207  *
 208  * This is a callback function which is called by the in kernel device sync
 209  * functions (like __dev_uc_sync, __dev_mc_sync, etc). This function only
 210  * populates the tmp_sync_list, which is later used by ice_add_mac to add the
 211  * mac filters from the hardware.
 212  */
 213 static int ice_add_mac_to_sync_list(struct net_device *netdev, const u8 *addr)
 214 {
 215         struct ice_netdev_priv *np = netdev_priv(netdev);
 216         struct ice_vsi *vsi = np->vsi;
 217
 218         if (ice_add_mac_to_list(vsi, &vsi->tmp_sync_list, addr))
 219                 return -EINVAL;
 220
 221         return 0;
 222 }
 223
 224 /**
 225  * ice_add_mac_to_unsync_list - creates list of mac addresses to be unsynced
 226  * @netdev: the net device on which the unsync is happening
 227  * @addr: mac address to unsync
 228  *
 229  * This is a callback function which is called by the in kernel device unsync
 230  * functions (like __dev_uc_unsync, __dev_mc_unsync, etc). This function only
 231  * populates the tmp_unsync_list, which is later used by ice_remove_mac to
 232  * delete the mac filters from the hardware.
 233  */
 234 static int ice_add_mac_to_unsync_list(struct net_device *netdev, const u8 *addr)
 235 {
 236         struct ice_netdev_priv *np = netdev_priv(netdev);
 237         struct ice_vsi *vsi = np->vsi;
 238
 239         if (ice_add_mac_to_list(vsi, &vsi->tmp_unsync_list, addr))
 240                 return -EINVAL;
 241
 242         return 0;
 243 }
 244
 245 /**
 246  * ice_free_fltr_list - free filter lists helper
 247  * @dev: pointer to the device struct
 248  * @h: pointer to the list head to be freed
 249  *
 250  * Helper function to free filter lists previously created using
 251  * ice_add_mac_to_list
 252  */
 253 static void ice_free_fltr_list(struct device *dev, struct list_head *h)
 254 {
 255         struct ice_fltr_list_entry *e, *tmp;
 256
 257         list_for_each_entry_safe(e, tmp, h, list_entry) {
 258                 list_del(&e->list_entry);
 259                 devm_kfree(dev, e);
 260         }
 261 }
 262
 263 /**
 264  * ice_vsi_fltr_changed - check if filter state changed
 265  * @vsi: VSI to be checked
 266  *
 267  * returns true if filter state has changed, false otherwise.
 268  */
 269 static bool ice_vsi_fltr_changed(struct ice_vsi *vsi)
 270 {
 271         return test_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags) ||
 272                test_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags) ||
 273                test_bit(ICE_VSI_FLAG_VLAN_FLTR_CHANGED, vsi->flags);
 274 }
 275
 276 /**
 277  * ice_vsi_sync_fltr - Update the VSI filter list to the HW
 278  * @vsi: ptr to the VSI
 279  *
 280  * Push any outstanding VSI filter changes through the AdminQ.
 281  */
 282 static int ice_vsi_sync_fltr(struct ice_vsi *vsi)
 283 {
 284         struct device *dev = &vsi->back->pdev->dev;
 285         struct net_device *netdev = vsi->netdev;
 286         bool promisc_forced_on = false;
 287         struct ice_pf *pf = vsi->back;
 288         struct ice_hw *hw = &pf->hw;
 289         enum ice_status status = 0;
 290         u32 changed_flags = 0;
 291         int err = 0;
 292
 293         if (!vsi->netdev)
 294                 return -EINVAL;
 295
 296         while (test_and_set_bit(__ICE_CFG_BUSY, vsi->state))
 297                 usleep_range(1000, 2000);
 298
 299         changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
 300         vsi->current_netdev_flags = vsi->netdev->flags;
 301
 302         INIT_LIST_HEAD(&vsi->tmp_sync_list);
 303         INIT_LIST_HEAD(&vsi->tmp_unsync_list);
 304
 305         if (ice_vsi_fltr_changed(vsi)) {
 306                 clear_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags);
 307                 clear_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags);
 308                 clear_bit(ICE_VSI_FLAG_VLAN_FLTR_CHANGED, vsi->flags);
 309
 310                 /* grab the netdev's addr_list_lock */
 311                 netif_addr_lock_bh(netdev);
 312                 __dev_uc_sync(netdev, ice_add_mac_to_sync_list,
 313                               ice_add_mac_to_unsync_list);
 314                 __dev_mc_sync(netdev, ice_add_mac_to_sync_list,
 315                               ice_add_mac_to_unsync_list);
 316                 /* our temp lists are populated. release lock */
 317                 netif_addr_unlock_bh(netdev);
 318         }
 319
 320         /* Remove mac addresses in the unsync list */
 321         status = ice_remove_mac(hw, &vsi->tmp_unsync_list);
 322         ice_free_fltr_list(dev, &vsi->tmp_unsync_list);
 323         if (status) {
 324                 netdev_err(netdev, "Failed to delete MAC filters\n");
 325                 /* if we failed because of alloc failures, just bail */
 326                 if (status == ICE_ERR_NO_MEMORY) {
 327                         err = -ENOMEM;
 328                         goto out;
 329                 }
 330         }
 331
 332         /* Add mac addresses in the sync list */
 333         status = ice_add_mac(hw, &vsi->tmp_sync_list);
 334         ice_free_fltr_list(dev, &vsi->tmp_sync_list);
 335         if (status) {
 336                 netdev_err(netdev, "Failed to add MAC filters\n");
 337                 /* If there is no more space for new umac filters, vsi
 338                  * should go into promiscuous mode. There should be some
 339                  * space reserved for promiscuous filters.
 340                  */
 341                 if (hw->adminq.sq_last_status == ICE_AQ_RC_ENOSPC &&
 342                     !test_and_set_bit(__ICE_FLTR_OVERFLOW_PROMISC,
 343                                       vsi->state)) {
 344                         promisc_forced_on = true;
 345                         netdev_warn(netdev,
 346                                     "Reached MAC filter limit, forcing promisc mode on VSI %d\n",
 347                                     vsi->vsi_num);
 348                 } else {
 349                         err = -EIO;
 350                         goto out;
 351                 }
 352         }
 353         /* check for changes in promiscuous modes */
 354         if (changed_flags & IFF_ALLMULTI)
 355                 netdev_warn(netdev, "Unsupported configuration\n");
 356
 357         if (((changed_flags & IFF_PROMISC) || promisc_forced_on) ||
 358             test_bit(ICE_VSI_FLAG_PROMISC_CHANGED, vsi->flags)) {
 359                 clear_bit(ICE_VSI_FLAG_PROMISC_CHANGED, vsi->flags);
 360                 if (vsi->current_netdev_flags & IFF_PROMISC) {
 361                         /* Apply TX filter rule to get traffic from VMs */
 362                         status = ice_cfg_dflt_vsi(hw, vsi->vsi_num, true,
 363                                                   ICE_FLTR_TX);
 364                         if (status) {
 365                                 netdev_err(netdev, "Error setting default VSI %i tx rule\n",
 366                                            vsi->vsi_num);
 367                                 vsi->current_netdev_flags &= ~IFF_PROMISC;
 368                                 err = -EIO;
 369                                 goto out_promisc;
 370                         }
 371                         /* Apply RX filter rule to get traffic from wire */
 372                         status = ice_cfg_dflt_vsi(hw, vsi->vsi_num, true,
 373                                                   ICE_FLTR_RX);
 374                         if (status) {
 375                                 netdev_err(netdev, "Error setting default VSI %i rx rule\n",
 376                                            vsi->vsi_num);
 377                                 vsi->current_netdev_flags &= ~IFF_PROMISC;
 378                                 err = -EIO;
 379                                 goto out_promisc;
 380                         }
 381                 } else {
 382                         /* Clear TX filter rule to stop traffic from VMs */
 383                         status = ice_cfg_dflt_vsi(hw, vsi->vsi_num, false,
 384                                                   ICE_FLTR_TX);
 385                         if (status) {
 386                                 netdev_err(netdev, "Error clearing default VSI %i tx rule\n",
 387                                            vsi->vsi_num);
 388                                 vsi->current_netdev_flags |= IFF_PROMISC;
 389                                 err = -EIO;
 390                                 goto out_promisc;
 391                         }
 392                         /* Clear filter RX to remove traffic from wire */
 393                         status = ice_cfg_dflt_vsi(hw, vsi->vsi_num, false,
 394                                                   ICE_FLTR_RX);
 395                         if (status) {
 396                                 netdev_err(netdev, "Error clearing default VSI %i rx rule\n",
 397                                            vsi->vsi_num);
 398                                 vsi->current_netdev_flags |= IFF_PROMISC;
 399                                 err = -EIO;
 400                                 goto out_promisc;
 401                         }
 402                 }
 403         }
 404         goto exit;
 405
 406 out_promisc:
 407         set_bit(ICE_VSI_FLAG_PROMISC_CHANGED, vsi->flags);
 408         goto exit;
 409 out:
 410         /* if something went wrong then set the changed flag so we try again */
 411         set_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags);
 412         set_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags);
 413 exit:
 414         clear_bit(__ICE_CFG_BUSY, vsi->state);
 415         return err;
 416 }
 417
 418 /**
 419  * ice_sync_fltr_subtask - Sync the VSI filter list with HW
 420  * @pf: board private structure
 421  */
 422 static void ice_sync_fltr_subtask(struct ice_pf *pf)
 423 {
 424         int v;
 425
 426         if (!pf || !(test_bit(ICE_FLAG_FLTR_SYNC, pf->flags)))
 427                 return;
 428
 429         clear_bit(ICE_FLAG_FLTR_SYNC, pf->flags);
 430
 431         for (v = 0; v < pf->num_alloc_vsi; v++)
 432                 if (pf->vsi[v] && ice_vsi_fltr_changed(pf->vsi[v]) &&
 433                     ice_vsi_sync_fltr(pf->vsi[v])) {
 434                         /* come back and try again later */
 435                         set_bit(ICE_FLAG_FLTR_SYNC, pf->flags);
 436                         break;
 437                 }
 438 }
 439
 440 /**
 441  * ice_is_reset_recovery_pending - schedule a reset
 442  * @state: pf state field
 443  */
 444 static bool ice_is_reset_recovery_pending(unsigned long int *state)
 445 {
 446         return test_bit(__ICE_RESET_RECOVERY_PENDING, state);
 447 }
 448
 449 /**
 450  * ice_prepare_for_reset - prep for the core to reset
 451  * @pf: board private structure
 452  *
 453  * Inform or close all dependent features in prep for reset.
 454  */
 455 static void
 456 ice_prepare_for_reset(struct ice_pf *pf)
 457 {
 458         struct ice_hw *hw = &pf->hw;
 459         u32 v;
 460
 461         ice_for_each_vsi(pf, v)
 462                 if (pf->vsi[v])
 463                         ice_remove_vsi_fltr(hw, pf->vsi[v]->vsi_num);
 464
 465         dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
 466
 467         /* disable the VSIs and their queues that are not already DOWN */
 468         /* pf_dis_all_vsi modifies netdev structures -rtnl_lock needed */
 469         ice_pf_dis_all_vsi(pf);
 470
 471         ice_for_each_vsi(pf, v)
 472                 if (pf->vsi[v])
 473                         pf->vsi[v]->vsi_num = 0;
 474
 475         ice_shutdown_all_ctrlq(hw);
 476 }
 477
 478 /**
 479  * ice_do_reset - Initiate one of many types of resets
 480  * @pf: board private structure
 481  * @reset_type: reset type requested
 482  * before this function was called.
 483  */
 484 static void ice_do_reset(struct ice_pf *pf, enum ice_reset_req reset_type)
 485 {
 486         struct device *dev = &pf->pdev->dev;
 487         struct ice_hw *hw = &pf->hw;
 488
 489         dev_dbg(dev, "reset_type 0x%x requested\n", reset_type);
 490         WARN_ON(in_interrupt());
 491
 492         /* PFR is a bit of a special case because it doesn't result in an OICR
 493          * interrupt. So for PFR, we prepare for reset, issue the reset and
 494          * rebuild sequentially.
 495          */
 496         if (reset_type == ICE_RESET_PFR) {
 497                 set_bit(__ICE_RESET_RECOVERY_PENDING, pf->state);
 498                 ice_prepare_for_reset(pf);
 499         }
 500
 501         /* trigger the reset */
 502         if (ice_reset(hw, reset_type)) {
 503                 dev_err(dev, "reset %d failed\n", reset_type);
 504                 set_bit(__ICE_RESET_FAILED, pf->state);
 505                 clear_bit(__ICE_RESET_RECOVERY_PENDING, pf->state);
 506                 return;
 507         }
 508
 509         if (reset_type == ICE_RESET_PFR) {
 510                 pf->pfr_count++;
 511                 ice_rebuild(pf);
 512                 clear_bit(__ICE_RESET_RECOVERY_PENDING, pf->state);
 513         }
 514 }
 515
 516 /**
 517  * ice_reset_subtask - Set up for resetting the device and driver
 518  * @pf: board private structure
 519  */
 520 static void ice_reset_subtask(struct ice_pf *pf)
 521 {
 522         enum ice_reset_req reset_type;
 523
 524         rtnl_lock();
 525
 526         /* When a CORER/GLOBR/EMPR is about to happen, the hardware triggers an
 527          * OICR interrupt. The OICR handler (ice_misc_intr) determines what
 528          * type of reset happened and sets __ICE_RESET_RECOVERY_PENDING bit in
 529          * pf->state. So if reset/recovery is pending (as indicated by this bit)
 530          * we do a rebuild and return.
 531          */
 532         if (ice_is_reset_recovery_pending(pf->state)) {
 533                 clear_bit(__ICE_GLOBR_RECV, pf->state);
 534                 clear_bit(__ICE_CORER_RECV, pf->state);
 535                 ice_prepare_for_reset(pf);
 536
 537                 /* make sure we are ready to rebuild */
 538                 if (ice_check_reset(&pf->hw))
 539                         set_bit(__ICE_RESET_FAILED, pf->state);
 540                 else
 541                         ice_rebuild(pf);
 542                 clear_bit(__ICE_RESET_RECOVERY_PENDING, pf->state);
 543                 goto unlock;
 544         }
 545
 546         /* No pending resets to finish processing. Check for new resets */
 547         if (test_and_clear_bit(__ICE_GLOBR_REQ, pf->state))
 548                 reset_type = ICE_RESET_GLOBR;
 549         else if (test_and_clear_bit(__ICE_CORER_REQ, pf->state))
 550                 reset_type = ICE_RESET_CORER;
 551         else if (test_and_clear_bit(__ICE_PFR_REQ, pf->state))
 552                 reset_type = ICE_RESET_PFR;
 553         else
 554                 goto unlock;
 555
 556         /* reset if not already down or resetting */
 557         if (!test_bit(__ICE_DOWN, pf->state) &&
 558             !test_bit(__ICE_CFG_BUSY, pf->state)) {
 559                 ice_do_reset(pf, reset_type);
 560         }
 561
 562 unlock:
 563         rtnl_unlock();
 564 }
 565
 566 /**
 567  * ice_watchdog_subtask - periodic tasks not using event driven scheduling
 568  * @pf: board private structure
 569  */
 570 static void ice_watchdog_subtask(struct ice_pf *pf)
 571 {
 572         int i;
 573
 574         /* if interface is down do nothing */
 575         if (test_bit(__ICE_DOWN, pf->state) ||
 576             test_bit(__ICE_CFG_BUSY, pf->state))
 577                 return;
 578
 579         /* make sure we don't do these things too often */
 580         if (time_before(jiffies,
 581                         pf->serv_tmr_prev + pf->serv_tmr_period))
 582                 return;
 583
 584         pf->serv_tmr_prev = jiffies;
 585
 586         /* Update the stats for active netdevs so the network stack
 587          * can look at updated numbers whenever it cares to
 588          */
 589         ice_update_pf_stats(pf);
 590         for (i = 0; i < pf->num_alloc_vsi; i++)
 591                 if (pf->vsi[i] && pf->vsi[i]->netdev)
 592                         ice_update_vsi_stats(pf->vsi[i]);
 593 }
 594
 595 /**
 596  * ice_print_link_msg - print link up or down message
 597  * @vsi: the VSI whose link status is being queried
 598  * @isup: boolean for if the link is now up or down
 599  */
 600 void ice_print_link_msg(struct ice_vsi *vsi, bool isup)
 601 {
 602         const char *speed;
 603         const char *fc;
 604
 605         if (vsi->current_isup == isup)
 606                 return;
 607
 608         vsi->current_isup = isup;
 609
 610         if (!isup) {
 611                 netdev_info(vsi->netdev, "NIC Link is Down\n");
 612                 return;
 613         }
 614
 615         switch (vsi->port_info->phy.link_info.link_speed) {
 616         case ICE_AQ_LINK_SPEED_40GB:
 617                 speed = "40 G";
 618                 break;
 619         case ICE_AQ_LINK_SPEED_25GB:
 620                 speed = "25 G";
 621                 break;
 622         case ICE_AQ_LINK_SPEED_20GB:
 623                 speed = "20 G";
 624                 break;
 625         case ICE_AQ_LINK_SPEED_10GB:
 626                 speed = "10 G";
 627                 break;
 628         case ICE_AQ_LINK_SPEED_5GB:
 629                 speed = "5 G";
 630                 break;
 631         case ICE_AQ_LINK_SPEED_2500MB:
 632                 speed = "2.5 G";
 633                 break;
 634         case ICE_AQ_LINK_SPEED_1000MB:
 635                 speed = "1 G";
 636                 break;
 637         case ICE_AQ_LINK_SPEED_100MB:
 638                 speed = "100 M";
 639                 break;
 640         default:
 641                 speed = "Unknown";
 642                 break;
 643         }
 644
 645         switch (vsi->port_info->fc.current_mode) {
 646         case ICE_FC_FULL:
 647                 fc = "RX/TX";
 648                 break;
 649         case ICE_FC_TX_PAUSE:
 650                 fc = "TX";
 651                 break;
 652         case ICE_FC_RX_PAUSE:
 653                 fc = "RX";
 654                 break;
 655         default:
 656                 fc = "Unknown";
 657                 break;
 658         }
 659
 660         netdev_info(vsi->netdev, "NIC Link is up %sbps, Flow Control: %s\n",
 661                     speed, fc);
 662 }
 663
 664 /**
 665  * ice_init_link_events - enable/initialize link events
 666  * @pi: pointer to the port_info instance
 667  *
 668  * Returns -EIO on failure, 0 on success
 669  */
 670 static int ice_init_link_events(struct ice_port_info *pi)
 671 {
 672         u16 mask;
 673
 674         mask = ~((u16)(ICE_AQ_LINK_EVENT_UPDOWN | ICE_AQ_LINK_EVENT_MEDIA_NA |
 675                        ICE_AQ_LINK_EVENT_MODULE_QUAL_FAIL));
 676
 677         if (ice_aq_set_event_mask(pi->hw, pi->lport, mask, NULL)) {
 678                 dev_dbg(ice_hw_to_dev(pi->hw),
 679                         "Failed to set link event mask for port %d\n",
 680                         pi->lport);
 681                 return -EIO;
 682         }
 683
 684         if (ice_aq_get_link_info(pi, true, NULL, NULL)) {
 685                 dev_dbg(ice_hw_to_dev(pi->hw),
 686                         "Failed to enable link events for port %d\n",
 687                         pi->lport);
 688                 return -EIO;
 689         }
 690
 691         return 0;
 692 }
 693
 694 /**
 695  * ice_vsi_link_event - update the vsi's netdev
 696  * @vsi: the vsi on which the link event occurred
 697  * @link_up: whether or not the vsi needs to be set up or down
 698  */
 699 static void ice_vsi_link_event(struct ice_vsi *vsi, bool link_up)
 700 {
 701         if (!vsi || test_bit(__ICE_DOWN, vsi->state))
 702                 return;
 703
 704         if (vsi->type == ICE_VSI_PF) {
 705                 if (!vsi->netdev) {
 706                         dev_dbg(&vsi->back->pdev->dev,
 707                                 "vsi->netdev is not initialized!\n");
 708                         return;
 709                 }
 710                 if (link_up) {
 711                         netif_carrier_on(vsi->netdev);
 712                         netif_tx_wake_all_queues(vsi->netdev);
 713                 } else {
 714                         netif_carrier_off(vsi->netdev);
 715                         netif_tx_stop_all_queues(vsi->netdev);
 716                 }
 717         }
 718 }
 719
 720 /**
 721  * ice_link_event - process the link event
 722  * @pf: pf that the link event is associated with
 723  * @pi: port_info for the port that the link event is associated with
 724  *
 725  * Returns -EIO if ice_get_link_status() fails
 726  * Returns 0 on success
 727  */
 728 static int
 729 ice_link_event(struct ice_pf *pf, struct ice_port_info *pi)
 730 {
 731         u8 new_link_speed, old_link_speed;
 732         struct ice_phy_info *phy_info;
 733         bool new_link_same_as_old;
 734         bool new_link, old_link;
 735         u8 lport;
 736         u16 v;
 737
 738         phy_info = &pi->phy;
 739         phy_info->link_info_old = phy_info->link_info;
 740         /* Force ice_get_link_status() to update link info */
 741         phy_info->get_link_info = true;
 742
 743         old_link = (phy_info->link_info_old.link_info & ICE_AQ_LINK_UP);
 744         old_link_speed = phy_info->link_info_old.link_speed;
 745
 746         lport = pi->lport;
 747         if (ice_get_link_status(pi, &new_link)) {
 748                 dev_dbg(&pf->pdev->dev,
 749                         "Could not get link status for port %d\n", lport);
 750                 return -EIO;
 751         }
 752
 753         new_link_speed = phy_info->link_info.link_speed;
 754
 755         new_link_same_as_old = (new_link == old_link &&
 756                                 new_link_speed == old_link_speed);
 757
 758         ice_for_each_vsi(pf, v) {
 759                 struct ice_vsi *vsi = pf->vsi[v];
 760
 761                 if (!vsi || !vsi->port_info)
 762                         continue;
 763
 764                 if (new_link_same_as_old &&
 765                     (test_bit(__ICE_DOWN, vsi->state) ||
 766                     new_link == netif_carrier_ok(vsi->netdev)))
 767                         continue;
 768
 769                 if (vsi->port_info->lport == lport) {
 770                         ice_print_link_msg(vsi, new_link);
 771                         ice_vsi_link_event(vsi, new_link);
 772                 }
 773         }
 774
 775         return 0;
 776 }
 777
 778 /**
 779  * ice_handle_link_event - handle link event via ARQ
 780  * @pf: pf that the link event is associated with
 781  *
 782  * Return -EINVAL if port_info is null
 783  * Return status on succes
 784  */
 785 static int ice_handle_link_event(struct ice_pf *pf)
 786 {
 787         struct ice_port_info *port_info;
 788         int status;
 789
 790         port_info = pf->hw.port_info;
 791         if (!port_info)
 792                 return -EINVAL;
 793
 794         status = ice_link_event(pf, port_info);
 795         if (status)
 796                 dev_dbg(&pf->pdev->dev,
 797                         "Could not process link event, error %d\n", status);
 798
 799         return status;
 800 }
 801
 802 /**
 803  * __ice_clean_ctrlq - helper function to clean controlq rings
 804  * @pf: ptr to struct ice_pf
 805  * @q_type: specific Control queue type
 806  */
 807 static int __ice_clean_ctrlq(struct ice_pf *pf, enum ice_ctl_q q_type)
 808 {
 809         struct ice_rq_event_info event;
 810         struct ice_hw *hw = &pf->hw;
 811         struct ice_ctl_q_info *cq;
 812         u16 pending, i = 0;
 813         const char *qtype;
 814         u32 oldval, val;
 815
 816         /* Do not clean control queue if/when PF reset fails */
 817         if (test_bit(__ICE_RESET_FAILED, pf->state))
 818                 return 0;
 819
 820         switch (q_type) {
 821         case ICE_CTL_Q_ADMIN:
 822                 cq = &hw->adminq;
 823                 qtype = "Admin";
 824                 break;
 825         default:
 826                 dev_warn(&pf->pdev->dev, "Unknown control queue type 0x%x\n",
 827                          q_type);
 828                 return 0;
 829         }
 830
 831         /* check for error indications - PF_xx_AxQLEN register layout for
 832          * FW/MBX/SB are identical so just use defines for PF_FW_AxQLEN.
 833          */
 834         val = rd32(hw, cq->rq.len);
 835         if (val & (PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M |
 836                    PF_FW_ARQLEN_ARQCRIT_M)) {
 837                 oldval = val;
 838                 if (val & PF_FW_ARQLEN_ARQVFE_M)
 839                         dev_dbg(&pf->pdev->dev,
 840                                 "%s Receive Queue VF Error detected\n", qtype);
 841                 if (val & PF_FW_ARQLEN_ARQOVFL_M) {
 842                         dev_dbg(&pf->pdev->dev,
 843                                 "%s Receive Queue Overflow Error detected\n",
 844                                 qtype);
 845                 }
 846                 if (val & PF_FW_ARQLEN_ARQCRIT_M)
 847                         dev_dbg(&pf->pdev->dev,
 848                                 "%s Receive Queue Critical Error detected\n",
 849                                 qtype);
 850                 val &= ~(PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M |
 851                          PF_FW_ARQLEN_ARQCRIT_M);
 852                 if (oldval != val)
 853                         wr32(hw, cq->rq.len, val);
 854         }
 855
 856         val = rd32(hw, cq->sq.len);
 857         if (val & (PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M |
 858                    PF_FW_ATQLEN_ATQCRIT_M)) {
 859                 oldval = val;
 860                 if (val & PF_FW_ATQLEN_ATQVFE_M)
 861                         dev_dbg(&pf->pdev->dev,
 862                                 "%s Send Queue VF Error detected\n", qtype);
 863                 if (val & PF_FW_ATQLEN_ATQOVFL_M) {
 864                         dev_dbg(&pf->pdev->dev,
 865                                 "%s Send Queue Overflow Error detected\n",
 866                                 qtype);
 867                 }
 868                 if (val & PF_FW_ATQLEN_ATQCRIT_M)
 869                         dev_dbg(&pf->pdev->dev,
 870                                 "%s Send Queue Critical Error detected\n",
 871                                 qtype);
 872                 val &= ~(PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M |
 873                          PF_FW_ATQLEN_ATQCRIT_M);
 874                 if (oldval != val)
 875                         wr32(hw, cq->sq.len, val);
 876         }
 877
 878         event.buf_len = cq->rq_buf_size;
 879         event.msg_buf = devm_kzalloc(&pf->pdev->dev, event.buf_len,
 880                                      GFP_KERNEL);
 881         if (!event.msg_buf)
 882                 return 0;
 883
 884         do {
 885                 enum ice_status ret;
 886                 u16 opcode;
 887
 888                 ret = ice_clean_rq_elem(hw, cq, &event, &pending);
 889                 if (ret == ICE_ERR_AQ_NO_WORK)
 890                         break;
 891                 if (ret) {
 892                         dev_err(&pf->pdev->dev,
 893                                 "%s Receive Queue event error %d\n", qtype,
 894                                 ret);
 895                         break;
 896                 }
 897
 898                 opcode = le16_to_cpu(event.desc.opcode);
 899
 900                 switch (opcode) {
 901                 case ice_aqc_opc_get_link_status:
 902                         if (ice_handle_link_event(pf))
 903                                 dev_err(&pf->pdev->dev,
 904                                         "Could not handle link event\n");
 905                         break;
 906                 default:
 907                         dev_dbg(&pf->pdev->dev,
 908                                 "%s Receive Queue unknown event 0x%04x ignored\n",
 909                                 qtype, opcode);
 910                         break;
 911                 }
 912         } while (pending && (i++ < ICE_DFLT_IRQ_WORK));
 913
 914         devm_kfree(&pf->pdev->dev, event.msg_buf);
 915
 916         return pending && (i == ICE_DFLT_IRQ_WORK);
 917 }
 918
 919 /**
 920  * ice_ctrlq_pending - check if there is a difference between ntc and ntu
 921  * @hw: pointer to hardware info
 922  * @cq: control queue information
 923  *
 924  * returns true if there are pending messages in a queue, false if there aren't
 925  */
 926 static bool ice_ctrlq_pending(struct ice_hw *hw, struct ice_ctl_q_info *cq)
 927 {
 928         u16 ntu;
 929
 930         ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask);
 931         return cq->rq.next_to_clean != ntu;
 932 }
 933
 934 /**
 935  * ice_clean_adminq_subtask - clean the AdminQ rings
 936  * @pf: board private structure
 937  */
 938 static void ice_clean_adminq_subtask(struct ice_pf *pf)
 939 {
 940         struct ice_hw *hw = &pf->hw;
 941
 942         if (!test_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state))
 943                 return;
 944
 945         if (__ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN))
 946                 return;
 947
 948         clear_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state);
 949
 950         /* There might be a situation where new messages arrive to a control
 951          * queue between processing the last message and clearing the
 952          * EVENT_PENDING bit. So before exiting, check queue head again (using
 953          * ice_ctrlq_pending) and process new messages if any.
 954          */
 955         if (ice_ctrlq_pending(hw, &hw->adminq))
 956                 __ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN);
 957
 958         ice_flush(hw);
 959 }
 960
 961 /**
 962  * ice_service_task_schedule - schedule the service task to wake up
 963  * @pf: board private structure
 964  *
 965  * If not already scheduled, this puts the task into the work queue.
 966  */
 967 static void ice_service_task_schedule(struct ice_pf *pf)
 968 {
 969         if (!test_bit(__ICE_DOWN, pf->state) &&
 970             !test_and_set_bit(__ICE_SERVICE_SCHED, pf->state))
 971                 queue_work(ice_wq, &pf->serv_task);
 972 }
 973
 974 /**
 975  * ice_service_task_complete - finish up the service task
 976  * @pf: board private structure
 977  */
 978 static void ice_service_task_complete(struct ice_pf *pf)
 979 {
 980         WARN_ON(!test_bit(__ICE_SERVICE_SCHED, pf->state));
 981
 982         /* force memory (pf->state) to sync before next service task */
 983         smp_mb__before_atomic();
 984         clear_bit(__ICE_SERVICE_SCHED, pf->state);
 985 }
 986
 987 /**
 988  * ice_service_timer - timer callback to schedule service task
 989  * @t: pointer to timer_list
 990  */
 991 static void ice_service_timer(struct timer_list *t)
 992 {
 993         struct ice_pf *pf = from_timer(pf, t, serv_tmr);
 994
 995         mod_timer(&pf->serv_tmr, round_jiffies(pf->serv_tmr_period + jiffies));
 996         ice_service_task_schedule(pf);
 997 }
 998
 999 /**
1000  * ice_service_task - manage and run subtasks
1001  * @work: pointer to work_struct contained by the PF struct
1002  */
1003 static void ice_service_task(struct work_struct *work)
1004 {
1005         struct ice_pf *pf = container_of(work, struct ice_pf, serv_task);
1006         unsigned long start_time = jiffies;
1007
1008         /* subtasks */
1009
1010         /* process reset requests first */
1011         ice_reset_subtask(pf);
1012
1013         /* bail if a reset/recovery cycle is pending */
1014         if (ice_is_reset_recovery_pending(pf->state) ||
1015             test_bit(__ICE_SUSPENDED, pf->state)) {
1016                 ice_service_task_complete(pf);
1017                 return;
1018         }
1019
1020         ice_sync_fltr_subtask(pf);
1021         ice_watchdog_subtask(pf);
1022         ice_clean_adminq_subtask(pf);
1023
1024         /* Clear __ICE_SERVICE_SCHED flag to allow scheduling next event */
1025         ice_service_task_complete(pf);
1026
1027         /* If the tasks have taken longer than one service timer period
1028          * or there is more work to be done, reset the service timer to
1029          * schedule the service task now.
1030          */
1031         if (time_after(jiffies, (start_time + pf->serv_tmr_period)) ||
1032             test_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state))
1033                 mod_timer(&pf->serv_tmr, jiffies);
1034 }
1035
1036 /**
1037  * ice_set_ctrlq_len - helper function to set controlq length
1038  * @hw: pointer to the hw instance
1039  */
1040 static void ice_set_ctrlq_len(struct ice_hw *hw)
1041 {
1042         hw->adminq.num_rq_entries = ICE_AQ_LEN;
1043         hw->adminq.num_sq_entries = ICE_AQ_LEN;
1044         hw->adminq.rq_buf_size = ICE_AQ_MAX_BUF_LEN;
1045         hw->adminq.sq_buf_size = ICE_AQ_MAX_BUF_LEN;
1046 }
1047
1048 /**
1049  * ice_irq_affinity_notify - Callback for affinity changes
1050  * @notify: context as to what irq was changed
1051  * @mask: the new affinity mask
1052  *
1053  * This is a callback function used by the irq_set_affinity_notifier function
1054  * so that we may register to receive changes to the irq affinity masks.
1055  */
1056 static void ice_irq_affinity_notify(struct irq_affinity_notify *notify,
1057                                     const cpumask_t *mask)
1058 {
1059         struct ice_q_vector *q_vector =
1060                 container_of(notify, struct ice_q_vector, affinity_notify);
1061
1062         cpumask_copy(&q_vector->affinity_mask, mask);
1063 }
1064
1065 /**
1066  * ice_irq_affinity_release - Callback for affinity notifier release
1067  * @ref: internal core kernel usage
1068  *
1069  * This is a callback function used by the irq_set_affinity_notifier function
1070  * to inform the current notification subscriber that they will no longer
1071  * receive notifications.
1072  */
1073 static void ice_irq_affinity_release(struct kref __always_unused *ref) {}
1074
1075 /**
1076  * ice_vsi_dis_irq - Mask off queue interrupt generation on the VSI
1077  * @vsi: the VSI being un-configured
1078  */
1079 static void ice_vsi_dis_irq(struct ice_vsi *vsi)
1080 {
1081         struct ice_pf *pf = vsi->back;
1082         struct ice_hw *hw = &pf->hw;
1083         int base = vsi->base_vector;
1084         u32 val;
1085         int i;
1086
1087         /* disable interrupt causation from each queue */
1088         if (vsi->tx_rings) {
1089                 ice_for_each_txq(vsi, i) {
1090                         if (vsi->tx_rings[i]) {
1091                                 u16 reg;
1092
1093                                 reg = vsi->tx_rings[i]->reg_idx;
1094                                 val = rd32(hw, QINT_TQCTL(reg));
1095                                 val &= ~QINT_TQCTL_CAUSE_ENA_M;
1096                                 wr32(hw, QINT_TQCTL(reg), val);
1097                         }
1098                 }
1099         }
1100
1101         if (vsi->rx_rings) {
1102                 ice_for_each_rxq(vsi, i) {
1103                         if (vsi->rx_rings[i]) {
1104                                 u16 reg;
1105
1106                                 reg = vsi->rx_rings[i]->reg_idx;
1107                                 val = rd32(hw, QINT_RQCTL(reg));
1108                                 val &= ~QINT_RQCTL_CAUSE_ENA_M;
1109                                 wr32(hw, QINT_RQCTL(reg), val);
1110                         }
1111                 }
1112         }
1113
1114         /* disable each interrupt */
1115         if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) {
1116                 for (i = vsi->base_vector;
1117                      i < (vsi->num_q_vectors + vsi->base_vector); i++)
1118                         wr32(hw, GLINT_DYN_CTL(i), 0);
1119
1120                 ice_flush(hw);
1121                 for (i = 0; i < vsi->num_q_vectors; i++)
1122                         synchronize_irq(pf->msix_entries[i + base].vector);
1123         }
1124 }
1125
1126 /**
1127  * ice_vsi_ena_irq - Enable IRQ for the given VSI
1128  * @vsi: the VSI being configured
1129  */
1130 static int ice_vsi_ena_irq(struct ice_vsi *vsi)
1131 {
1132         struct ice_pf *pf = vsi->back;
1133         struct ice_hw *hw = &pf->hw;
1134
1135         if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) {
1136                 int i;
1137
1138                 for (i = 0; i < vsi->num_q_vectors; i++)
1139                         ice_irq_dynamic_ena(hw, vsi, vsi->q_vectors[i]);
1140         }
1141
1142         ice_flush(hw);
1143         return 0;
1144 }
1145
1146 /**
1147  * ice_vsi_delete - delete a VSI from the switch
1148  * @vsi: pointer to VSI being removed
1149  */
1150 static void ice_vsi_delete(struct ice_vsi *vsi)
1151 {
1152         struct ice_pf *pf = vsi->back;
1153         struct ice_vsi_ctx ctxt;
1154         enum ice_status status;
1155
1156         ctxt.vsi_num = vsi->vsi_num;
1157
1158         memcpy(&ctxt.info, &vsi->info, sizeof(struct ice_aqc_vsi_props));
1159
1160         status = ice_aq_free_vsi(&pf->hw, &ctxt, false, NULL);
1161         if (status)
1162                 dev_err(&pf->pdev->dev, "Failed to delete VSI %i in FW\n",
1163                         vsi->vsi_num);
1164 }
1165
1166 /**
1167  * ice_vsi_req_irq_msix - get MSI-X vectors from the OS for the VSI
1168  * @vsi: the VSI being configured
1169  * @basename: name for the vector
1170  */
1171 static int ice_vsi_req_irq_msix(struct ice_vsi *vsi, char *basename)
1172 {
1173         int q_vectors = vsi->num_q_vectors;
1174         struct ice_pf *pf = vsi->back;
1175         int base = vsi->base_vector;
1176         int rx_int_idx = 0;
1177         int tx_int_idx = 0;
1178         int vector, err;
1179         int irq_num;
1180
1181         for (vector = 0; vector < q_vectors; vector++) {
1182                 struct ice_q_vector *q_vector = vsi->q_vectors[vector];
1183
1184                 irq_num = pf->msix_entries[base + vector].vector;
1185
1186                 if (q_vector->tx.ring && q_vector->rx.ring) {
1187                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
1188                                  "%s-%s-%d", basename, "TxRx", rx_int_idx++);
1189                         tx_int_idx++;
1190                 } else if (q_vector->rx.ring) {
1191                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
1192                                  "%s-%s-%d", basename, "rx", rx_int_idx++);
1193                 } else if (q_vector->tx.ring) {
1194                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
1195                                  "%s-%s-%d", basename, "tx", tx_int_idx++);
1196                 } else {
1197                         /* skip this unused q_vector */
1198                         continue;
1199                 }
1200                 err = devm_request_irq(&pf->pdev->dev,
1201                                        pf->msix_entries[base + vector].vector,
1202                                        vsi->irq_handler, 0, q_vector->name,
1203                                        q_vector);
1204                 if (err) {
1205                         netdev_err(vsi->netdev,
1206                                    "MSIX request_irq failed, error: %d\n", err);
1207                         goto free_q_irqs;
1208                 }
1209
1210                 /* register for affinity change notifications */
1211                 q_vector->affinity_notify.notify = ice_irq_affinity_notify;
1212                 q_vector->affinity_notify.release = ice_irq_affinity_release;
1213                 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
1214
1215                 /* assign the mask for this irq */
1216                 irq_set_affinity_hint(irq_num, &q_vector->affinity_mask);
1217         }
1218
1219         vsi->irqs_ready = true;
1220         return 0;
1221
1222 free_q_irqs:
1223         while (vector) {
1224                 vector--;
1225                 irq_num = pf->msix_entries[base + vector].vector,
1226                 irq_set_affinity_notifier(irq_num, NULL);
1227                 irq_set_affinity_hint(irq_num, NULL);
1228                 devm_free_irq(&pf->pdev->dev, irq_num, &vsi->q_vectors[vector]);
1229         }
1230         return err;
1231 }
1232
1233 /**
1234  * ice_vsi_set_rss_params - Setup RSS capabilities per VSI type
1235  * @vsi: the VSI being configured
1236  */
1237 static void ice_vsi_set_rss_params(struct ice_vsi *vsi)
1238 {
1239         struct ice_hw_common_caps *cap;
1240         struct ice_pf *pf = vsi->back;
1241
1242         if (!test_bit(ICE_FLAG_RSS_ENA, pf->flags)) {
1243                 vsi->rss_size = 1;
1244                 return;
1245         }
1246
1247         cap = &pf->hw.func_caps.common_cap;
1248         switch (vsi->type) {
1249         case ICE_VSI_PF:
1250                 /* PF VSI will inherit RSS instance of PF */
1251                 vsi->rss_table_size = cap->rss_table_size;
1252                 vsi->rss_size = min_t(int, num_online_cpus(),
1253                                       BIT(cap->rss_table_entry_width));
1254                 vsi->rss_lut_type = ICE_AQC_GSET_RSS_LUT_TABLE_TYPE_PF;
1255                 break;
1256         default:
1257                 dev_warn(&pf->pdev->dev, "Unknown VSI type %d\n", vsi->type);
1258                 break;
1259         }
1260 }
1261
1262 /**
1263  * ice_vsi_setup_q_map - Setup a VSI queue map
1264  * @vsi: the VSI being configured
1265  * @ctxt: VSI context structure
1266  */
1267 static void ice_vsi_setup_q_map(struct ice_vsi *vsi, struct ice_vsi_ctx *ctxt)
1268 {
1269         u16 offset = 0, qmap = 0, numq_tc;
1270         u16 pow = 0, max_rss = 0, qcount;
1271         u16 qcount_tx = vsi->alloc_txq;
1272         u16 qcount_rx = vsi->alloc_rxq;
1273         bool ena_tc0 = false;
1274         int i;
1275
1276         /* at least TC0 should be enabled by default */
1277         if (vsi->tc_cfg.numtc) {
1278                 if (!(vsi->tc_cfg.ena_tc & BIT(0)))
1279                         ena_tc0 =  true;
1280         } else {
1281                 ena_tc0 =  true;
1282         }
1283
1284         if (ena_tc0) {
1285                 vsi->tc_cfg.numtc++;
1286                 vsi->tc_cfg.ena_tc |= 1;
1287         }
1288
1289         numq_tc = qcount_rx / vsi->tc_cfg.numtc;
1290
1291         /* TC mapping is a function of the number of Rx queues assigned to the
1292          * VSI for each traffic class and the offset of these queues.
1293          * The first 10 bits are for queue offset for TC0, next 4 bits for no:of
1294          * queues allocated to TC0. No:of queues is a power-of-2.
1295          *
1296          * If TC is not enabled, the queue offset is set to 0, and allocate one
1297          * queue, this way, traffic for the given TC will be sent to the default
1298          * queue.
1299          *
1300          * Setup number and offset of Rx queues for all TCs for the VSI
1301          */
1302
1303         /* qcount will change if RSS is enabled */
1304         if (test_bit(ICE_FLAG_RSS_ENA, vsi->back->flags)) {
1305                 if (vsi->type == ICE_VSI_PF)
1306                         max_rss = ICE_MAX_LG_RSS_QS;
1307                 else
1308                         max_rss = ICE_MAX_SMALL_RSS_QS;
1309
1310                 qcount = min_t(int, numq_tc, max_rss);
1311                 qcount = min_t(int, qcount, vsi->rss_size);
1312         } else {
1313                 qcount = numq_tc;
1314         }
1315
1316         /* find the (rounded up) power-of-2 of qcount */
1317         pow = order_base_2(qcount);
1318
1319         for (i = 0; i < ICE_MAX_TRAFFIC_CLASS; i++) {
1320                 if (!(vsi->tc_cfg.ena_tc & BIT(i))) {
1321                         /* TC is not enabled */
1322                         vsi->tc_cfg.tc_info[i].qoffset = 0;
1323                         vsi->tc_cfg.tc_info[i].qcount = 1;
1324                         ctxt->info.tc_mapping[i] = 0;
1325                         continue;
1326                 }
1327
1328                 /* TC is enabled */
1329                 vsi->tc_cfg.tc_info[i].qoffset = offset;
1330                 vsi->tc_cfg.tc_info[i].qcount = qcount;
1331
1332                 qmap = ((offset << ICE_AQ_VSI_TC_Q_OFFSET_S) &
1333                         ICE_AQ_VSI_TC_Q_OFFSET_M) |
1334                         ((pow << ICE_AQ_VSI_TC_Q_NUM_S) &
1335                          ICE_AQ_VSI_TC_Q_NUM_M);
1336                 offset += qcount;
1337                 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
1338         }
1339
1340         vsi->num_txq = qcount_tx;
1341         vsi->num_rxq = offset;
1342
1343         /* Rx queue mapping */
1344         ctxt->info.mapping_flags |= cpu_to_le16(ICE_AQ_VSI_Q_MAP_CONTIG);
1345         /* q_mapping buffer holds the info for the first queue allocated for
1346          * this VSI in the PF space and also the number of queues associated
1347          * with this VSI.
1348          */
1349         ctxt->info.q_mapping[0] = cpu_to_le16(vsi->rxq_map[0]);
1350         ctxt->info.q_mapping[1] = cpu_to_le16(vsi->num_rxq);
1351 }
1352
1353 /**
1354  * ice_set_dflt_vsi_ctx - Set default VSI context before adding a VSI
1355  * @ctxt: the VSI context being set
1356  *
1357  * This initializes a default VSI context for all sections except the Queues.
1358  */
1359 static void ice_set_dflt_vsi_ctx(struct ice_vsi_ctx *ctxt)
1360 {
1361         u32 table = 0;
1362
1363         memset(&ctxt->info, 0, sizeof(ctxt->info));
1364         /* VSI's should be allocated from shared pool */
1365         ctxt->alloc_from_pool = true;
1366         /* Src pruning enabled by default */
1367         ctxt->info.sw_flags = ICE_AQ_VSI_SW_FLAG_SRC_PRUNE;
1368         /* Traffic from VSI can be sent to LAN */
1369         ctxt->info.sw_flags2 = ICE_AQ_VSI_SW_FLAG_LAN_ENA;
1370
1371         /* By default bits 3 and 4 in vlan_flags are 0's which results in legacy
1372          * behavior (show VLAN, DEI, and UP) in descriptor. Also, allow all
1373          * packets untagged/tagged.
1374          */
1375         ctxt->info.vlan_flags = ((ICE_AQ_VSI_VLAN_MODE_ALL &
1376                                   ICE_AQ_VSI_VLAN_MODE_M) >>
1377                                  ICE_AQ_VSI_VLAN_MODE_S);
1378
1379         /* Have 1:1 UP mapping for both ingress/egress tables */
1380         table |= ICE_UP_TABLE_TRANSLATE(0, 0);
1381         table |= ICE_UP_TABLE_TRANSLATE(1, 1);
1382         table |= ICE_UP_TABLE_TRANSLATE(2, 2);
1383         table |= ICE_UP_TABLE_TRANSLATE(3, 3);
1384         table |= ICE_UP_TABLE_TRANSLATE(4, 4);
1385         table |= ICE_UP_TABLE_TRANSLATE(5, 5);
1386         table |= ICE_UP_TABLE_TRANSLATE(6, 6);
1387         table |= ICE_UP_TABLE_TRANSLATE(7, 7);
1388         ctxt->info.ingress_table = cpu_to_le32(table);
1389         ctxt->info.egress_table = cpu_to_le32(table);
1390         /* Have 1:1 UP mapping for outer to inner UP table */
1391         ctxt->info.outer_up_table = cpu_to_le32(table);
1392         /* No Outer tag support outer_tag_flags remains to zero */
1393 }
1394
1395 /**
1396  * ice_set_rss_vsi_ctx - Set RSS VSI context before adding a VSI
1397  * @ctxt: the VSI context being set
1398  * @vsi: the VSI being configured
1399  */
1400 static void ice_set_rss_vsi_ctx(struct ice_vsi_ctx *ctxt, struct ice_vsi *vsi)
1401 {
1402         u8 lut_type, hash_type;
1403
1404         switch (vsi->type) {
1405         case ICE_VSI_PF:
1406                 /* PF VSI will inherit RSS instance of PF */
1407                 lut_type = ICE_AQ_VSI_Q_OPT_RSS_LUT_PF;
1408                 hash_type = ICE_AQ_VSI_Q_OPT_RSS_TPLZ;
1409                 break;
1410         default:
1411                 dev_warn(&vsi->back->pdev->dev, "Unknown VSI type %d\n",
1412                          vsi->type);
1413                 return;
1414         }
1415
1416         ctxt->info.q_opt_rss = ((lut_type << ICE_AQ_VSI_Q_OPT_RSS_LUT_S) &
1417                                 ICE_AQ_VSI_Q_OPT_RSS_LUT_M) |
1418                                 ((hash_type << ICE_AQ_VSI_Q_OPT_RSS_HASH_S) &
1419                                  ICE_AQ_VSI_Q_OPT_RSS_HASH_M);
1420 }
1421
1422 /**
1423  * ice_vsi_add - Create a new VSI or fetch preallocated VSI
1424  * @vsi: the VSI being configured
1425  *
1426  * This initializes a VSI context depending on the VSI type to be added and
1427  * passes it down to the add_vsi aq command to create a new VSI.
1428  */
1429 static int ice_vsi_add(struct ice_vsi *vsi)
1430 {
1431         struct ice_vsi_ctx ctxt = { 0 };
1432         struct ice_pf *pf = vsi->back;
1433         struct ice_hw *hw = &pf->hw;
1434         int ret = 0;
1435
1436         switch (vsi->type) {
1437         case ICE_VSI_PF:
1438                 ctxt.flags = ICE_AQ_VSI_TYPE_PF;
1439                 break;
1440         default:
1441                 return -ENODEV;
1442         }
1443
1444         ice_set_dflt_vsi_ctx(&ctxt);
1445         /* if the switch is in VEB mode, allow VSI loopback */
1446         if (vsi->vsw->bridge_mode == BRIDGE_MODE_VEB)
1447                 ctxt.info.sw_flags |= ICE_AQ_VSI_SW_FLAG_ALLOW_LB;
1448
1449         /* Set LUT type and HASH type if RSS is enabled */
1450         if (test_bit(ICE_FLAG_RSS_ENA, pf->flags))
1451                 ice_set_rss_vsi_ctx(&ctxt, vsi);
1452
1453         ctxt.info.sw_id = vsi->port_info->sw_id;
1454         ice_vsi_setup_q_map(vsi, &ctxt);
1455
1456         ret = ice_aq_add_vsi(hw, &ctxt, NULL);
1457         if (ret) {
1458                 dev_err(&vsi->back->pdev->dev,
1459                         "Add VSI AQ call failed, err %d\n", ret);
1460                 return -EIO;
1461         }
1462         vsi->info = ctxt.info;
1463         vsi->vsi_num = ctxt.vsi_num;
1464
1465         return ret;
1466 }
1467
1468 /**
1469  * ice_vsi_release_msix - Clear the queue to Interrupt mapping in HW
1470  * @vsi: the VSI being cleaned up
1471  */
1472 static void ice_vsi_release_msix(struct ice_vsi *vsi)
1473 {
1474         struct ice_pf *pf = vsi->back;
1475         u16 vector = vsi->base_vector;
1476         struct ice_hw *hw = &pf->hw;
1477         u32 txq = 0;
1478         u32 rxq = 0;
1479         int i, q;
1480
1481         for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
1482                 struct ice_q_vector *q_vector = vsi->q_vectors[i];
1483
1484                 wr32(hw, GLINT_ITR(ICE_RX_ITR, vector), 0);
1485                 wr32(hw, GLINT_ITR(ICE_TX_ITR, vector), 0);
1486                 for (q = 0; q < q_vector->num_ring_tx; q++) {
1487                         wr32(hw, QINT_TQCTL(vsi->txq_map[txq]), 0);
1488                         txq++;
1489                 }
1490
1491                 for (q = 0; q < q_vector->num_ring_rx; q++) {
1492                         wr32(hw, QINT_RQCTL(vsi->rxq_map[rxq]), 0);
1493                         rxq++;
1494                 }
1495         }
1496
1497         ice_flush(hw);
1498 }
1499
1500 /**
1501  * ice_vsi_clear_rings - Deallocates the Tx and Rx rings for VSI
1502  * @vsi: the VSI having rings deallocated
1503  */
1504 static void ice_vsi_clear_rings(struct ice_vsi *vsi)
1505 {
1506         int i;
1507
1508         if (vsi->tx_rings) {
1509                 for (i = 0; i < vsi->alloc_txq; i++) {
1510                         if (vsi->tx_rings[i]) {
1511                                 kfree_rcu(vsi->tx_rings[i], rcu);
1512                                 vsi->tx_rings[i] = NULL;
1513                         }
1514                 }
1515         }
1516         if (vsi->rx_rings) {
1517                 for (i = 0; i < vsi->alloc_rxq; i++) {
1518                         if (vsi->rx_rings[i]) {
1519                                 kfree_rcu(vsi->rx_rings[i], rcu);
1520                                 vsi->rx_rings[i] = NULL;
1521                         }
1522                 }
1523         }
1524 }
1525
1526 /**
1527  * ice_vsi_alloc_rings - Allocates Tx and Rx rings for the VSI
1528  * @vsi: VSI which is having rings allocated
1529  */
1530 static int ice_vsi_alloc_rings(struct ice_vsi *vsi)
1531 {
1532         struct ice_pf *pf = vsi->back;
1533         int i;
1534
1535         /* Allocate tx_rings */
1536         for (i = 0; i < vsi->alloc_txq; i++) {
1537                 struct ice_ring *ring;
1538
1539                 /* allocate with kzalloc(), free with kfree_rcu() */
1540                 ring = kzalloc(sizeof(*ring), GFP_KERNEL);
1541
1542                 if (!ring)
1543                         goto err_out;
1544
1545                 ring->q_index = i;
1546                 ring->reg_idx = vsi->txq_map[i];
1547                 ring->ring_active = false;
1548                 ring->vsi = vsi;
1549                 ring->netdev = vsi->netdev;
1550                 ring->dev = &pf->pdev->dev;
1551                 ring->count = vsi->num_desc;
1552
1553                 vsi->tx_rings[i] = ring;
1554         }
1555
1556         /* Allocate rx_rings */
1557         for (i = 0; i < vsi->alloc_rxq; i++) {
1558                 struct ice_ring *ring;
1559
1560                 /* allocate with kzalloc(), free with kfree_rcu() */
1561                 ring = kzalloc(sizeof(*ring), GFP_KERNEL);
1562                 if (!ring)
1563                         goto err_out;
1564
1565                 ring->q_index = i;
1566                 ring->reg_idx = vsi->rxq_map[i];
1567                 ring->ring_active = false;
1568                 ring->vsi = vsi;
1569                 ring->netdev = vsi->netdev;
1570                 ring->dev = &pf->pdev->dev;
1571                 ring->count = vsi->num_desc;
1572                 vsi->rx_rings[i] = ring;
1573         }
1574
1575         return 0;
1576
1577 err_out:
1578         ice_vsi_clear_rings(vsi);
1579         return -ENOMEM;
1580 }
1581
1582 /**
1583  * ice_vsi_free_irq - Free the irq association with the OS
1584  * @vsi: the VSI being configured
1585  */
1586 static void ice_vsi_free_irq(struct ice_vsi *vsi)
1587 {
1588         struct ice_pf *pf = vsi->back;
1589         int base = vsi->base_vector;
1590
1591         if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) {
1592                 int i;
1593
1594                 if (!vsi->q_vectors || !vsi->irqs_ready)
1595                         return;
1596
1597                 vsi->irqs_ready = false;
1598                 for (i = 0; i < vsi->num_q_vectors; i++) {
1599                         u16 vector = i + base;
1600                         int irq_num;
1601
1602                         irq_num = pf->msix_entries[vector].vector;
1603
1604                         /* free only the irqs that were actually requested */
1605                         if (!vsi->q_vectors[i] ||
1606                             !(vsi->q_vectors[i]->num_ring_tx ||
1607                               vsi->q_vectors[i]->num_ring_rx))
1608                                 continue;
1609
1610                         /* clear the affinity notifier in the IRQ descriptor */
1611                         irq_set_affinity_notifier(irq_num, NULL);
1612
1613                         /* clear the affinity_mask in the IRQ descriptor */
1614                         irq_set_affinity_hint(irq_num, NULL);
1615                         synchronize_irq(irq_num);
1616                         devm_free_irq(&pf->pdev->dev, irq_num,
1617                                       vsi->q_vectors[i]);
1618                 }
1619                 ice_vsi_release_msix(vsi);
1620         }
1621 }
1622
1623 /**
1624  * ice_vsi_cfg_msix - MSIX mode Interrupt Config in the HW
1625  * @vsi: the VSI being configured
1626  */
1627 static void ice_vsi_cfg_msix(struct ice_vsi *vsi)
1628 {
1629         struct ice_pf *pf = vsi->back;
1630         u16 vector = vsi->base_vector;
1631         struct ice_hw *hw = &pf->hw;
1632         u32 txq = 0, rxq = 0;
1633         int i, q, itr;
1634         u8 itr_gran;
1635
1636         for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
1637                 struct ice_q_vector *q_vector = vsi->q_vectors[i];
1638
1639                 itr_gran = hw->itr_gran_200;
1640
1641                 if (q_vector->num_ring_rx) {
1642                         q_vector->rx.itr =
1643                                 ITR_TO_REG(vsi->rx_rings[rxq]->rx_itr_setting,
1644                                            itr_gran);
1645                         q_vector->rx.latency_range = ICE_LOW_LATENCY;
1646                 }
1647
1648                 if (q_vector->num_ring_tx) {
1649                         q_vector->tx.itr =
1650                                 ITR_TO_REG(vsi->tx_rings[txq]->tx_itr_setting,
1651                                            itr_gran);
1652                         q_vector->tx.latency_range = ICE_LOW_LATENCY;
1653                 }
1654                 wr32(hw, GLINT_ITR(ICE_RX_ITR, vector), q_vector->rx.itr);
1655                 wr32(hw, GLINT_ITR(ICE_TX_ITR, vector), q_vector->tx.itr);
1656
1657                 /* Both Transmit Queue Interrupt Cause Control register
1658                  * and Receive Queue Interrupt Cause control register
1659                  * expects MSIX_INDX field to be the vector index
1660                  * within the function space and not the absolute
1661                  * vector index across PF or across device.
1662                  * For SR-IOV VF VSIs queue vector index always starts
1663                  * with 1 since first vector index(0) is used for OICR
1664                  * in VF space. Since VMDq and other PF VSIs are withtin
1665                  * the PF function space, use the vector index thats
1666                  * tracked for this PF.
1667                  */
1668                 for (q = 0; q < q_vector->num_ring_tx; q++) {
1669                         u32 val;
1670
1671                         itr = ICE_TX_ITR;
1672                         val = QINT_TQCTL_CAUSE_ENA_M |
1673                               (itr << QINT_TQCTL_ITR_INDX_S)  |
1674                               (vector << QINT_TQCTL_MSIX_INDX_S);
1675                         wr32(hw, QINT_TQCTL(vsi->txq_map[txq]), val);
1676                         txq++;
1677                 }
1678
1679                 for (q = 0; q < q_vector->num_ring_rx; q++) {
1680                         u32 val;
1681
1682                         itr = ICE_RX_ITR;
1683                         val = QINT_RQCTL_CAUSE_ENA_M |
1684                               (itr << QINT_RQCTL_ITR_INDX_S)  |
1685                               (vector << QINT_RQCTL_MSIX_INDX_S);
1686                         wr32(hw, QINT_RQCTL(vsi->rxq_map[rxq]), val);
1687                         rxq++;
1688                 }
1689         }
1690
1691         ice_flush(hw);
1692 }
1693
1694 /**
1695  * ice_ena_misc_vector - enable the non-queue interrupts
1696  * @pf: board private structure
1697  */
1698 static void ice_ena_misc_vector(struct ice_pf *pf)
1699 {
1700         struct ice_hw *hw = &pf->hw;
1701         u32 val;
1702
1703         /* clear things first */
1704         wr32(hw, PFINT_OICR_ENA, 0);    /* disable all */
1705         rd32(hw, PFINT_OICR);           /* read to clear */
1706
1707         val = (PFINT_OICR_ECC_ERR_M |
1708                PFINT_OICR_MAL_DETECT_M |
1709                PFINT_OICR_GRST_M |
1710                PFINT_OICR_PCI_EXCEPTION_M |
1711                PFINT_OICR_HMC_ERR_M |
1712                PFINT_OICR_PE_CRITERR_M);
1713
1714         wr32(hw, PFINT_OICR_ENA, val);
1715
1716         /* SW_ITR_IDX = 0, but don't change INTENA */
1717         wr32(hw, GLINT_DYN_CTL(pf->oicr_idx),
1718              GLINT_DYN_CTL_SW_ITR_INDX_M | GLINT_DYN_CTL_INTENA_MSK_M);
1719 }
1720
1721 /**
1722  * ice_misc_intr - misc interrupt handler
1723  * @irq: interrupt number
1724  * @data: pointer to a q_vector
1725  */
1726 static irqreturn_t ice_misc_intr(int __always_unused irq, void *data)
1727 {
1728         struct ice_pf *pf = (struct ice_pf *)data;
1729         struct ice_hw *hw = &pf->hw;
1730         irqreturn_t ret = IRQ_NONE;
1731         u32 oicr, ena_mask;
1732
1733         set_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state);
1734
1735         oicr = rd32(hw, PFINT_OICR);
1736         ena_mask = rd32(hw, PFINT_OICR_ENA);
1737
1738         if (oicr & PFINT_OICR_GRST_M) {
1739                 u32 reset;
1740                 /* we have a reset warning */
1741                 ena_mask &= ~PFINT_OICR_GRST_M;
1742                 reset = (rd32(hw, GLGEN_RSTAT) & GLGEN_RSTAT_RESET_TYPE_M) >>
1743                         GLGEN_RSTAT_RESET_TYPE_S;
1744
1745                 if (reset == ICE_RESET_CORER)
1746                         pf->corer_count++;
1747                 else if (reset == ICE_RESET_GLOBR)
1748                         pf->globr_count++;
1749                 else
1750                         pf->empr_count++;
1751
1752                 /* If a reset cycle isn't already in progress, we set a bit in
1753                  * pf->state so that the service task can start a reset/rebuild.
1754                  * We also make note of which reset happened so that peer
1755                  * devices/drivers can be informed.
1756                  */
1757                 if (!test_bit(__ICE_RESET_RECOVERY_PENDING, pf->state)) {
1758                         if (reset == ICE_RESET_CORER)
1759                                 set_bit(__ICE_CORER_RECV, pf->state);
1760                         else if (reset == ICE_RESET_GLOBR)
1761                                 set_bit(__ICE_GLOBR_RECV, pf->state);
1762                         else
1763                                 set_bit(__ICE_EMPR_RECV, pf->state);
1764
1765                         set_bit(__ICE_RESET_RECOVERY_PENDING, pf->state);
1766                 }
1767         }
1768
1769         if (oicr & PFINT_OICR_HMC_ERR_M) {
1770                 ena_mask &= ~PFINT_OICR_HMC_ERR_M;
1771                 dev_dbg(&pf->pdev->dev,
1772                         "HMC Error interrupt - info 0x%x, data 0x%x\n",
1773                         rd32(hw, PFHMC_ERRORINFO),
1774                         rd32(hw, PFHMC_ERRORDATA));
1775         }
1776
1777         /* Report and mask off any remaining unexpected interrupts */
1778         oicr &= ena_mask;
1779         if (oicr) {
1780                 dev_dbg(&pf->pdev->dev, "unhandled interrupt oicr=0x%08x\n",
1781                         oicr);
1782                 /* If a critical error is pending there is no choice but to
1783                  * reset the device.
1784                  */
1785                 if (oicr & (PFINT_OICR_PE_CRITERR_M |
1786                             PFINT_OICR_PCI_EXCEPTION_M |
1787                             PFINT_OICR_ECC_ERR_M)) {
1788                         set_bit(__ICE_PFR_REQ, pf->state);
1789                         ice_service_task_schedule(pf);
1790                 }
1791                 ena_mask &= ~oicr;
1792         }
1793         ret = IRQ_HANDLED;
1794
1795         /* re-enable interrupt causes that are not handled during this pass */
1796         wr32(hw, PFINT_OICR_ENA, ena_mask);
1797         if (!test_bit(__ICE_DOWN, pf->state)) {
1798                 ice_service_task_schedule(pf);
1799                 ice_irq_dynamic_ena(hw, NULL, NULL);
1800         }
1801
1802         return ret;
1803 }
1804
1805 /**
1806  * ice_vsi_map_rings_to_vectors - Map VSI rings to interrupt vectors
1807  * @vsi: the VSI being configured
1808  *
1809  * This function maps descriptor rings to the queue-specific vectors allotted
1810  * through the MSI-X enabling code. On a constrained vector budget, we map Tx
1811  * and Rx rings to the vector as "efficiently" as possible.
1812  */
1813 static void ice_vsi_map_rings_to_vectors(struct ice_vsi *vsi)
1814 {
1815         int q_vectors = vsi->num_q_vectors;
1816         int tx_rings_rem, rx_rings_rem;
1817         int v_id;
1818
1819         /* initially assigning remaining rings count to VSIs num queue value */
1820         tx_rings_rem = vsi->num_txq;
1821         rx_rings_rem = vsi->num_rxq;
1822
1823         for (v_id = 0; v_id < q_vectors; v_id++) {
1824                 struct ice_q_vector *q_vector = vsi->q_vectors[v_id];
1825                 int tx_rings_per_v, rx_rings_per_v, q_id, q_base;
1826
1827                 /* Tx rings mapping to vector */
1828                 tx_rings_per_v = DIV_ROUND_UP(tx_rings_rem, q_vectors - v_id);
1829                 q_vector->num_ring_tx = tx_rings_per_v;
1830                 q_vector->tx.ring = NULL;
1831                 q_base = vsi->num_txq - tx_rings_rem;
1832
1833                 for (q_id = q_base; q_id < (q_base + tx_rings_per_v); q_id++) {
1834                         struct ice_ring *tx_ring = vsi->tx_rings[q_id];
1835
1836                         tx_ring->q_vector = q_vector;
1837                         tx_ring->next = q_vector->tx.ring;
1838                         q_vector->tx.ring = tx_ring;
1839                 }
1840                 tx_rings_rem -= tx_rings_per_v;
1841
1842                 /* Rx rings mapping to vector */
1843                 rx_rings_per_v = DIV_ROUND_UP(rx_rings_rem, q_vectors - v_id);
1844                 q_vector->num_ring_rx = rx_rings_per_v;
1845                 q_vector->rx.ring = NULL;
1846                 q_base = vsi->num_rxq - rx_rings_rem;
1847
1848                 for (q_id = q_base; q_id < (q_base + rx_rings_per_v); q_id++) {
1849                         struct ice_ring *rx_ring = vsi->rx_rings[q_id];
1850
1851                         rx_ring->q_vector = q_vector;
1852                         rx_ring->next = q_vector->rx.ring;
1853                         q_vector->rx.ring = rx_ring;
1854                 }
1855                 rx_rings_rem -= rx_rings_per_v;
1856         }
1857 }
1858
1859 /**
1860  * ice_vsi_set_num_qs - Set num queues, descriptors and vectors for a VSI
1861  * @vsi: the VSI being configured
1862  *
1863  * Return 0 on success and a negative value on error
1864  */
1865 static void ice_vsi_set_num_qs(struct ice_vsi *vsi)
1866 {
1867         struct ice_pf *pf = vsi->back;
1868
1869         switch (vsi->type) {
1870         case ICE_VSI_PF:
1871                 vsi->alloc_txq = pf->num_lan_tx;
1872                 vsi->alloc_rxq = pf->num_lan_rx;
1873                 vsi->num_desc = ALIGN(ICE_DFLT_NUM_DESC, ICE_REQ_DESC_MULTIPLE);
1874                 vsi->num_q_vectors = max_t(int, pf->num_lan_rx, pf->num_lan_tx);
1875                 break;
1876         default:
1877                 dev_warn(&vsi->back->pdev->dev, "Unknown VSI type %d\n",
1878                          vsi->type);
1879                 break;
1880         }
1881 }
1882
1883 /**
1884  * ice_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
1885  * @vsi: VSI pointer
1886  * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
1887  *
1888  * On error: returns error code (negative)
1889  * On success: returns 0
1890  */
1891 static int ice_vsi_alloc_arrays(struct ice_vsi *vsi, bool alloc_qvectors)
1892 {
1893         struct ice_pf *pf = vsi->back;
1894
1895         /* allocate memory for both Tx and Rx ring pointers */
1896         vsi->tx_rings = devm_kcalloc(&pf->pdev->dev, vsi->alloc_txq,
1897                                      sizeof(struct ice_ring *), GFP_KERNEL);
1898         if (!vsi->tx_rings)
1899                 goto err_txrings;
1900
1901         vsi->rx_rings = devm_kcalloc(&pf->pdev->dev, vsi->alloc_rxq,
1902                                      sizeof(struct ice_ring *), GFP_KERNEL);
1903         if (!vsi->rx_rings)
1904                 goto err_rxrings;
1905
1906         if (alloc_qvectors) {
1907                 /* allocate memory for q_vector pointers */
1908                 vsi->q_vectors = devm_kcalloc(&pf->pdev->dev,
1909                                               vsi->num_q_vectors,
1910                                               sizeof(struct ice_q_vector *),
1911                                               GFP_KERNEL);
1912                 if (!vsi->q_vectors)
1913                         goto err_vectors;
1914         }
1915
1916         return 0;
1917
1918 err_vectors:
1919         devm_kfree(&pf->pdev->dev, vsi->rx_rings);
1920 err_rxrings:
1921         devm_kfree(&pf->pdev->dev, vsi->tx_rings);
1922 err_txrings:
1923         return -ENOMEM;
1924 }
1925
1926 /**
1927  * ice_msix_clean_rings - MSIX mode Interrupt Handler
1928  * @irq: interrupt number
1929  * @data: pointer to a q_vector
1930  */
1931 static irqreturn_t ice_msix_clean_rings(int __always_unused irq, void *data)
1932 {
1933         struct ice_q_vector *q_vector = (struct ice_q_vector *)data;
1934
1935         if (!q_vector->tx.ring && !q_vector->rx.ring)
1936                 return IRQ_HANDLED;
1937
1938         napi_schedule(&q_vector->napi);
1939
1940         return IRQ_HANDLED;
1941 }
1942
1943 /**
1944  * ice_vsi_alloc - Allocates the next available struct vsi in the PF
1945  * @pf: board private structure
1946  * @type: type of VSI
1947  *
1948  * returns a pointer to a VSI on success, NULL on failure.
1949  */
1950 static struct ice_vsi *ice_vsi_alloc(struct ice_pf *pf, enum ice_vsi_type type)
1951 {
1952         struct ice_vsi *vsi = NULL;
1953
1954         /* Need to protect the allocation of the VSIs at the PF level */
1955         mutex_lock(&pf->sw_mutex);
1956
1957         /* If we have already allocated our maximum number of VSIs,
1958          * pf->next_vsi will be ICE_NO_VSI. If not, pf->next_vsi index
1959          * is available to be populated
1960          */
1961         if (pf->next_vsi == ICE_NO_VSI) {
1962                 dev_dbg(&pf->pdev->dev, "out of VSI slots!\n");
1963                 goto unlock_pf;
1964         }
1965
1966         vsi = devm_kzalloc(&pf->pdev->dev, sizeof(*vsi), GFP_KERNEL);
1967         if (!vsi)
1968                 goto unlock_pf;
1969
1970         vsi->type = type;
1971         vsi->back = pf;
1972         set_bit(__ICE_DOWN, vsi->state);
1973         vsi->idx = pf->next_vsi;
1974         vsi->work_lmt = ICE_DFLT_IRQ_WORK;
1975
1976         ice_vsi_set_num_qs(vsi);
1977
1978         switch (vsi->type) {
1979         case ICE_VSI_PF:
1980                 if (ice_vsi_alloc_arrays(vsi, true))
1981                         goto err_rings;
1982
1983                 /* Setup default MSIX irq handler for VSI */
1984                 vsi->irq_handler = ice_msix_clean_rings;
1985                 break;
1986         default:
1987                 dev_warn(&pf->pdev->dev, "Unknown VSI type %d\n", vsi->type);
1988                 goto unlock_pf;
1989         }
1990
1991         /* fill VSI slot in the PF struct */
1992         pf->vsi[pf->next_vsi] = vsi;
1993
1994         /* prepare pf->next_vsi for next use */
1995         pf->next_vsi = ice_get_free_slot(pf->vsi, pf->num_alloc_vsi,
1996                                          pf->next_vsi);
1997         goto unlock_pf;
1998
1999 err_rings:
2000         devm_kfree(&pf->pdev->dev, vsi);
2001         vsi = NULL;
2002 unlock_pf:
2003         mutex_unlock(&pf->sw_mutex);
2004         return vsi;
2005 }
2006
2007 /**
2008  * ice_free_irq_msix_misc - Unroll misc vector setup
2009  * @pf: board private structure
2010  */
2011 static void ice_free_irq_msix_misc(struct ice_pf *pf)
2012 {
2013         /* disable OICR interrupt */
2014         wr32(&pf->hw, PFINT_OICR_ENA, 0);
2015         ice_flush(&pf->hw);
2016
2017         if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags) && pf->msix_entries) {
2018                 synchronize_irq(pf->msix_entries[pf->oicr_idx].vector);
2019                 devm_free_irq(&pf->pdev->dev,
2020                               pf->msix_entries[pf->oicr_idx].vector, pf);
2021         }
2022
2023         ice_free_res(pf->irq_tracker, pf->oicr_idx, ICE_RES_MISC_VEC_ID);
2024 }
2025
2026 /**
2027  * ice_req_irq_msix_misc - Setup the misc vector to handle non queue events
2028  * @pf: board private structure
2029  *
2030  * This sets up the handler for MSIX 0, which is used to manage the
2031  * non-queue interrupts, e.g. AdminQ and errors.  This is not used
2032  * when in MSI or Legacy interrupt mode.
2033  */
2034 static int ice_req_irq_msix_misc(struct ice_pf *pf)
2035 {
2036         struct ice_hw *hw = &pf->hw;
2037         int oicr_idx, err = 0;
2038         u8 itr_gran;
2039         u32 val;
2040
2041         if (!pf->int_name[0])
2042                 snprintf(pf->int_name, sizeof(pf->int_name) - 1, "%s-%s:misc",
2043                          dev_driver_string(&pf->pdev->dev),
2044                          dev_name(&pf->pdev->dev));
2045
2046         /* Do not request IRQ but do enable OICR interrupt since settings are
2047          * lost during reset. Note that this function is called only during
2048          * rebuild path and not while reset is in progress.
2049          */
2050         if (ice_is_reset_recovery_pending(pf->state))
2051                 goto skip_req_irq;
2052
2053         /* reserve one vector in irq_tracker for misc interrupts */
2054         oicr_idx = ice_get_res(pf, pf->irq_tracker, 1, ICE_RES_MISC_VEC_ID);
2055         if (oicr_idx < 0)
2056                 return oicr_idx;
2057
2058         pf->oicr_idx = oicr_idx;
2059
2060         err = devm_request_irq(&pf->pdev->dev,
2061                                pf->msix_entries[pf->oicr_idx].vector,
2062                                ice_misc_intr, 0, pf->int_name, pf);
2063         if (err) {
2064                 dev_err(&pf->pdev->dev,
2065                         "devm_request_irq for %s failed: %d\n",
2066                         pf->int_name, err);
2067                 ice_free_res(pf->irq_tracker, 1, ICE_RES_MISC_VEC_ID);
2068                 return err;
2069         }
2070
2071 skip_req_irq:
2072         ice_ena_misc_vector(pf);
2073
2074         val = ((pf->oicr_idx & PFINT_OICR_CTL_MSIX_INDX_M) |
2075                PFINT_OICR_CTL_CAUSE_ENA_M);
2076         wr32(hw, PFINT_OICR_CTL, val);
2077
2078         /* This enables Admin queue Interrupt causes */
2079         val = ((pf->oicr_idx & PFINT_FW_CTL_MSIX_INDX_M) |
2080                PFINT_FW_CTL_CAUSE_ENA_M);
2081         wr32(hw, PFINT_FW_CTL, val);
2082
2083         itr_gran = hw->itr_gran_200;
2084
2085         wr32(hw, GLINT_ITR(ICE_RX_ITR, pf->oicr_idx),
2086              ITR_TO_REG(ICE_ITR_8K, itr_gran));
2087
2088         ice_flush(hw);
2089         ice_irq_dynamic_ena(hw, NULL, NULL);
2090
2091         return 0;
2092 }
2093
2094 /**
2095  * ice_vsi_get_qs_contig - Assign a contiguous chunk of queues to VSI
2096  * @vsi: the VSI getting queues
2097  *
2098  * Return 0 on success and a negative value on error
2099  */
2100 static int ice_vsi_get_qs_contig(struct ice_vsi *vsi)
2101 {
2102         struct ice_pf *pf = vsi->back;
2103         int offset, ret = 0;
2104
2105         mutex_lock(&pf->avail_q_mutex);
2106         /* look for contiguous block of queues for tx */
2107         offset = bitmap_find_next_zero_area(pf->avail_txqs, ICE_MAX_TXQS,
2108                                             0, vsi->alloc_txq, 0);
2109         if (offset < ICE_MAX_TXQS) {
2110                 int i;
2111
2112                 bitmap_set(pf->avail_txqs, offset, vsi->alloc_txq);
2113                 for (i = 0; i < vsi->alloc_txq; i++)
2114                         vsi->txq_map[i] = i + offset;
2115         } else {
2116                 ret = -ENOMEM;
2117                 vsi->tx_mapping_mode = ICE_VSI_MAP_SCATTER;
2118         }
2119
2120         /* look for contiguous block of queues for rx */
2121         offset = bitmap_find_next_zero_area(pf->avail_rxqs, ICE_MAX_RXQS,
2122                                             0, vsi->alloc_rxq, 0);
2123         if (offset < ICE_MAX_RXQS) {
2124                 int i;
2125
2126                 bitmap_set(pf->avail_rxqs, offset, vsi->alloc_rxq);
2127                 for (i = 0; i < vsi->alloc_rxq; i++)
2128                         vsi->rxq_map[i] = i + offset;
2129         } else {
2130                 ret = -ENOMEM;
2131                 vsi->rx_mapping_mode = ICE_VSI_MAP_SCATTER;
2132         }
2133         mutex_unlock(&pf->avail_q_mutex);
2134
2135         return ret;
2136 }
2137
2138 /**
2139  * ice_vsi_get_qs_scatter - Assign a scattered queues to VSI
2140  * @vsi: the VSI getting queues
2141  *
2142  * Return 0 on success and a negative value on error
2143  */
2144 static int ice_vsi_get_qs_scatter(struct ice_vsi *vsi)
2145 {
2146         struct ice_pf *pf = vsi->back;
2147         int i, index = 0;
2148
2149         mutex_lock(&pf->avail_q_mutex);
2150
2151         if (vsi->tx_mapping_mode == ICE_VSI_MAP_SCATTER) {
2152                 for (i = 0; i < vsi->alloc_txq; i++) {
2153                         index = find_next_zero_bit(pf->avail_txqs,
2154                                                    ICE_MAX_TXQS, index);
2155                         if (index < ICE_MAX_TXQS) {
2156                                 set_bit(index, pf->avail_txqs);
2157                                 vsi->txq_map[i] = index;
2158                         } else {
2159                                 goto err_scatter_tx;
2160                         }
2161                 }
2162         }
2163
2164         if (vsi->rx_mapping_mode == ICE_VSI_MAP_SCATTER) {
2165                 for (i = 0; i < vsi->alloc_rxq; i++) {
2166                         index = find_next_zero_bit(pf->avail_rxqs,
2167                                                    ICE_MAX_RXQS, index);
2168                         if (index < ICE_MAX_RXQS) {
2169                                 set_bit(index, pf->avail_rxqs);
2170                                 vsi->rxq_map[i] = index;
2171                         } else {
2172                                 goto err_scatter_rx;
2173                         }
2174                 }
2175         }
2176
2177         mutex_unlock(&pf->avail_q_mutex);
2178         return 0;
2179
2180 err_scatter_rx:
2181         /* unflag any queues we have grabbed (i is failed position) */
2182         for (index = 0; index < i; index++) {
2183                 clear_bit(vsi->rxq_map[index], pf->avail_rxqs);
2184                 vsi->rxq_map[index] = 0;
2185         }
2186         i = vsi->alloc_txq;
2187 err_scatter_tx:
2188         /* i is either position of failed attempt or vsi->alloc_txq */
2189         for (index = 0; index < i; index++) {
2190                 clear_bit(vsi->txq_map[index], pf->avail_txqs);
2191                 vsi->txq_map[index] = 0;
2192         }
2193
2194         mutex_unlock(&pf->avail_q_mutex);
2195         return -ENOMEM;
2196 }
2197
2198 /**
2199  * ice_vsi_get_qs - Assign queues from PF to VSI
2200  * @vsi: the VSI to assign queues to
2201  *
2202  * Returns 0 on success and a negative value on error
2203  */
2204 static int ice_vsi_get_qs(struct ice_vsi *vsi)
2205 {
2206         int ret = 0;
2207
2208         vsi->tx_mapping_mode = ICE_VSI_MAP_CONTIG;
2209         vsi->rx_mapping_mode = ICE_VSI_MAP_CONTIG;
2210
2211         /* NOTE: ice_vsi_get_qs_contig() will set the rx/tx mapping
2212          * modes individually to scatter if assigning contiguous queues
2213          * to rx or tx fails
2214          */
2215         ret = ice_vsi_get_qs_contig(vsi);
2216         if (ret < 0) {
2217                 if (vsi->tx_mapping_mode == ICE_VSI_MAP_SCATTER)
2218                         vsi->alloc_txq = max_t(u16, vsi->alloc_txq,
2219                                                ICE_MAX_SCATTER_TXQS);
2220                 if (vsi->rx_mapping_mode == ICE_VSI_MAP_SCATTER)
2221                         vsi->alloc_rxq = max_t(u16, vsi->alloc_rxq,
2222                                                ICE_MAX_SCATTER_RXQS);
2223                 ret = ice_vsi_get_qs_scatter(vsi);
2224         }
2225
2226         return ret;
2227 }
2228
2229 /**
2230  * ice_vsi_put_qs - Release queues from VSI to PF
2231  * @vsi: the VSI thats going to release queues
2232  */
2233 static void ice_vsi_put_qs(struct ice_vsi *vsi)
2234 {
2235         struct ice_pf *pf = vsi->back;
2236         int i;
2237
2238         mutex_lock(&pf->avail_q_mutex);
2239
2240         for (i = 0; i < vsi->alloc_txq; i++) {
2241                 clear_bit(vsi->txq_map[i], pf->avail_txqs);
2242                 vsi->txq_map[i] = ICE_INVAL_Q_INDEX;
2243         }
2244
2245         for (i = 0; i < vsi->alloc_rxq; i++) {
2246                 clear_bit(vsi->rxq_map[i], pf->avail_rxqs);
2247                 vsi->rxq_map[i] = ICE_INVAL_Q_INDEX;
2248         }
2249
2250         mutex_unlock(&pf->avail_q_mutex);
2251 }
2252
2253 /**
2254  * ice_free_q_vector - Free memory allocated for a specific interrupt vector
2255  * @vsi: VSI having the memory freed
2256  * @v_idx: index of the vector to be freed
2257  */
2258 static void ice_free_q_vector(struct ice_vsi *vsi, int v_idx)
2259 {
2260         struct ice_q_vector *q_vector;
2261         struct ice_ring *ring;
2262
2263         if (!vsi->q_vectors[v_idx]) {
2264                 dev_dbg(&vsi->back->pdev->dev, "Queue vector at index %d not found\n",
2265                         v_idx);
2266                 return;
2267         }
2268         q_vector = vsi->q_vectors[v_idx];
2269
2270         ice_for_each_ring(ring, q_vector->tx)
2271                 ring->q_vector = NULL;
2272         ice_for_each_ring(ring, q_vector->rx)
2273                 ring->q_vector = NULL;
2274
2275         /* only VSI with an associated netdev is set up with NAPI */
2276         if (vsi->netdev)
2277                 netif_napi_del(&q_vector->napi);
2278
2279         devm_kfree(&vsi->back->pdev->dev, q_vector);
2280         vsi->q_vectors[v_idx] = NULL;
2281 }
2282
2283 /**
2284  * ice_vsi_free_q_vectors - Free memory allocated for interrupt vectors
2285  * @vsi: the VSI having memory freed
2286  */
2287 static void ice_vsi_free_q_vectors(struct ice_vsi *vsi)
2288 {
2289         int v_idx;
2290
2291         for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
2292                 ice_free_q_vector(vsi, v_idx);
2293 }
2294
2295 /**
2296  * ice_cfg_netdev - Setup the netdev flags
2297  * @vsi: the VSI being configured
2298  *
2299  * Returns 0 on success, negative value on failure
2300  */
2301 static int ice_cfg_netdev(struct ice_vsi *vsi)
2302 {
2303         netdev_features_t csumo_features;
2304         netdev_features_t vlano_features;
2305         netdev_features_t dflt_features;
2306         netdev_features_t tso_features;
2307         struct ice_netdev_priv *np;
2308         struct net_device *netdev;
2309         u8 mac_addr[ETH_ALEN];
2310
2311         netdev = alloc_etherdev_mqs(sizeof(struct ice_netdev_priv),
2312                                     vsi->alloc_txq, vsi->alloc_rxq);
2313         if (!netdev)
2314                 return -ENOMEM;
2315
2316         vsi->netdev = netdev;
2317         np = netdev_priv(netdev);
2318         np->vsi = vsi;
2319
2320         dflt_features = NETIF_F_SG      |
2321                         NETIF_F_HIGHDMA |
2322                         NETIF_F_RXHASH;
2323
2324         csumo_features = NETIF_F_RXCSUM   |
2325                          NETIF_F_IP_CSUM  |
2326                          NETIF_F_IPV6_CSUM;
2327
2328         vlano_features = NETIF_F_HW_VLAN_CTAG_FILTER |
2329                          NETIF_F_HW_VLAN_CTAG_TX     |
2330                          NETIF_F_HW_VLAN_CTAG_RX;
2331
2332         tso_features = NETIF_F_TSO;
2333
2334         /* set features that user can change */
2335         netdev->hw_features = dflt_features | csumo_features |
2336                               vlano_features | tso_features;
2337
2338         /* enable features */
2339         netdev->features |= netdev->hw_features;
2340         /* encap and VLAN devices inherit default, csumo and tso features */
2341         netdev->hw_enc_features |= dflt_features | csumo_features |
2342                                    tso_features;
2343         netdev->vlan_features |= dflt_features | csumo_features |
2344                                  tso_features;
2345
2346         if (vsi->type == ICE_VSI_PF) {
2347                 SET_NETDEV_DEV(netdev, &vsi->back->pdev->dev);
2348                 ether_addr_copy(mac_addr, vsi->port_info->mac.perm_addr);
2349
2350                 ether_addr_copy(netdev->dev_addr, mac_addr);
2351                 ether_addr_copy(netdev->perm_addr, mac_addr);
2352         }
2353
2354         netdev->priv_flags |= IFF_UNICAST_FLT;
2355
2356         /* assign netdev_ops */
2357         netdev->netdev_ops = &ice_netdev_ops;
2358
2359         /* setup watchdog timeout value to be 5 second */
2360         netdev->watchdog_timeo = 5 * HZ;
2361
2362         ice_set_ethtool_ops(netdev);
2363
2364         netdev->min_mtu = ETH_MIN_MTU;
2365         netdev->max_mtu = ICE_MAX_MTU;
2366
2367         return 0;
2368 }
2369
2370 /**
2371  * ice_vsi_free_arrays - clean up vsi resources
2372  * @vsi: pointer to VSI being cleared
2373  * @free_qvectors: bool to specify if q_vectors should be deallocated
2374  */
2375 static void ice_vsi_free_arrays(struct ice_vsi *vsi, bool free_qvectors)
2376 {
2377         struct ice_pf *pf = vsi->back;
2378
2379         /* free the ring and vector containers */
2380         if (free_qvectors && vsi->q_vectors) {
2381                 devm_kfree(&pf->pdev->dev, vsi->q_vectors);
2382                 vsi->q_vectors = NULL;
2383         }
2384         if (vsi->tx_rings) {
2385                 devm_kfree(&pf->pdev->dev, vsi->tx_rings);
2386                 vsi->tx_rings = NULL;
2387         }
2388         if (vsi->rx_rings) {
2389                 devm_kfree(&pf->pdev->dev, vsi->rx_rings);
2390                 vsi->rx_rings = NULL;
2391         }
2392 }
2393
2394 /**
2395  * ice_vsi_clear - clean up and deallocate the provided vsi
2396  * @vsi: pointer to VSI being cleared
2397  *
2398  * This deallocates the vsi's queue resources, removes it from the PF's
2399  * VSI array if necessary, and deallocates the VSI
2400  *
2401  * Returns 0 on success, negative on failure
2402  */
2403 static int ice_vsi_clear(struct ice_vsi *vsi)
2404 {
2405         struct ice_pf *pf = NULL;
2406
2407         if (!vsi)
2408                 return 0;
2409
2410         if (!vsi->back)
2411                 return -EINVAL;
2412
2413         pf = vsi->back;
2414
2415         if (!pf->vsi[vsi->idx] || pf->vsi[vsi->idx] != vsi) {
2416                 dev_dbg(&pf->pdev->dev, "vsi does not exist at pf->vsi[%d]\n",
2417                         vsi->idx);
2418                 return -EINVAL;
2419         }
2420
2421         mutex_lock(&pf->sw_mutex);
2422         /* updates the PF for this cleared vsi */
2423
2424         pf->vsi[vsi->idx] = NULL;
2425         if (vsi->idx < pf->next_vsi)
2426                 pf->next_vsi = vsi->idx;
2427
2428         ice_vsi_free_arrays(vsi, true);
2429         mutex_unlock(&pf->sw_mutex);
2430         devm_kfree(&pf->pdev->dev, vsi);
2431
2432         return 0;
2433 }
2434
2435 /**
2436  * ice_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
2437  * @vsi: the VSI being configured
2438  * @v_idx: index of the vector in the vsi struct
2439  *
2440  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
2441  */
2442 static int ice_vsi_alloc_q_vector(struct ice_vsi *vsi, int v_idx)
2443 {
2444         struct ice_pf *pf = vsi->back;
2445         struct ice_q_vector *q_vector;
2446
2447         /* allocate q_vector */
2448         q_vector = devm_kzalloc(&pf->pdev->dev, sizeof(*q_vector), GFP_KERNEL);
2449         if (!q_vector)
2450                 return -ENOMEM;
2451
2452         q_vector->vsi = vsi;
2453         q_vector->v_idx = v_idx;
2454         /* only set affinity_mask if the CPU is online */
2455         if (cpu_online(v_idx))
2456                 cpumask_set_cpu(v_idx, &q_vector->affinity_mask);
2457
2458         if (vsi->netdev)
2459                 netif_napi_add(vsi->netdev, &q_vector->napi, ice_napi_poll,
2460                                NAPI_POLL_WEIGHT);
2461         /* tie q_vector and vsi together */
2462         vsi->q_vectors[v_idx] = q_vector;
2463
2464         return 0;
2465 }
2466
2467 /**
2468  * ice_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
2469  * @vsi: the VSI being configured
2470  *
2471  * We allocate one q_vector per queue interrupt.  If allocation fails we
2472  * return -ENOMEM.
2473  */
2474 static int ice_vsi_alloc_q_vectors(struct ice_vsi *vsi)
2475 {
2476         struct ice_pf *pf = vsi->back;
2477         int v_idx = 0, num_q_vectors;
2478         int err;
2479
2480         if (vsi->q_vectors[0]) {
2481                 dev_dbg(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
2482                         vsi->vsi_num);
2483                 return -EEXIST;
2484         }
2485
2486         if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) {
2487                 num_q_vectors = vsi->num_q_vectors;
2488         } else {
2489                 err = -EINVAL;
2490                 goto err_out;
2491         }
2492
2493         for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
2494                 err = ice_vsi_alloc_q_vector(vsi, v_idx);
2495                 if (err)
2496                         goto err_out;
2497         }
2498
2499         return 0;
2500
2501 err_out:
2502         while (v_idx--)
2503                 ice_free_q_vector(vsi, v_idx);
2504
2505         dev_err(&pf->pdev->dev,
2506                 "Failed to allocate %d q_vector for VSI %d, ret=%d\n",
2507                 vsi->num_q_vectors, vsi->vsi_num, err);
2508         vsi->num_q_vectors = 0;
2509         return err;
2510 }
2511
2512 /**
2513  * ice_vsi_setup_vector_base - Set up the base vector for the given VSI
2514  * @vsi: ptr to the VSI
2515  *
2516  * This should only be called after ice_vsi_alloc() which allocates the
2517  * corresponding SW VSI structure and initializes num_queue_pairs for the
2518  * newly allocated VSI.
2519  *
2520  * Returns 0 on success or negative on failure
2521  */
2522 static int ice_vsi_setup_vector_base(struct ice_vsi *vsi)
2523 {
2524         struct ice_pf *pf = vsi->back;
2525         int num_q_vectors = 0;
2526
2527         if (vsi->base_vector) {
2528                 dev_dbg(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
2529                         vsi->vsi_num, vsi->base_vector);
2530                 return -EEXIST;
2531         }
2532
2533         if (!test_bit(ICE_FLAG_MSIX_ENA, pf->flags))
2534                 return -ENOENT;
2535
2536         switch (vsi->type) {
2537         case ICE_VSI_PF:
2538                 num_q_vectors = vsi->num_q_vectors;
2539                 break;
2540         default:
2541                 dev_warn(&vsi->back->pdev->dev, "Unknown VSI type %d\n",
2542                          vsi->type);
2543                 break;
2544         }
2545
2546         if (num_q_vectors)
2547                 vsi->base_vector = ice_get_res(pf, pf->irq_tracker,
2548                                                num_q_vectors, vsi->idx);
2549
2550         if (vsi->base_vector < 0) {
2551                 dev_err(&pf->pdev->dev,
2552                         "Failed to get tracking for %d vectors for VSI %d, err=%d\n",
2553                         num_q_vectors, vsi->vsi_num, vsi->base_vector);
2554                 return -ENOENT;
2555         }
2556
2557         return 0;
2558 }
2559
2560 /**
2561  * ice_fill_rss_lut - Fill the RSS lookup table with default values
2562  * @lut: Lookup table
2563  * @rss_table_size: Lookup table size
2564  * @rss_size: Range of queue number for hashing
2565  */
2566 void ice_fill_rss_lut(u8 *lut, u16 rss_table_size, u16 rss_size)
2567 {
2568         u16 i;
2569
2570         for (i = 0; i < rss_table_size; i++)
2571                 lut[i] = i % rss_size;
2572 }
2573
2574 /**
2575  * ice_vsi_cfg_rss - Configure RSS params for a VSI
2576  * @vsi: VSI to be configured
2577  */
2578 static int ice_vsi_cfg_rss(struct ice_vsi *vsi)
2579 {
2580         u8 seed[ICE_AQC_GET_SET_RSS_KEY_DATA_RSS_KEY_SIZE];
2581         struct ice_aqc_get_set_rss_keys *key;
2582         struct ice_pf *pf = vsi->back;
2583         enum ice_status status;
2584         int err = 0;
2585         u8 *lut;
2586
2587         vsi->rss_size = min_t(int, vsi->rss_size, vsi->num_rxq);
2588
2589         lut = devm_kzalloc(&pf->pdev->dev, vsi->rss_table_size, GFP_KERNEL);
2590         if (!lut)
2591                 return -ENOMEM;
2592
2593         if (vsi->rss_lut_user)
2594                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
2595         else
2596                 ice_fill_rss_lut(lut, vsi->rss_table_size, vsi->rss_size);
2597
2598         status = ice_aq_set_rss_lut(&pf->hw, vsi->vsi_num, vsi->rss_lut_type,
2599                                     lut, vsi->rss_table_size);
2600
2601         if (status) {
2602                 dev_err(&vsi->back->pdev->dev,
2603                         "set_rss_lut failed, error %d\n", status);
2604                 err = -EIO;
2605                 goto ice_vsi_cfg_rss_exit;
2606         }
2607
2608         key = devm_kzalloc(&vsi->back->pdev->dev, sizeof(*key), GFP_KERNEL);
2609         if (!key) {
2610                 err = -ENOMEM;
2611                 goto ice_vsi_cfg_rss_exit;
2612         }
2613
2614         if (vsi->rss_hkey_user)
2615                 memcpy(seed, vsi->rss_hkey_user,
2616                        ICE_AQC_GET_SET_RSS_KEY_DATA_RSS_KEY_SIZE);
2617         else
2618                 netdev_rss_key_fill((void *)seed,
2619                                     ICE_AQC_GET_SET_RSS_KEY_DATA_RSS_KEY_SIZE);
2620         memcpy(&key->standard_rss_key, seed,
2621                ICE_AQC_GET_SET_RSS_KEY_DATA_RSS_KEY_SIZE);
2622
2623         status = ice_aq_set_rss_key(&pf->hw, vsi->vsi_num, key);
2624
2625         if (status) {
2626                 dev_err(&vsi->back->pdev->dev, "set_rss_key failed, error %d\n",
2627                         status);
2628                 err = -EIO;
2629         }
2630
2631         devm_kfree(&pf->pdev->dev, key);
2632 ice_vsi_cfg_rss_exit:
2633         devm_kfree(&pf->pdev->dev, lut);
2634         return err;
2635 }
2636
2637 /**
2638  * ice_vsi_reinit_setup - return resource and reallocate resource for a VSI
2639  * @vsi: pointer to the ice_vsi
2640  *
2641  * This reallocates the VSIs queue resources
2642  *
2643  * Returns 0 on success and negative value on failure
2644  */
2645 static int ice_vsi_reinit_setup(struct ice_vsi *vsi)
2646 {
2647         u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 };
2648         int ret, i;
2649
2650         if (!vsi)
2651                 return -EINVAL;
2652
2653         ice_vsi_free_q_vectors(vsi);
2654         ice_free_res(vsi->back->irq_tracker, vsi->base_vector, vsi->idx);
2655         vsi->base_vector = 0;
2656         ice_vsi_clear_rings(vsi);
2657         ice_vsi_free_arrays(vsi, false);
2658         ice_vsi_set_num_qs(vsi);
2659
2660         /* Initialize VSI struct elements and create VSI in FW */
2661         ret = ice_vsi_add(vsi);
2662         if (ret < 0)
2663                 goto err_vsi;
2664
2665         ret = ice_vsi_alloc_arrays(vsi, false);
2666         if (ret < 0)
2667                 goto err_vsi;
2668
2669         switch (vsi->type) {
2670         case ICE_VSI_PF:
2671                 if (!vsi->netdev) {
2672                         ret = ice_cfg_netdev(vsi);
2673                         if (ret)
2674                                 goto err_rings;
2675
2676                         ret = register_netdev(vsi->netdev);
2677                         if (ret)
2678                                 goto err_rings;
2679
2680                         netif_carrier_off(vsi->netdev);
2681                         netif_tx_stop_all_queues(vsi->netdev);
2682                 }
2683
2684                 ret = ice_vsi_alloc_q_vectors(vsi);
2685                 if (ret)
2686                         goto err_rings;
2687
2688                 ret = ice_vsi_setup_vector_base(vsi);
2689                 if (ret)
2690                         goto err_vectors;
2691
2692                 ret = ice_vsi_alloc_rings(vsi);
2693                 if (ret)
2694                         goto err_vectors;
2695
2696                 ice_vsi_map_rings_to_vectors(vsi);
2697                 break;
2698         default:
2699                 break;
2700         }
2701
2702         ice_vsi_set_tc_cfg(vsi);
2703
2704         /* configure VSI nodes based on number of queues and TC's */
2705         for (i = 0; i < vsi->tc_cfg.numtc; i++)
2706                 max_txqs[i] = vsi->num_txq;
2707
2708         ret = ice_cfg_vsi_lan(vsi->port_info, vsi->vsi_num,
2709                               vsi->tc_cfg.ena_tc, max_txqs);
2710         if (ret) {
2711                 dev_info(&vsi->back->pdev->dev,
2712                          "Failed VSI lan queue config\n");
2713                 goto err_vectors;
2714         }
2715         return 0;
2716
2717 err_vectors:
2718         ice_vsi_free_q_vectors(vsi);
2719 err_rings:
2720         if (vsi->netdev) {
2721                 vsi->current_netdev_flags = 0;
2722                 unregister_netdev(vsi->netdev);
2723                 free_netdev(vsi->netdev);
2724                 vsi->netdev = NULL;
2725         }
2726 err_vsi:
2727         ice_vsi_clear(vsi);
2728         set_bit(__ICE_RESET_FAILED, vsi->back->state);
2729         return ret;
2730 }
2731
2732 /**
2733  * ice_vsi_setup - Set up a VSI by a given type
2734  * @pf: board private structure
2735  * @type: VSI type
2736  * @pi: pointer to the port_info instance
2737  *
2738  * This allocates the sw VSI structure and its queue resources.
2739  *
2740  * Returns pointer to the successfully allocated and configure VSI sw struct on
2741  * success, otherwise returns NULL on failure.
2742  */
2743 static struct ice_vsi *
2744 ice_vsi_setup(struct ice_pf *pf, enum ice_vsi_type type,
2745               struct ice_port_info *pi)
2746 {
2747         u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 };
2748         struct device *dev = &pf->pdev->dev;
2749         struct ice_vsi_ctx ctxt = { 0 };
2750         struct ice_vsi *vsi;
2751         int ret, i;
2752
2753         vsi = ice_vsi_alloc(pf, type);
2754         if (!vsi) {
2755                 dev_err(dev, "could not allocate VSI\n");
2756                 return NULL;
2757         }
2758
2759         vsi->port_info = pi;
2760         vsi->vsw = pf->first_sw;
2761
2762         if (ice_vsi_get_qs(vsi)) {
2763                 dev_err(dev, "Failed to allocate queues. vsi->idx = %d\n",
2764                         vsi->idx);
2765                 goto err_get_qs;
2766         }
2767
2768         /* set RSS capabilities */
2769         ice_vsi_set_rss_params(vsi);
2770
2771         /* create the VSI */
2772         ret = ice_vsi_add(vsi);
2773         if (ret)
2774                 goto err_vsi;
2775
2776         ctxt.vsi_num = vsi->vsi_num;
2777
2778         switch (vsi->type) {
2779         case ICE_VSI_PF:
2780                 ret = ice_cfg_netdev(vsi);
2781                 if (ret)
2782                         goto err_cfg_netdev;
2783
2784                 ret = register_netdev(vsi->netdev);
2785                 if (ret)
2786                         goto err_register_netdev;
2787
2788                 netif_carrier_off(vsi->netdev);
2789
2790                 /* make sure transmit queues start off as stopped */
2791                 netif_tx_stop_all_queues(vsi->netdev);
2792                 ret = ice_vsi_alloc_q_vectors(vsi);
2793                 if (ret)
2794                         goto err_msix;
2795
2796                 ret = ice_vsi_setup_vector_base(vsi);
2797                 if (ret)
2798                         goto err_rings;
2799
2800                 ret = ice_vsi_alloc_rings(vsi);
2801                 if (ret)
2802                         goto err_rings;
2803
2804                 ice_vsi_map_rings_to_vectors(vsi);
2805
2806                 /* Do not exit if configuring RSS had an issue, at least
2807                  * receive traffic on first queue. Hence no need to capture
2808                  * return value
2809                  */
2810                 if (test_bit(ICE_FLAG_RSS_ENA, pf->flags))
2811                         ice_vsi_cfg_rss(vsi);
2812                 break;
2813         default:
2814                 /* if vsi type is not recognized, clean up the resources and
2815                  * exit
2816                  */
2817                 goto err_rings;
2818         }
2819
2820         ice_vsi_set_tc_cfg(vsi);
2821
2822         /* configure VSI nodes based on number of queues and TC's */
2823         for (i = 0; i < vsi->tc_cfg.numtc; i++)
2824                 max_txqs[i] = vsi->num_txq;
2825
2826         ret = ice_cfg_vsi_lan(vsi->port_info, vsi->vsi_num,
2827                               vsi->tc_cfg.ena_tc, max_txqs);
2828         if (ret) {
2829                 dev_info(&pf->pdev->dev, "Failed VSI lan queue config\n");
2830                 goto err_rings;
2831         }
2832
2833         return vsi;
2834
2835 err_rings:
2836         ice_vsi_free_q_vectors(vsi);
2837 err_msix:
2838         if (vsi->netdev && vsi->netdev->reg_state == NETREG_REGISTERED)
2839                 unregister_netdev(vsi->netdev);
2840 err_register_netdev:
2841         if (vsi->netdev) {
2842                 free_netdev(vsi->netdev);
2843                 vsi->netdev = NULL;
2844         }
2845 err_cfg_netdev:
2846         ret = ice_aq_free_vsi(&pf->hw, &ctxt, false, NULL);
2847         if (ret)
2848                 dev_err(&vsi->back->pdev->dev,
2849                         "Free VSI AQ call failed, err %d\n", ret);
2850 err_vsi:
2851         ice_vsi_put_qs(vsi);
2852 err_get_qs:
2853         pf->q_left_tx += vsi->alloc_txq;
2854         pf->q_left_rx += vsi->alloc_rxq;
2855         ice_vsi_clear(vsi);
2856
2857         return NULL;
2858 }
2859
2860 /**
2861  * ice_vsi_add_vlan - Add vsi membership for given vlan
2862  * @vsi: the vsi being configured
2863  * @vid: vlan id to be added
2864  */
2865 static int ice_vsi_add_vlan(struct ice_vsi *vsi, u16 vid)
2866 {
2867         struct ice_fltr_list_entry *tmp;
2868         struct ice_pf *pf = vsi->back;
2869         LIST_HEAD(tmp_add_list);
2870         enum ice_status status;
2871         int err = 0;
2872
2873         tmp = devm_kzalloc(&pf->pdev->dev, sizeof(*tmp), GFP_KERNEL);
2874         if (!tmp)
2875                 return -ENOMEM;
2876
2877         tmp->fltr_info.lkup_type = ICE_SW_LKUP_VLAN;
2878         tmp->fltr_info.fltr_act = ICE_FWD_TO_VSI;
2879         tmp->fltr_info.flag = ICE_FLTR_TX;
2880         tmp->fltr_info.src = vsi->vsi_num;
2881         tmp->fltr_info.fwd_id.vsi_id = vsi->vsi_num;
2882         tmp->fltr_info.l_data.vlan.vlan_id = vid;
2883
2884         INIT_LIST_HEAD(&tmp->list_entry);
2885         list_add(&tmp->list_entry, &tmp_add_list);
2886
2887         status = ice_add_vlan(&pf->hw, &tmp_add_list);
2888         if (status) {
2889                 err = -ENODEV;
2890                 dev_err(&pf->pdev->dev, "Failure Adding VLAN %d on VSI %i\n",
2891                         vid, vsi->vsi_num);
2892         }
2893
2894         ice_free_fltr_list(&pf->pdev->dev, &tmp_add_list);
2895         return err;
2896 }
2897
2898 /**
2899  * ice_vlan_rx_add_vid - Add a vlan id filter to HW offload
2900  * @netdev: network interface to be adjusted
2901  * @proto: unused protocol
2902  * @vid: vlan id to be added
2903  *
2904  * net_device_ops implementation for adding vlan ids
2905  */
2906 static int ice_vlan_rx_add_vid(struct net_device *netdev,
2907                                __always_unused __be16 proto, u16 vid)
2908 {
2909         struct ice_netdev_priv *np = netdev_priv(netdev);
2910         struct ice_vsi *vsi = np->vsi;
2911         int ret = 0;
2912
2913         if (vid >= VLAN_N_VID) {
2914                 netdev_err(netdev, "VLAN id requested %d is out of range %d\n",
2915                            vid, VLAN_N_VID);
2916                 return -EINVAL;
2917         }
2918
2919         if (vsi->info.pvid)
2920                 return -EINVAL;
2921
2922         /* Add all VLAN ids including 0 to the switch filter. VLAN id 0 is
2923          * needed to continue allowing all untagged packets since VLAN prune
2924          * list is applied to all packets by the switch
2925          */
2926         ret = ice_vsi_add_vlan(vsi, vid);
2927
2928         if (!ret)
2929                 set_bit(vid, vsi->active_vlans);
2930
2931         return ret;
2932 }
2933
2934 /**
2935  * ice_vsi_kill_vlan - Remove VSI membership for a given VLAN
2936  * @vsi: the VSI being configured
2937  * @vid: VLAN id to be removed
2938  */
2939 static void ice_vsi_kill_vlan(struct ice_vsi *vsi, u16 vid)
2940 {
2941         struct ice_fltr_list_entry *list;
2942         struct ice_pf *pf = vsi->back;
2943         LIST_HEAD(tmp_add_list);
2944
2945         list = devm_kzalloc(&pf->pdev->dev, sizeof(*list), GFP_KERNEL);
2946         if (!list)
2947                 return;
2948
2949         list->fltr_info.lkup_type = ICE_SW_LKUP_VLAN;
2950         list->fltr_info.fwd_id.vsi_id = vsi->vsi_num;
2951         list->fltr_info.fltr_act = ICE_FWD_TO_VSI;
2952         list->fltr_info.l_data.vlan.vlan_id = vid;
2953         list->fltr_info.flag = ICE_FLTR_TX;
2954         list->fltr_info.src = vsi->vsi_num;
2955
2956         INIT_LIST_HEAD(&list->list_entry);
2957         list_add(&list->list_entry, &tmp_add_list);
2958
2959         if (ice_remove_vlan(&pf->hw, &tmp_add_list))
2960                 dev_err(&pf->pdev->dev, "Error removing VLAN %d on vsi %i\n",
2961                         vid, vsi->vsi_num);
2962
2963         ice_free_fltr_list(&pf->pdev->dev, &tmp_add_list);
2964 }
2965
2966 /**
2967  * ice_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
2968  * @netdev: network interface to be adjusted
2969  * @proto: unused protocol
2970  * @vid: vlan id to be removed
2971  *
2972  * net_device_ops implementation for removing vlan ids
2973  */
2974 static int ice_vlan_rx_kill_vid(struct net_device *netdev,
2975                                 __always_unused __be16 proto, u16 vid)
2976 {
2977         struct ice_netdev_priv *np = netdev_priv(netdev);
2978         struct ice_vsi *vsi = np->vsi;
2979
2980         if (vsi->info.pvid)
2981                 return -EINVAL;
2982
2983         /* return code is ignored as there is nothing a user
2984          * can do about failure to remove and a log message was
2985          * already printed from the other function
2986          */
2987         ice_vsi_kill_vlan(vsi, vid);
2988
2989         clear_bit(vid, vsi->active_vlans);
2990
2991         return 0;
2992 }
2993
2994 /**
2995  * ice_setup_pf_sw - Setup the HW switch on startup or after reset
2996  * @pf: board private structure
2997  *
2998  * Returns 0 on success, negative value on failure
2999  */
3000 static int ice_setup_pf_sw(struct ice_pf *pf)
3001 {
3002         LIST_HEAD(tmp_add_list);
3003         u8 broadcast[ETH_ALEN];
3004         struct ice_vsi *vsi;
3005         int status = 0;
3006
3007         if (!ice_is_reset_recovery_pending(pf->state)) {
3008                 vsi = ice_vsi_setup(pf, ICE_VSI_PF, pf->hw.port_info);
3009                 if (!vsi) {
3010                         status = -ENOMEM;
3011                         goto error_exit;
3012                 }
3013         } else {
3014                 vsi = pf->vsi[0];
3015                 status = ice_vsi_reinit_setup(vsi);
3016                 if (status < 0)
3017                         return -EIO;
3018         }
3019
3020         /* tmp_add_list contains a list of MAC addresses for which MAC
3021          * filters need to be programmed. Add the VSI's unicast MAC to
3022          * this list
3023          */
3024         status = ice_add_mac_to_list(vsi, &tmp_add_list,
3025                                      vsi->port_info->mac.perm_addr);
3026         if (status)
3027                 goto error_exit;
3028
3029         /* VSI needs to receive broadcast traffic, so add the broadcast
3030          * MAC address to the list.
3031          */
3032         eth_broadcast_addr(broadcast);
3033         status = ice_add_mac_to_list(vsi, &tmp_add_list, broadcast);
3034         if (status)
3035                 goto error_exit;
3036
3037         /* program MAC filters for entries in tmp_add_list */
3038         status = ice_add_mac(&pf->hw, &tmp_add_list);
3039         if (status) {
3040                 dev_err(&pf->pdev->dev, "Could not add MAC filters\n");
3041                 status = -ENOMEM;
3042                 goto error_exit;
3043         }
3044
3045         ice_free_fltr_list(&pf->pdev->dev, &tmp_add_list);
3046         return status;
3047
3048 error_exit:
3049         ice_free_fltr_list(&pf->pdev->dev, &tmp_add_list);
3050
3051         if (vsi) {
3052                 ice_vsi_free_q_vectors(vsi);
3053                 if (vsi->netdev && vsi->netdev->reg_state == NETREG_REGISTERED)
3054                         unregister_netdev(vsi->netdev);
3055                 if (vsi->netdev) {
3056                         free_netdev(vsi->netdev);
3057                         vsi->netdev = NULL;
3058                 }
3059
3060                 ice_vsi_delete(vsi);
3061                 ice_vsi_put_qs(vsi);
3062                 pf->q_left_tx += vsi->alloc_txq;
3063                 pf->q_left_rx += vsi->alloc_rxq;
3064                 ice_vsi_clear(vsi);
3065         }
3066         return status;
3067 }
3068
3069 /**
3070  * ice_determine_q_usage - Calculate queue distribution
3071  * @pf: board private structure
3072  *
3073  * Return -ENOMEM if we don't get enough queues for all ports
3074  */
3075 static void ice_determine_q_usage(struct ice_pf *pf)
3076 {
3077         u16 q_left_tx, q_left_rx;
3078
3079         q_left_tx = pf->hw.func_caps.common_cap.num_txq;
3080         q_left_rx = pf->hw.func_caps.common_cap.num_rxq;
3081
3082         pf->num_lan_tx = min_t(int, q_left_tx, num_online_cpus());
3083
3084         /* only 1 rx queue unless RSS is enabled */
3085         if (!test_bit(ICE_FLAG_RSS_ENA, pf->flags))
3086                 pf->num_lan_rx = 1;
3087         else
3088                 pf->num_lan_rx = min_t(int, q_left_rx, num_online_cpus());
3089
3090         pf->q_left_tx = q_left_tx - pf->num_lan_tx;
3091         pf->q_left_rx = q_left_rx - pf->num_lan_rx;
3092 }
3093
3094 /**
3095  * ice_deinit_pf - Unrolls initialziations done by ice_init_pf
3096  * @pf: board private structure to initialize
3097  */
3098 static void ice_deinit_pf(struct ice_pf *pf)
3099 {
3100         if (pf->serv_tmr.function)
3101                 del_timer_sync(&pf->serv_tmr);
3102         if (pf->serv_task.func)
3103                 cancel_work_sync(&pf->serv_task);
3104         mutex_destroy(&pf->sw_mutex);
3105         mutex_destroy(&pf->avail_q_mutex);
3106 }
3107
3108 /**
3109  * ice_init_pf - Initialize general software structures (struct ice_pf)
3110  * @pf: board private structure to initialize
3111  */
3112 static void ice_init_pf(struct ice_pf *pf)
3113 {
3114         bitmap_zero(pf->flags, ICE_PF_FLAGS_NBITS);
3115         set_bit(ICE_FLAG_MSIX_ENA, pf->flags);
3116
3117         mutex_init(&pf->sw_mutex);
3118         mutex_init(&pf->avail_q_mutex);
3119
3120         /* Clear avail_[t|r]x_qs bitmaps (set all to avail) */
3121         mutex_lock(&pf->avail_q_mutex);
3122         bitmap_zero(pf->avail_txqs, ICE_MAX_TXQS);
3123         bitmap_zero(pf->avail_rxqs, ICE_MAX_RXQS);
3124         mutex_unlock(&pf->avail_q_mutex);
3125
3126         if (pf->hw.func_caps.common_cap.rss_table_size)
3127                 set_bit(ICE_FLAG_RSS_ENA, pf->flags);
3128
3129         /* setup service timer and periodic service task */
3130         timer_setup(&pf->serv_tmr, ice_service_timer, 0);
3131         pf->serv_tmr_period = HZ;
3132         INIT_WORK(&pf->serv_task, ice_service_task);
3133         clear_bit(__ICE_SERVICE_SCHED, pf->state);
3134 }
3135
3136 /**
3137  * ice_ena_msix_range - Request a range of MSIX vectors from the OS
3138  * @pf: board private structure
3139  *
3140  * compute the number of MSIX vectors required (v_budget) and request from
3141  * the OS. Return the number of vectors reserved or negative on failure
3142  */
3143 static int ice_ena_msix_range(struct ice_pf *pf)
3144 {
3145         int v_left, v_actual, v_budget = 0;
3146         int needed, err, i;
3147
3148         v_left = pf->hw.func_caps.common_cap.num_msix_vectors;
3149
3150         /* reserve one vector for miscellaneous handler */
3151         needed = 1;
3152         v_budget += needed;
3153         v_left -= needed;
3154
3155         /* reserve vectors for LAN traffic */
3156         pf->num_lan_msix = min_t(int, num_online_cpus(), v_left);
3157         v_budget += pf->num_lan_msix;
3158
3159         pf->msix_entries = devm_kcalloc(&pf->pdev->dev, v_budget,
3160                                         sizeof(struct msix_entry), GFP_KERNEL);
3161
3162         if (!pf->msix_entries) {
3163                 err = -ENOMEM;
3164                 goto exit_err;
3165         }
3166
3167         for (i = 0; i < v_budget; i++)
3168                 pf->msix_entries[i].entry = i;
3169
3170         /* actually reserve the vectors */
3171         v_actual = pci_enable_msix_range(pf->pdev, pf->msix_entries,
3172                                          ICE_MIN_MSIX, v_budget);
3173
3174         if (v_actual < 0) {
3175                 dev_err(&pf->pdev->dev, "unable to reserve MSI-X vectors\n");
3176                 err = v_actual;
3177                 goto msix_err;
3178         }
3179
3180         if (v_actual < v_budget) {
3181                 dev_warn(&pf->pdev->dev,
3182                          "not enough vectors. requested = %d, obtained = %d\n",
3183                          v_budget, v_actual);
3184                 if (v_actual >= (pf->num_lan_msix + 1)) {
3185                         pf->num_avail_msix = v_actual - (pf->num_lan_msix + 1);
3186                 } else if (v_actual >= 2) {
3187                         pf->num_lan_msix = 1;
3188                         pf->num_avail_msix = v_actual - 2;
3189                 } else {
3190                         pci_disable_msix(pf->pdev);
3191                         err = -ERANGE;
3192                         goto msix_err;
3193                 }
3194         }
3195
3196         return v_actual;
3197
3198 msix_err:
3199         devm_kfree(&pf->pdev->dev, pf->msix_entries);
3200         goto exit_err;
3201
3202 exit_err:
3203         pf->num_lan_msix = 0;
3204         clear_bit(ICE_FLAG_MSIX_ENA, pf->flags);
3205         return err;
3206 }
3207
3208 /**
3209  * ice_dis_msix - Disable MSI-X interrupt setup in OS
3210  * @pf: board private structure
3211  */
3212 static void ice_dis_msix(struct ice_pf *pf)
3213 {
3214         pci_disable_msix(pf->pdev);
3215         devm_kfree(&pf->pdev->dev, pf->msix_entries);
3216         pf->msix_entries = NULL;
3217         clear_bit(ICE_FLAG_MSIX_ENA, pf->flags);
3218 }
3219
3220 /**
3221  * ice_init_interrupt_scheme - Determine proper interrupt scheme
3222  * @pf: board private structure to initialize
3223  */
3224 static int ice_init_interrupt_scheme(struct ice_pf *pf)
3225 {
3226         int vectors = 0;
3227         ssize_t size;
3228
3229         if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags))
3230                 vectors = ice_ena_msix_range(pf);
3231         else
3232                 return -ENODEV;
3233
3234         if (vectors < 0)
3235                 return vectors;
3236
3237         /* set up vector assignment tracking */
3238         size = sizeof(struct ice_res_tracker) + (sizeof(u16) * vectors);
3239
3240         pf->irq_tracker = devm_kzalloc(&pf->pdev->dev, size, GFP_KERNEL);
3241         if (!pf->irq_tracker) {
3242                 ice_dis_msix(pf);
3243                 return -ENOMEM;
3244         }
3245
3246         pf->irq_tracker->num_entries = vectors;
3247
3248         return 0;
3249 }
3250
3251 /**
3252  * ice_clear_interrupt_scheme - Undo things done by ice_init_interrupt_scheme
3253  * @pf: board private structure
3254  */
3255 static void ice_clear_interrupt_scheme(struct ice_pf *pf)
3256 {
3257         if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags))
3258                 ice_dis_msix(pf);
3259
3260         if (pf->irq_tracker) {
3261                 devm_kfree(&pf->pdev->dev, pf->irq_tracker);
3262                 pf->irq_tracker = NULL;
3263         }
3264 }
3265
3266 /**
3267  * ice_probe - Device initialization routine
3268  * @pdev: PCI device information struct
3269  * @ent: entry in ice_pci_tbl
3270  *
3271  * Returns 0 on success, negative on failure
3272  */
3273 static int ice_probe(struct pci_dev *pdev,
3274                      const struct pci_device_id __always_unused *ent)
3275 {
3276         struct ice_pf *pf;
3277         struct ice_hw *hw;
3278         int err;
3279
3280         /* this driver uses devres, see Documentation/driver-model/devres.txt */
3281         err = pcim_enable_device(pdev);
3282         if (err)
3283                 return err;
3284
3285         err = pcim_iomap_regions(pdev, BIT(ICE_BAR0), pci_name(pdev));
3286         if (err) {
3287                 dev_err(&pdev->dev, "BAR0 I/O map error %d\n", err);
3288                 return err;
3289         }
3290
3291         pf = devm_kzalloc(&pdev->dev, sizeof(*pf), GFP_KERNEL);
3292         if (!pf)
3293                 return -ENOMEM;
3294
3295         /* set up for high or low dma */
3296         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3297         if (err)
3298                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
3299         if (err) {
3300                 dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err);
3301                 return err;
3302         }
3303
3304         pci_enable_pcie_error_reporting(pdev);
3305         pci_set_master(pdev);
3306
3307         pf->pdev = pdev;
3308         pci_set_drvdata(pdev, pf);
3309         set_bit(__ICE_DOWN, pf->state);
3310
3311         hw = &pf->hw;
3312         hw->hw_addr = pcim_iomap_table(pdev)[ICE_BAR0];
3313         hw->back = pf;
3314         hw->vendor_id = pdev->vendor;
3315         hw->device_id = pdev->device;
3316         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
3317         hw->subsystem_vendor_id = pdev->subsystem_vendor;
3318         hw->subsystem_device_id = pdev->subsystem_device;
3319         hw->bus.device = PCI_SLOT(pdev->devfn);
3320         hw->bus.func = PCI_FUNC(pdev->devfn);
3321         ice_set_ctrlq_len(hw);
3322
3323         pf->msg_enable = netif_msg_init(debug, ICE_DFLT_NETIF_M);
3324
3325 #ifndef CONFIG_DYNAMIC_DEBUG
3326         if (debug < -1)
3327                 hw->debug_mask = debug;
3328 #endif
3329
3330         err = ice_init_hw(hw);
3331         if (err) {
3332                 dev_err(&pdev->dev, "ice_init_hw failed: %d\n", err);
3333                 err = -EIO;
3334                 goto err_exit_unroll;
3335         }
3336
3337         dev_info(&pdev->dev, "firmware %d.%d.%05d api %d.%d\n",
3338                  hw->fw_maj_ver, hw->fw_min_ver, hw->fw_build,
3339                  hw->api_maj_ver, hw->api_min_ver);
3340
3341         ice_init_pf(pf);
3342
3343         ice_determine_q_usage(pf);
3344
3345         pf->num_alloc_vsi = min_t(u16, ICE_MAX_VSI_ALLOC,
3346                                   hw->func_caps.guaranteed_num_vsi);
3347         if (!pf->num_alloc_vsi) {
3348                 err = -EIO;
3349                 goto err_init_pf_unroll;
3350         }
3351
3352         pf->vsi = devm_kcalloc(&pdev->dev, pf->num_alloc_vsi,
3353                                sizeof(struct ice_vsi *), GFP_KERNEL);
3354         if (!pf->vsi) {
3355                 err = -ENOMEM;
3356                 goto err_init_pf_unroll;
3357         }
3358
3359         err = ice_init_interrupt_scheme(pf);
3360         if (err) {
3361                 dev_err(&pdev->dev,
3362                         "ice_init_interrupt_scheme failed: %d\n", err);
3363                 err = -EIO;
3364                 goto err_init_interrupt_unroll;
3365         }
3366
3367         /* In case of MSIX we are going to setup the misc vector right here
3368          * to handle admin queue events etc. In case of legacy and MSI
3369          * the misc functionality and queue processing is combined in
3370          * the same vector and that gets setup at open.
3371          */
3372         if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) {
3373                 err = ice_req_irq_msix_misc(pf);
3374                 if (err) {
3375                         dev_err(&pdev->dev,
3376                                 "setup of misc vector failed: %d\n", err);
3377                         goto err_init_interrupt_unroll;
3378                 }
3379         }
3380
3381         /* create switch struct for the switch element created by FW on boot */
3382         pf->first_sw = devm_kzalloc(&pdev->dev, sizeof(struct ice_sw),
3383                                     GFP_KERNEL);
3384         if (!pf->first_sw) {
3385                 err = -ENOMEM;
3386                 goto err_msix_misc_unroll;
3387         }
3388
3389         pf->first_sw->bridge_mode = BRIDGE_MODE_VEB;
3390         pf->first_sw->pf = pf;
3391
3392         /* record the sw_id available for later use */
3393         pf->first_sw->sw_id = hw->port_info->sw_id;
3394
3395         err = ice_setup_pf_sw(pf);
3396         if (err) {
3397                 dev_err(&pdev->dev,
3398                         "probe failed due to setup pf switch:%d\n", err);
3399                 goto err_alloc_sw_unroll;
3400         }
3401
3402         /* Driver is mostly up */
3403         clear_bit(__ICE_DOWN, pf->state);
3404
3405         /* since everything is good, start the service timer */
3406         mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period));
3407
3408         err = ice_init_link_events(pf->hw.port_info);
3409         if (err) {
3410                 dev_err(&pdev->dev, "ice_init_link_events failed: %d\n", err);
3411                 goto err_alloc_sw_unroll;
3412         }
3413
3414         return 0;
3415
3416 err_alloc_sw_unroll:
3417         set_bit(__ICE_DOWN, pf->state);
3418         devm_kfree(&pf->pdev->dev, pf->first_sw);
3419 err_msix_misc_unroll:
3420         ice_free_irq_msix_misc(pf);
3421 err_init_interrupt_unroll:
3422         ice_clear_interrupt_scheme(pf);
3423         devm_kfree(&pdev->dev, pf->vsi);
3424 err_init_pf_unroll:
3425         ice_deinit_pf(pf);
3426         ice_deinit_hw(hw);
3427 err_exit_unroll:
3428         pci_disable_pcie_error_reporting(pdev);
3429         return err;
3430 }
3431
3432 /**
3433  * ice_remove - Device removal routine
3434  * @pdev: PCI device information struct
3435  */
3436 static void ice_remove(struct pci_dev *pdev)
3437 {
3438         struct ice_pf *pf = pci_get_drvdata(pdev);
3439         int i = 0;
3440         int err;
3441
3442         if (!pf)
3443                 return;
3444
3445         set_bit(__ICE_DOWN, pf->state);
3446
3447         for (i = 0; i < pf->num_alloc_vsi; i++) {
3448                 if (!pf->vsi[i])
3449                         continue;
3450
3451                 err = ice_vsi_release(pf->vsi[i]);
3452                 if (err)
3453                         dev_dbg(&pf->pdev->dev, "Failed to release VSI index %d (err %d)\n",
3454                                 i, err);
3455         }
3456
3457         ice_free_irq_msix_misc(pf);
3458         ice_clear_interrupt_scheme(pf);
3459         ice_deinit_pf(pf);
3460         ice_deinit_hw(&pf->hw);
3461         pci_disable_pcie_error_reporting(pdev);
3462 }
3463
3464 /* ice_pci_tbl - PCI Device ID Table
3465  *
3466  * Wildcard entries (PCI_ANY_ID) should come last
3467  * Last entry must be all 0s
3468  *
3469  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
3470  *   Class, Class Mask, private data (not used) }
3471  */
3472 static const struct pci_device_id ice_pci_tbl[] = {
3473         { PCI_VDEVICE(INTEL, ICE_DEV_ID_C810_BACKPLANE), 0 },
3474         { PCI_VDEVICE(INTEL, ICE_DEV_ID_C810_QSFP), 0 },
3475         { PCI_VDEVICE(INTEL, ICE_DEV_ID_C810_SFP), 0 },
3476         { PCI_VDEVICE(INTEL, ICE_DEV_ID_C810_10G_BASE_T), 0 },
3477         { PCI_VDEVICE(INTEL, ICE_DEV_ID_C810_SGMII), 0 },
3478         /* required last entry */
3479         { 0, }
3480 };
3481 MODULE_DEVICE_TABLE(pci, ice_pci_tbl);
3482
3483 static struct pci_driver ice_driver = {
3484         .name = KBUILD_MODNAME,
3485         .id_table = ice_pci_tbl,
3486         .probe = ice_probe,
3487         .remove = ice_remove,
3488 };
3489
3490 /**
3491  * ice_module_init - Driver registration routine
3492  *
3493  * ice_module_init is the first routine called when the driver is
3494  * loaded. All it does is register with the PCI subsystem.
3495  */
3496 static int __init ice_module_init(void)
3497 {
3498         int status;
3499
3500         pr_info("%s - version %s\n", ice_driver_string, ice_drv_ver);
3501         pr_info("%s\n", ice_copyright);
3502
3503         ice_wq = alloc_ordered_workqueue("%s", WQ_MEM_RECLAIM, KBUILD_MODNAME);
3504         if (!ice_wq) {
3505                 pr_err("Failed to create workqueue\n");
3506                 return -ENOMEM;
3507         }
3508
3509         status = pci_register_driver(&ice_driver);
3510         if (status) {
3511                 pr_err("failed to register pci driver, err %d\n", status);
3512                 destroy_workqueue(ice_wq);
3513         }
3514
3515         return status;
3516 }
3517 module_init(ice_module_init);
3518
3519 /**
3520  * ice_module_exit - Driver exit cleanup routine
3521  *
3522  * ice_module_exit is called just before the driver is removed
3523  * from memory.
3524  */
3525 static void __exit ice_module_exit(void)
3526 {
3527         pci_unregister_driver(&ice_driver);
3528         destroy_workqueue(ice_wq);
3529         pr_info("module unloaded\n");
3530 }
3531 module_exit(ice_module_exit);
3532
3533 /**
3534  * ice_set_mac_address - NDO callback to set mac address
3535  * @netdev: network interface device structure
3536  * @pi: pointer to an address structure
3537  *
3538  * Returns 0 on success, negative on failure
3539  */
3540 static int ice_set_mac_address(struct net_device *netdev, void *pi)
3541 {
3542         struct ice_netdev_priv *np = netdev_priv(netdev);
3543         struct ice_vsi *vsi = np->vsi;
3544         struct ice_pf *pf = vsi->back;
3545         struct ice_hw *hw = &pf->hw;
3546         struct sockaddr *addr = pi;
3547         enum ice_status status;
3548         LIST_HEAD(a_mac_list);
3549         LIST_HEAD(r_mac_list);
3550         u8 flags = 0;
3551         int err;
3552         u8 *mac;
3553
3554         mac = (u8 *)addr->sa_data;
3555
3556         if (!is_valid_ether_addr(mac))
3557                 return -EADDRNOTAVAIL;
3558
3559         if (ether_addr_equal(netdev->dev_addr, mac)) {
3560                 netdev_warn(netdev, "already using mac %pM\n", mac);
3561                 return 0;
3562         }
3563
3564         if (test_bit(__ICE_DOWN, pf->state) ||
3565             ice_is_reset_recovery_pending(pf->state)) {
3566                 netdev_err(netdev, "can't set mac %pM. device not ready\n",
3567                            mac);
3568                 return -EBUSY;
3569         }
3570
3571         /* When we change the mac address we also have to change the mac address
3572          * based filter rules that were created previously for the old mac
3573          * address. So first, we remove the old filter rule using ice_remove_mac
3574          * and then create a new filter rule using ice_add_mac. Note that for
3575          * both these operations, we first need to form a "list" of mac
3576          * addresses (even though in this case, we have only 1 mac address to be
3577          * added/removed) and this done using ice_add_mac_to_list. Depending on
3578          * the ensuing operation this "list" of mac addresses is either to be
3579          * added or removed from the filter.
3580          */
3581         err = ice_add_mac_to_list(vsi, &r_mac_list, netdev->dev_addr);
3582         if (err) {
3583                 err = -EADDRNOTAVAIL;
3584                 goto free_lists;
3585         }
3586
3587         status = ice_remove_mac(hw, &r_mac_list);
3588         if (status) {
3589                 err = -EADDRNOTAVAIL;
3590                 goto free_lists;
3591         }
3592
3593         err = ice_add_mac_to_list(vsi, &a_mac_list, mac);
3594         if (err) {
3595                 err = -EADDRNOTAVAIL;
3596                 goto free_lists;
3597         }
3598
3599         status = ice_add_mac(hw, &a_mac_list);
3600         if (status) {
3601                 err = -EADDRNOTAVAIL;
3602                 goto free_lists;
3603         }
3604
3605 free_lists:
3606         /* free list entries */
3607         ice_free_fltr_list(&pf->pdev->dev, &r_mac_list);
3608         ice_free_fltr_list(&pf->pdev->dev, &a_mac_list);
3609
3610         if (err) {
3611                 netdev_err(netdev, "can't set mac %pM. filter update failed\n",
3612                            mac);
3613                 return err;
3614         }
3615
3616         /* change the netdev's mac address */
3617         memcpy(netdev->dev_addr, mac, netdev->addr_len);
3618         netdev_dbg(vsi->netdev, "updated mac address to %pM\n",
3619                    netdev->dev_addr);
3620
3621         /* write new mac address to the firmware */
3622         flags = ICE_AQC_MAN_MAC_UPDATE_LAA_WOL;
3623         status = ice_aq_manage_mac_write(hw, mac, flags, NULL);
3624         if (status) {
3625                 netdev_err(netdev, "can't set mac %pM. write to firmware failed.\n",
3626                            mac);
3627         }
3628         return 0;
3629 }
3630
3631 /**
3632  * ice_set_rx_mode - NDO callback to set the netdev filters
3633  * @netdev: network interface device structure
3634  */
3635 static void ice_set_rx_mode(struct net_device *netdev)
3636 {
3637         struct ice_netdev_priv *np = netdev_priv(netdev);
3638         struct ice_vsi *vsi = np->vsi;
3639
3640         if (!vsi)
3641                 return;
3642
3643         /* Set the flags to synchronize filters
3644          * ndo_set_rx_mode may be triggered even without a change in netdev
3645          * flags
3646          */
3647         set_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags);
3648         set_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags);
3649         set_bit(ICE_FLAG_FLTR_SYNC, vsi->back->flags);
3650
3651         /* schedule our worker thread which will take care of
3652          * applying the new filter changes
3653          */
3654         ice_service_task_schedule(vsi->back);
3655 }
3656
3657 /**
3658  * ice_fdb_add - add an entry to the hardware database
3659  * @ndm: the input from the stack
3660  * @tb: pointer to array of nladdr (unused)
3661  * @dev: the net device pointer
3662  * @addr: the MAC address entry being added
3663  * @vid: VLAN id
3664  * @flags: instructions from stack about fdb operation
3665  */
3666 static int ice_fdb_add(struct ndmsg *ndm, struct nlattr __always_unused *tb[],
3667                        struct net_device *dev, const unsigned char *addr,
3668                        u16 vid, u16 flags)
3669 {
3670         int err;
3671
3672         if (vid) {
3673                 netdev_err(dev, "VLANs aren't supported yet for dev_uc|mc_add()\n");
3674                 return -EINVAL;
3675         }
3676         if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
3677                 netdev_err(dev, "FDB only supports static addresses\n");
3678                 return -EINVAL;
3679         }
3680
3681         if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
3682                 err = dev_uc_add_excl(dev, addr);
3683         else if (is_multicast_ether_addr(addr))
3684                 err = dev_mc_add_excl(dev, addr);
3685         else
3686                 err = -EINVAL;
3687
3688         /* Only return duplicate errors if NLM_F_EXCL is set */
3689         if (err == -EEXIST && !(flags & NLM_F_EXCL))
3690                 err = 0;
3691
3692         return err;
3693 }
3694
3695 /**
3696  * ice_fdb_del - delete an entry from the hardware database
3697  * @ndm: the input from the stack
3698  * @tb: pointer to array of nladdr (unused)
3699  * @dev: the net device pointer
3700  * @addr: the MAC address entry being added
3701  * @vid: VLAN id
3702  */
3703 static int ice_fdb_del(struct ndmsg *ndm, __always_unused struct nlattr *tb[],
3704                        struct net_device *dev, const unsigned char *addr,
3705                        __always_unused u16 vid)
3706 {
3707         int err;
3708
3709         if (ndm->ndm_state & NUD_PERMANENT) {
3710                 netdev_err(dev, "FDB only supports static addresses\n");
3711                 return -EINVAL;
3712         }
3713
3714         if (is_unicast_ether_addr(addr))
3715                 err = dev_uc_del(dev, addr);
3716         else if (is_multicast_ether_addr(addr))
3717                 err = dev_mc_del(dev, addr);
3718         else
3719                 err = -EINVAL;
3720
3721         return err;
3722 }
3723
3724 /**
3725  * ice_vsi_manage_vlan_insertion - Manage VLAN insertion for the VSI for Tx
3726  * @vsi: the vsi being changed
3727  */
3728 static int ice_vsi_manage_vlan_insertion(struct ice_vsi *vsi)
3729 {
3730         struct device *dev = &vsi->back->pdev->dev;
3731         struct ice_hw *hw = &vsi->back->hw;
3732         struct ice_vsi_ctx ctxt = { 0 };
3733         enum ice_status status;
3734
3735         /* Here we are configuring the VSI to let the driver add VLAN tags by
3736          * setting vlan_flags to ICE_AQ_VSI_VLAN_MODE_ALL. The actual VLAN tag
3737          * insertion happens in the Tx hot path, in ice_tx_map.
3738          */
3739         ctxt.info.vlan_flags = ICE_AQ_VSI_VLAN_MODE_ALL;
3740
3741         ctxt.info.valid_sections = cpu_to_le16(ICE_AQ_VSI_PROP_VLAN_VALID);
3742         ctxt.vsi_num = vsi->vsi_num;
3743
3744         status = ice_aq_update_vsi(hw, &ctxt, NULL);
3745         if (status) {
3746                 dev_err(dev, "update VSI for VLAN insert failed, err %d aq_err %d\n",
3747                         status, hw->adminq.sq_last_status);
3748                 return -EIO;
3749         }
3750
3751         vsi->info.vlan_flags = ctxt.info.vlan_flags;
3752         return 0;
3753 }
3754
3755 /**
3756  * ice_vsi_manage_vlan_stripping - Manage VLAN stripping for the VSI for Rx
3757  * @vsi: the vsi being changed
3758  * @ena: boolean value indicating if this is a enable or disable request
3759  */
3760 static int ice_vsi_manage_vlan_stripping(struct ice_vsi *vsi, bool ena)
3761 {
3762         struct device *dev = &vsi->back->pdev->dev;
3763         struct ice_hw *hw = &vsi->back->hw;
3764         struct ice_vsi_ctx ctxt = { 0 };
3765         enum ice_status status;
3766
3767         /* Here we are configuring what the VSI should do with the VLAN tag in
3768          * the Rx packet. We can either leave the tag in the packet or put it in
3769          * the Rx descriptor.
3770          */
3771         if (ena) {
3772                 /* Strip VLAN tag from Rx packet and put it in the desc */
3773                 ctxt.info.vlan_flags = ICE_AQ_VSI_VLAN_EMOD_STR_BOTH;
3774         } else {
3775                 /* Disable stripping. Leave tag in packet */
3776                 ctxt.info.vlan_flags = ICE_AQ_VSI_VLAN_EMOD_NOTHING;
3777         }
3778
3779         /* Allow all packets untagged/tagged */
3780         ctxt.info.vlan_flags |= ICE_AQ_VSI_VLAN_MODE_ALL;
3781
3782         ctxt.info.valid_sections = cpu_to_le16(ICE_AQ_VSI_PROP_VLAN_VALID);
3783         ctxt.vsi_num = vsi->vsi_num;
3784
3785         status = ice_aq_update_vsi(hw, &ctxt, NULL);
3786         if (status) {
3787                 dev_err(dev, "update VSI for VALN strip failed, ena = %d err %d aq_err %d\n",
3788                         ena, status, hw->adminq.sq_last_status);
3789                 return -EIO;
3790         }
3791
3792         vsi->info.vlan_flags = ctxt.info.vlan_flags;
3793         return 0;
3794 }
3795
3796 /**
3797  * ice_set_features - set the netdev feature flags
3798  * @netdev: ptr to the netdev being adjusted
3799  * @features: the feature set that the stack is suggesting
3800  */
3801 static int ice_set_features(struct net_device *netdev,
3802                             netdev_features_t features)
3803 {
3804         struct ice_netdev_priv *np = netdev_priv(netdev);
3805         struct ice_vsi *vsi = np->vsi;
3806         int ret = 0;
3807
3808         if ((features & NETIF_F_HW_VLAN_CTAG_RX) &&
3809             !(netdev->features & NETIF_F_HW_VLAN_CTAG_RX))
3810                 ret = ice_vsi_manage_vlan_stripping(vsi, true);
3811         else if (!(features & NETIF_F_HW_VLAN_CTAG_RX) &&
3812                  (netdev->features & NETIF_F_HW_VLAN_CTAG_RX))
3813                 ret = ice_vsi_manage_vlan_stripping(vsi, false);
3814         else if ((features & NETIF_F_HW_VLAN_CTAG_TX) &&
3815                  !(netdev->features & NETIF_F_HW_VLAN_CTAG_TX))
3816                 ret = ice_vsi_manage_vlan_insertion(vsi);
3817         else if (!(features & NETIF_F_HW_VLAN_CTAG_TX) &&
3818                  (netdev->features & NETIF_F_HW_VLAN_CTAG_TX))
3819                 ret = ice_vsi_manage_vlan_insertion(vsi);
3820
3821         return ret;
3822 }
3823
3824 /**
3825  * ice_vsi_vlan_setup - Setup vlan offload properties on a VSI
3826  * @vsi: VSI to setup vlan properties for
3827  */
3828 static int ice_vsi_vlan_setup(struct ice_vsi *vsi)
3829 {
3830         int ret = 0;
3831
3832         if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
3833                 ret = ice_vsi_manage_vlan_stripping(vsi, true);
3834         if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_TX)
3835                 ret = ice_vsi_manage_vlan_insertion(vsi);
3836
3837         return ret;
3838 }
3839
3840 /**
3841  * ice_restore_vlan - Reinstate VLANs when vsi/netdev comes back up
3842  * @vsi: the VSI being brought back up
3843  */
3844 static int ice_restore_vlan(struct ice_vsi *vsi)
3845 {
3846         int err;
3847         u16 vid;
3848
3849         if (!vsi->netdev)
3850                 return -EINVAL;
3851
3852         err = ice_vsi_vlan_setup(vsi);
3853         if (err)
3854                 return err;
3855
3856         for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID) {
3857                 err = ice_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q), vid);
3858                 if (err)
3859                         break;
3860         }
3861
3862         return err;
3863 }
3864
3865 /**
3866  * ice_setup_tx_ctx - setup a struct ice_tlan_ctx instance
3867  * @ring: The Tx ring to configure
3868  * @tlan_ctx: Pointer to the Tx LAN queue context structure to be initialized
3869  * @pf_q: queue index in the PF space
3870  *
3871  * Configure the Tx descriptor ring in TLAN context.
3872  */
3873 static void
3874 ice_setup_tx_ctx(struct ice_ring *ring, struct ice_tlan_ctx *tlan_ctx, u16 pf_q)
3875 {
3876         struct ice_vsi *vsi = ring->vsi;
3877         struct ice_hw *hw = &vsi->back->hw;
3878
3879         tlan_ctx->base = ring->dma >> ICE_TLAN_CTX_BASE_S;
3880
3881         tlan_ctx->port_num = vsi->port_info->lport;
3882
3883         /* Transmit Queue Length */
3884         tlan_ctx->qlen = ring->count;
3885
3886         /* PF number */
3887         tlan_ctx->pf_num = hw->pf_id;
3888
3889         /* queue belongs to a specific VSI type
3890          * VF / VM index should be programmed per vmvf_type setting:
3891          * for vmvf_type = VF, it is VF number between 0-256
3892          * for vmvf_type = VM, it is VM number between 0-767
3893          * for PF or EMP this field should be set to zero
3894          */
3895         switch (vsi->type) {
3896         case ICE_VSI_PF:
3897                 tlan_ctx->vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_PF;
3898                 break;
3899         default:
3900                 return;
3901         }
3902
3903         /* make sure the context is associated with the right VSI */
3904         tlan_ctx->src_vsi = vsi->vsi_num;
3905
3906         tlan_ctx->tso_ena = ICE_TX_LEGACY;
3907         tlan_ctx->tso_qnum = pf_q;
3908
3909         /* Legacy or Advanced Host Interface:
3910          * 0: Advanced Host Interface
3911          * 1: Legacy Host Interface
3912          */
3913         tlan_ctx->legacy_int = ICE_TX_LEGACY;
3914 }
3915
3916 /**
3917  * ice_vsi_cfg_txqs - Configure the VSI for Tx
3918  * @vsi: the VSI being configured
3919  *
3920  * Return 0 on success and a negative value on error
3921  * Configure the Tx VSI for operation.
3922  */
3923 static int ice_vsi_cfg_txqs(struct ice_vsi *vsi)
3924 {
3925         struct ice_aqc_add_tx_qgrp *qg_buf;
3926         struct ice_aqc_add_txqs_perq *txq;
3927         struct ice_pf *pf = vsi->back;
3928         enum ice_status status;
3929         u16 buf_len, i, pf_q;
3930         int err = 0, tc = 0;
3931         u8 num_q_grps;
3932
3933         buf_len = sizeof(struct ice_aqc_add_tx_qgrp);
3934         qg_buf = devm_kzalloc(&pf->pdev->dev, buf_len, GFP_KERNEL);
3935         if (!qg_buf)
3936                 return -ENOMEM;
3937
3938         if (vsi->num_txq > ICE_MAX_TXQ_PER_TXQG) {
3939                 err = -EINVAL;
3940                 goto err_cfg_txqs;
3941         }
3942         qg_buf->num_txqs = 1;
3943         num_q_grps = 1;
3944
3945         /* set up and configure the tx queues */
3946         ice_for_each_txq(vsi, i) {
3947                 struct ice_tlan_ctx tlan_ctx = { 0 };
3948
3949                 pf_q = vsi->txq_map[i];
3950                 ice_setup_tx_ctx(vsi->tx_rings[i], &tlan_ctx, pf_q);
3951                 /* copy context contents into the qg_buf */
3952                 qg_buf->txqs[0].txq_id = cpu_to_le16(pf_q);
3953                 ice_set_ctx((u8 *)&tlan_ctx, qg_buf->txqs[0].txq_ctx,
3954                             ice_tlan_ctx_info);
3955
3956                 /* init queue specific tail reg. It is referred as transmit
3957                  * comm scheduler queue doorbell.
3958                  */
3959                 vsi->tx_rings[i]->tail = pf->hw.hw_addr + QTX_COMM_DBELL(pf_q);
3960                 status = ice_ena_vsi_txq(vsi->port_info, vsi->vsi_num, tc,
3961                                          num_q_grps, qg_buf, buf_len, NULL);
3962                 if (status) {
3963                         dev_err(&vsi->back->pdev->dev,
3964                                 "Failed to set LAN Tx queue context, error: %d\n",
3965                                 status);
3966                         err = -ENODEV;
3967                         goto err_cfg_txqs;
3968                 }
3969
3970                 /* Add Tx Queue TEID into the VSI tx ring from the response
3971                  * This will complete configuring and enabling the queue.
3972                  */
3973                 txq = &qg_buf->txqs[0];
3974                 if (pf_q == le16_to_cpu(txq->txq_id))
3975                         vsi->tx_rings[i]->txq_teid =
3976                                 le32_to_cpu(txq->q_teid);
3977         }
3978 err_cfg_txqs:
3979         devm_kfree(&pf->pdev->dev, qg_buf);
3980         return err;
3981 }
3982
3983 /**
3984  * ice_setup_rx_ctx - Configure a receive ring context
3985  * @ring: The Rx ring to configure
3986  *
3987  * Configure the Rx descriptor ring in RLAN context.
3988  */
3989 static int ice_setup_rx_ctx(struct ice_ring *ring)
3990 {
3991         struct ice_vsi *vsi = ring->vsi;
3992         struct ice_hw *hw = &vsi->back->hw;
3993         u32 rxdid = ICE_RXDID_FLEX_NIC;
3994         struct ice_rlan_ctx rlan_ctx;
3995         u32 regval;
3996         u16 pf_q;
3997         int err;
3998
3999         /* what is RX queue number in global space of 2K rx queues */
4000         pf_q = vsi->rxq_map[ring->q_index];
4001
4002         /* clear the context structure first */
4003         memset(&rlan_ctx, 0, sizeof(rlan_ctx));
4004
4005         rlan_ctx.base = ring->dma >> ICE_RLAN_BASE_S;
4006
4007         rlan_ctx.qlen = ring->count;
4008
4009         /* Receive Packet Data Buffer Size.
4010          * The Packet Data Buffer Size is defined in 128 byte units.
4011          */
4012         rlan_ctx.dbuf = vsi->rx_buf_len >> ICE_RLAN_CTX_DBUF_S;
4013
4014         /* use 32 byte descriptors */
4015         rlan_ctx.dsize = 1;
4016
4017         /* Strip the Ethernet CRC bytes before the packet is posted to host
4018          * memory.
4019          */
4020         rlan_ctx.crcstrip = 1;
4021
4022         /* L2TSEL flag defines the reported L2 Tags in the receive descriptor */
4023         rlan_ctx.l2tsel = 1;
4024
4025         rlan_ctx.dtype = ICE_RX_DTYPE_NO_SPLIT;
4026         rlan_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_NO_SPLIT;
4027         rlan_ctx.hsplit_1 = ICE_RLAN_RX_HSPLIT_1_NO_SPLIT;
4028
4029         /* This controls whether VLAN is stripped from inner headers
4030          * The VLAN in the inner L2 header is stripped to the receive
4031          * descriptor if enabled by this flag.
4032          */
4033         rlan_ctx.showiv = 0;
4034
4035         /* Max packet size for this queue - must not be set to a larger value
4036          * than 5 x DBUF
4037          */
4038         rlan_ctx.rxmax = min_t(u16, vsi->max_frame,
4039                                ICE_MAX_CHAINED_RX_BUFS * vsi->rx_buf_len);
4040
4041         /* Rx queue threshold in units of 64 */
4042         rlan_ctx.lrxqthresh = 1;
4043
4044          /* Enable Flexible Descriptors in the queue context which
4045           * allows this driver to select a specific receive descriptor format
4046           */
4047         regval = rd32(hw, QRXFLXP_CNTXT(pf_q));
4048         regval |= (rxdid << QRXFLXP_CNTXT_RXDID_IDX_S) &
4049                 QRXFLXP_CNTXT_RXDID_IDX_M;
4050
4051         /* increasing context priority to pick up profile id;
4052          * default is 0x01; setting to 0x03 to ensure profile
4053          * is programming if prev context is of same priority
4054          */
4055         regval |= (0x03 << QRXFLXP_CNTXT_RXDID_PRIO_S) &
4056                 QRXFLXP_CNTXT_RXDID_PRIO_M;
4057
4058         wr32(hw, QRXFLXP_CNTXT(pf_q), regval);
4059
4060         /* Absolute queue number out of 2K needs to be passed */
4061         err = ice_write_rxq_ctx(hw, &rlan_ctx, pf_q);
4062         if (err) {
4063                 dev_err(&vsi->back->pdev->dev,
4064                         "Failed to set LAN Rx queue context for absolute Rx queue %d error: %d\n",
4065                         pf_q, err);
4066                 return -EIO;
4067         }
4068
4069         /* init queue specific tail register */
4070         ring->tail = hw->hw_addr + QRX_TAIL(pf_q);
4071         writel(0, ring->tail);
4072         ice_alloc_rx_bufs(ring, ICE_DESC_UNUSED(ring));
4073
4074         return 0;
4075 }
4076
4077 /**
4078  * ice_vsi_cfg_rxqs - Configure the VSI for Rx
4079  * @vsi: the VSI being configured
4080  *
4081  * Return 0 on success and a negative value on error
4082  * Configure the Rx VSI for operation.
4083  */
4084 static int ice_vsi_cfg_rxqs(struct ice_vsi *vsi)
4085 {
4086         int err = 0;
4087         u16 i;
4088
4089         if (vsi->netdev && vsi->netdev->mtu > ETH_DATA_LEN)
4090                 vsi->max_frame = vsi->netdev->mtu +
4091                         ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
4092         else
4093                 vsi->max_frame = ICE_RXBUF_2048;
4094
4095         vsi->rx_buf_len = ICE_RXBUF_2048;
4096         /* set up individual rings */
4097         for (i = 0; i < vsi->num_rxq && !err; i++)
4098                 err = ice_setup_rx_ctx(vsi->rx_rings[i]);
4099
4100         if (err) {
4101                 dev_err(&vsi->back->pdev->dev, "ice_setup_rx_ctx failed\n");
4102                 return -EIO;
4103         }
4104         return err;
4105 }
4106
4107 /**
4108  * ice_vsi_cfg - Setup the VSI
4109  * @vsi: the VSI being configured
4110  *
4111  * Return 0 on success and negative value on error
4112  */
4113 static int ice_vsi_cfg(struct ice_vsi *vsi)
4114 {
4115         int err;
4116
4117         if (vsi->netdev) {
4118                 ice_set_rx_mode(vsi->netdev);
4119                 err = ice_restore_vlan(vsi);
4120                 if (err)
4121                         return err;
4122         }
4123
4124         err = ice_vsi_cfg_txqs(vsi);
4125         if (!err)
4126                 err = ice_vsi_cfg_rxqs(vsi);
4127
4128         return err;
4129 }
4130
4131 /**
4132  * ice_vsi_stop_tx_rings - Disable Tx rings
4133  * @vsi: the VSI being configured
4134  */
4135 static int ice_vsi_stop_tx_rings(struct ice_vsi *vsi)
4136 {
4137         struct ice_pf *pf = vsi->back;
4138         struct ice_hw *hw = &pf->hw;
4139         enum ice_status status;
4140         u32 *q_teids, val;
4141         u16 *q_ids, i;
4142         int err = 0;
4143
4144         if (vsi->num_txq > ICE_LAN_TXQ_MAX_QDIS)
4145                 return -EINVAL;
4146
4147         q_teids = devm_kcalloc(&pf->pdev->dev, vsi->num_txq, sizeof(*q_teids),
4148                                GFP_KERNEL);
4149         if (!q_teids)
4150                 return -ENOMEM;
4151
4152         q_ids = devm_kcalloc(&pf->pdev->dev, vsi->num_txq, sizeof(*q_ids),
4153                              GFP_KERNEL);
4154         if (!q_ids) {
4155                 err = -ENOMEM;
4156                 goto err_alloc_q_ids;
4157         }
4158
4159         /* set up the tx queue list to be disabled */
4160         ice_for_each_txq(vsi, i) {
4161                 u16 v_idx;
4162
4163                 if (!vsi->tx_rings || !vsi->tx_rings[i]) {
4164                         err = -EINVAL;
4165                         goto err_out;
4166                 }
4167
4168                 q_ids[i] = vsi->txq_map[i];
4169                 q_teids[i] = vsi->tx_rings[i]->txq_teid;
4170
4171                 /* clear cause_ena bit for disabled queues */
4172                 val = rd32(hw, QINT_TQCTL(vsi->tx_rings[i]->reg_idx));
4173                 val &= ~QINT_TQCTL_CAUSE_ENA_M;
4174                 wr32(hw, QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val);
4175
4176                 /* software is expected to wait for 100 ns */
4177                 ndelay(100);
4178
4179                 /* trigger a software interrupt for the vector associated to
4180                  * the queue to schedule napi handler
4181                  */
4182                 v_idx = vsi->tx_rings[i]->q_vector->v_idx;
4183                 wr32(hw, GLINT_DYN_CTL(vsi->base_vector + v_idx),
4184                      GLINT_DYN_CTL_SWINT_TRIG_M | GLINT_DYN_CTL_INTENA_MSK_M);
4185         }
4186         status = ice_dis_vsi_txq(vsi->port_info, vsi->num_txq, q_ids, q_teids,
4187                                  NULL);
4188         if (status) {
4189                 dev_err(&pf->pdev->dev,
4190                         "Failed to disable LAN Tx queues, error: %d\n",
4191                         status);
4192                 err = -ENODEV;
4193         }
4194
4195 err_out:
4196         devm_kfree(&pf->pdev->dev, q_ids);
4197
4198 err_alloc_q_ids:
4199         devm_kfree(&pf->pdev->dev, q_teids);
4200
4201         return err;
4202 }
4203
4204 /**
4205  * ice_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
4206  * @pf: the PF being configured
4207  * @pf_q: the PF queue
4208  * @ena: enable or disable state of the queue
4209  *
4210  * This routine will wait for the given Rx queue of the PF to reach the
4211  * enabled or disabled state.
4212  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4213  * multiple retries; else will return 0 in case of success.
4214  */
4215 static int ice_pf_rxq_wait(struct ice_pf *pf, int pf_q, bool ena)
4216 {
4217         int i;
4218
4219         for (i = 0; i < ICE_Q_WAIT_RETRY_LIMIT; i++) {
4220                 u32 rx_reg = rd32(&pf->hw, QRX_CTRL(pf_q));
4221
4222                 if (ena == !!(rx_reg & QRX_CTRL_QENA_STAT_M))
4223                         break;
4224
4225                 usleep_range(10, 20);
4226         }
4227         if (i >= ICE_Q_WAIT_RETRY_LIMIT)
4228                 return -ETIMEDOUT;
4229
4230         return 0;
4231 }
4232
4233 /**
4234  * ice_vsi_ctrl_rx_rings - Start or stop a VSI's rx rings
4235  * @vsi: the VSI being configured
4236  * @ena: start or stop the rx rings
4237  */
4238 static int ice_vsi_ctrl_rx_rings(struct ice_vsi *vsi, bool ena)
4239 {
4240         struct ice_pf *pf = vsi->back;
4241         struct ice_hw *hw = &pf->hw;
4242         int i, j, ret = 0;
4243
4244         for (i = 0; i < vsi->num_rxq; i++) {
4245                 int pf_q = vsi->rxq_map[i];
4246                 u32 rx_reg;
4247
4248                 for (j = 0; j < ICE_Q_WAIT_MAX_RETRY; j++) {
4249                         rx_reg = rd32(hw, QRX_CTRL(pf_q));
4250                         if (((rx_reg >> QRX_CTRL_QENA_REQ_S) & 1) ==
4251                             ((rx_reg >> QRX_CTRL_QENA_STAT_S) & 1))
4252                                 break;
4253                         usleep_range(1000, 2000);
4254                 }
4255
4256                 /* Skip if the queue is already in the requested state */
4257                 if (ena == !!(rx_reg & QRX_CTRL_QENA_STAT_M))
4258                         continue;
4259
4260                 /* turn on/off the queue */
4261                 if (ena)
4262                         rx_reg |= QRX_CTRL_QENA_REQ_M;
4263                 else
4264                         rx_reg &= ~QRX_CTRL_QENA_REQ_M;
4265                 wr32(hw, QRX_CTRL(pf_q), rx_reg);
4266
4267                 /* wait for the change to finish */
4268                 ret = ice_pf_rxq_wait(pf, pf_q, ena);
4269                 if (ret) {
4270                         dev_err(&pf->pdev->dev,
4271                                 "VSI idx %d Rx ring %d %sable timeout\n",
4272                                 vsi->idx, pf_q, (ena ? "en" : "dis"));
4273                         break;
4274                 }
4275         }
4276
4277         return ret;
4278 }
4279
4280 /**
4281  * ice_vsi_start_rx_rings - start VSI's rx rings
4282  * @vsi: the VSI whose rings are to be started
4283  *
4284  * Returns 0 on success and a negative value on error
4285  */
4286 static int ice_vsi_start_rx_rings(struct ice_vsi *vsi)
4287 {
4288         return ice_vsi_ctrl_rx_rings(vsi, true);
4289 }
4290
4291 /**
4292  * ice_vsi_stop_rx_rings - stop VSI's rx rings
4293  * @vsi: the VSI
4294  *
4295  * Returns 0 on success and a negative value on error
4296  */
4297 static int ice_vsi_stop_rx_rings(struct ice_vsi *vsi)
4298 {
4299         return ice_vsi_ctrl_rx_rings(vsi, false);
4300 }
4301
4302 /**
4303  * ice_vsi_stop_tx_rx_rings - stop VSI's tx and rx rings
4304  * @vsi: the VSI
4305  * Returns 0 on success and a negative value on error
4306  */
4307 static int ice_vsi_stop_tx_rx_rings(struct ice_vsi *vsi)
4308 {
4309         int err_tx, err_rx;
4310
4311         err_tx = ice_vsi_stop_tx_rings(vsi);
4312         if (err_tx)
4313                 dev_dbg(&vsi->back->pdev->dev, "Failed to disable Tx rings\n");
4314
4315         err_rx = ice_vsi_stop_rx_rings(vsi);
4316         if (err_rx)
4317                 dev_dbg(&vsi->back->pdev->dev, "Failed to disable Rx rings\n");
4318
4319         if (err_tx || err_rx)
4320                 return -EIO;
4321
4322         return 0;
4323 }
4324
4325 /**
4326  * ice_napi_enable_all - Enable NAPI for all q_vectors in the VSI
4327  * @vsi: the VSI being configured
4328  */
4329 static void ice_napi_enable_all(struct ice_vsi *vsi)
4330 {
4331         int q_idx;
4332
4333         if (!vsi->netdev)
4334                 return;
4335
4336         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++)
4337                 napi_enable(&vsi->q_vectors[q_idx]->napi);
4338 }
4339
4340 /**
4341  * ice_up_complete - Finish the last steps of bringing up a connection
4342  * @vsi: The VSI being configured
4343  *
4344  * Return 0 on success and negative value on error
4345  */
4346 static int ice_up_complete(struct ice_vsi *vsi)
4347 {
4348         struct ice_pf *pf = vsi->back;
4349         int err;
4350
4351         if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags))
4352                 ice_vsi_cfg_msix(vsi);
4353         else
4354                 return -ENOTSUPP;
4355
4356         /* Enable only Rx rings, Tx rings were enabled by the FW when the
4357          * Tx queue group list was configured and the context bits were
4358          * programmed using ice_vsi_cfg_txqs
4359          */
4360         err = ice_vsi_start_rx_rings(vsi);
4361         if (err)
4362                 return err;
4363
4364         clear_bit(__ICE_DOWN, vsi->state);
4365         ice_napi_enable_all(vsi);
4366         ice_vsi_ena_irq(vsi);
4367
4368         if (vsi->port_info &&
4369             (vsi->port_info->phy.link_info.link_info & ICE_AQ_LINK_UP) &&
4370             vsi->netdev) {
4371                 ice_print_link_msg(vsi, true);
4372                 netif_tx_start_all_queues(vsi->netdev);
4373                 netif_carrier_on(vsi->netdev);
4374         }
4375
4376         ice_service_task_schedule(pf);
4377
4378         return err;
4379 }
4380
4381 /**
4382  * ice_up - Bring the connection back up after being down
4383  * @vsi: VSI being configured
4384  */
4385 int ice_up(struct ice_vsi *vsi)
4386 {
4387         int err;
4388
4389         err = ice_vsi_cfg(vsi);
4390         if (!err)
4391                 err = ice_up_complete(vsi);
4392
4393         return err;
4394 }
4395
4396 /**
4397  * ice_fetch_u64_stats_per_ring - get packets and bytes stats per ring
4398  * @ring: Tx or Rx ring to read stats from
4399  * @pkts: packets stats counter
4400  * @bytes: bytes stats counter
4401  *
4402  * This function fetches stats from the ring considering the atomic operations
4403  * that needs to be performed to read u64 values in 32 bit machine.
4404  */
4405 static void ice_fetch_u64_stats_per_ring(struct ice_ring *ring, u64 *pkts,
4406                                          u64 *bytes)
4407 {
4408         unsigned int start;
4409         *pkts = 0;
4410         *bytes = 0;
4411
4412         if (!ring)
4413                 return;
4414         do {
4415                 start = u64_stats_fetch_begin_irq(&ring->syncp);
4416                 *pkts = ring->stats.pkts;
4417                 *bytes = ring->stats.bytes;
4418         } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
4419 }
4420
4421 /**
4422  * ice_stat_update40 - read 40 bit stat from the chip and update stat values
4423  * @hw: ptr to the hardware info
4424  * @hireg: high 32 bit HW register to read from
4425  * @loreg: low 32 bit HW register to read from
4426  * @prev_stat_loaded: bool to specify if previous stats are loaded
4427  * @prev_stat: ptr to previous loaded stat value
4428  * @cur_stat: ptr to current stat value
4429  */
4430 static void ice_stat_update40(struct ice_hw *hw, u32 hireg, u32 loreg,
4431                               bool prev_stat_loaded, u64 *prev_stat,
4432                               u64 *cur_stat)
4433 {
4434         u64 new_data;
4435
4436         new_data = rd32(hw, loreg);
4437         new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
4438
4439         /* device stats are not reset at PFR, they likely will not be zeroed
4440          * when the driver starts. So save the first values read and use them as
4441          * offsets to be subtracted from the raw values in order to report stats
4442          * that count from zero.
4443          */
4444         if (!prev_stat_loaded)
4445                 *prev_stat = new_data;
4446         if (likely(new_data >= *prev_stat))
4447                 *cur_stat = new_data - *prev_stat;
4448         else
4449                 /* to manage the potential roll-over */
4450                 *cur_stat = (new_data + BIT_ULL(40)) - *prev_stat;
4451         *cur_stat &= 0xFFFFFFFFFFULL;
4452 }
4453
4454 /**
4455  * ice_stat_update32 - read 32 bit stat from the chip and update stat values
4456  * @hw: ptr to the hardware info
4457  * @reg: HW register to read from
4458  * @prev_stat_loaded: bool to specify if previous stats are loaded
4459  * @prev_stat: ptr to previous loaded stat value
4460  * @cur_stat: ptr to current stat value
4461  */
4462 static void ice_stat_update32(struct ice_hw *hw, u32 reg, bool prev_stat_loaded,
4463                               u64 *prev_stat, u64 *cur_stat)
4464 {
4465         u32 new_data;
4466
4467         new_data = rd32(hw, reg);
4468
4469         /* device stats are not reset at PFR, they likely will not be zeroed
4470          * when the driver starts. So save the first values read and use them as
4471          * offsets to be subtracted from the raw values in order to report stats
4472          * that count from zero.
4473          */
4474         if (!prev_stat_loaded)
4475                 *prev_stat = new_data;
4476         if (likely(new_data >= *prev_stat))
4477                 *cur_stat = new_data - *prev_stat;
4478         else
4479                 /* to manage the potential roll-over */
4480                 *cur_stat = (new_data + BIT_ULL(32)) - *prev_stat;
4481 }
4482
4483 /**
4484  * ice_update_eth_stats - Update VSI-specific ethernet statistics counters
4485  * @vsi: the VSI to be updated
4486  */
4487 static void ice_update_eth_stats(struct ice_vsi *vsi)
4488 {
4489         struct ice_eth_stats *prev_es, *cur_es;
4490         struct ice_hw *hw = &vsi->back->hw;
4491         u16 vsi_num = vsi->vsi_num;    /* HW absolute index of a VSI */
4492
4493         prev_es = &vsi->eth_stats_prev;
4494         cur_es = &vsi->eth_stats;
4495
4496         ice_stat_update40(hw, GLV_GORCH(vsi_num), GLV_GORCL(vsi_num),
4497                           vsi->stat_offsets_loaded, &prev_es->rx_bytes,
4498                           &cur_es->rx_bytes);
4499
4500         ice_stat_update40(hw, GLV_UPRCH(vsi_num), GLV_UPRCL(vsi_num),
4501                           vsi->stat_offsets_loaded, &prev_es->rx_unicast,
4502                           &cur_es->rx_unicast);
4503
4504         ice_stat_update40(hw, GLV_MPRCH(vsi_num), GLV_MPRCL(vsi_num),
4505                           vsi->stat_offsets_loaded, &prev_es->rx_multicast,
4506                           &cur_es->rx_multicast);
4507
4508         ice_stat_update40(hw, GLV_BPRCH(vsi_num), GLV_BPRCL(vsi_num),
4509                           vsi->stat_offsets_loaded, &prev_es->rx_broadcast,
4510                           &cur_es->rx_broadcast);
4511
4512         ice_stat_update32(hw, GLV_RDPC(vsi_num), vsi->stat_offsets_loaded,
4513                           &prev_es->rx_discards, &cur_es->rx_discards);
4514
4515         ice_stat_update40(hw, GLV_GOTCH(vsi_num), GLV_GOTCL(vsi_num),
4516                           vsi->stat_offsets_loaded, &prev_es->tx_bytes,
4517                           &cur_es->tx_bytes);
4518
4519         ice_stat_update40(hw, GLV_UPTCH(vsi_num), GLV_UPTCL(vsi_num),
4520                           vsi->stat_offsets_loaded, &prev_es->tx_unicast,
4521                           &cur_es->tx_unicast);
4522
4523         ice_stat_update40(hw, GLV_MPTCH(vsi_num), GLV_MPTCL(vsi_num),
4524                           vsi->stat_offsets_loaded, &prev_es->tx_multicast,
4525                           &cur_es->tx_multicast);
4526
4527         ice_stat_update40(hw, GLV_BPTCH(vsi_num), GLV_BPTCL(vsi_num),
4528                           vsi->stat_offsets_loaded, &prev_es->tx_broadcast,
4529                           &cur_es->tx_broadcast);
4530
4531         ice_stat_update32(hw, GLV_TEPC(vsi_num), vsi->stat_offsets_loaded,
4532                           &prev_es->tx_errors, &cur_es->tx_errors);
4533
4534         vsi->stat_offsets_loaded = true;
4535 }
4536
4537 /**
4538  * ice_update_vsi_ring_stats - Update VSI stats counters
4539  * @vsi: the VSI to be updated
4540  */
4541 static void ice_update_vsi_ring_stats(struct ice_vsi *vsi)
4542 {
4543         struct rtnl_link_stats64 *vsi_stats = &vsi->net_stats;
4544         struct ice_ring *ring;
4545         u64 pkts, bytes;
4546         int i;
4547
4548         /* reset netdev stats */
4549         vsi_stats->tx_packets = 0;
4550         vsi_stats->tx_bytes = 0;
4551         vsi_stats->rx_packets = 0;
4552         vsi_stats->rx_bytes = 0;
4553
4554         /* reset non-netdev (extended) stats */
4555         vsi->tx_restart = 0;
4556         vsi->tx_busy = 0;
4557         vsi->tx_linearize = 0;
4558         vsi->rx_buf_failed = 0;
4559         vsi->rx_page_failed = 0;
4560
4561         rcu_read_lock();
4562
4563         /* update Tx rings counters */
4564         ice_for_each_txq(vsi, i) {
4565                 ring = READ_ONCE(vsi->tx_rings[i]);
4566                 ice_fetch_u64_stats_per_ring(ring, &pkts, &bytes);
4567                 vsi_stats->tx_packets += pkts;
4568                 vsi_stats->tx_bytes += bytes;
4569                 vsi->tx_restart += ring->tx_stats.restart_q;
4570                 vsi->tx_busy += ring->tx_stats.tx_busy;
4571                 vsi->tx_linearize += ring->tx_stats.tx_linearize;
4572         }
4573
4574         /* update Rx rings counters */
4575         ice_for_each_rxq(vsi, i) {
4576                 ring = READ_ONCE(vsi->rx_rings[i]);
4577                 ice_fetch_u64_stats_per_ring(ring, &pkts, &bytes);
4578                 vsi_stats->rx_packets += pkts;
4579                 vsi_stats->rx_bytes += bytes;
4580                 vsi->rx_buf_failed += ring->rx_stats.alloc_buf_failed;
4581                 vsi->rx_page_failed += ring->rx_stats.alloc_page_failed;
4582         }
4583
4584         rcu_read_unlock();
4585 }
4586
4587 /**
4588  * ice_update_vsi_stats - Update VSI stats counters
4589  * @vsi: the VSI to be updated
4590  */
4591 static void ice_update_vsi_stats(struct ice_vsi *vsi)
4592 {
4593         struct rtnl_link_stats64 *cur_ns = &vsi->net_stats;
4594         struct ice_eth_stats *cur_es = &vsi->eth_stats;
4595         struct ice_pf *pf = vsi->back;
4596
4597         if (test_bit(__ICE_DOWN, vsi->state) ||
4598             test_bit(__ICE_CFG_BUSY, pf->state))
4599                 return;
4600
4601         /* get stats as recorded by Tx/Rx rings */
4602         ice_update_vsi_ring_stats(vsi);
4603
4604         /* get VSI stats as recorded by the hardware */
4605         ice_update_eth_stats(vsi);
4606
4607         cur_ns->tx_errors = cur_es->tx_errors;
4608         cur_ns->rx_dropped = cur_es->rx_discards;
4609         cur_ns->tx_dropped = cur_es->tx_discards;
4610         cur_ns->multicast = cur_es->rx_multicast;
4611
4612         /* update some more netdev stats if this is main VSI */
4613         if (vsi->type == ICE_VSI_PF) {
4614                 cur_ns->rx_crc_errors = pf->stats.crc_errors;
4615                 cur_ns->rx_errors = pf->stats.crc_errors +
4616                                     pf->stats.illegal_bytes;
4617                 cur_ns->rx_length_errors = pf->stats.rx_len_errors;
4618         }
4619 }
4620
4621 /**
4622  * ice_update_pf_stats - Update PF port stats counters
4623  * @pf: PF whose stats needs to be updated
4624  */
4625 static void ice_update_pf_stats(struct ice_pf *pf)
4626 {
4627         struct ice_hw_port_stats *prev_ps, *cur_ps;
4628         struct ice_hw *hw = &pf->hw;
4629         u8 pf_id;
4630
4631         prev_ps = &pf->stats_prev;
4632         cur_ps = &pf->stats;
4633         pf_id = hw->pf_id;
4634
4635         ice_stat_update40(hw, GLPRT_GORCH(pf_id), GLPRT_GORCL(pf_id),
4636                           pf->stat_prev_loaded, &prev_ps->eth.rx_bytes,
4637                           &cur_ps->eth.rx_bytes);
4638
4639         ice_stat_update40(hw, GLPRT_UPRCH(pf_id), GLPRT_UPRCL(pf_id),
4640                           pf->stat_prev_loaded, &prev_ps->eth.rx_unicast,
4641                           &cur_ps->eth.rx_unicast);
4642
4643         ice_stat_update40(hw, GLPRT_MPRCH(pf_id), GLPRT_MPRCL(pf_id),
4644                           pf->stat_prev_loaded, &prev_ps->eth.rx_multicast,
4645                           &cur_ps->eth.rx_multicast);
4646
4647         ice_stat_update40(hw, GLPRT_BPRCH(pf_id), GLPRT_BPRCL(pf_id),
4648                           pf->stat_prev_loaded, &prev_ps->eth.rx_broadcast,
4649                           &cur_ps->eth.rx_broadcast);
4650
4651         ice_stat_update40(hw, GLPRT_GOTCH(pf_id), GLPRT_GOTCL(pf_id),
4652                           pf->stat_prev_loaded, &prev_ps->eth.tx_bytes,
4653                           &cur_ps->eth.tx_bytes);
4654
4655         ice_stat_update40(hw, GLPRT_UPTCH(pf_id), GLPRT_UPTCL(pf_id),
4656                           pf->stat_prev_loaded, &prev_ps->eth.tx_unicast,
4657                           &cur_ps->eth.tx_unicast);
4658
4659         ice_stat_update40(hw, GLPRT_MPTCH(pf_id), GLPRT_MPTCL(pf_id),
4660                           pf->stat_prev_loaded, &prev_ps->eth.tx_multicast,
4661                           &cur_ps->eth.tx_multicast);
4662
4663         ice_stat_update40(hw, GLPRT_BPTCH(pf_id), GLPRT_BPTCL(pf_id),
4664                           pf->stat_prev_loaded, &prev_ps->eth.tx_broadcast,
4665                           &cur_ps->eth.tx_broadcast);
4666
4667         ice_stat_update32(hw, GLPRT_TDOLD(pf_id), pf->stat_prev_loaded,
4668                           &prev_ps->tx_dropped_link_down,
4669                           &cur_ps->tx_dropped_link_down);
4670
4671         ice_stat_update40(hw, GLPRT_PRC64H(pf_id), GLPRT_PRC64L(pf_id),
4672                           pf->stat_prev_loaded, &prev_ps->rx_size_64,
4673                           &cur_ps->rx_size_64);
4674
4675         ice_stat_update40(hw, GLPRT_PRC127H(pf_id), GLPRT_PRC127L(pf_id),
4676                           pf->stat_prev_loaded, &prev_ps->rx_size_127,
4677                           &cur_ps->rx_size_127);
4678
4679         ice_stat_update40(hw, GLPRT_PRC255H(pf_id), GLPRT_PRC255L(pf_id),
4680                           pf->stat_prev_loaded, &prev_ps->rx_size_255,
4681                           &cur_ps->rx_size_255);
4682
4683         ice_stat_update40(hw, GLPRT_PRC511H(pf_id), GLPRT_PRC511L(pf_id),
4684                           pf->stat_prev_loaded, &prev_ps->rx_size_511,
4685                           &cur_ps->rx_size_511);
4686
4687         ice_stat_update40(hw, GLPRT_PRC1023H(pf_id),
4688                           GLPRT_PRC1023L(pf_id), pf->stat_prev_loaded,
4689                           &prev_ps->rx_size_1023, &cur_ps->rx_size_1023);
4690
4691         ice_stat_update40(hw, GLPRT_PRC1522H(pf_id),
4692                           GLPRT_PRC1522L(pf_id), pf->stat_prev_loaded,
4693                           &prev_ps->rx_size_1522, &cur_ps->rx_size_1522);
4694
4695         ice_stat_update40(hw, GLPRT_PRC9522H(pf_id),
4696                           GLPRT_PRC9522L(pf_id), pf->stat_prev_loaded,
4697                           &prev_ps->rx_size_big, &cur_ps->rx_size_big);
4698
4699         ice_stat_update40(hw, GLPRT_PTC64H(pf_id), GLPRT_PTC64L(pf_id),
4700                           pf->stat_prev_loaded, &prev_ps->tx_size_64,
4701                           &cur_ps->tx_size_64);
4702
4703         ice_stat_update40(hw, GLPRT_PTC127H(pf_id), GLPRT_PTC127L(pf_id),
4704                           pf->stat_prev_loaded, &prev_ps->tx_size_127,
4705                           &cur_ps->tx_size_127);
4706
4707         ice_stat_update40(hw, GLPRT_PTC255H(pf_id), GLPRT_PTC255L(pf_id),
4708                           pf->stat_prev_loaded, &prev_ps->tx_size_255,
4709                           &cur_ps->tx_size_255);
4710
4711         ice_stat_update40(hw, GLPRT_PTC511H(pf_id), GLPRT_PTC511L(pf_id),
4712                           pf->stat_prev_loaded, &prev_ps->tx_size_511,
4713                           &cur_ps->tx_size_511);
4714
4715         ice_stat_update40(hw, GLPRT_PTC1023H(pf_id),
4716                           GLPRT_PTC1023L(pf_id), pf->stat_prev_loaded,
4717                           &prev_ps->tx_size_1023, &cur_ps->tx_size_1023);
4718
4719         ice_stat_update40(hw, GLPRT_PTC1522H(pf_id),
4720                           GLPRT_PTC1522L(pf_id), pf->stat_prev_loaded,
4721                           &prev_ps->tx_size_1522, &cur_ps->tx_size_1522);
4722
4723         ice_stat_update40(hw, GLPRT_PTC9522H(pf_id),
4724                           GLPRT_PTC9522L(pf_id), pf->stat_prev_loaded,
4725                           &prev_ps->tx_size_big, &cur_ps->tx_size_big);
4726
4727         ice_stat_update32(hw, GLPRT_LXONRXC(pf_id), pf->stat_prev_loaded,
4728                           &prev_ps->link_xon_rx, &cur_ps->link_xon_rx);
4729
4730         ice_stat_update32(hw, GLPRT_LXOFFRXC(pf_id), pf->stat_prev_loaded,
4731                           &prev_ps->link_xoff_rx, &cur_ps->link_xoff_rx);
4732
4733         ice_stat_update32(hw, GLPRT_LXONTXC(pf_id), pf->stat_prev_loaded,
4734                           &prev_ps->link_xon_tx, &cur_ps->link_xon_tx);
4735
4736         ice_stat_update32(hw, GLPRT_LXOFFTXC(pf_id), pf->stat_prev_loaded,
4737                           &prev_ps->link_xoff_tx, &cur_ps->link_xoff_tx);
4738
4739         ice_stat_update32(hw, GLPRT_CRCERRS(pf_id), pf->stat_prev_loaded,
4740                           &prev_ps->crc_errors, &cur_ps->crc_errors);
4741
4742         ice_stat_update32(hw, GLPRT_ILLERRC(pf_id), pf->stat_prev_loaded,
4743                           &prev_ps->illegal_bytes, &cur_ps->illegal_bytes);
4744
4745         ice_stat_update32(hw, GLPRT_MLFC(pf_id), pf->stat_prev_loaded,
4746                           &prev_ps->mac_local_faults,
4747                           &cur_ps->mac_local_faults);
4748
4749         ice_stat_update32(hw, GLPRT_MRFC(pf_id), pf->stat_prev_loaded,
4750                           &prev_ps->mac_remote_faults,
4751                           &cur_ps->mac_remote_faults);
4752
4753         ice_stat_update32(hw, GLPRT_RLEC(pf_id), pf->stat_prev_loaded,
4754                           &prev_ps->rx_len_errors, &cur_ps->rx_len_errors);
4755
4756         ice_stat_update32(hw, GLPRT_RUC(pf_id), pf->stat_prev_loaded,
4757                           &prev_ps->rx_undersize, &cur_ps->rx_undersize);
4758
4759         ice_stat_update32(hw, GLPRT_RFC(pf_id), pf->stat_prev_loaded,
4760                           &prev_ps->rx_fragments, &cur_ps->rx_fragments);
4761
4762         ice_stat_update32(hw, GLPRT_ROC(pf_id), pf->stat_prev_loaded,
4763                           &prev_ps->rx_oversize, &cur_ps->rx_oversize);
4764
4765         ice_stat_update32(hw, GLPRT_RJC(pf_id), pf->stat_prev_loaded,
4766                           &prev_ps->rx_jabber, &cur_ps->rx_jabber);
4767
4768         pf->stat_prev_loaded = true;
4769 }
4770
4771 /**
4772  * ice_get_stats64 - get statistics for network device structure
4773  * @netdev: network interface device structure
4774  * @stats: main device statistics structure
4775  */
4776 static
4777 void ice_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats)
4778 {
4779         struct ice_netdev_priv *np = netdev_priv(netdev);
4780         struct rtnl_link_stats64 *vsi_stats;
4781         struct ice_vsi *vsi = np->vsi;
4782
4783         vsi_stats = &vsi->net_stats;
4784
4785         if (test_bit(__ICE_DOWN, vsi->state) || !vsi->num_txq || !vsi->num_rxq)
4786                 return;
4787         /* netdev packet/byte stats come from ring counter. These are obtained
4788          * by summing up ring counters (done by ice_update_vsi_ring_stats).
4789          */
4790         ice_update_vsi_ring_stats(vsi);
4791         stats->tx_packets = vsi_stats->tx_packets;
4792         stats->tx_bytes = vsi_stats->tx_bytes;
4793         stats->rx_packets = vsi_stats->rx_packets;
4794         stats->rx_bytes = vsi_stats->rx_bytes;
4795
4796         /* The rest of the stats can be read from the hardware but instead we
4797          * just return values that the watchdog task has already obtained from
4798          * the hardware.
4799          */
4800         stats->multicast = vsi_stats->multicast;
4801         stats->tx_errors = vsi_stats->tx_errors;
4802         stats->tx_dropped = vsi_stats->tx_dropped;
4803         stats->rx_errors = vsi_stats->rx_errors;
4804         stats->rx_dropped = vsi_stats->rx_dropped;
4805         stats->rx_crc_errors = vsi_stats->rx_crc_errors;
4806         stats->rx_length_errors = vsi_stats->rx_length_errors;
4807 }
4808
4809 #ifdef CONFIG_NET_POLL_CONTROLLER
4810 /**
4811  * ice_netpoll - polling "interrupt" handler
4812  * @netdev: network interface device structure
4813  *
4814  * Used by netconsole to send skbs without having to re-enable interrupts.
4815  * This is not called in the normal interrupt path.
4816  */
4817 static void ice_netpoll(struct net_device *netdev)
4818 {
4819         struct ice_netdev_priv *np = netdev_priv(netdev);
4820         struct ice_vsi *vsi = np->vsi;
4821         struct ice_pf *pf = vsi->back;
4822         int i;
4823
4824         if (test_bit(__ICE_DOWN, vsi->state) ||
4825             !test_bit(ICE_FLAG_MSIX_ENA, pf->flags))
4826                 return;
4827
4828         for (i = 0; i < vsi->num_q_vectors; i++)
4829                 ice_msix_clean_rings(0, vsi->q_vectors[i]);
4830 }
4831 #endif /* CONFIG_NET_POLL_CONTROLLER */
4832
4833 /**
4834  * ice_napi_disable_all - Disable NAPI for all q_vectors in the VSI
4835  * @vsi: VSI having NAPI disabled
4836  */
4837 static void ice_napi_disable_all(struct ice_vsi *vsi)
4838 {
4839         int q_idx;
4840
4841         if (!vsi->netdev)
4842                 return;
4843
4844         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++)
4845                 napi_disable(&vsi->q_vectors[q_idx]->napi);
4846 }
4847
4848 /**
4849  * ice_down - Shutdown the connection
4850  * @vsi: The VSI being stopped
4851  */
4852 int ice_down(struct ice_vsi *vsi)
4853 {
4854         int i, err;
4855
4856         /* Caller of this function is expected to set the
4857          * vsi->state __ICE_DOWN bit
4858          */
4859         if (vsi->netdev) {
4860                 netif_carrier_off(vsi->netdev);
4861                 netif_tx_disable(vsi->netdev);
4862         }
4863
4864         ice_vsi_dis_irq(vsi);
4865         err = ice_vsi_stop_tx_rx_rings(vsi);
4866         ice_napi_disable_all(vsi);
4867
4868         ice_for_each_txq(vsi, i)
4869                 ice_clean_tx_ring(vsi->tx_rings[i]);
4870
4871         ice_for_each_rxq(vsi, i)
4872                 ice_clean_rx_ring(vsi->rx_rings[i]);
4873
4874         if (err)
4875                 netdev_err(vsi->netdev, "Failed to close VSI 0x%04X on switch 0x%04X\n",
4876                            vsi->vsi_num, vsi->vsw->sw_id);
4877         return err;
4878 }
4879
4880 /**
4881  * ice_vsi_setup_tx_rings - Allocate VSI Tx queue resources
4882  * @vsi: VSI having resources allocated
4883  *
4884  * Return 0 on success, negative on failure
4885  */
4886 static int ice_vsi_setup_tx_rings(struct ice_vsi *vsi)
4887 {
4888         int i, err = 0;
4889
4890         if (!vsi->num_txq) {
4891                 dev_err(&vsi->back->pdev->dev, "VSI %d has 0 Tx queues\n",
4892                         vsi->vsi_num);
4893                 return -EINVAL;
4894         }
4895
4896         ice_for_each_txq(vsi, i) {
4897                 err = ice_setup_tx_ring(vsi->tx_rings[i]);
4898                 if (err)
4899                         break;
4900         }
4901
4902         return err;
4903 }
4904
4905 /**
4906  * ice_vsi_setup_rx_rings - Allocate VSI Rx queue resources
4907  * @vsi: VSI having resources allocated
4908  *
4909  * Return 0 on success, negative on failure
4910  */
4911 static int ice_vsi_setup_rx_rings(struct ice_vsi *vsi)
4912 {
4913         int i, err = 0;
4914
4915         if (!vsi->num_rxq) {
4916                 dev_err(&vsi->back->pdev->dev, "VSI %d has 0 Rx queues\n",
4917                         vsi->vsi_num);
4918                 return -EINVAL;
4919         }
4920
4921         ice_for_each_rxq(vsi, i) {
4922                 err = ice_setup_rx_ring(vsi->rx_rings[i]);
4923                 if (err)
4924                         break;
4925         }
4926
4927         return err;
4928 }
4929
4930 /**
4931  * ice_vsi_req_irq - Request IRQ from the OS
4932  * @vsi: The VSI IRQ is being requested for
4933  * @basename: name for the vector
4934  *
4935  * Return 0 on success and a negative value on error
4936  */
4937 static int ice_vsi_req_irq(struct ice_vsi *vsi, char *basename)
4938 {
4939         struct ice_pf *pf = vsi->back;
4940         int err = -EINVAL;
4941
4942         if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags))
4943                 err = ice_vsi_req_irq_msix(vsi, basename);
4944
4945         return err;
4946 }
4947
4948 /**
4949  * ice_vsi_free_tx_rings - Free Tx resources for VSI queues
4950  * @vsi: the VSI having resources freed
4951  */
4952 static void ice_vsi_free_tx_rings(struct ice_vsi *vsi)
4953 {
4954         int i;
4955
4956         if (!vsi->tx_rings)
4957                 return;
4958
4959         ice_for_each_txq(vsi, i)
4960                 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
4961                         ice_free_tx_ring(vsi->tx_rings[i]);
4962 }
4963
4964 /**
4965  * ice_vsi_free_rx_rings - Free Rx resources for VSI queues
4966  * @vsi: the VSI having resources freed
4967  */
4968 static void ice_vsi_free_rx_rings(struct ice_vsi *vsi)
4969 {
4970         int i;
4971
4972         if (!vsi->rx_rings)
4973                 return;
4974
4975         ice_for_each_rxq(vsi, i)
4976                 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
4977                         ice_free_rx_ring(vsi->rx_rings[i]);
4978 }
4979
4980 /**
4981  * ice_vsi_open - Called when a network interface is made active
4982  * @vsi: the VSI to open
4983  *
4984  * Initialization of the VSI
4985  *
4986  * Returns 0 on success, negative value on error
4987  */
4988 static int ice_vsi_open(struct ice_vsi *vsi)
4989 {
4990         char int_name[ICE_INT_NAME_STR_LEN];
4991         struct ice_pf *pf = vsi->back;
4992         int err;
4993
4994         /* allocate descriptors */
4995         err = ice_vsi_setup_tx_rings(vsi);
4996         if (err)
4997                 goto err_setup_tx;
4998
4999         err = ice_vsi_setup_rx_rings(vsi);
5000         if (err)
5001                 goto err_setup_rx;
5002
5003         err = ice_vsi_cfg(vsi);
5004         if (err)
5005                 goto err_setup_rx;
5006
5007         snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
5008                  dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
5009         err = ice_vsi_req_irq(vsi, int_name);
5010         if (err)
5011                 goto err_setup_rx;
5012
5013         /* Notify the stack of the actual queue counts. */
5014         err = netif_set_real_num_tx_queues(vsi->netdev, vsi->num_txq);
5015         if (err)
5016                 goto err_set_qs;
5017
5018         err = netif_set_real_num_rx_queues(vsi->netdev, vsi->num_rxq);
5019         if (err)
5020                 goto err_set_qs;
5021
5022         err = ice_up_complete(vsi);
5023         if (err)
5024                 goto err_up_complete;
5025
5026         return 0;
5027
5028 err_up_complete:
5029         ice_down(vsi);
5030 err_set_qs:
5031         ice_vsi_free_irq(vsi);
5032 err_setup_rx:
5033         ice_vsi_free_rx_rings(vsi);
5034 err_setup_tx:
5035         ice_vsi_free_tx_rings(vsi);
5036
5037         return err;
5038 }
5039
5040 /**
5041  * ice_vsi_close - Shut down a VSI
5042  * @vsi: the VSI being shut down
5043  */
5044 static void ice_vsi_close(struct ice_vsi *vsi)
5045 {
5046         if (!test_and_set_bit(__ICE_DOWN, vsi->state))
5047                 ice_down(vsi);
5048
5049         ice_vsi_free_irq(vsi);
5050         ice_vsi_free_tx_rings(vsi);
5051         ice_vsi_free_rx_rings(vsi);
5052 }
5053
5054 /**
5055  * ice_rss_clean - Delete RSS related VSI structures that hold user inputs
5056  * @vsi: the VSI being removed
5057  */
5058 static void ice_rss_clean(struct ice_vsi *vsi)
5059 {
5060         struct ice_pf *pf;
5061
5062         pf = vsi->back;
5063
5064         if (vsi->rss_hkey_user)
5065                 devm_kfree(&pf->pdev->dev, vsi->rss_hkey_user);
5066         if (vsi->rss_lut_user)
5067                 devm_kfree(&pf->pdev->dev, vsi->rss_lut_user);
5068 }
5069
5070 /**
5071  * ice_vsi_release - Delete a VSI and free its resources
5072  * @vsi: the VSI being removed
5073  *
5074  * Returns 0 on success or < 0 on error
5075  */
5076 static int ice_vsi_release(struct ice_vsi *vsi)
5077 {
5078         struct ice_pf *pf;
5079
5080         if (!vsi->back)
5081                 return -ENODEV;
5082         pf = vsi->back;
5083
5084         if (vsi->netdev) {
5085                 unregister_netdev(vsi->netdev);
5086                 free_netdev(vsi->netdev);
5087                 vsi->netdev = NULL;
5088         }
5089
5090         if (test_bit(ICE_FLAG_RSS_ENA, pf->flags))
5091                 ice_rss_clean(vsi);
5092
5093         /* Disable VSI and free resources */
5094         ice_vsi_dis_irq(vsi);
5095         ice_vsi_close(vsi);
5096
5097         /* reclaim interrupt vectors back to PF */
5098         ice_free_res(vsi->back->irq_tracker, vsi->base_vector, vsi->idx);
5099         pf->num_avail_msix += vsi->num_q_vectors;
5100
5101         ice_remove_vsi_fltr(&pf->hw, vsi->vsi_num);
5102         ice_vsi_delete(vsi);
5103         ice_vsi_free_q_vectors(vsi);
5104         ice_vsi_clear_rings(vsi);
5105
5106         ice_vsi_put_qs(vsi);
5107         pf->q_left_tx += vsi->alloc_txq;
5108         pf->q_left_rx += vsi->alloc_rxq;
5109
5110         ice_vsi_clear(vsi);
5111
5112         return 0;
5113 }
5114
5115 /**
5116  * ice_dis_vsi - pause a VSI
5117  * @vsi: the VSI being paused
5118  */
5119 static void ice_dis_vsi(struct ice_vsi *vsi)
5120 {
5121         if (test_bit(__ICE_DOWN, vsi->state))
5122                 return;
5123
5124         set_bit(__ICE_NEEDS_RESTART, vsi->state);
5125
5126         if (vsi->netdev && netif_running(vsi->netdev) &&
5127             vsi->type == ICE_VSI_PF)
5128                 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
5129
5130         ice_vsi_close(vsi);
5131 }
5132
5133 /**
5134  * ice_ena_vsi - resume a VSI
5135  * @vsi: the VSI being resume
5136  */
5137 static void ice_ena_vsi(struct ice_vsi *vsi)
5138 {
5139         if (!test_and_clear_bit(__ICE_NEEDS_RESTART, vsi->state))
5140                 return;
5141
5142         if (vsi->netdev && netif_running(vsi->netdev))
5143                 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
5144         else if (ice_vsi_open(vsi))
5145                 /* this clears the DOWN bit */
5146                 dev_dbg(&vsi->back->pdev->dev, "Failed open VSI 0x%04X on switch 0x%04X\n",
5147                         vsi->vsi_num, vsi->vsw->sw_id);
5148 }
5149
5150 /**
5151  * ice_pf_dis_all_vsi - Pause all VSIs on a PF
5152  * @pf: the PF
5153  */
5154 static void ice_pf_dis_all_vsi(struct ice_pf *pf)
5155 {
5156         int v;
5157
5158         ice_for_each_vsi(pf, v)
5159                 if (pf->vsi[v])
5160                         ice_dis_vsi(pf->vsi[v]);
5161 }
5162
5163 /**
5164  * ice_pf_ena_all_vsi - Resume all VSIs on a PF
5165  * @pf: the PF
5166  */
5167 static void ice_pf_ena_all_vsi(struct ice_pf *pf)
5168 {
5169         int v;
5170
5171         ice_for_each_vsi(pf, v)
5172                 if (pf->vsi[v])
5173                         ice_ena_vsi(pf->vsi[v]);
5174 }
5175
5176 /**
5177  * ice_rebuild - rebuild after reset
5178  * @pf: pf to rebuild
5179  */
5180 static void ice_rebuild(struct ice_pf *pf)
5181 {
5182         struct device *dev = &pf->pdev->dev;
5183         struct ice_hw *hw = &pf->hw;
5184         enum ice_status ret;
5185         int err;
5186
5187         if (test_bit(__ICE_DOWN, pf->state))
5188                 goto clear_recovery;
5189
5190         dev_dbg(dev, "rebuilding pf\n");
5191
5192         ret = ice_init_all_ctrlq(hw);
5193         if (ret) {
5194                 dev_err(dev, "control queues init failed %d\n", ret);
5195                 goto fail_reset;
5196         }
5197
5198         ret = ice_clear_pf_cfg(hw);
5199         if (ret) {
5200                 dev_err(dev, "clear PF configuration failed %d\n", ret);
5201                 goto fail_reset;
5202         }
5203
5204         ice_clear_pxe_mode(hw);
5205
5206         ret = ice_get_caps(hw);
5207         if (ret) {
5208                 dev_err(dev, "ice_get_caps failed %d\n", ret);
5209                 goto fail_reset;
5210         }
5211
5212         /* basic nic switch setup */
5213         err = ice_setup_pf_sw(pf);
5214         if (err) {
5215                 dev_err(dev, "ice_setup_pf_sw failed\n");
5216                 goto fail_reset;
5217         }
5218
5219         /* start misc vector */
5220         if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) {
5221                 err = ice_req_irq_msix_misc(pf);
5222                 if (err) {
5223                         dev_err(dev, "misc vector setup failed: %d\n", err);
5224                         goto fail_reset;
5225                 }
5226         }
5227
5228         /* restart the VSIs that were rebuilt and running before the reset */
5229         ice_pf_ena_all_vsi(pf);
5230
5231         return;
5232
5233 fail_reset:
5234         ice_shutdown_all_ctrlq(hw);
5235         set_bit(__ICE_RESET_FAILED, pf->state);
5236 clear_recovery:
5237         set_bit(__ICE_RESET_RECOVERY_PENDING, pf->state);
5238 }
5239
5240 /**
5241  * ice_change_mtu - NDO callback to change the MTU
5242  * @netdev: network interface device structure
5243  * @new_mtu: new value for maximum frame size
5244  *
5245  * Returns 0 on success, negative on failure
5246  */
5247 static int ice_change_mtu(struct net_device *netdev, int new_mtu)
5248 {
5249         struct ice_netdev_priv *np = netdev_priv(netdev);
5250         struct ice_vsi *vsi = np->vsi;
5251         struct ice_pf *pf = vsi->back;
5252         u8 count = 0;
5253
5254         if (new_mtu == netdev->mtu) {
5255                 netdev_warn(netdev, "mtu is already %u\n", netdev->mtu);
5256                 return 0;
5257         }
5258
5259         if (new_mtu < netdev->min_mtu) {
5260                 netdev_err(netdev, "new mtu invalid. min_mtu is %d\n",
5261                            netdev->min_mtu);
5262                 return -EINVAL;
5263         } else if (new_mtu > netdev->max_mtu) {
5264                 netdev_err(netdev, "new mtu invalid. max_mtu is %d\n",
5265                            netdev->min_mtu);
5266                 return -EINVAL;
5267         }
5268         /* if a reset is in progress, wait for some time for it to complete */
5269         do {
5270                 if (ice_is_reset_recovery_pending(pf->state)) {
5271                         count++;
5272                         usleep_range(1000, 2000);
5273                 } else {
5274                         break;
5275                 }
5276
5277         } while (count < 100);
5278
5279         if (count == 100) {
5280                 netdev_err(netdev, "can't change mtu. Device is busy\n");
5281                 return -EBUSY;
5282         }
5283
5284         netdev->mtu = new_mtu;
5285
5286         /* if VSI is up, bring it down and then back up */
5287         if (!test_and_set_bit(__ICE_DOWN, vsi->state)) {
5288                 int err;
5289
5290                 err = ice_down(vsi);
5291                 if (err) {
5292                         netdev_err(netdev, "change mtu if_up err %d\n", err);
5293                         return err;
5294                 }
5295
5296                 err = ice_up(vsi);
5297                 if (err) {
5298                         netdev_err(netdev, "change mtu if_up err %d\n", err);
5299                         return err;
5300                 }
5301         }
5302
5303         netdev_dbg(netdev, "changed mtu to %d\n", new_mtu);
5304         return 0;
5305 }
5306
5307 /**
5308  * ice_set_rss - Set RSS keys and lut
5309  * @vsi: Pointer to VSI structure
5310  * @seed: RSS hash seed
5311  * @lut: Lookup table
5312  * @lut_size: Lookup table size
5313  *
5314  * Returns 0 on success, negative on failure
5315  */
5316 int ice_set_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
5317 {
5318         struct ice_pf *pf = vsi->back;
5319         struct ice_hw *hw = &pf->hw;
5320         enum ice_status status;
5321
5322         if (seed) {
5323                 struct ice_aqc_get_set_rss_keys *buf =
5324                                   (struct ice_aqc_get_set_rss_keys *)seed;
5325
5326                 status = ice_aq_set_rss_key(hw, vsi->vsi_num, buf);
5327
5328                 if (status) {
5329                         dev_err(&pf->pdev->dev,
5330                                 "Cannot set RSS key, err %d aq_err %d\n",
5331                                 status, hw->adminq.rq_last_status);
5332                         return -EIO;
5333                 }
5334         }
5335
5336         if (lut) {
5337                 status = ice_aq_set_rss_lut(hw, vsi->vsi_num,
5338                                             vsi->rss_lut_type, lut, lut_size);
5339                 if (status) {
5340                         dev_err(&pf->pdev->dev,
5341                                 "Cannot set RSS lut, err %d aq_err %d\n",
5342                                 status, hw->adminq.rq_last_status);
5343                         return -EIO;
5344                 }
5345         }
5346
5347         return 0;
5348 }
5349
5350 /**
5351  * ice_get_rss - Get RSS keys and lut
5352  * @vsi: Pointer to VSI structure
5353  * @seed: Buffer to store the keys
5354  * @lut: Buffer to store the lookup table entries
5355  * @lut_size: Size of buffer to store the lookup table entries
5356  *
5357  * Returns 0 on success, negative on failure
5358  */
5359 int ice_get_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
5360 {
5361         struct ice_pf *pf = vsi->back;
5362         struct ice_hw *hw = &pf->hw;
5363         enum ice_status status;
5364
5365         if (seed) {
5366                 struct ice_aqc_get_set_rss_keys *buf =
5367                                   (struct ice_aqc_get_set_rss_keys *)seed;
5368
5369                 status = ice_aq_get_rss_key(hw, vsi->vsi_num, buf);
5370                 if (status) {
5371                         dev_err(&pf->pdev->dev,
5372                                 "Cannot get RSS key, err %d aq_err %d\n",
5373                                 status, hw->adminq.rq_last_status);
5374                         return -EIO;
5375                 }
5376         }
5377
5378         if (lut) {
5379                 status = ice_aq_get_rss_lut(hw, vsi->vsi_num,
5380                                             vsi->rss_lut_type, lut, lut_size);
5381                 if (status) {
5382                         dev_err(&pf->pdev->dev,
5383                                 "Cannot get RSS lut, err %d aq_err %d\n",
5384                                 status, hw->adminq.rq_last_status);
5385                         return -EIO;
5386                 }
5387         }
5388
5389         return 0;
5390 }
5391
5392 /**
5393  * ice_open - Called when a network interface becomes active
5394  * @netdev: network interface device structure
5395  *
5396  * The open entry point is called when a network interface is made
5397  * active by the system (IFF_UP).  At this point all resources needed
5398  * for transmit and receive operations are allocated, the interrupt
5399  * handler is registered with the OS, the netdev watchdog is enabled,
5400  * and the stack is notified that the interface is ready.
5401  *
5402  * Returns 0 on success, negative value on failure
5403  */
5404 static int ice_open(struct net_device *netdev)
5405 {
5406         struct ice_netdev_priv *np = netdev_priv(netdev);
5407         struct ice_vsi *vsi = np->vsi;
5408         int err;
5409
5410         netif_carrier_off(netdev);
5411
5412         err = ice_vsi_open(vsi);
5413
5414         if (err)
5415                 netdev_err(netdev, "Failed to open VSI 0x%04X on switch 0x%04X\n",
5416                            vsi->vsi_num, vsi->vsw->sw_id);
5417         return err;
5418 }
5419
5420 /**
5421  * ice_stop - Disables a network interface
5422  * @netdev: network interface device structure
5423  *
5424  * The stop entry point is called when an interface is de-activated by the OS,
5425  * and the netdevice enters the DOWN state.  The hardware is still under the
5426  * driver's control, but the netdev interface is disabled.
5427  *
5428  * Returns success only - not allowed to fail
5429  */
5430 static int ice_stop(struct net_device *netdev)
5431 {
5432         struct ice_netdev_priv *np = netdev_priv(netdev);
5433         struct ice_vsi *vsi = np->vsi;
5434
5435         ice_vsi_close(vsi);
5436
5437         return 0;
5438 }
5439
5440 /**
5441  * ice_features_check - Validate encapsulated packet conforms to limits
5442  * @skb: skb buffer
5443  * @netdev: This port's netdev
5444  * @features: Offload features that the stack believes apply
5445  */
5446 static netdev_features_t
5447 ice_features_check(struct sk_buff *skb,
5448                    struct net_device __always_unused *netdev,
5449                    netdev_features_t features)
5450 {
5451         size_t len;
5452
5453         /* No point in doing any of this if neither checksum nor GSO are
5454          * being requested for this frame.  We can rule out both by just
5455          * checking for CHECKSUM_PARTIAL
5456          */
5457         if (skb->ip_summed != CHECKSUM_PARTIAL)
5458                 return features;
5459
5460         /* We cannot support GSO if the MSS is going to be less than
5461          * 64 bytes.  If it is then we need to drop support for GSO.
5462          */
5463         if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
5464                 features &= ~NETIF_F_GSO_MASK;
5465
5466         len = skb_network_header(skb) - skb->data;
5467         if (len & ~(ICE_TXD_MACLEN_MAX))
5468                 goto out_rm_features;
5469
5470         len = skb_transport_header(skb) - skb_network_header(skb);
5471         if (len & ~(ICE_TXD_IPLEN_MAX))
5472                 goto out_rm_features;
5473
5474         if (skb->encapsulation) {
5475                 len = skb_inner_network_header(skb) - skb_transport_header(skb);
5476                 if (len & ~(ICE_TXD_L4LEN_MAX))
5477                         goto out_rm_features;
5478
5479                 len = skb_inner_transport_header(skb) -
5480                       skb_inner_network_header(skb);
5481                 if (len & ~(ICE_TXD_IPLEN_MAX))
5482                         goto out_rm_features;
5483         }
5484
5485         return features;
5486 out_rm_features:
5487         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
5488 }
5489
5490 static const struct net_device_ops ice_netdev_ops = {
5491         .ndo_open = ice_open,
5492         .ndo_stop = ice_stop,
5493         .ndo_start_xmit = ice_start_xmit,
5494         .ndo_features_check = ice_features_check,
5495         .ndo_set_rx_mode = ice_set_rx_mode,
5496         .ndo_set_mac_address = ice_set_mac_address,
5497         .ndo_validate_addr = eth_validate_addr,
5498         .ndo_change_mtu = ice_change_mtu,
5499         .ndo_get_stats64 = ice_get_stats64,
5500 #ifdef CONFIG_NET_POLL_CONTROLLER
5501         .ndo_poll_controller = ice_netpoll,
5502 #endif /* CONFIG_NET_POLL_CONTROLLER */
5503         .ndo_vlan_rx_add_vid = ice_vlan_rx_add_vid,
5504         .ndo_vlan_rx_kill_vid = ice_vlan_rx_kill_vid,
5505         .ndo_set_features = ice_set_features,
5506         .ndo_fdb_add = ice_fdb_add,
5507         .ndo_fdb_del = ice_fdb_del,
5508 };