drivers/net/ethernet/microsoft/mana/shm_channel.c

   1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
   2 /* Copyright (c) 2021, Microsoft Corporation. */
   3
   4 #include <linux/delay.h>
   5 #include <linux/device.h>
   6 #include <linux/io.h>
   7 #include <linux/mm.h>
   8
   9 #include <net/mana/gdma.h>
  10 #include <net/mana/shm_channel.h>
  11
  12 #define PAGE_FRAME_L48_WIDTH_BYTES 6
  13 #define PAGE_FRAME_L48_WIDTH_BITS (PAGE_FRAME_L48_WIDTH_BYTES * 8)
  14 #define PAGE_FRAME_L48_MASK 0x0000FFFFFFFFFFFF
  15 #define PAGE_FRAME_H4_WIDTH_BITS 4
  16 #define VECTOR_MASK 0xFFFF
  17 #define SHMEM_VF_RESET_STATE ((u32)-1)
  18
  19 #define SMC_MSG_TYPE_ESTABLISH_HWC 1
  20 #define SMC_MSG_TYPE_ESTABLISH_HWC_VERSION 0
  21
  22 #define SMC_MSG_TYPE_DESTROY_HWC 2
  23 #define SMC_MSG_TYPE_DESTROY_HWC_VERSION 0
  24
  25 #define SMC_MSG_DIRECTION_REQUEST 0
  26 #define SMC_MSG_DIRECTION_RESPONSE 1
  27
  28 /* Structures labeled with "HW DATA" are exchanged with the hardware. All of
  29  * them are naturally aligned and hence don't need __packed.
  30  */
  31
  32 /* Shared memory channel protocol header
  33  *
  34  * msg_type: set on request and response; response matches request.
  35  * msg_version: newer PF writes back older response (matching request)
  36  *  older PF acts on latest version known and sets that version in result
  37  *  (less than request).
  38  * direction: 0 for request, VF->PF; 1 for response, PF->VF.
  39  * status: 0 on request,
  40  *   operation result on response (success = 0, failure = 1 or greater).
  41  * reset_vf: If set on either establish or destroy request, indicates perform
  42  *  FLR before/after the operation.
  43  * owner_is_pf: 1 indicates PF owned, 0 indicates VF owned.
  44  */
  45 union smc_proto_hdr {
  46         u32 as_uint32;
  47
  48         struct {
  49                 u8 msg_type     : 3;
  50                 u8 msg_version  : 3;
  51                 u8 reserved_1   : 1;
  52                 u8 direction    : 1;
  53
  54                 u8 status;
  55
  56                 u8 reserved_2;
  57
  58                 u8 reset_vf     : 1;
  59                 u8 reserved_3   : 6;
  60                 u8 owner_is_pf  : 1;
  61         };
  62 }; /* HW DATA */
  63
  64 #define SMC_APERTURE_BITS 256
  65 #define SMC_BASIC_UNIT (sizeof(u32))
  66 #define SMC_APERTURE_DWORDS (SMC_APERTURE_BITS / (SMC_BASIC_UNIT * 8))
  67 #define SMC_LAST_DWORD (SMC_APERTURE_DWORDS - 1)
  68
  69 static int mana_smc_poll_register(void __iomem *base, bool reset)
  70 {
  71         void __iomem *ptr = base + SMC_LAST_DWORD * SMC_BASIC_UNIT;
  72         u32 last_dword;
  73         int i;
  74
  75         /* Poll the hardware for the ownership bit. This should be pretty fast,
  76          * but let's do it in a loop just in case the hardware or the PF
  77          * driver are temporarily busy.
  78          */
  79         for (i = 0; i < 20 * 1000; i++)  {
  80                 last_dword = readl(ptr);
  81
  82                 /* shmem reads as 0xFFFFFFFF in the reset case */
  83                 if (reset && last_dword == SHMEM_VF_RESET_STATE)
  84                         return 0;
  85
  86                 /* If bit_31 is set, the PF currently owns the SMC. */
  87                 if (!(last_dword & BIT(31)))
  88                         return 0;
  89
  90                 usleep_range(1000, 2000);
  91         }
  92
  93         return -ETIMEDOUT;
  94 }
  95
  96 static int mana_smc_read_response(struct shm_channel *sc, u32 msg_type,
  97                                   u32 msg_version, bool reset_vf)
  98 {
  99         void __iomem *base = sc->base;
 100         union smc_proto_hdr hdr;
 101         int err;
 102
 103         /* Wait for PF to respond. */
 104         err = mana_smc_poll_register(base, reset_vf);
 105         if (err)
 106                 return err;
 107
 108         hdr.as_uint32 = readl(base + SMC_LAST_DWORD * SMC_BASIC_UNIT);
 109
 110         if (reset_vf && hdr.as_uint32 == SHMEM_VF_RESET_STATE)
 111                 return 0;
 112
 113         /* Validate protocol fields from the PF driver */
 114         if (hdr.msg_type != msg_type || hdr.msg_version > msg_version ||
 115             hdr.direction != SMC_MSG_DIRECTION_RESPONSE) {
 116                 dev_err(sc->dev, "Wrong SMC response 0x%x, type=%d, ver=%d\n",
 117                         hdr.as_uint32, msg_type, msg_version);
 118                 return -EPROTO;
 119         }
 120
 121         /* Validate the operation result */
 122         if (hdr.status != 0) {
 123                 dev_err(sc->dev, "SMC operation failed: 0x%x\n", hdr.status);
 124                 return -EPROTO;
 125         }
 126
 127         return 0;
 128 }
 129
 130 void mana_smc_init(struct shm_channel *sc, struct device *dev,
 131                    void __iomem *base)
 132 {
 133         sc->dev = dev;
 134         sc->base = base;
 135 }
 136
 137 int mana_smc_setup_hwc(struct shm_channel *sc, bool reset_vf, u64 eq_addr,
 138                        u64 cq_addr, u64 rq_addr, u64 sq_addr,
 139                        u32 eq_msix_index)
 140 {
 141         union smc_proto_hdr *hdr;
 142         u16 all_addr_h4bits = 0;
 143         u16 frame_addr_seq = 0;
 144         u64 frame_addr = 0;
 145         u8 shm_buf[32];
 146         u64 *shmem;
 147         u32 *dword;
 148         u8 *ptr;
 149         int err;
 150         int i;
 151
 152         /* Ensure VF already has possession of shared memory */
 153         err = mana_smc_poll_register(sc->base, false);
 154         if (err) {
 155                 dev_err(sc->dev, "Timeout when setting up HWC: %d\n", err);
 156                 return err;
 157         }
 158
 159         if (!MANA_PAGE_ALIGNED(eq_addr) || !MANA_PAGE_ALIGNED(cq_addr) ||
 160             !MANA_PAGE_ALIGNED(rq_addr) || !MANA_PAGE_ALIGNED(sq_addr))
 161                 return -EINVAL;
 162
 163         if ((eq_msix_index & VECTOR_MASK) != eq_msix_index)
 164                 return -EINVAL;
 165
 166         /* Scheme for packing four addresses and extra info into 256 bits.
 167          *
 168          * Addresses must be page frame aligned, so only frame address bits
 169          * are transferred.
 170          *
 171          * 52-bit frame addresses are split into the lower 48 bits and upper
 172          * 4 bits. Lower 48 bits of 4 address are written sequentially from
 173          * the start of the 256-bit shared memory region followed by 16 bits
 174          * containing the upper 4 bits of the 4 addresses in sequence.
 175          *
 176          * A 16 bit EQ vector number fills out the next-to-last 32-bit dword.
 177          *
 178          * The final 32-bit dword is used for protocol control information as
 179          * defined in smc_proto_hdr.
 180          */
 181
 182         memset(shm_buf, 0, sizeof(shm_buf));
 183         ptr = shm_buf;
 184
 185         /* EQ addr: low 48 bits of frame address */
 186         shmem = (u64 *)ptr;
 187         frame_addr = MANA_PFN(eq_addr);
 188         *shmem = frame_addr & PAGE_FRAME_L48_MASK;
 189         all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
 190                 (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
 191         ptr += PAGE_FRAME_L48_WIDTH_BYTES;
 192
 193         /* CQ addr: low 48 bits of frame address */
 194         shmem = (u64 *)ptr;
 195         frame_addr = MANA_PFN(cq_addr);
 196         *shmem = frame_addr & PAGE_FRAME_L48_MASK;
 197         all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
 198                 (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
 199         ptr += PAGE_FRAME_L48_WIDTH_BYTES;
 200
 201         /* RQ addr: low 48 bits of frame address */
 202         shmem = (u64 *)ptr;
 203         frame_addr = MANA_PFN(rq_addr);
 204         *shmem = frame_addr & PAGE_FRAME_L48_MASK;
 205         all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
 206                 (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
 207         ptr += PAGE_FRAME_L48_WIDTH_BYTES;
 208
 209         /* SQ addr: low 48 bits of frame address */
 210         shmem = (u64 *)ptr;
 211         frame_addr = MANA_PFN(sq_addr);
 212         *shmem = frame_addr & PAGE_FRAME_L48_MASK;
 213         all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
 214                 (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
 215         ptr += PAGE_FRAME_L48_WIDTH_BYTES;
 216
 217         /* High 4 bits of the four frame addresses */
 218         *((u16 *)ptr) = all_addr_h4bits;
 219         ptr += sizeof(u16);
 220
 221         /* EQ MSIX vector number */
 222         *((u16 *)ptr) = (u16)eq_msix_index;
 223         ptr += sizeof(u16);
 224
 225         /* 32-bit protocol header in final dword */
 226         *((u32 *)ptr) = 0;
 227
 228         hdr = (union smc_proto_hdr *)ptr;
 229         hdr->msg_type = SMC_MSG_TYPE_ESTABLISH_HWC;
 230         hdr->msg_version = SMC_MSG_TYPE_ESTABLISH_HWC_VERSION;
 231         hdr->direction = SMC_MSG_DIRECTION_REQUEST;
 232         hdr->reset_vf = reset_vf;
 233
 234         /* Write 256-message buffer to shared memory (final 32-bit write
 235          * triggers HW to set possession bit to PF).
 236          */
 237         dword = (u32 *)shm_buf;
 238         for (i = 0; i < SMC_APERTURE_DWORDS; i++)
 239                 writel(*dword++, sc->base + i * SMC_BASIC_UNIT);
 240
 241         /* Read shmem response (polling for VF possession) and validate.
 242          * For setup, waiting for response on shared memory is not strictly
 243          * necessary, since wait occurs later for results to appear in EQE's.
 244          */
 245         err = mana_smc_read_response(sc, SMC_MSG_TYPE_ESTABLISH_HWC,
 246                                      SMC_MSG_TYPE_ESTABLISH_HWC_VERSION,
 247                                      reset_vf);
 248         if (err) {
 249                 dev_err(sc->dev, "Error when setting up HWC: %d\n", err);
 250                 return err;
 251         }
 252
 253         return 0;
 254 }
 255
 256 int mana_smc_teardown_hwc(struct shm_channel *sc, bool reset_vf)
 257 {
 258         union smc_proto_hdr hdr = {};
 259         int err;
 260
 261         /* Ensure already has possession of shared memory */
 262         err = mana_smc_poll_register(sc->base, false);
 263         if (err) {
 264                 dev_err(sc->dev, "Timeout when tearing down HWC\n");
 265                 return err;
 266         }
 267
 268         /* Set up protocol header for HWC destroy message */
 269         hdr.msg_type = SMC_MSG_TYPE_DESTROY_HWC;
 270         hdr.msg_version = SMC_MSG_TYPE_DESTROY_HWC_VERSION;
 271         hdr.direction = SMC_MSG_DIRECTION_REQUEST;
 272         hdr.reset_vf = reset_vf;
 273
 274         /* Write message in high 32 bits of 256-bit shared memory, causing HW
 275          * to set possession bit to PF.
 276          */
 277         writel(hdr.as_uint32, sc->base + SMC_LAST_DWORD * SMC_BASIC_UNIT);
 278
 279         /* Read shmem response (polling for VF possession) and validate.
 280          * For teardown, waiting for response is required to ensure hardware
 281          * invalidates MST entries before software frees memory.
 282          */
 283         err = mana_smc_read_response(sc, SMC_MSG_TYPE_DESTROY_HWC,
 284                                      SMC_MSG_TYPE_DESTROY_HWC_VERSION,
 285                                      reset_vf);
 286         if (err) {
 287                 dev_err(sc->dev, "Error when tearing down HWC: %d\n", err);
 288                 return err;
 289         }
 290
 291         return 0;
 292 }