drivers/vdpa/solidrun/snet_ctrl.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * SolidRun DPU driver for control plane
   4  *
   5  * Copyright (C) 2022-2023 SolidRun
   6  *
   7  * Author: Alvaro Karsz <alvaro.karsz@solid-run.com>
   8  *
   9  */
  10
  11 #include <linux/iopoll.h>
  12
  13 #include "snet_vdpa.h"
  14
  15 enum snet_ctrl_opcodes {
  16         SNET_CTRL_OP_DESTROY = 1,
  17         SNET_CTRL_OP_READ_VQ_STATE,
  18         SNET_CTRL_OP_SUSPEND,
  19         SNET_CTRL_OP_RESUME,
  20 };
  21
  22 #define SNET_CTRL_TIMEOUT               2000000
  23
  24 #define SNET_CTRL_DATA_SIZE_MASK        0x0000FFFF
  25 #define SNET_CTRL_IN_PROCESS_MASK       0x00010000
  26 #define SNET_CTRL_CHUNK_RDY_MASK        0x00020000
  27 #define SNET_CTRL_ERROR_MASK            0x0FFC0000
  28
  29 #define SNET_VAL_TO_ERR(val)            (-(((val) & SNET_CTRL_ERROR_MASK) >> 18))
  30 #define SNET_EMPTY_CTRL(val)            (((val) & SNET_CTRL_ERROR_MASK) || \
  31                                                 !((val) & SNET_CTRL_IN_PROCESS_MASK))
  32 #define SNET_DATA_READY(val)            ((val) & (SNET_CTRL_ERROR_MASK | SNET_CTRL_CHUNK_RDY_MASK))
  33
  34 /* Control register used to read data from the DPU */
  35 struct snet_ctrl_reg_ctrl {
  36         /* Chunk size in 4B words */
  37         u16 data_size;
  38         /* We are in the middle of a command */
  39         u16 in_process:1;
  40         /* A data chunk is ready and can be consumed */
  41         u16 chunk_ready:1;
  42         /* Error code */
  43         u16 error:10;
  44         /* Saved for future usage */
  45         u16 rsvd:4;
  46 };
  47
  48 /* Opcode register */
  49 struct snet_ctrl_reg_op {
  50         u16 opcode;
  51         /* Only if VQ index is relevant for the command */
  52         u16 vq_idx;
  53 };
  54
  55 struct snet_ctrl_regs {
  56         struct snet_ctrl_reg_op op;
  57         struct snet_ctrl_reg_ctrl ctrl;
  58         u32 rsvd;
  59         u32 data[];
  60 };
  61
  62 static struct snet_ctrl_regs __iomem *snet_get_ctrl(struct snet *snet)
  63 {
  64         return snet->bar + snet->psnet->cfg.ctrl_off;
  65 }
  66
  67 static int snet_wait_for_empty_ctrl(struct snet_ctrl_regs __iomem *regs)
  68 {
  69         u32 val;
  70
  71         return readx_poll_timeout(ioread32, &regs->ctrl, val, SNET_EMPTY_CTRL(val), 10,
  72                                   SNET_CTRL_TIMEOUT);
  73 }
  74
  75 static int snet_wait_for_empty_op(struct snet_ctrl_regs __iomem *regs)
  76 {
  77         u32 val;
  78
  79         return readx_poll_timeout(ioread32, &regs->op, val, !val, 10, SNET_CTRL_TIMEOUT);
  80 }
  81
  82 static int snet_wait_for_data(struct snet_ctrl_regs __iomem *regs)
  83 {
  84         u32 val;
  85
  86         return readx_poll_timeout(ioread32, &regs->ctrl, val, SNET_DATA_READY(val), 10,
  87                                   SNET_CTRL_TIMEOUT);
  88 }
  89
  90 static u32 snet_read32_word(struct snet_ctrl_regs __iomem *ctrl_regs, u16 word_idx)
  91 {
  92         return ioread32(&ctrl_regs->data[word_idx]);
  93 }
  94
  95 static u32 snet_read_ctrl(struct snet_ctrl_regs __iomem *ctrl_regs)
  96 {
  97         return ioread32(&ctrl_regs->ctrl);
  98 }
  99
 100 static void snet_write_ctrl(struct snet_ctrl_regs __iomem *ctrl_regs, u32 val)
 101 {
 102         iowrite32(val, &ctrl_regs->ctrl);
 103 }
 104
 105 static void snet_write_op(struct snet_ctrl_regs __iomem *ctrl_regs, u32 val)
 106 {
 107         iowrite32(val, &ctrl_regs->op);
 108 }
 109
 110 static int snet_wait_for_dpu_completion(struct snet_ctrl_regs __iomem *ctrl_regs)
 111 {
 112         /* Wait until the DPU finishes completely.
 113          * It will clear the opcode register.
 114          */
 115         return snet_wait_for_empty_op(ctrl_regs);
 116 }
 117
 118 /* Reading ctrl from the DPU:
 119  * buf_size must be 4B aligned
 120  *
 121  * Steps:
 122  *
 123  * (1) Verify that the DPU is not in the middle of another operation by
 124  *     reading the in_process and error bits in the control register.
 125  * (2) Write the request opcode and the VQ idx in the opcode register
 126  *     and write the buffer size in the control register.
 127  * (3) Start readind chunks of data, chunk_ready bit indicates that a
 128  *     data chunk is available, we signal that we read the data by clearing the bit.
 129  * (4) Detect that the transfer is completed when the in_process bit
 130  *     in the control register is cleared or when the an error appears.
 131  */
 132 static int snet_ctrl_read_from_dpu(struct snet *snet, u16 opcode, u16 vq_idx, void *buffer,
 133                                    u32 buf_size)
 134 {
 135         struct pci_dev *pdev = snet->pdev;
 136         struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
 137         u32 *bfr_ptr = (u32 *)buffer;
 138         u32 val;
 139         u16 buf_words;
 140         int ret;
 141         u16 words, i, tot_words = 0;
 142
 143         /* Supported for config 2+ */
 144         if (!SNET_CFG_VER(snet, 2))
 145                 return -EOPNOTSUPP;
 146
 147         if (!IS_ALIGNED(buf_size, 4))
 148                 return -EINVAL;
 149
 150         mutex_lock(&snet->ctrl_lock);
 151
 152         buf_words = buf_size / 4;
 153
 154         /* Make sure control register is empty */
 155         ret = snet_wait_for_empty_ctrl(regs);
 156         if (ret) {
 157                 SNET_WARN(pdev, "Timeout waiting for previous control data to be consumed\n");
 158                 goto exit;
 159         }
 160
 161         /* We need to write the buffer size in the control register, and the opcode + vq index in
 162          * the opcode register.
 163          * We use a spinlock to serialize the writes.
 164          */
 165         spin_lock(&snet->ctrl_spinlock);
 166
 167         snet_write_ctrl(regs, buf_words);
 168         snet_write_op(regs, opcode | (vq_idx << 16));
 169
 170         spin_unlock(&snet->ctrl_spinlock);
 171
 172         while (buf_words != tot_words) {
 173                 ret = snet_wait_for_data(regs);
 174                 if (ret) {
 175                         SNET_WARN(pdev, "Timeout waiting for control data\n");
 176                         goto exit;
 177                 }
 178
 179                 val = snet_read_ctrl(regs);
 180
 181                 /* Error? */
 182                 if (val & SNET_CTRL_ERROR_MASK) {
 183                         ret = SNET_VAL_TO_ERR(val);
 184                         SNET_WARN(pdev, "Error while reading control data from DPU, err %d\n", ret);
 185                         goto exit;
 186                 }
 187
 188                 words = min_t(u16, val & SNET_CTRL_DATA_SIZE_MASK, buf_words - tot_words);
 189
 190                 for (i = 0; i < words; i++) {
 191                         *bfr_ptr = snet_read32_word(regs, i);
 192                         bfr_ptr++;
 193                 }
 194
 195                 tot_words += words;
 196
 197                 /* Is the job completed? */
 198                 if (!(val & SNET_CTRL_IN_PROCESS_MASK))
 199                         break;
 200
 201                 /* Clear the chunk ready bit and continue */
 202                 val &= ~SNET_CTRL_CHUNK_RDY_MASK;
 203                 snet_write_ctrl(regs, val);
 204         }
 205
 206         ret = snet_wait_for_dpu_completion(regs);
 207         if (ret)
 208                 SNET_WARN(pdev, "Timeout waiting for the DPU to complete a control command\n");
 209
 210 exit:
 211         mutex_unlock(&snet->ctrl_lock);
 212         return ret;
 213 }
 214
 215 /* Send a control message to the DPU using the old mechanism
 216  * used with config version 1.
 217  */
 218 static int snet_send_ctrl_msg_old(struct snet *snet, u32 opcode)
 219 {
 220         struct pci_dev *pdev = snet->pdev;
 221         struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
 222         int ret;
 223
 224         mutex_lock(&snet->ctrl_lock);
 225
 226         /* Old mechanism uses just 1 register, the opcode register.
 227          * Make sure that the opcode register is empty, and that the DPU isn't
 228          * processing an old message.
 229          */
 230         ret = snet_wait_for_empty_op(regs);
 231         if (ret) {
 232                 SNET_WARN(pdev, "Timeout waiting for previous control message to be ACKed\n");
 233                 goto exit;
 234         }
 235
 236         /* Write the message */
 237         snet_write_op(regs, opcode);
 238
 239         /* DPU ACKs the message by clearing the opcode register */
 240         ret = snet_wait_for_empty_op(regs);
 241         if (ret)
 242                 SNET_WARN(pdev, "Timeout waiting for a control message to be ACKed\n");
 243
 244 exit:
 245         mutex_unlock(&snet->ctrl_lock);
 246         return ret;
 247 }
 248
 249 /* Send a control message to the DPU.
 250  * A control message is a message without payload.
 251  */
 252 static int snet_send_ctrl_msg(struct snet *snet, u16 opcode, u16 vq_idx)
 253 {
 254         struct pci_dev *pdev = snet->pdev;
 255         struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
 256         u32 val;
 257         int ret;
 258
 259         /* If config version is not 2+, use the old mechanism */
 260         if (!SNET_CFG_VER(snet, 2))
 261                 return snet_send_ctrl_msg_old(snet, opcode);
 262
 263         mutex_lock(&snet->ctrl_lock);
 264
 265         /* Make sure control register is empty */
 266         ret = snet_wait_for_empty_ctrl(regs);
 267         if (ret) {
 268                 SNET_WARN(pdev, "Timeout waiting for previous control data to be consumed\n");
 269                 goto exit;
 270         }
 271
 272         /* We need to clear the control register and write the opcode + vq index in the opcode
 273          * register.
 274          * We use a spinlock to serialize the writes.
 275          */
 276         spin_lock(&snet->ctrl_spinlock);
 277
 278         snet_write_ctrl(regs, 0);
 279         snet_write_op(regs, opcode | (vq_idx << 16));
 280
 281         spin_unlock(&snet->ctrl_spinlock);
 282
 283         /* The DPU ACKs control messages by setting the chunk ready bit
 284          * without data.
 285          */
 286         ret = snet_wait_for_data(regs);
 287         if (ret) {
 288                 SNET_WARN(pdev, "Timeout waiting for control message to be ACKed\n");
 289                 goto exit;
 290         }
 291
 292         /* Check for errors */
 293         val = snet_read_ctrl(regs);
 294         ret = SNET_VAL_TO_ERR(val);
 295
 296         /* Clear the chunk ready bit */
 297         val &= ~SNET_CTRL_CHUNK_RDY_MASK;
 298         snet_write_ctrl(regs, val);
 299
 300         ret = snet_wait_for_dpu_completion(regs);
 301         if (ret)
 302                 SNET_WARN(pdev, "Timeout waiting for DPU to complete a control command, err %d\n",
 303                           ret);
 304
 305 exit:
 306         mutex_unlock(&snet->ctrl_lock);
 307         return ret;
 308 }
 309
 310 void snet_ctrl_clear(struct snet *snet)
 311 {
 312         struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
 313
 314         snet_write_op(regs, 0);
 315 }
 316
 317 int snet_destroy_dev(struct snet *snet)
 318 {
 319         return snet_send_ctrl_msg(snet, SNET_CTRL_OP_DESTROY, 0);
 320 }
 321
 322 int snet_read_vq_state(struct snet *snet, u16 idx, struct vdpa_vq_state *state)
 323 {
 324         return snet_ctrl_read_from_dpu(snet, SNET_CTRL_OP_READ_VQ_STATE, idx, state,
 325                                        sizeof(*state));
 326 }
 327
 328 int snet_suspend_dev(struct snet *snet)
 329 {
 330         return snet_send_ctrl_msg(snet, SNET_CTRL_OP_SUSPEND, 0);
 331 }
 332
 333 int snet_resume_dev(struct snet *snet)
 334 {
 335         return snet_send_ctrl_msg(snet, SNET_CTRL_OP_RESUME, 0);
 336 }