drivers/dma/ptdma/ptdma-dev.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * AMD Passthru DMA device driver
   4  * -- Based on the CCP driver
   5  *
   6  * Copyright (C) 2016,2021 Advanced Micro Devices, Inc.
   7  *
   8  * Author: Sanjay R Mehta <sanju.mehta@amd.com>
   9  * Author: Gary R Hook <gary.hook@amd.com>
  10  */
  11
  12 #include <linux/bitfield.h>
  13 #include <linux/dma-mapping.h>
  14 #include <linux/debugfs.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/kernel.h>
  17 #include <linux/module.h>
  18 #include <linux/pci.h>
  19
  20 #include "ptdma.h"
  21
  22 /* Human-readable error strings */
  23 static char *pt_error_codes[] = {
  24         "",
  25         "ERR 01: ILLEGAL_ENGINE",
  26         "ERR 03: ILLEGAL_FUNCTION_TYPE",
  27         "ERR 04: ILLEGAL_FUNCTION_MODE",
  28         "ERR 06: ILLEGAL_FUNCTION_SIZE",
  29         "ERR 08: ILLEGAL_FUNCTION_RSVD",
  30         "ERR 09: ILLEGAL_BUFFER_LENGTH",
  31         "ERR 10: VLSB_FAULT",
  32         "ERR 11: ILLEGAL_MEM_ADDR",
  33         "ERR 12: ILLEGAL_MEM_SEL",
  34         "ERR 13: ILLEGAL_CONTEXT_ID",
  35         "ERR 15: 0xF Reserved",
  36         "ERR 18: CMD_TIMEOUT",
  37         "ERR 19: IDMA0_AXI_SLVERR",
  38         "ERR 20: IDMA0_AXI_DECERR",
  39         "ERR 21: 0x15 Reserved",
  40         "ERR 22: IDMA1_AXI_SLAVE_FAULT",
  41         "ERR 23: IDMA1_AIXI_DECERR",
  42         "ERR 24: 0x18 Reserved",
  43         "ERR 27: 0x1B Reserved",
  44         "ERR 38: ODMA0_AXI_SLVERR",
  45         "ERR 39: ODMA0_AXI_DECERR",
  46         "ERR 40: 0x28 Reserved",
  47         "ERR 41: ODMA1_AXI_SLVERR",
  48         "ERR 42: ODMA1_AXI_DECERR",
  49         "ERR 43: LSB_PARITY_ERR",
  50 };
  51
  52 static void pt_log_error(struct pt_device *d, int e)
  53 {
  54         dev_err(d->dev, "PTDMA error: %s (0x%x)\n", pt_error_codes[e], e);
  55 }
  56
  57 void pt_start_queue(struct pt_cmd_queue *cmd_q)
  58 {
  59         /* Turn on the run bit */
  60         iowrite32(cmd_q->qcontrol | CMD_Q_RUN, cmd_q->reg_control);
  61 }
  62
  63 void pt_stop_queue(struct pt_cmd_queue *cmd_q)
  64 {
  65         /* Turn off the run bit */
  66         iowrite32(cmd_q->qcontrol & ~CMD_Q_RUN, cmd_q->reg_control);
  67 }
  68
  69 static int pt_core_execute_cmd(struct ptdma_desc *desc, struct pt_cmd_queue *cmd_q)
  70 {
  71         bool soc = FIELD_GET(DWORD0_SOC, desc->dw0);
  72         u8 *q_desc = (u8 *)&cmd_q->qbase[cmd_q->qidx];
  73         u32 tail;
  74         unsigned long flags;
  75
  76         if (soc) {
  77                 desc->dw0 |= FIELD_PREP(DWORD0_IOC, desc->dw0);
  78                 desc->dw0 &= ~DWORD0_SOC;
  79         }
  80         spin_lock_irqsave(&cmd_q->q_lock, flags);
  81
  82         /* Copy 32-byte command descriptor to hw queue. */
  83         memcpy(q_desc, desc, 32);
  84         cmd_q->qidx = (cmd_q->qidx + 1) % CMD_Q_LEN;
  85
  86         /* The data used by this command must be flushed to memory */
  87         wmb();
  88
  89         /* Write the new tail address back to the queue register */
  90         tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);
  91         iowrite32(tail, cmd_q->reg_control + 0x0004);
  92
  93         /* Turn the queue back on using our cached control register */
  94         pt_start_queue(cmd_q);
  95         spin_unlock_irqrestore(&cmd_q->q_lock, flags);
  96
  97         return 0;
  98 }
  99
 100 int pt_core_perform_passthru(struct pt_cmd_queue *cmd_q,
 101                              struct pt_passthru_engine *pt_engine)
 102 {
 103         struct ptdma_desc desc;
 104         struct pt_device *pt = container_of(cmd_q, struct pt_device, cmd_q);
 105
 106         cmd_q->cmd_error = 0;
 107         cmd_q->total_pt_ops++;
 108         memset(&desc, 0, sizeof(desc));
 109         desc.dw0 = CMD_DESC_DW0_VAL;
 110         desc.length = pt_engine->src_len;
 111         desc.src_lo = lower_32_bits(pt_engine->src_dma);
 112         desc.dw3.src_hi = upper_32_bits(pt_engine->src_dma);
 113         desc.dst_lo = lower_32_bits(pt_engine->dst_dma);
 114         desc.dw5.dst_hi = upper_32_bits(pt_engine->dst_dma);
 115
 116         if (cmd_q->int_en)
 117                 pt_core_enable_queue_interrupts(pt);
 118         else
 119                 pt_core_disable_queue_interrupts(pt);
 120
 121         return pt_core_execute_cmd(&desc, cmd_q);
 122 }
 123
 124 static void pt_do_cmd_complete(unsigned long data)
 125 {
 126         struct pt_tasklet_data *tdata = (struct pt_tasklet_data *)data;
 127         struct pt_cmd *cmd = tdata->cmd;
 128         struct pt_cmd_queue *cmd_q = &cmd->pt->cmd_q;
 129         u32 tail;
 130
 131         if (cmd_q->cmd_error) {
 132                /*
 133                 * Log the error and flush the queue by
 134                 * moving the head pointer
 135                 */
 136                 tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);
 137                 pt_log_error(cmd_q->pt, cmd_q->cmd_error);
 138                 iowrite32(tail, cmd_q->reg_control + 0x0008);
 139         }
 140
 141         cmd->pt_cmd_callback(cmd->data, cmd->ret);
 142 }
 143
 144 void pt_check_status_trans(struct pt_device *pt, struct pt_cmd_queue *cmd_q)
 145 {
 146         u32 status;
 147
 148         status = ioread32(cmd_q->reg_control + 0x0010);
 149         if (status) {
 150                 cmd_q->int_status = status;
 151                 cmd_q->q_status = ioread32(cmd_q->reg_control + 0x0100);
 152                 cmd_q->q_int_status = ioread32(cmd_q->reg_control + 0x0104);
 153
 154                 /* On error, only save the first error value */
 155                 if ((status & INT_ERROR) && !cmd_q->cmd_error)
 156                         cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);
 157
 158                 /* Acknowledge the completion */
 159                 iowrite32(status, cmd_q->reg_control + 0x0010);
 160                 pt_do_cmd_complete((ulong)&pt->tdata);
 161         }
 162 }
 163
 164 static irqreturn_t pt_core_irq_handler(int irq, void *data)
 165 {
 166         struct pt_device *pt = data;
 167         struct pt_cmd_queue *cmd_q = &pt->cmd_q;
 168
 169         pt_core_disable_queue_interrupts(pt);
 170         pt->total_interrupts++;
 171         pt_check_status_trans(pt, cmd_q);
 172         pt_core_enable_queue_interrupts(pt);
 173         return IRQ_HANDLED;
 174 }
 175
 176 int pt_core_init(struct pt_device *pt)
 177 {
 178         char dma_pool_name[MAX_DMAPOOL_NAME_LEN];
 179         struct pt_cmd_queue *cmd_q = &pt->cmd_q;
 180         u32 dma_addr_lo, dma_addr_hi;
 181         struct device *dev = pt->dev;
 182         struct dma_pool *dma_pool;
 183         int ret;
 184
 185         /* Allocate a dma pool for the queue */
 186         snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q", dev_name(pt->dev));
 187
 188         dma_pool = dma_pool_create(dma_pool_name, dev,
 189                                    PT_DMAPOOL_MAX_SIZE,
 190                                    PT_DMAPOOL_ALIGN, 0);
 191         if (!dma_pool)
 192                 return -ENOMEM;
 193
 194         /* ptdma core initialisation */
 195         iowrite32(CMD_CONFIG_VHB_EN, pt->io_regs + CMD_CONFIG_OFFSET);
 196         iowrite32(CMD_QUEUE_PRIO, pt->io_regs + CMD_QUEUE_PRIO_OFFSET);
 197         iowrite32(CMD_TIMEOUT_DISABLE, pt->io_regs + CMD_TIMEOUT_OFFSET);
 198         iowrite32(CMD_CLK_GATE_CONFIG, pt->io_regs + CMD_CLK_GATE_CTL_OFFSET);
 199         iowrite32(CMD_CONFIG_REQID, pt->io_regs + CMD_REQID_CONFIG_OFFSET);
 200
 201         cmd_q->pt = pt;
 202         cmd_q->dma_pool = dma_pool;
 203         spin_lock_init(&cmd_q->q_lock);
 204
 205         /* Page alignment satisfies our needs for N <= 128 */
 206         cmd_q->qsize = Q_SIZE(Q_DESC_SIZE);
 207         cmd_q->qbase = dma_alloc_coherent(dev, cmd_q->qsize,
 208                                           &cmd_q->qbase_dma,
 209                                           GFP_KERNEL);
 210         if (!cmd_q->qbase) {
 211                 dev_err(dev, "unable to allocate command queue\n");
 212                 ret = -ENOMEM;
 213                 goto e_destroy_pool;
 214         }
 215
 216         cmd_q->qidx = 0;
 217
 218         /* Preset some register values */
 219         cmd_q->reg_control = pt->io_regs + CMD_Q_STATUS_INCR;
 220
 221         /* Turn off the queues and disable interrupts until ready */
 222         pt_core_disable_queue_interrupts(pt);
 223
 224         cmd_q->qcontrol = 0; /* Start with nothing */
 225         iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
 226
 227         ioread32(cmd_q->reg_control + 0x0104);
 228         ioread32(cmd_q->reg_control + 0x0100);
 229
 230         /* Clear the interrupt status */
 231         iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010);
 232
 233         /* Request an irq */
 234         ret = request_irq(pt->pt_irq, pt_core_irq_handler, 0, dev_name(pt->dev), pt);
 235         if (ret) {
 236                 dev_err(dev, "unable to allocate an IRQ\n");
 237                 goto e_free_dma;
 238         }
 239
 240         /* Update the device registers with queue information. */
 241         cmd_q->qcontrol &= ~CMD_Q_SIZE;
 242         cmd_q->qcontrol |= FIELD_PREP(CMD_Q_SIZE, QUEUE_SIZE_VAL);
 243
 244         cmd_q->qdma_tail = cmd_q->qbase_dma;
 245         dma_addr_lo = lower_32_bits(cmd_q->qdma_tail);
 246         iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0004);
 247         iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0008);
 248
 249         dma_addr_hi = upper_32_bits(cmd_q->qdma_tail);
 250         cmd_q->qcontrol |= (dma_addr_hi << 16);
 251         iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
 252
 253         pt_core_enable_queue_interrupts(pt);
 254
 255         /* Register the DMA engine support */
 256         ret = pt_dmaengine_register(pt);
 257         if (ret)
 258                 goto e_free_irq;
 259
 260         /* Set up debugfs entries */
 261         ptdma_debugfs_setup(pt);
 262
 263         return 0;
 264
 265 e_free_irq:
 266         free_irq(pt->pt_irq, pt);
 267
 268 e_free_dma:
 269         dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, cmd_q->qbase_dma);
 270
 271 e_destroy_pool:
 272         dma_pool_destroy(pt->cmd_q.dma_pool);
 273
 274         return ret;
 275 }
 276
 277 void pt_core_destroy(struct pt_device *pt)
 278 {
 279         struct device *dev = pt->dev;
 280         struct pt_cmd_queue *cmd_q = &pt->cmd_q;
 281         struct pt_cmd *cmd;
 282
 283         /* Unregister the DMA engine */
 284         pt_dmaengine_unregister(pt);
 285
 286         /* Disable and clear interrupts */
 287         pt_core_disable_queue_interrupts(pt);
 288
 289         /* Turn off the run bit */
 290         pt_stop_queue(cmd_q);
 291
 292         /* Clear the interrupt status */
 293         iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010);
 294         ioread32(cmd_q->reg_control + 0x0104);
 295         ioread32(cmd_q->reg_control + 0x0100);
 296
 297         free_irq(pt->pt_irq, pt);
 298
 299         dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase,
 300                           cmd_q->qbase_dma);
 301
 302         /* Flush the cmd queue */
 303         while (!list_empty(&pt->cmd)) {
 304                 /* Invoke the callback directly with an error code */
 305                 cmd = list_first_entry(&pt->cmd, struct pt_cmd, entry);
 306                 list_del(&cmd->entry);
 307                 cmd->pt_cmd_callback(cmd->data, -ENODEV);
 308         }
 309 }