drivers/dma/idxd/idxd.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
   3 #ifndef _IDXD_H_
   4 #define _IDXD_H_
   5
   6 #include <linux/sbitmap.h>
   7 #include <linux/dmaengine.h>
   8 #include <linux/percpu-rwsem.h>
   9 #include <linux/wait.h>
  10 #include <linux/cdev.h>
  11 #include "registers.h"
  12
  13 #define IDXD_DRIVER_VERSION     "1.00"
  14
  15 extern struct kmem_cache *idxd_desc_pool;
  16
  17 #define IDXD_REG_TIMEOUT        50
  18 #define IDXD_DRAIN_TIMEOUT      5000
  19
  20 enum idxd_type {
  21         IDXD_TYPE_UNKNOWN = -1,
  22         IDXD_TYPE_DSA = 0,
  23         IDXD_TYPE_IAX,
  24         IDXD_TYPE_MAX,
  25 };
  26
  27 #define IDXD_NAME_SIZE          128
  28
  29 struct idxd_device_driver {
  30         struct device_driver drv;
  31 };
  32
  33 struct idxd_irq_entry {
  34         struct idxd_device *idxd;
  35         int id;
  36         struct llist_head pending_llist;
  37         struct list_head work_list;
  38         /*
  39          * Lock to protect access between irq thread process descriptor
  40          * and irq thread processing error descriptor.
  41          */
  42         spinlock_t list_lock;
  43 };
  44
  45 struct idxd_group {
  46         struct device conf_dev;
  47         struct idxd_device *idxd;
  48         struct grpcfg grpcfg;
  49         int id;
  50         int num_engines;
  51         int num_wqs;
  52         bool use_token_limit;
  53         u8 tokens_allowed;
  54         u8 tokens_reserved;
  55         int tc_a;
  56         int tc_b;
  57 };
  58
  59 #define IDXD_MAX_PRIORITY       0xf
  60
  61 enum idxd_wq_state {
  62         IDXD_WQ_DISABLED = 0,
  63         IDXD_WQ_ENABLED,
  64 };
  65
  66 enum idxd_wq_flag {
  67         WQ_FLAG_DEDICATED = 0,
  68         WQ_FLAG_BLOCK_ON_FAULT,
  69 };
  70
  71 enum idxd_wq_type {
  72         IDXD_WQT_NONE = 0,
  73         IDXD_WQT_KERNEL,
  74         IDXD_WQT_USER,
  75 };
  76
  77 struct idxd_cdev {
  78         struct cdev cdev;
  79         struct device *dev;
  80         int minor;
  81         struct wait_queue_head err_queue;
  82 };
  83
  84 #define IDXD_ALLOCATED_BATCH_SIZE       128U
  85 #define WQ_NAME_SIZE   1024
  86 #define WQ_TYPE_SIZE   10
  87
  88 enum idxd_op_type {
  89         IDXD_OP_BLOCK = 0,
  90         IDXD_OP_NONBLOCK = 1,
  91 };
  92
  93 enum idxd_complete_type {
  94         IDXD_COMPLETE_NORMAL = 0,
  95         IDXD_COMPLETE_ABORT,
  96         IDXD_COMPLETE_DEV_FAIL,
  97 };
  98
  99 struct idxd_wq {
 100         void __iomem *portal;
 101         struct device conf_dev;
 102         struct idxd_cdev idxd_cdev;
 103         struct idxd_device *idxd;
 104         int id;
 105         enum idxd_wq_type type;
 106         struct idxd_group *group;
 107         int client_count;
 108         struct mutex wq_lock;   /* mutex for workqueue */
 109         u32 size;
 110         u32 threshold;
 111         u32 priority;
 112         enum idxd_wq_state state;
 113         unsigned long flags;
 114         union wqcfg *wqcfg;
 115         u32 vec_ptr;            /* interrupt steering */
 116         struct dsa_hw_desc **hw_descs;
 117         int num_descs;
 118         union {
 119                 struct dsa_completion_record *compls;
 120                 struct iax_completion_record *iax_compls;
 121         };
 122         void *compls_raw;
 123         dma_addr_t compls_addr;
 124         dma_addr_t compls_addr_raw;
 125         int compls_size;
 126         struct idxd_desc **descs;
 127         struct sbitmap_queue sbq;
 128         struct dma_chan dma_chan;
 129         char name[WQ_NAME_SIZE + 1];
 130         u64 max_xfer_bytes;
 131         u32 max_batch_size;
 132         bool ats_dis;
 133 };
 134
 135 struct idxd_engine {
 136         struct device conf_dev;
 137         int id;
 138         struct idxd_group *group;
 139         struct idxd_device *idxd;
 140 };
 141
 142 /* shadow registers */
 143 struct idxd_hw {
 144         u32 version;
 145         union gen_cap_reg gen_cap;
 146         union wq_cap_reg wq_cap;
 147         union group_cap_reg group_cap;
 148         union engine_cap_reg engine_cap;
 149         struct opcap opcap;
 150 };
 151
 152 enum idxd_device_state {
 153         IDXD_DEV_HALTED = -1,
 154         IDXD_DEV_DISABLED = 0,
 155         IDXD_DEV_CONF_READY,
 156         IDXD_DEV_ENABLED,
 157 };
 158
 159 enum idxd_device_flag {
 160         IDXD_FLAG_CONFIGURABLE = 0,
 161         IDXD_FLAG_CMD_RUNNING,
 162         IDXD_FLAG_PASID_ENABLED,
 163 };
 164
 165 struct idxd_device {
 166         enum idxd_type type;
 167         struct device conf_dev;
 168         struct list_head list;
 169         struct idxd_hw hw;
 170         enum idxd_device_state state;
 171         unsigned long flags;
 172         int id;
 173         int major;
 174         u8 cmd_status;
 175
 176         struct pci_dev *pdev;
 177         void __iomem *reg_base;
 178
 179         spinlock_t dev_lock;    /* spinlock for device */
 180         struct completion *cmd_done;
 181         struct idxd_group *groups;
 182         struct idxd_wq *wqs;
 183         struct idxd_engine *engines;
 184
 185         struct iommu_sva *sva;
 186         unsigned int pasid;
 187
 188         int num_groups;
 189
 190         u32 msix_perm_offset;
 191         u32 wqcfg_offset;
 192         u32 grpcfg_offset;
 193         u32 perfmon_offset;
 194
 195         u64 max_xfer_bytes;
 196         u32 max_batch_size;
 197         int max_groups;
 198         int max_engines;
 199         int max_tokens;
 200         int max_wqs;
 201         int max_wq_size;
 202         int token_limit;
 203         int nr_tokens;          /* non-reserved tokens */
 204         unsigned int wqcfg_size;
 205         int compl_size;
 206
 207         union sw_err_reg sw_err;
 208         wait_queue_head_t cmd_waitq;
 209         struct msix_entry *msix_entries;
 210         int num_wq_irqs;
 211         struct idxd_irq_entry *irq_entries;
 212
 213         struct dma_device dma_dev;
 214         struct workqueue_struct *wq;
 215         struct work_struct work;
 216 };
 217
 218 /* IDXD software descriptor */
 219 struct idxd_desc {
 220         union {
 221                 struct dsa_hw_desc *hw;
 222                 struct iax_hw_desc *iax_hw;
 223         };
 224         dma_addr_t desc_dma;
 225         union {
 226                 struct dsa_completion_record *completion;
 227                 struct iax_completion_record *iax_completion;
 228         };
 229         dma_addr_t compl_dma;
 230         struct dma_async_tx_descriptor txd;
 231         struct llist_node llnode;
 232         struct list_head list;
 233         int id;
 234         int cpu;
 235         struct idxd_wq *wq;
 236 };
 237
 238 #define confdev_to_idxd(dev) container_of(dev, struct idxd_device, conf_dev)
 239 #define confdev_to_wq(dev) container_of(dev, struct idxd_wq, conf_dev)
 240
 241 extern struct bus_type dsa_bus_type;
 242 extern struct bus_type iax_bus_type;
 243
 244 extern bool support_enqcmd;
 245
 246 static inline bool wq_dedicated(struct idxd_wq *wq)
 247 {
 248         return test_bit(WQ_FLAG_DEDICATED, &wq->flags);
 249 }
 250
 251 static inline bool wq_shared(struct idxd_wq *wq)
 252 {
 253         return !test_bit(WQ_FLAG_DEDICATED, &wq->flags);
 254 }
 255
 256 static inline bool device_pasid_enabled(struct idxd_device *idxd)
 257 {
 258         return test_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
 259 }
 260
 261 static inline bool device_swq_supported(struct idxd_device *idxd)
 262 {
 263         return (support_enqcmd && device_pasid_enabled(idxd));
 264 }
 265
 266 enum idxd_portal_prot {
 267         IDXD_PORTAL_UNLIMITED = 0,
 268         IDXD_PORTAL_LIMITED,
 269 };
 270
 271 static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot)
 272 {
 273         return prot * 0x1000;
 274 }
 275
 276 static inline int idxd_get_wq_portal_full_offset(int wq_id,
 277                                                  enum idxd_portal_prot prot)
 278 {
 279         return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot);
 280 }
 281
 282 static inline void idxd_set_type(struct idxd_device *idxd)
 283 {
 284         struct pci_dev *pdev = idxd->pdev;
 285
 286         if (pdev->device == PCI_DEVICE_ID_INTEL_DSA_SPR0)
 287                 idxd->type = IDXD_TYPE_DSA;
 288         else if (pdev->device == PCI_DEVICE_ID_INTEL_IAX_SPR0)
 289                 idxd->type = IDXD_TYPE_IAX;
 290         else
 291                 idxd->type = IDXD_TYPE_UNKNOWN;
 292 }
 293
 294 static inline void idxd_wq_get(struct idxd_wq *wq)
 295 {
 296         wq->client_count++;
 297 }
 298
 299 static inline void idxd_wq_put(struct idxd_wq *wq)
 300 {
 301         wq->client_count--;
 302 }
 303
 304 static inline int idxd_wq_refcount(struct idxd_wq *wq)
 305 {
 306         return wq->client_count;
 307 };
 308
 309 const char *idxd_get_dev_name(struct idxd_device *idxd);
 310 int idxd_register_bus_type(void);
 311 void idxd_unregister_bus_type(void);
 312 int idxd_setup_sysfs(struct idxd_device *idxd);
 313 void idxd_cleanup_sysfs(struct idxd_device *idxd);
 314 int idxd_register_driver(void);
 315 void idxd_unregister_driver(void);
 316 struct bus_type *idxd_get_bus_type(struct idxd_device *idxd);
 317
 318 /* device interrupt control */
 319 irqreturn_t idxd_irq_handler(int vec, void *data);
 320 irqreturn_t idxd_misc_thread(int vec, void *data);
 321 irqreturn_t idxd_wq_thread(int irq, void *data);
 322 void idxd_mask_error_interrupts(struct idxd_device *idxd);
 323 void idxd_unmask_error_interrupts(struct idxd_device *idxd);
 324 void idxd_mask_msix_vectors(struct idxd_device *idxd);
 325 void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id);
 326 void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id);
 327
 328 /* device control */
 329 void idxd_device_init_reset(struct idxd_device *idxd);
 330 int idxd_device_enable(struct idxd_device *idxd);
 331 int idxd_device_disable(struct idxd_device *idxd);
 332 void idxd_device_reset(struct idxd_device *idxd);
 333 void idxd_device_cleanup(struct idxd_device *idxd);
 334 int idxd_device_config(struct idxd_device *idxd);
 335 void idxd_device_wqs_clear_state(struct idxd_device *idxd);
 336 void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid);
 337
 338 /* work queue control */
 339 int idxd_wq_alloc_resources(struct idxd_wq *wq);
 340 void idxd_wq_free_resources(struct idxd_wq *wq);
 341 int idxd_wq_enable(struct idxd_wq *wq);
 342 int idxd_wq_disable(struct idxd_wq *wq);
 343 void idxd_wq_drain(struct idxd_wq *wq);
 344 int idxd_wq_map_portal(struct idxd_wq *wq);
 345 void idxd_wq_unmap_portal(struct idxd_wq *wq);
 346 void idxd_wq_disable_cleanup(struct idxd_wq *wq);
 347 int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid);
 348 int idxd_wq_disable_pasid(struct idxd_wq *wq);
 349
 350 /* submission */
 351 int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc);
 352 struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype);
 353 void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc);
 354
 355 /* dmaengine */
 356 int idxd_register_dma_device(struct idxd_device *idxd);
 357 void idxd_unregister_dma_device(struct idxd_device *idxd);
 358 int idxd_register_dma_channel(struct idxd_wq *wq);
 359 void idxd_unregister_dma_channel(struct idxd_wq *wq);
 360 void idxd_parse_completion_status(u8 status, enum dmaengine_tx_result *res);
 361 void idxd_dma_complete_txd(struct idxd_desc *desc,
 362                            enum idxd_complete_type comp_type);
 363 dma_cookie_t idxd_dma_tx_submit(struct dma_async_tx_descriptor *tx);
 364
 365 /* cdev */
 366 int idxd_cdev_register(void);
 367 void idxd_cdev_remove(void);
 368 int idxd_cdev_get_major(struct idxd_device *idxd);
 369 int idxd_wq_add_cdev(struct idxd_wq *wq);
 370 void idxd_wq_del_cdev(struct idxd_wq *wq);
 371
 372 #endif