drivers/misc/pci_endpoint_test.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Host side test driver to test endpoint functionality
   4  *
   5  * Copyright (C) 2017 Texas Instruments
   6  * Author: Kishon Vijay Abraham I <kishon@ti.com>
   7  */
   8
   9 #include <linux/crc32.h>
  10 #include <linux/cleanup.h>
  11 #include <linux/delay.h>
  12 #include <linux/fs.h>
  13 #include <linux/io.h>
  14 #include <linux/interrupt.h>
  15 #include <linux/irq.h>
  16 #include <linux/miscdevice.h>
  17 #include <linux/module.h>
  18 #include <linux/mutex.h>
  19 #include <linux/random.h>
  20 #include <linux/slab.h>
  21 #include <linux/uaccess.h>
  22 #include <linux/pci.h>
  23 #include <linux/pci_ids.h>
  24
  25 #include <linux/pci_regs.h>
  26
  27 #include <uapi/linux/pcitest.h>
  28
  29 #define DRV_MODULE_NAME                         "pci-endpoint-test"
  30
  31 #define IRQ_TYPE_UNDEFINED                      -1
  32 #define IRQ_TYPE_INTX                           0
  33 #define IRQ_TYPE_MSI                            1
  34 #define IRQ_TYPE_MSIX                           2
  35
  36 #define PCI_ENDPOINT_TEST_MAGIC                 0x0
  37
  38 #define PCI_ENDPOINT_TEST_COMMAND               0x4
  39 #define COMMAND_RAISE_INTX_IRQ                  BIT(0)
  40 #define COMMAND_RAISE_MSI_IRQ                   BIT(1)
  41 #define COMMAND_RAISE_MSIX_IRQ                  BIT(2)
  42 #define COMMAND_READ                            BIT(3)
  43 #define COMMAND_WRITE                           BIT(4)
  44 #define COMMAND_COPY                            BIT(5)
  45
  46 #define PCI_ENDPOINT_TEST_STATUS                0x8
  47 #define STATUS_READ_SUCCESS                     BIT(0)
  48 #define STATUS_READ_FAIL                        BIT(1)
  49 #define STATUS_WRITE_SUCCESS                    BIT(2)
  50 #define STATUS_WRITE_FAIL                       BIT(3)
  51 #define STATUS_COPY_SUCCESS                     BIT(4)
  52 #define STATUS_COPY_FAIL                        BIT(5)
  53 #define STATUS_IRQ_RAISED                       BIT(6)
  54 #define STATUS_SRC_ADDR_INVALID                 BIT(7)
  55 #define STATUS_DST_ADDR_INVALID                 BIT(8)
  56
  57 #define PCI_ENDPOINT_TEST_LOWER_SRC_ADDR        0x0c
  58 #define PCI_ENDPOINT_TEST_UPPER_SRC_ADDR        0x10
  59
  60 #define PCI_ENDPOINT_TEST_LOWER_DST_ADDR        0x14
  61 #define PCI_ENDPOINT_TEST_UPPER_DST_ADDR        0x18
  62
  63 #define PCI_ENDPOINT_TEST_SIZE                  0x1c
  64 #define PCI_ENDPOINT_TEST_CHECKSUM              0x20
  65
  66 #define PCI_ENDPOINT_TEST_IRQ_TYPE              0x24
  67 #define PCI_ENDPOINT_TEST_IRQ_NUMBER            0x28
  68
  69 #define PCI_ENDPOINT_TEST_FLAGS                 0x2c
  70 #define FLAG_USE_DMA                            BIT(0)
  71
  72 #define PCI_DEVICE_ID_TI_AM654                  0xb00c
  73 #define PCI_DEVICE_ID_TI_J7200                  0xb00f
  74 #define PCI_DEVICE_ID_TI_AM64                   0xb010
  75 #define PCI_DEVICE_ID_TI_J721S2         0xb013
  76 #define PCI_DEVICE_ID_LS1088A                   0x80c0
  77 #define PCI_DEVICE_ID_IMX8                      0x0808
  78
  79 #define is_am654_pci_dev(pdev)          \
  80                 ((pdev)->device == PCI_DEVICE_ID_TI_AM654)
  81
  82 #define PCI_DEVICE_ID_RENESAS_R8A774A1          0x0028
  83 #define PCI_DEVICE_ID_RENESAS_R8A774B1          0x002b
  84 #define PCI_DEVICE_ID_RENESAS_R8A774C0          0x002d
  85 #define PCI_DEVICE_ID_RENESAS_R8A774E1          0x0025
  86 #define PCI_DEVICE_ID_RENESAS_R8A779F0          0x0031
  87
  88 #define PCI_VENDOR_ID_ROCKCHIP                  0x1d87
  89 #define PCI_DEVICE_ID_ROCKCHIP_RK3588           0x3588
  90
  91 static DEFINE_IDA(pci_endpoint_test_ida);
  92
  93 #define to_endpoint_test(priv) container_of((priv), struct pci_endpoint_test, \
  94                                             miscdev)
  95
  96 static bool no_msi;
  97 module_param(no_msi, bool, 0444);
  98 MODULE_PARM_DESC(no_msi, "Disable MSI interrupt in pci_endpoint_test");
  99
 100 static int irq_type = IRQ_TYPE_MSI;
 101 module_param(irq_type, int, 0444);
 102 MODULE_PARM_DESC(irq_type, "IRQ mode selection in pci_endpoint_test (0 - Legacy, 1 - MSI, 2 - MSI-X)");
 103
 104 enum pci_barno {
 105         BAR_0,
 106         BAR_1,
 107         BAR_2,
 108         BAR_3,
 109         BAR_4,
 110         BAR_5,
 111 };
 112
 113 struct pci_endpoint_test {
 114         struct pci_dev  *pdev;
 115         void __iomem    *base;
 116         void __iomem    *bar[PCI_STD_NUM_BARS];
 117         struct completion irq_raised;
 118         int             last_irq;
 119         int             num_irqs;
 120         int             irq_type;
 121         /* mutex to protect the ioctls */
 122         struct mutex    mutex;
 123         struct miscdevice miscdev;
 124         enum pci_barno test_reg_bar;
 125         size_t alignment;
 126         const char *name;
 127 };
 128
 129 struct pci_endpoint_test_data {
 130         enum pci_barno test_reg_bar;
 131         size_t alignment;
 132         int irq_type;
 133 };
 134
 135 static inline u32 pci_endpoint_test_readl(struct pci_endpoint_test *test,
 136                                           u32 offset)
 137 {
 138         return readl(test->base + offset);
 139 }
 140
 141 static inline void pci_endpoint_test_writel(struct pci_endpoint_test *test,
 142                                             u32 offset, u32 value)
 143 {
 144         writel(value, test->base + offset);
 145 }
 146
 147 static irqreturn_t pci_endpoint_test_irqhandler(int irq, void *dev_id)
 148 {
 149         struct pci_endpoint_test *test = dev_id;
 150         u32 reg;
 151
 152         reg = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_STATUS);
 153         if (reg & STATUS_IRQ_RAISED) {
 154                 test->last_irq = irq;
 155                 complete(&test->irq_raised);
 156         }
 157
 158         return IRQ_HANDLED;
 159 }
 160
 161 static void pci_endpoint_test_free_irq_vectors(struct pci_endpoint_test *test)
 162 {
 163         struct pci_dev *pdev = test->pdev;
 164
 165         pci_free_irq_vectors(pdev);
 166         test->irq_type = IRQ_TYPE_UNDEFINED;
 167 }
 168
 169 static bool pci_endpoint_test_alloc_irq_vectors(struct pci_endpoint_test *test,
 170                                                 int type)
 171 {
 172         int irq = -1;
 173         struct pci_dev *pdev = test->pdev;
 174         struct device *dev = &pdev->dev;
 175         bool res = true;
 176
 177         switch (type) {
 178         case IRQ_TYPE_INTX:
 179                 irq = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_INTX);
 180                 if (irq < 0)
 181                         dev_err(dev, "Failed to get Legacy interrupt\n");
 182                 break;
 183         case IRQ_TYPE_MSI:
 184                 irq = pci_alloc_irq_vectors(pdev, 1, 32, PCI_IRQ_MSI);
 185                 if (irq < 0)
 186                         dev_err(dev, "Failed to get MSI interrupts\n");
 187                 break;
 188         case IRQ_TYPE_MSIX:
 189                 irq = pci_alloc_irq_vectors(pdev, 1, 2048, PCI_IRQ_MSIX);
 190                 if (irq < 0)
 191                         dev_err(dev, "Failed to get MSI-X interrupts\n");
 192                 break;
 193         default:
 194                 dev_err(dev, "Invalid IRQ type selected\n");
 195         }
 196
 197         if (irq < 0) {
 198                 irq = 0;
 199                 res = false;
 200         }
 201
 202         test->irq_type = type;
 203         test->num_irqs = irq;
 204
 205         return res;
 206 }
 207
 208 static void pci_endpoint_test_release_irq(struct pci_endpoint_test *test)
 209 {
 210         int i;
 211         struct pci_dev *pdev = test->pdev;
 212         struct device *dev = &pdev->dev;
 213
 214         for (i = 0; i < test->num_irqs; i++)
 215                 devm_free_irq(dev, pci_irq_vector(pdev, i), test);
 216
 217         test->num_irqs = 0;
 218 }
 219
 220 static bool pci_endpoint_test_request_irq(struct pci_endpoint_test *test)
 221 {
 222         int i;
 223         int err;
 224         struct pci_dev *pdev = test->pdev;
 225         struct device *dev = &pdev->dev;
 226
 227         for (i = 0; i < test->num_irqs; i++) {
 228                 err = devm_request_irq(dev, pci_irq_vector(pdev, i),
 229                                        pci_endpoint_test_irqhandler,
 230                                        IRQF_SHARED, test->name, test);
 231                 if (err)
 232                         goto fail;
 233         }
 234
 235         return true;
 236
 237 fail:
 238         switch (irq_type) {
 239         case IRQ_TYPE_INTX:
 240                 dev_err(dev, "Failed to request IRQ %d for Legacy\n",
 241                         pci_irq_vector(pdev, i));
 242                 break;
 243         case IRQ_TYPE_MSI:
 244                 dev_err(dev, "Failed to request IRQ %d for MSI %d\n",
 245                         pci_irq_vector(pdev, i),
 246                         i + 1);
 247                 break;
 248         case IRQ_TYPE_MSIX:
 249                 dev_err(dev, "Failed to request IRQ %d for MSI-X %d\n",
 250                         pci_irq_vector(pdev, i),
 251                         i + 1);
 252                 break;
 253         }
 254
 255         return false;
 256 }
 257
 258 static const u32 bar_test_pattern[] = {
 259         0xA0A0A0A0,
 260         0xA1A1A1A1,
 261         0xA2A2A2A2,
 262         0xA3A3A3A3,
 263         0xA4A4A4A4,
 264         0xA5A5A5A5,
 265 };
 266
 267 static int pci_endpoint_test_bar_memcmp(struct pci_endpoint_test *test,
 268                                         enum pci_barno barno, int offset,
 269                                         void *write_buf, void *read_buf,
 270                                         int size)
 271 {
 272         memset(write_buf, bar_test_pattern[barno], size);
 273         memcpy_toio(test->bar[barno] + offset, write_buf, size);
 274
 275         memcpy_fromio(read_buf, test->bar[barno] + offset, size);
 276
 277         return memcmp(write_buf, read_buf, size);
 278 }
 279
 280 static bool pci_endpoint_test_bar(struct pci_endpoint_test *test,
 281                                   enum pci_barno barno)
 282 {
 283         int j, bar_size, buf_size, iters, remain;
 284         void *write_buf __free(kfree) = NULL;
 285         void *read_buf __free(kfree) = NULL;
 286         struct pci_dev *pdev = test->pdev;
 287
 288         if (!test->bar[barno])
 289                 return false;
 290
 291         bar_size = pci_resource_len(pdev, barno);
 292
 293         if (barno == test->test_reg_bar)
 294                 bar_size = 0x4;
 295
 296         /*
 297          * Allocate a buffer of max size 1MB, and reuse that buffer while
 298          * iterating over the whole BAR size (which might be much larger).
 299          */
 300         buf_size = min(SZ_1M, bar_size);
 301
 302         write_buf = kmalloc(buf_size, GFP_KERNEL);
 303         if (!write_buf)
 304                 return false;
 305
 306         read_buf = kmalloc(buf_size, GFP_KERNEL);
 307         if (!read_buf)
 308                 return false;
 309
 310         iters = bar_size / buf_size;
 311         for (j = 0; j < iters; j++)
 312                 if (pci_endpoint_test_bar_memcmp(test, barno, buf_size * j,
 313                                                  write_buf, read_buf, buf_size))
 314                         return false;
 315
 316         remain = bar_size % buf_size;
 317         if (remain)
 318                 if (pci_endpoint_test_bar_memcmp(test, barno, buf_size * iters,
 319                                                  write_buf, read_buf, remain))
 320                         return false;
 321
 322         return true;
 323 }
 324
 325 static bool pci_endpoint_test_intx_irq(struct pci_endpoint_test *test)
 326 {
 327         u32 val;
 328
 329         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE,
 330                                  IRQ_TYPE_INTX);
 331         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, 0);
 332         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
 333                                  COMMAND_RAISE_INTX_IRQ);
 334         val = wait_for_completion_timeout(&test->irq_raised,
 335                                           msecs_to_jiffies(1000));
 336         if (!val)
 337                 return false;
 338
 339         return true;
 340 }
 341
 342 static bool pci_endpoint_test_msi_irq(struct pci_endpoint_test *test,
 343                                        u16 msi_num, bool msix)
 344 {
 345         u32 val;
 346         struct pci_dev *pdev = test->pdev;
 347
 348         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE,
 349                                  msix ? IRQ_TYPE_MSIX : IRQ_TYPE_MSI);
 350         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, msi_num);
 351         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
 352                                  msix ? COMMAND_RAISE_MSIX_IRQ :
 353                                  COMMAND_RAISE_MSI_IRQ);
 354         val = wait_for_completion_timeout(&test->irq_raised,
 355                                           msecs_to_jiffies(1000));
 356         if (!val)
 357                 return false;
 358
 359         return pci_irq_vector(pdev, msi_num - 1) == test->last_irq;
 360 }
 361
 362 static int pci_endpoint_test_validate_xfer_params(struct device *dev,
 363                 struct pci_endpoint_test_xfer_param *param, size_t alignment)
 364 {
 365         if (!param->size) {
 366                 dev_dbg(dev, "Data size is zero\n");
 367                 return -EINVAL;
 368         }
 369
 370         if (param->size > SIZE_MAX - alignment) {
 371                 dev_dbg(dev, "Maximum transfer data size exceeded\n");
 372                 return -EINVAL;
 373         }
 374
 375         return 0;
 376 }
 377
 378 static bool pci_endpoint_test_copy(struct pci_endpoint_test *test,
 379                                    unsigned long arg)
 380 {
 381         struct pci_endpoint_test_xfer_param param;
 382         bool ret = false;
 383         void *src_addr;
 384         void *dst_addr;
 385         u32 flags = 0;
 386         bool use_dma;
 387         size_t size;
 388         dma_addr_t src_phys_addr;
 389         dma_addr_t dst_phys_addr;
 390         struct pci_dev *pdev = test->pdev;
 391         struct device *dev = &pdev->dev;
 392         void *orig_src_addr;
 393         dma_addr_t orig_src_phys_addr;
 394         void *orig_dst_addr;
 395         dma_addr_t orig_dst_phys_addr;
 396         size_t offset;
 397         size_t alignment = test->alignment;
 398         int irq_type = test->irq_type;
 399         u32 src_crc32;
 400         u32 dst_crc32;
 401         int err;
 402
 403         err = copy_from_user(&param, (void __user *)arg, sizeof(param));
 404         if (err) {
 405                 dev_err(dev, "Failed to get transfer param\n");
 406                 return false;
 407         }
 408
 409         err = pci_endpoint_test_validate_xfer_params(dev, &param, alignment);
 410         if (err)
 411                 return false;
 412
 413         size = param.size;
 414
 415         use_dma = !!(param.flags & PCITEST_FLAGS_USE_DMA);
 416         if (use_dma)
 417                 flags |= FLAG_USE_DMA;
 418
 419         if (irq_type < IRQ_TYPE_INTX || irq_type > IRQ_TYPE_MSIX) {
 420                 dev_err(dev, "Invalid IRQ type option\n");
 421                 goto err;
 422         }
 423
 424         orig_src_addr = kzalloc(size + alignment, GFP_KERNEL);
 425         if (!orig_src_addr) {
 426                 dev_err(dev, "Failed to allocate source buffer\n");
 427                 ret = false;
 428                 goto err;
 429         }
 430
 431         get_random_bytes(orig_src_addr, size + alignment);
 432         orig_src_phys_addr = dma_map_single(dev, orig_src_addr,
 433                                             size + alignment, DMA_TO_DEVICE);
 434         if (dma_mapping_error(dev, orig_src_phys_addr)) {
 435                 dev_err(dev, "failed to map source buffer address\n");
 436                 ret = false;
 437                 goto err_src_phys_addr;
 438         }
 439
 440         if (alignment && !IS_ALIGNED(orig_src_phys_addr, alignment)) {
 441                 src_phys_addr = PTR_ALIGN(orig_src_phys_addr, alignment);
 442                 offset = src_phys_addr - orig_src_phys_addr;
 443                 src_addr = orig_src_addr + offset;
 444         } else {
 445                 src_phys_addr = orig_src_phys_addr;
 446                 src_addr = orig_src_addr;
 447         }
 448
 449         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_LOWER_SRC_ADDR,
 450                                  lower_32_bits(src_phys_addr));
 451
 452         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_UPPER_SRC_ADDR,
 453                                  upper_32_bits(src_phys_addr));
 454
 455         src_crc32 = crc32_le(~0, src_addr, size);
 456
 457         orig_dst_addr = kzalloc(size + alignment, GFP_KERNEL);
 458         if (!orig_dst_addr) {
 459                 dev_err(dev, "Failed to allocate destination address\n");
 460                 ret = false;
 461                 goto err_dst_addr;
 462         }
 463
 464         orig_dst_phys_addr = dma_map_single(dev, orig_dst_addr,
 465                                             size + alignment, DMA_FROM_DEVICE);
 466         if (dma_mapping_error(dev, orig_dst_phys_addr)) {
 467                 dev_err(dev, "failed to map destination buffer address\n");
 468                 ret = false;
 469                 goto err_dst_phys_addr;
 470         }
 471
 472         if (alignment && !IS_ALIGNED(orig_dst_phys_addr, alignment)) {
 473                 dst_phys_addr = PTR_ALIGN(orig_dst_phys_addr, alignment);
 474                 offset = dst_phys_addr - orig_dst_phys_addr;
 475                 dst_addr = orig_dst_addr + offset;
 476         } else {
 477                 dst_phys_addr = orig_dst_phys_addr;
 478                 dst_addr = orig_dst_addr;
 479         }
 480
 481         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_LOWER_DST_ADDR,
 482                                  lower_32_bits(dst_phys_addr));
 483         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_UPPER_DST_ADDR,
 484                                  upper_32_bits(dst_phys_addr));
 485
 486         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_SIZE,
 487                                  size);
 488
 489         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_FLAGS, flags);
 490         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE, irq_type);
 491         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, 1);
 492         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
 493                                  COMMAND_COPY);
 494
 495         wait_for_completion(&test->irq_raised);
 496
 497         dma_unmap_single(dev, orig_dst_phys_addr, size + alignment,
 498                          DMA_FROM_DEVICE);
 499
 500         dst_crc32 = crc32_le(~0, dst_addr, size);
 501         if (dst_crc32 == src_crc32)
 502                 ret = true;
 503
 504 err_dst_phys_addr:
 505         kfree(orig_dst_addr);
 506
 507 err_dst_addr:
 508         dma_unmap_single(dev, orig_src_phys_addr, size + alignment,
 509                          DMA_TO_DEVICE);
 510
 511 err_src_phys_addr:
 512         kfree(orig_src_addr);
 513
 514 err:
 515         return ret;
 516 }
 517
 518 static bool pci_endpoint_test_write(struct pci_endpoint_test *test,
 519                                     unsigned long arg)
 520 {
 521         struct pci_endpoint_test_xfer_param param;
 522         bool ret = false;
 523         u32 flags = 0;
 524         bool use_dma;
 525         u32 reg;
 526         void *addr;
 527         dma_addr_t phys_addr;
 528         struct pci_dev *pdev = test->pdev;
 529         struct device *dev = &pdev->dev;
 530         void *orig_addr;
 531         dma_addr_t orig_phys_addr;
 532         size_t offset;
 533         size_t alignment = test->alignment;
 534         int irq_type = test->irq_type;
 535         size_t size;
 536         u32 crc32;
 537         int err;
 538
 539         err = copy_from_user(&param, (void __user *)arg, sizeof(param));
 540         if (err != 0) {
 541                 dev_err(dev, "Failed to get transfer param\n");
 542                 return false;
 543         }
 544
 545         err = pci_endpoint_test_validate_xfer_params(dev, &param, alignment);
 546         if (err)
 547                 return false;
 548
 549         size = param.size;
 550
 551         use_dma = !!(param.flags & PCITEST_FLAGS_USE_DMA);
 552         if (use_dma)
 553                 flags |= FLAG_USE_DMA;
 554
 555         if (irq_type < IRQ_TYPE_INTX || irq_type > IRQ_TYPE_MSIX) {
 556                 dev_err(dev, "Invalid IRQ type option\n");
 557                 goto err;
 558         }
 559
 560         orig_addr = kzalloc(size + alignment, GFP_KERNEL);
 561         if (!orig_addr) {
 562                 dev_err(dev, "Failed to allocate address\n");
 563                 ret = false;
 564                 goto err;
 565         }
 566
 567         get_random_bytes(orig_addr, size + alignment);
 568
 569         orig_phys_addr = dma_map_single(dev, orig_addr, size + alignment,
 570                                         DMA_TO_DEVICE);
 571         if (dma_mapping_error(dev, orig_phys_addr)) {
 572                 dev_err(dev, "failed to map source buffer address\n");
 573                 ret = false;
 574                 goto err_phys_addr;
 575         }
 576
 577         if (alignment && !IS_ALIGNED(orig_phys_addr, alignment)) {
 578                 phys_addr =  PTR_ALIGN(orig_phys_addr, alignment);
 579                 offset = phys_addr - orig_phys_addr;
 580                 addr = orig_addr + offset;
 581         } else {
 582                 phys_addr = orig_phys_addr;
 583                 addr = orig_addr;
 584         }
 585
 586         crc32 = crc32_le(~0, addr, size);
 587         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_CHECKSUM,
 588                                  crc32);
 589
 590         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_LOWER_SRC_ADDR,
 591                                  lower_32_bits(phys_addr));
 592         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_UPPER_SRC_ADDR,
 593                                  upper_32_bits(phys_addr));
 594
 595         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_SIZE, size);
 596
 597         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_FLAGS, flags);
 598         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE, irq_type);
 599         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, 1);
 600         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
 601                                  COMMAND_READ);
 602
 603         wait_for_completion(&test->irq_raised);
 604
 605         reg = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_STATUS);
 606         if (reg & STATUS_READ_SUCCESS)
 607                 ret = true;
 608
 609         dma_unmap_single(dev, orig_phys_addr, size + alignment,
 610                          DMA_TO_DEVICE);
 611
 612 err_phys_addr:
 613         kfree(orig_addr);
 614
 615 err:
 616         return ret;
 617 }
 618
 619 static bool pci_endpoint_test_read(struct pci_endpoint_test *test,
 620                                    unsigned long arg)
 621 {
 622         struct pci_endpoint_test_xfer_param param;
 623         bool ret = false;
 624         u32 flags = 0;
 625         bool use_dma;
 626         size_t size;
 627         void *addr;
 628         dma_addr_t phys_addr;
 629         struct pci_dev *pdev = test->pdev;
 630         struct device *dev = &pdev->dev;
 631         void *orig_addr;
 632         dma_addr_t orig_phys_addr;
 633         size_t offset;
 634         size_t alignment = test->alignment;
 635         int irq_type = test->irq_type;
 636         u32 crc32;
 637         int err;
 638
 639         err = copy_from_user(&param, (void __user *)arg, sizeof(param));
 640         if (err) {
 641                 dev_err(dev, "Failed to get transfer param\n");
 642                 return false;
 643         }
 644
 645         err = pci_endpoint_test_validate_xfer_params(dev, &param, alignment);
 646         if (err)
 647                 return false;
 648
 649         size = param.size;
 650
 651         use_dma = !!(param.flags & PCITEST_FLAGS_USE_DMA);
 652         if (use_dma)
 653                 flags |= FLAG_USE_DMA;
 654
 655         if (irq_type < IRQ_TYPE_INTX || irq_type > IRQ_TYPE_MSIX) {
 656                 dev_err(dev, "Invalid IRQ type option\n");
 657                 goto err;
 658         }
 659
 660         orig_addr = kzalloc(size + alignment, GFP_KERNEL);
 661         if (!orig_addr) {
 662                 dev_err(dev, "Failed to allocate destination address\n");
 663                 ret = false;
 664                 goto err;
 665         }
 666
 667         orig_phys_addr = dma_map_single(dev, orig_addr, size + alignment,
 668                                         DMA_FROM_DEVICE);
 669         if (dma_mapping_error(dev, orig_phys_addr)) {
 670                 dev_err(dev, "failed to map source buffer address\n");
 671                 ret = false;
 672                 goto err_phys_addr;
 673         }
 674
 675         if (alignment && !IS_ALIGNED(orig_phys_addr, alignment)) {
 676                 phys_addr = PTR_ALIGN(orig_phys_addr, alignment);
 677                 offset = phys_addr - orig_phys_addr;
 678                 addr = orig_addr + offset;
 679         } else {
 680                 phys_addr = orig_phys_addr;
 681                 addr = orig_addr;
 682         }
 683
 684         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_LOWER_DST_ADDR,
 685                                  lower_32_bits(phys_addr));
 686         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_UPPER_DST_ADDR,
 687                                  upper_32_bits(phys_addr));
 688
 689         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_SIZE, size);
 690
 691         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_FLAGS, flags);
 692         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE, irq_type);
 693         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, 1);
 694         pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
 695                                  COMMAND_WRITE);
 696
 697         wait_for_completion(&test->irq_raised);
 698
 699         dma_unmap_single(dev, orig_phys_addr, size + alignment,
 700                          DMA_FROM_DEVICE);
 701
 702         crc32 = crc32_le(~0, addr, size);
 703         if (crc32 == pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_CHECKSUM))
 704                 ret = true;
 705
 706 err_phys_addr:
 707         kfree(orig_addr);
 708 err:
 709         return ret;
 710 }
 711
 712 static bool pci_endpoint_test_clear_irq(struct pci_endpoint_test *test)
 713 {
 714         pci_endpoint_test_release_irq(test);
 715         pci_endpoint_test_free_irq_vectors(test);
 716         return true;
 717 }
 718
 719 static bool pci_endpoint_test_set_irq(struct pci_endpoint_test *test,
 720                                       int req_irq_type)
 721 {
 722         struct pci_dev *pdev = test->pdev;
 723         struct device *dev = &pdev->dev;
 724
 725         if (req_irq_type < IRQ_TYPE_INTX || req_irq_type > IRQ_TYPE_MSIX) {
 726                 dev_err(dev, "Invalid IRQ type option\n");
 727                 return false;
 728         }
 729
 730         if (test->irq_type == req_irq_type)
 731                 return true;
 732
 733         pci_endpoint_test_release_irq(test);
 734         pci_endpoint_test_free_irq_vectors(test);
 735
 736         if (!pci_endpoint_test_alloc_irq_vectors(test, req_irq_type))
 737                 goto err;
 738
 739         if (!pci_endpoint_test_request_irq(test))
 740                 goto err;
 741
 742         return true;
 743
 744 err:
 745         pci_endpoint_test_free_irq_vectors(test);
 746         return false;
 747 }
 748
 749 static long pci_endpoint_test_ioctl(struct file *file, unsigned int cmd,
 750                                     unsigned long arg)
 751 {
 752         int ret = -EINVAL;
 753         enum pci_barno bar;
 754         struct pci_endpoint_test *test = to_endpoint_test(file->private_data);
 755         struct pci_dev *pdev = test->pdev;
 756
 757         mutex_lock(&test->mutex);
 758
 759         reinit_completion(&test->irq_raised);
 760         test->last_irq = -ENODATA;
 761
 762         switch (cmd) {
 763         case PCITEST_BAR:
 764                 bar = arg;
 765                 if (bar > BAR_5)
 766                         goto ret;
 767                 if (is_am654_pci_dev(pdev) && bar == BAR_0)
 768                         goto ret;
 769                 ret = pci_endpoint_test_bar(test, bar);
 770                 break;
 771         case PCITEST_INTX_IRQ:
 772                 ret = pci_endpoint_test_intx_irq(test);
 773                 break;
 774         case PCITEST_MSI:
 775         case PCITEST_MSIX:
 776                 ret = pci_endpoint_test_msi_irq(test, arg, cmd == PCITEST_MSIX);
 777                 break;
 778         case PCITEST_WRITE:
 779                 ret = pci_endpoint_test_write(test, arg);
 780                 break;
 781         case PCITEST_READ:
 782                 ret = pci_endpoint_test_read(test, arg);
 783                 break;
 784         case PCITEST_COPY:
 785                 ret = pci_endpoint_test_copy(test, arg);
 786                 break;
 787         case PCITEST_SET_IRQTYPE:
 788                 ret = pci_endpoint_test_set_irq(test, arg);
 789                 break;
 790         case PCITEST_GET_IRQTYPE:
 791                 ret = irq_type;
 792                 break;
 793         case PCITEST_CLEAR_IRQ:
 794                 ret = pci_endpoint_test_clear_irq(test);
 795                 break;
 796         }
 797
 798 ret:
 799         mutex_unlock(&test->mutex);
 800         return ret;
 801 }
 802
 803 static const struct file_operations pci_endpoint_test_fops = {
 804         .owner = THIS_MODULE,
 805         .unlocked_ioctl = pci_endpoint_test_ioctl,
 806 };
 807
 808 static int pci_endpoint_test_probe(struct pci_dev *pdev,
 809                                    const struct pci_device_id *ent)
 810 {
 811         int err;
 812         int id;
 813         char name[24];
 814         enum pci_barno bar;
 815         void __iomem *base;
 816         struct device *dev = &pdev->dev;
 817         struct pci_endpoint_test *test;
 818         struct pci_endpoint_test_data *data;
 819         enum pci_barno test_reg_bar = BAR_0;
 820         struct miscdevice *misc_device;
 821
 822         if (pci_is_bridge(pdev))
 823                 return -ENODEV;
 824
 825         test = devm_kzalloc(dev, sizeof(*test), GFP_KERNEL);
 826         if (!test)
 827                 return -ENOMEM;
 828
 829         test->test_reg_bar = 0;
 830         test->alignment = 0;
 831         test->pdev = pdev;
 832         test->irq_type = IRQ_TYPE_UNDEFINED;
 833
 834         if (no_msi)
 835                 irq_type = IRQ_TYPE_INTX;
 836
 837         data = (struct pci_endpoint_test_data *)ent->driver_data;
 838         if (data) {
 839                 test_reg_bar = data->test_reg_bar;
 840                 test->test_reg_bar = test_reg_bar;
 841                 test->alignment = data->alignment;
 842                 irq_type = data->irq_type;
 843         }
 844
 845         init_completion(&test->irq_raised);
 846         mutex_init(&test->mutex);
 847
 848         dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48));
 849
 850         err = pci_enable_device(pdev);
 851         if (err) {
 852                 dev_err(dev, "Cannot enable PCI device\n");
 853                 return err;
 854         }
 855
 856         err = pci_request_regions(pdev, DRV_MODULE_NAME);
 857         if (err) {
 858                 dev_err(dev, "Cannot obtain PCI resources\n");
 859                 goto err_disable_pdev;
 860         }
 861
 862         pci_set_master(pdev);
 863
 864         if (!pci_endpoint_test_alloc_irq_vectors(test, irq_type)) {
 865                 err = -EINVAL;
 866                 goto err_disable_irq;
 867         }
 868
 869         for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) {
 870                 if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) {
 871                         base = pci_ioremap_bar(pdev, bar);
 872                         if (!base) {
 873                                 dev_err(dev, "Failed to read BAR%d\n", bar);
 874                                 WARN_ON(bar == test_reg_bar);
 875                         }
 876                         test->bar[bar] = base;
 877                 }
 878         }
 879
 880         test->base = test->bar[test_reg_bar];
 881         if (!test->base) {
 882                 err = -ENOMEM;
 883                 dev_err(dev, "Cannot perform PCI test without BAR%d\n",
 884                         test_reg_bar);
 885                 goto err_iounmap;
 886         }
 887
 888         pci_set_drvdata(pdev, test);
 889
 890         id = ida_alloc(&pci_endpoint_test_ida, GFP_KERNEL);
 891         if (id < 0) {
 892                 err = id;
 893                 dev_err(dev, "Unable to get id\n");
 894                 goto err_iounmap;
 895         }
 896
 897         snprintf(name, sizeof(name), DRV_MODULE_NAME ".%d", id);
 898         test->name = kstrdup(name, GFP_KERNEL);
 899         if (!test->name) {
 900                 err = -ENOMEM;
 901                 goto err_ida_remove;
 902         }
 903
 904         if (!pci_endpoint_test_request_irq(test)) {
 905                 err = -EINVAL;
 906                 goto err_kfree_test_name;
 907         }
 908
 909         misc_device = &test->miscdev;
 910         misc_device->minor = MISC_DYNAMIC_MINOR;
 911         misc_device->name = kstrdup(name, GFP_KERNEL);
 912         if (!misc_device->name) {
 913                 err = -ENOMEM;
 914                 goto err_release_irq;
 915         }
 916         misc_device->parent = &pdev->dev;
 917         misc_device->fops = &pci_endpoint_test_fops;
 918
 919         err = misc_register(misc_device);
 920         if (err) {
 921                 dev_err(dev, "Failed to register device\n");
 922                 goto err_kfree_name;
 923         }
 924
 925         return 0;
 926
 927 err_kfree_name:
 928         kfree(misc_device->name);
 929
 930 err_release_irq:
 931         pci_endpoint_test_release_irq(test);
 932
 933 err_kfree_test_name:
 934         kfree(test->name);
 935
 936 err_ida_remove:
 937         ida_free(&pci_endpoint_test_ida, id);
 938
 939 err_iounmap:
 940         for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) {
 941                 if (test->bar[bar])
 942                         pci_iounmap(pdev, test->bar[bar]);
 943         }
 944
 945 err_disable_irq:
 946         pci_endpoint_test_free_irq_vectors(test);
 947         pci_release_regions(pdev);
 948
 949 err_disable_pdev:
 950         pci_disable_device(pdev);
 951
 952         return err;
 953 }
 954
 955 static void pci_endpoint_test_remove(struct pci_dev *pdev)
 956 {
 957         int id;
 958         enum pci_barno bar;
 959         struct pci_endpoint_test *test = pci_get_drvdata(pdev);
 960         struct miscdevice *misc_device = &test->miscdev;
 961
 962         if (sscanf(misc_device->name, DRV_MODULE_NAME ".%d", &id) != 1)
 963                 return;
 964         if (id < 0)
 965                 return;
 966
 967         pci_endpoint_test_release_irq(test);
 968         pci_endpoint_test_free_irq_vectors(test);
 969
 970         misc_deregister(&test->miscdev);
 971         kfree(misc_device->name);
 972         kfree(test->name);
 973         ida_free(&pci_endpoint_test_ida, id);
 974         for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) {
 975                 if (test->bar[bar])
 976                         pci_iounmap(pdev, test->bar[bar]);
 977         }
 978
 979         pci_release_regions(pdev);
 980         pci_disable_device(pdev);
 981 }
 982
 983 static const struct pci_endpoint_test_data default_data = {
 984         .test_reg_bar = BAR_0,
 985         .alignment = SZ_4K,
 986         .irq_type = IRQ_TYPE_MSI,
 987 };
 988
 989 static const struct pci_endpoint_test_data am654_data = {
 990         .test_reg_bar = BAR_2,
 991         .alignment = SZ_64K,
 992         .irq_type = IRQ_TYPE_MSI,
 993 };
 994
 995 static const struct pci_endpoint_test_data j721e_data = {
 996         .alignment = 256,
 997         .irq_type = IRQ_TYPE_MSI,
 998 };
 999
1000 static const struct pci_endpoint_test_data rk3588_data = {
1001         .alignment = SZ_64K,
1002         .irq_type = IRQ_TYPE_MSI,
1003 };
1004
1005 /*
1006  * If the controller's Vendor/Device ID are programmable, you may be able to
1007  * use one of the existing entries for testing instead of adding a new one.
1008  */
1009 static const struct pci_device_id pci_endpoint_test_tbl[] = {
1010         { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_DRA74x),
1011           .driver_data = (kernel_ulong_t)&default_data,
1012         },
1013         { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_DRA72x),
1014           .driver_data = (kernel_ulong_t)&default_data,
1015         },
1016         { PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, 0x81c0),
1017           .driver_data = (kernel_ulong_t)&default_data,
1018         },
1019         { PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, PCI_DEVICE_ID_IMX8),},
1020         { PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, PCI_DEVICE_ID_LS1088A),
1021           .driver_data = (kernel_ulong_t)&default_data,
1022         },
1023         { PCI_DEVICE_DATA(SYNOPSYS, EDDA, NULL) },
1024         { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_AM654),
1025           .driver_data = (kernel_ulong_t)&am654_data
1026         },
1027         { PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A774A1),},
1028         { PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A774B1),},
1029         { PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A774C0),},
1030         { PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A774E1),},
1031         { PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A779F0),
1032           .driver_data = (kernel_ulong_t)&default_data,
1033         },
1034         { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_J721E),
1035           .driver_data = (kernel_ulong_t)&j721e_data,
1036         },
1037         { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_J7200),
1038           .driver_data = (kernel_ulong_t)&j721e_data,
1039         },
1040         { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_AM64),
1041           .driver_data = (kernel_ulong_t)&j721e_data,
1042         },
1043         { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_J721S2),
1044           .driver_data = (kernel_ulong_t)&j721e_data,
1045         },
1046         { PCI_DEVICE(PCI_VENDOR_ID_ROCKCHIP, PCI_DEVICE_ID_ROCKCHIP_RK3588),
1047           .driver_data = (kernel_ulong_t)&rk3588_data,
1048         },
1049         { }
1050 };
1051 MODULE_DEVICE_TABLE(pci, pci_endpoint_test_tbl);
1052
1053 static struct pci_driver pci_endpoint_test_driver = {
1054         .name           = DRV_MODULE_NAME,
1055         .id_table       = pci_endpoint_test_tbl,
1056         .probe          = pci_endpoint_test_probe,
1057         .remove         = pci_endpoint_test_remove,
1058         .sriov_configure = pci_sriov_configure_simple,
1059 };
1060 module_pci_driver(pci_endpoint_test_driver);
1061
1062 MODULE_DESCRIPTION("PCI ENDPOINT TEST HOST DRIVER");
1063 MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>");
1064 MODULE_LICENSE("GPL v2");