?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
drm/virtio: Add drm_panic support
[drm/drm-misc.git] / drivers / pci / endpoint / functions / pci-epf-test.c
blobb94e205ae10b94c33c7a2c4f38f6c38e1f94443a
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Test driver to test endpoint functionality
4 *
5 * Copyright (C) 2017 Texas Instruments
6 * Author: Kishon Vijay Abraham I <kishon@ti.com>
7 */
8
9 #include <linux/crc32.h>
10 #include <linux/delay.h>
11 #include <linux/dmaengine.h>
12 #include <linux/io.h>
13 #include <linux/module.h>
14 #include <linux/slab.h>
15 #include <linux/pci_ids.h>
16 #include <linux/random.h>
17
18 #include <linux/pci-epc.h>
19 #include <linux/pci-epf.h>
20 #include <linux/pci_regs.h>
21
22 #define IRQ_TYPE_INTX 0
23 #define IRQ_TYPE_MSI 1
24 #define IRQ_TYPE_MSIX 2
25
26 #define COMMAND_RAISE_INTX_IRQ BIT(0)
27 #define COMMAND_RAISE_MSI_IRQ BIT(1)
28 #define COMMAND_RAISE_MSIX_IRQ BIT(2)
29 #define COMMAND_READ BIT(3)
30 #define COMMAND_WRITE BIT(4)
31 #define COMMAND_COPY BIT(5)
32
33 #define STATUS_READ_SUCCESS BIT(0)
34 #define STATUS_READ_FAIL BIT(1)
35 #define STATUS_WRITE_SUCCESS BIT(2)
36 #define STATUS_WRITE_FAIL BIT(3)
37 #define STATUS_COPY_SUCCESS BIT(4)
38 #define STATUS_COPY_FAIL BIT(5)
39 #define STATUS_IRQ_RAISED BIT(6)
40 #define STATUS_SRC_ADDR_INVALID BIT(7)
41 #define STATUS_DST_ADDR_INVALID BIT(8)
42
43 #define FLAG_USE_DMA BIT(0)
44
45 #define TIMER_RESOLUTION 1
46
47 #define CAP_UNALIGNED_ACCESS BIT(0)
48
49 static struct workqueue_struct *kpcitest_workqueue;
50
51 struct pci_epf_test {
52 void *reg[PCI_STD_NUM_BARS];
53 struct pci_epf *epf;
54 enum pci_barno test_reg_bar;
55 size_t msix_table_offset;
56 struct delayed_work cmd_handler;
57 struct dma_chan *dma_chan_tx;
58 struct dma_chan *dma_chan_rx;
59 struct dma_chan *transfer_chan;
60 dma_cookie_t transfer_cookie;
61 enum dma_status transfer_status;
62 struct completion transfer_complete;
63 bool dma_supported;
64 bool dma_private;
65 const struct pci_epc_features *epc_features;
66 };
67
68 struct pci_epf_test_reg {
69 u32 magic;
70 u32 command;
71 u32 status;
72 u64 src_addr;
73 u64 dst_addr;
74 u32 size;
75 u32 checksum;
76 u32 irq_type;
77 u32 irq_number;
78 u32 flags;
79 u32 caps;
80 } __packed;
81
82 static struct pci_epf_header test_header = {
83 .vendorid = PCI_ANY_ID,
84 .deviceid = PCI_ANY_ID,
85 .baseclass_code = PCI_CLASS_OTHERS,
86 .interrupt_pin = PCI_INTERRUPT_INTA,
87 };
88
89 static size_t bar_size[] = { 512, 512, 1024, 16384, 131072, 1048576 };
90
91 static void pci_epf_test_dma_callback(void *param)
92 {
93 struct pci_epf_test *epf_test = param;
94 struct dma_tx_state state;
95
96 epf_test->transfer_status =
97 dmaengine_tx_status(epf_test->transfer_chan,
98 epf_test->transfer_cookie, &state);
99 if (epf_test->transfer_status == DMA_COMPLETE ||
100 epf_test->transfer_status == DMA_ERROR)
101 complete(&epf_test->transfer_complete);
102 }
103
104 /**
105 * pci_epf_test_data_transfer() - Function that uses dmaengine API to transfer
106 * data between PCIe EP and remote PCIe RC
107 * @epf_test: the EPF test device that performs the data transfer operation
108 * @dma_dst: The destination address of the data transfer. It can be a physical
109 * address given by pci_epc_mem_alloc_addr or DMA mapping APIs.
110 * @dma_src: The source address of the data transfer. It can be a physical
111 * address given by pci_epc_mem_alloc_addr or DMA mapping APIs.
112 * @len: The size of the data transfer
113 * @dma_remote: remote RC physical address
114 * @dir: DMA transfer direction
115 *
116 * Function that uses dmaengine API to transfer data between PCIe EP and remote
117 * PCIe RC. The source and destination address can be a physical address given
118 * by pci_epc_mem_alloc_addr or the one obtained using DMA mapping APIs.
119 *
120 * The function returns '0' on success and negative value on failure.
121 */
122 static int pci_epf_test_data_transfer(struct pci_epf_test *epf_test,
123 dma_addr_t dma_dst, dma_addr_t dma_src,
124 size_t len, dma_addr_t dma_remote,
125 enum dma_transfer_direction dir)
126 {
127 struct dma_chan *chan = (dir == DMA_MEM_TO_DEV) ?
128 epf_test->dma_chan_tx : epf_test->dma_chan_rx;
129 dma_addr_t dma_local = (dir == DMA_MEM_TO_DEV) ? dma_src : dma_dst;
130 enum dma_ctrl_flags flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
131 struct pci_epf *epf = epf_test->epf;
132 struct dma_async_tx_descriptor *tx;
133 struct dma_slave_config sconf = {};
134 struct device *dev = &epf->dev;
135 int ret;
136
137 if (IS_ERR_OR_NULL(chan)) {
138 dev_err(dev, "Invalid DMA memcpy channel\n");
139 return -EINVAL;
140 }
141
142 if (epf_test->dma_private) {
143 sconf.direction = dir;
144 if (dir == DMA_MEM_TO_DEV)
145 sconf.dst_addr = dma_remote;
146 else
147 sconf.src_addr = dma_remote;
148
149 if (dmaengine_slave_config(chan, &sconf)) {
150 dev_err(dev, "DMA slave config fail\n");
151 return -EIO;
152 }
153 tx = dmaengine_prep_slave_single(chan, dma_local, len, dir,
154 flags);
155 } else {
156 tx = dmaengine_prep_dma_memcpy(chan, dma_dst, dma_src, len,
157 flags);
158 }
159
160 if (!tx) {
161 dev_err(dev, "Failed to prepare DMA memcpy\n");
162 return -EIO;
163 }
164
165 reinit_completion(&epf_test->transfer_complete);
166 epf_test->transfer_chan = chan;
167 tx->callback = pci_epf_test_dma_callback;
168 tx->callback_param = epf_test;
169 epf_test->transfer_cookie = dmaengine_submit(tx);
170
171 ret = dma_submit_error(epf_test->transfer_cookie);
172 if (ret) {
173 dev_err(dev, "Failed to do DMA tx_submit %d\n", ret);
174 goto terminate;
175 }
176
177 dma_async_issue_pending(chan);
178 ret = wait_for_completion_interruptible(&epf_test->transfer_complete);
179 if (ret < 0) {
180 dev_err(dev, "DMA wait_for_completion interrupted\n");
181 goto terminate;
182 }
183
184 if (epf_test->transfer_status == DMA_ERROR) {
185 dev_err(dev, "DMA transfer failed\n");
186 ret = -EIO;
187 }
188
189 terminate:
190 dmaengine_terminate_sync(chan);
191
192 return ret;
193 }
194
195 struct epf_dma_filter {
196 struct device *dev;
197 u32 dma_mask;
198 };
199
200 static bool epf_dma_filter_fn(struct dma_chan *chan, void *node)
201 {
202 struct epf_dma_filter *filter = node;
203 struct dma_slave_caps caps;
204
205 memset(&caps, 0, sizeof(caps));
206 dma_get_slave_caps(chan, &caps);
207
208 return chan->device->dev == filter->dev
209 && (filter->dma_mask & caps.directions);
210 }
211
212 /**
213 * pci_epf_test_init_dma_chan() - Function to initialize EPF test DMA channel
214 * @epf_test: the EPF test device that performs data transfer operation
215 *
216 * Function to initialize EPF test DMA channel.
217 */
218 static int pci_epf_test_init_dma_chan(struct pci_epf_test *epf_test)
219 {
220 struct pci_epf *epf = epf_test->epf;
221 struct device *dev = &epf->dev;
222 struct epf_dma_filter filter;
223 struct dma_chan *dma_chan;
224 dma_cap_mask_t mask;
225 int ret;
226
227 filter.dev = epf->epc->dev.parent;
228 filter.dma_mask = BIT(DMA_DEV_TO_MEM);
229
230 dma_cap_zero(mask);
231 dma_cap_set(DMA_SLAVE, mask);
232 dma_chan = dma_request_channel(mask, epf_dma_filter_fn, &filter);
233 if (!dma_chan) {
234 dev_info(dev, "Failed to get private DMA rx channel. Falling back to generic one\n");
235 goto fail_back_tx;
236 }
237
238 epf_test->dma_chan_rx = dma_chan;
239
240 filter.dma_mask = BIT(DMA_MEM_TO_DEV);
241 dma_chan = dma_request_channel(mask, epf_dma_filter_fn, &filter);
242
243 if (!dma_chan) {
244 dev_info(dev, "Failed to get private DMA tx channel. Falling back to generic one\n");
245 goto fail_back_rx;
246 }
247
248 epf_test->dma_chan_tx = dma_chan;
249 epf_test->dma_private = true;
250
251 init_completion(&epf_test->transfer_complete);
252
253 return 0;
254
255 fail_back_rx:
256 dma_release_channel(epf_test->dma_chan_rx);
257 epf_test->dma_chan_rx = NULL;
258
259 fail_back_tx:
260 dma_cap_zero(mask);
261 dma_cap_set(DMA_MEMCPY, mask);
262
263 dma_chan = dma_request_chan_by_mask(&mask);
264 if (IS_ERR(dma_chan)) {
265 ret = PTR_ERR(dma_chan);
266 if (ret != -EPROBE_DEFER)
267 dev_err(dev, "Failed to get DMA channel\n");
268 return ret;
269 }
270 init_completion(&epf_test->transfer_complete);
271
272 epf_test->dma_chan_tx = epf_test->dma_chan_rx = dma_chan;
273
274 return 0;
275 }
276
277 /**
278 * pci_epf_test_clean_dma_chan() - Function to cleanup EPF test DMA channel
279 * @epf_test: the EPF test device that performs data transfer operation
280 *
281 * Helper to cleanup EPF test DMA channel.
282 */
283 static void pci_epf_test_clean_dma_chan(struct pci_epf_test *epf_test)
284 {
285 if (!epf_test->dma_supported)
286 return;
287
288 dma_release_channel(epf_test->dma_chan_tx);
289 if (epf_test->dma_chan_tx == epf_test->dma_chan_rx) {
290 epf_test->dma_chan_tx = NULL;
291 epf_test->dma_chan_rx = NULL;
292 return;
293 }
294
295 dma_release_channel(epf_test->dma_chan_rx);
296 epf_test->dma_chan_rx = NULL;
297 }
298
299 static void pci_epf_test_print_rate(struct pci_epf_test *epf_test,
300 const char *op, u64 size,
301 struct timespec64 *start,
302 struct timespec64 *end, bool dma)
303 {
304 struct timespec64 ts = timespec64_sub(*end, *start);
305 u64 rate = 0, ns;
306
307 /* calculate the rate */
308 ns = timespec64_to_ns(&ts);
309 if (ns)
310 rate = div64_u64(size * NSEC_PER_SEC, ns * 1000);
311
312 dev_info(&epf_test->epf->dev,
313 "%s => Size: %llu B, DMA: %s, Time: %llu.%09u s, Rate: %llu KB/s\n",
314 op, size, dma ? "YES" : "NO",
315 (u64)ts.tv_sec, (u32)ts.tv_nsec, rate);
316 }
317
318 static void pci_epf_test_copy(struct pci_epf_test *epf_test,
319 struct pci_epf_test_reg *reg)
320 {
321 int ret = 0;
322 struct timespec64 start, end;
323 struct pci_epf *epf = epf_test->epf;
324 struct pci_epc *epc = epf->epc;
325 struct device *dev = &epf->dev;
326 struct pci_epc_map src_map, dst_map;
327 u64 src_addr = reg->src_addr;
328 u64 dst_addr = reg->dst_addr;
329 size_t copy_size = reg->size;
330 ssize_t map_size = 0;
331 void *copy_buf = NULL, *buf;
332
333 if (reg->flags & FLAG_USE_DMA) {
334 if (!dma_has_cap(DMA_MEMCPY, epf_test->dma_chan_tx->device->cap_mask)) {
335 dev_err(dev, "DMA controller doesn't support MEMCPY\n");
336 ret = -EINVAL;
337 goto set_status;
338 }
339 } else {
340 copy_buf = kzalloc(copy_size, GFP_KERNEL);
341 if (!copy_buf) {
342 ret = -ENOMEM;
343 goto set_status;
344 }
345 buf = copy_buf;
346 }
347
348 while (copy_size) {
349 ret = pci_epc_mem_map(epc, epf->func_no, epf->vfunc_no,
350 src_addr, copy_size, &src_map);
351 if (ret) {
352 dev_err(dev, "Failed to map source address\n");
353 reg->status = STATUS_SRC_ADDR_INVALID;
354 goto free_buf;
355 }
356
357 ret = pci_epc_mem_map(epf->epc, epf->func_no, epf->vfunc_no,
358 dst_addr, copy_size, &dst_map);
359 if (ret) {
360 dev_err(dev, "Failed to map destination address\n");
361 reg->status = STATUS_DST_ADDR_INVALID;
362 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no,
363 &src_map);
364 goto free_buf;
365 }
366
367 map_size = min_t(size_t, dst_map.pci_size, src_map.pci_size);
368
369 ktime_get_ts64(&start);
370 if (reg->flags & FLAG_USE_DMA) {
371 ret = pci_epf_test_data_transfer(epf_test,
372 dst_map.phys_addr, src_map.phys_addr,
373 map_size, 0, DMA_MEM_TO_MEM);
374 if (ret) {
375 dev_err(dev, "Data transfer failed\n");
376 goto unmap;
377 }
378 } else {
379 memcpy_fromio(buf, src_map.virt_addr, map_size);
380 memcpy_toio(dst_map.virt_addr, buf, map_size);
381 buf += map_size;
382 }
383 ktime_get_ts64(&end);
384
385 copy_size -= map_size;
386 src_addr += map_size;
387 dst_addr += map_size;
388
389 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &dst_map);
390 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &src_map);
391 map_size = 0;
392 }
393
394 pci_epf_test_print_rate(epf_test, "COPY", reg->size, &start,
395 &end, reg->flags & FLAG_USE_DMA);
396
397 unmap:
398 if (map_size) {
399 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &dst_map);
400 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &src_map);
401 }
402
403 free_buf:
404 kfree(copy_buf);
405
406 set_status:
407 if (!ret)
408 reg->status |= STATUS_COPY_SUCCESS;
409 else
410 reg->status |= STATUS_COPY_FAIL;
411 }
412
413 static void pci_epf_test_read(struct pci_epf_test *epf_test,
414 struct pci_epf_test_reg *reg)
415 {
416 int ret = 0;
417 void *src_buf, *buf;
418 u32 crc32;
419 struct pci_epc_map map;
420 phys_addr_t dst_phys_addr;
421 struct timespec64 start, end;
422 struct pci_epf *epf = epf_test->epf;
423 struct pci_epc *epc = epf->epc;
424 struct device *dev = &epf->dev;
425 struct device *dma_dev = epf->epc->dev.parent;
426 u64 src_addr = reg->src_addr;
427 size_t src_size = reg->size;
428 ssize_t map_size = 0;
429
430 src_buf = kzalloc(src_size, GFP_KERNEL);
431 if (!src_buf) {
432 ret = -ENOMEM;
433 goto set_status;
434 }
435 buf = src_buf;
436
437 while (src_size) {
438 ret = pci_epc_mem_map(epc, epf->func_no, epf->vfunc_no,
439 src_addr, src_size, &map);
440 if (ret) {
441 dev_err(dev, "Failed to map address\n");
442 reg->status = STATUS_SRC_ADDR_INVALID;
443 goto free_buf;
444 }
445
446 map_size = map.pci_size;
447 if (reg->flags & FLAG_USE_DMA) {
448 dst_phys_addr = dma_map_single(dma_dev, buf, map_size,
449 DMA_FROM_DEVICE);
450 if (dma_mapping_error(dma_dev, dst_phys_addr)) {
451 dev_err(dev,
452 "Failed to map destination buffer addr\n");
453 ret = -ENOMEM;
454 goto unmap;
455 }
456
457 ktime_get_ts64(&start);
458 ret = pci_epf_test_data_transfer(epf_test,
459 dst_phys_addr, map.phys_addr,
460 map_size, src_addr, DMA_DEV_TO_MEM);
461 if (ret)
462 dev_err(dev, "Data transfer failed\n");
463 ktime_get_ts64(&end);
464
465 dma_unmap_single(dma_dev, dst_phys_addr, map_size,
466 DMA_FROM_DEVICE);
467
468 if (ret)
469 goto unmap;
470 } else {
471 ktime_get_ts64(&start);
472 memcpy_fromio(buf, map.virt_addr, map_size);
473 ktime_get_ts64(&end);
474 }
475
476 src_size -= map_size;
477 src_addr += map_size;
478 buf += map_size;
479
480 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &map);
481 map_size = 0;
482 }
483
484 pci_epf_test_print_rate(epf_test, "READ", reg->size, &start,
485 &end, reg->flags & FLAG_USE_DMA);
486
487 crc32 = crc32_le(~0, src_buf, reg->size);
488 if (crc32 != reg->checksum)
489 ret = -EIO;
490
491 unmap:
492 if (map_size)
493 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &map);
494
495 free_buf:
496 kfree(src_buf);
497
498 set_status:
499 if (!ret)
500 reg->status |= STATUS_READ_SUCCESS;
501 else
502 reg->status |= STATUS_READ_FAIL;
503 }
504
505 static void pci_epf_test_write(struct pci_epf_test *epf_test,
506 struct pci_epf_test_reg *reg)
507 {
508 int ret = 0;
509 void *dst_buf, *buf;
510 struct pci_epc_map map;
511 phys_addr_t src_phys_addr;
512 struct timespec64 start, end;
513 struct pci_epf *epf = epf_test->epf;
514 struct pci_epc *epc = epf->epc;
515 struct device *dev = &epf->dev;
516 struct device *dma_dev = epf->epc->dev.parent;
517 u64 dst_addr = reg->dst_addr;
518 size_t dst_size = reg->size;
519 ssize_t map_size = 0;
520
521 dst_buf = kzalloc(dst_size, GFP_KERNEL);
522 if (!dst_buf) {
523 ret = -ENOMEM;
524 goto set_status;
525 }
526 get_random_bytes(dst_buf, dst_size);
527 reg->checksum = crc32_le(~0, dst_buf, dst_size);
528 buf = dst_buf;
529
530 while (dst_size) {
531 ret = pci_epc_mem_map(epc, epf->func_no, epf->vfunc_no,
532 dst_addr, dst_size, &map);
533 if (ret) {
534 dev_err(dev, "Failed to map address\n");
535 reg->status = STATUS_DST_ADDR_INVALID;
536 goto free_buf;
537 }
538
539 map_size = map.pci_size;
540 if (reg->flags & FLAG_USE_DMA) {
541 src_phys_addr = dma_map_single(dma_dev, buf, map_size,
542 DMA_TO_DEVICE);
543 if (dma_mapping_error(dma_dev, src_phys_addr)) {
544 dev_err(dev,
545 "Failed to map source buffer addr\n");
546 ret = -ENOMEM;
547 goto unmap;
548 }
549
550 ktime_get_ts64(&start);
551
552 ret = pci_epf_test_data_transfer(epf_test,
553 map.phys_addr, src_phys_addr,
554 map_size, dst_addr,
555 DMA_MEM_TO_DEV);
556 if (ret)
557 dev_err(dev, "Data transfer failed\n");
558 ktime_get_ts64(&end);
559
560 dma_unmap_single(dma_dev, src_phys_addr, map_size,
561 DMA_TO_DEVICE);
562
563 if (ret)
564 goto unmap;
565 } else {
566 ktime_get_ts64(&start);
567 memcpy_toio(map.virt_addr, buf, map_size);
568 ktime_get_ts64(&end);
569 }
570
571 dst_size -= map_size;
572 dst_addr += map_size;
573 buf += map_size;
574
575 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &map);
576 map_size = 0;
577 }
578
579 pci_epf_test_print_rate(epf_test, "WRITE", reg->size, &start,
580 &end, reg->flags & FLAG_USE_DMA);
581
582 /*
583 * wait 1ms inorder for the write to complete. Without this delay L3
584 * error in observed in the host system.
585 */
586 usleep_range(1000, 2000);
587
588 unmap:
589 if (map_size)
590 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &map);
591
592 free_buf:
593 kfree(dst_buf);
594
595 set_status:
596 if (!ret)
597 reg->status |= STATUS_WRITE_SUCCESS;
598 else
599 reg->status |= STATUS_WRITE_FAIL;
600 }
601
602 static void pci_epf_test_raise_irq(struct pci_epf_test *epf_test,
603 struct pci_epf_test_reg *reg)
604 {
605 struct pci_epf *epf = epf_test->epf;
606 struct device *dev = &epf->dev;
607 struct pci_epc *epc = epf->epc;
608 u32 status = reg->status | STATUS_IRQ_RAISED;
609 int count;
610
611 /*
612 * Set the status before raising the IRQ to ensure that the host sees
613 * the updated value when it gets the IRQ.
614 */
615 WRITE_ONCE(reg->status, status);
616
617 switch (reg->irq_type) {
618 case IRQ_TYPE_INTX:
619 pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no,
620 PCI_IRQ_INTX, 0);
621 break;
622 case IRQ_TYPE_MSI:
623 count = pci_epc_get_msi(epc, epf->func_no, epf->vfunc_no);
624 if (reg->irq_number > count || count <= 0) {
625 dev_err(dev, "Invalid MSI IRQ number %d / %d\n",
626 reg->irq_number, count);
627 return;
628 }
629 pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no,
630 PCI_IRQ_MSI, reg->irq_number);
631 break;
632 case IRQ_TYPE_MSIX:
633 count = pci_epc_get_msix(epc, epf->func_no, epf->vfunc_no);
634 if (reg->irq_number > count || count <= 0) {
635 dev_err(dev, "Invalid MSIX IRQ number %d / %d\n",
636 reg->irq_number, count);
637 return;
638 }
639 pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no,
640 PCI_IRQ_MSIX, reg->irq_number);
641 break;
642 default:
643 dev_err(dev, "Failed to raise IRQ, unknown type\n");
644 break;
645 }
646 }
647
648 static void pci_epf_test_cmd_handler(struct work_struct *work)
649 {
650 u32 command;
651 struct pci_epf_test *epf_test = container_of(work, struct pci_epf_test,
652 cmd_handler.work);
653 struct pci_epf *epf = epf_test->epf;
654 struct device *dev = &epf->dev;
655 enum pci_barno test_reg_bar = epf_test->test_reg_bar;
656 struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar];
657
658 command = READ_ONCE(reg->command);
659 if (!command)
660 goto reset_handler;
661
662 WRITE_ONCE(reg->command, 0);
663 WRITE_ONCE(reg->status, 0);
664
665 if ((READ_ONCE(reg->flags) & FLAG_USE_DMA) &&
666 !epf_test->dma_supported) {
667 dev_err(dev, "Cannot transfer data using DMA\n");
668 goto reset_handler;
669 }
670
671 if (reg->irq_type > IRQ_TYPE_MSIX) {
672 dev_err(dev, "Failed to detect IRQ type\n");
673 goto reset_handler;
674 }
675
676 switch (command) {
677 case COMMAND_RAISE_INTX_IRQ:
678 case COMMAND_RAISE_MSI_IRQ:
679 case COMMAND_RAISE_MSIX_IRQ:
680 pci_epf_test_raise_irq(epf_test, reg);
681 break;
682 case COMMAND_WRITE:
683 pci_epf_test_write(epf_test, reg);
684 pci_epf_test_raise_irq(epf_test, reg);
685 break;
686 case COMMAND_READ:
687 pci_epf_test_read(epf_test, reg);
688 pci_epf_test_raise_irq(epf_test, reg);
689 break;
690 case COMMAND_COPY:
691 pci_epf_test_copy(epf_test, reg);
692 pci_epf_test_raise_irq(epf_test, reg);
693 break;
694 default:
695 dev_err(dev, "Invalid command 0x%x\n", command);
696 break;
697 }
698
699 reset_handler:
700 queue_delayed_work(kpcitest_workqueue, &epf_test->cmd_handler,
701 msecs_to_jiffies(1));
702 }
703
704 static int pci_epf_test_set_bar(struct pci_epf *epf)
705 {
706 int bar, ret;
707 struct pci_epc *epc = epf->epc;
708 struct device *dev = &epf->dev;
709 struct pci_epf_test *epf_test = epf_get_drvdata(epf);
710 enum pci_barno test_reg_bar = epf_test->test_reg_bar;
711
712 for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) {
713 if (!epf_test->reg[bar])
714 continue;
715
716 ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no,
717 &epf->bar[bar]);
718 if (ret) {
719 pci_epf_free_space(epf, epf_test->reg[bar], bar,
720 PRIMARY_INTERFACE);
721 dev_err(dev, "Failed to set BAR%d\n", bar);
722 if (bar == test_reg_bar)
723 return ret;
724 }
725 }
726
727 return 0;
728 }
729
730 static void pci_epf_test_clear_bar(struct pci_epf *epf)
731 {
732 struct pci_epf_test *epf_test = epf_get_drvdata(epf);
733 struct pci_epc *epc = epf->epc;
734 int bar;
735
736 for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) {
737 if (!epf_test->reg[bar])
738 continue;
739
740 pci_epc_clear_bar(epc, epf->func_no, epf->vfunc_no,
741 &epf->bar[bar]);
742 }
743 }
744
745 static void pci_epf_test_set_capabilities(struct pci_epf *epf)
746 {
747 struct pci_epf_test *epf_test = epf_get_drvdata(epf);
748 enum pci_barno test_reg_bar = epf_test->test_reg_bar;
749 struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar];
750 struct pci_epc *epc = epf->epc;
751 u32 caps = 0;
752
753 if (epc->ops->align_addr)
754 caps |= CAP_UNALIGNED_ACCESS;
755
756 reg->caps = cpu_to_le32(caps);
757 }
758
759 static int pci_epf_test_epc_init(struct pci_epf *epf)
760 {
761 struct pci_epf_test *epf_test = epf_get_drvdata(epf);
762 struct pci_epf_header *header = epf->header;
763 const struct pci_epc_features *epc_features = epf_test->epc_features;
764 struct pci_epc *epc = epf->epc;
765 struct device *dev = &epf->dev;
766 bool linkup_notifier = false;
767 int ret;
768
769 epf_test->dma_supported = true;
770
771 ret = pci_epf_test_init_dma_chan(epf_test);
772 if (ret)
773 epf_test->dma_supported = false;
774
775 if (epf->vfunc_no <= 1) {
776 ret = pci_epc_write_header(epc, epf->func_no, epf->vfunc_no, header);
777 if (ret) {
778 dev_err(dev, "Configuration header write failed\n");
779 return ret;
780 }
781 }
782
783 pci_epf_test_set_capabilities(epf);
784
785 ret = pci_epf_test_set_bar(epf);
786 if (ret)
787 return ret;
788
789 if (epc_features->msi_capable) {
790 ret = pci_epc_set_msi(epc, epf->func_no, epf->vfunc_no,
791 epf->msi_interrupts);
792 if (ret) {
793 dev_err(dev, "MSI configuration failed\n");
794 return ret;
795 }
796 }
797
798 if (epc_features->msix_capable) {
799 ret = pci_epc_set_msix(epc, epf->func_no, epf->vfunc_no,
800 epf->msix_interrupts,
801 epf_test->test_reg_bar,
802 epf_test->msix_table_offset);
803 if (ret) {
804 dev_err(dev, "MSI-X configuration failed\n");
805 return ret;
806 }
807 }
808
809 linkup_notifier = epc_features->linkup_notifier;
810 if (!linkup_notifier)
811 queue_work(kpcitest_workqueue, &epf_test->cmd_handler.work);
812
813 return 0;
814 }
815
816 static void pci_epf_test_epc_deinit(struct pci_epf *epf)
817 {
818 struct pci_epf_test *epf_test = epf_get_drvdata(epf);
819
820 cancel_delayed_work_sync(&epf_test->cmd_handler);
821 pci_epf_test_clean_dma_chan(epf_test);
822 pci_epf_test_clear_bar(epf);
823 }
824
825 static int pci_epf_test_link_up(struct pci_epf *epf)
826 {
827 struct pci_epf_test *epf_test = epf_get_drvdata(epf);
828
829 queue_delayed_work(kpcitest_workqueue, &epf_test->cmd_handler,
830 msecs_to_jiffies(1));
831
832 return 0;
833 }
834
835 static int pci_epf_test_link_down(struct pci_epf *epf)
836 {
837 struct pci_epf_test *epf_test = epf_get_drvdata(epf);
838
839 cancel_delayed_work_sync(&epf_test->cmd_handler);
840
841 return 0;
842 }
843
844 static const struct pci_epc_event_ops pci_epf_test_event_ops = {
845 .epc_init = pci_epf_test_epc_init,
846 .epc_deinit = pci_epf_test_epc_deinit,
847 .link_up = pci_epf_test_link_up,
848 .link_down = pci_epf_test_link_down,
849 };
850
851 static int pci_epf_test_alloc_space(struct pci_epf *epf)
852 {
853 struct pci_epf_test *epf_test = epf_get_drvdata(epf);
854 struct device *dev = &epf->dev;
855 size_t msix_table_size = 0;
856 size_t test_reg_bar_size;
857 size_t pba_size = 0;
858 void *base;
859 enum pci_barno test_reg_bar = epf_test->test_reg_bar;
860 enum pci_barno bar;
861 const struct pci_epc_features *epc_features = epf_test->epc_features;
862 size_t test_reg_size;
863
864 test_reg_bar_size = ALIGN(sizeof(struct pci_epf_test_reg), 128);
865
866 if (epc_features->msix_capable) {
867 msix_table_size = PCI_MSIX_ENTRY_SIZE * epf->msix_interrupts;
868 epf_test->msix_table_offset = test_reg_bar_size;
869 /* Align to QWORD or 8 Bytes */
870 pba_size = ALIGN(DIV_ROUND_UP(epf->msix_interrupts, 8), 8);
871 }
872 test_reg_size = test_reg_bar_size + msix_table_size + pba_size;
873
874 base = pci_epf_alloc_space(epf, test_reg_size, test_reg_bar,
875 epc_features, PRIMARY_INTERFACE);
876 if (!base) {
877 dev_err(dev, "Failed to allocated register space\n");
878 return -ENOMEM;
879 }
880 epf_test->reg[test_reg_bar] = base;
881
882 for (bar = BAR_0; bar < PCI_STD_NUM_BARS; bar++) {
883 bar = pci_epc_get_next_free_bar(epc_features, bar);
884 if (bar == NO_BAR)
885 break;
886
887 if (bar == test_reg_bar)
888 continue;
889
890 base = pci_epf_alloc_space(epf, bar_size[bar], bar,
891 epc_features, PRIMARY_INTERFACE);
892 if (!base)
893 dev_err(dev, "Failed to allocate space for BAR%d\n",
894 bar);
895 epf_test->reg[bar] = base;
896 }
897
898 return 0;
899 }
900
901 static void pci_epf_test_free_space(struct pci_epf *epf)
902 {
903 struct pci_epf_test *epf_test = epf_get_drvdata(epf);
904 int bar;
905
906 for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) {
907 if (!epf_test->reg[bar])
908 continue;
909
910 pci_epf_free_space(epf, epf_test->reg[bar], bar,
911 PRIMARY_INTERFACE);
912 }
913 }
914
915 static int pci_epf_test_bind(struct pci_epf *epf)
916 {
917 int ret;
918 struct pci_epf_test *epf_test = epf_get_drvdata(epf);
919 const struct pci_epc_features *epc_features;
920 enum pci_barno test_reg_bar = BAR_0;
921 struct pci_epc *epc = epf->epc;
922
923 if (WARN_ON_ONCE(!epc))
924 return -EINVAL;
925
926 epc_features = pci_epc_get_features(epc, epf->func_no, epf->vfunc_no);
927 if (!epc_features) {
928 dev_err(&epf->dev, "epc_features not implemented\n");
929 return -EOPNOTSUPP;
930 }
931
932 test_reg_bar = pci_epc_get_first_free_bar(epc_features);
933 if (test_reg_bar < 0)
934 return -EINVAL;
935
936 epf_test->test_reg_bar = test_reg_bar;
937 epf_test->epc_features = epc_features;
938
939 ret = pci_epf_test_alloc_space(epf);
940 if (ret)
941 return ret;
942
943 return 0;
944 }
945
946 static void pci_epf_test_unbind(struct pci_epf *epf)
947 {
948 struct pci_epf_test *epf_test = epf_get_drvdata(epf);
949 struct pci_epc *epc = epf->epc;
950
951 cancel_delayed_work_sync(&epf_test->cmd_handler);
952 if (epc->init_complete) {
953 pci_epf_test_clean_dma_chan(epf_test);
954 pci_epf_test_clear_bar(epf);
955 }
956 pci_epf_test_free_space(epf);
957 }
958
959 static const struct pci_epf_device_id pci_epf_test_ids[] = {
960 {
961 .name = "pci_epf_test",
962 },
963 {},
964 };
965
966 static int pci_epf_test_probe(struct pci_epf *epf,
967 const struct pci_epf_device_id *id)
968 {
969 struct pci_epf_test *epf_test;
970 struct device *dev = &epf->dev;
971
972 epf_test = devm_kzalloc(dev, sizeof(*epf_test), GFP_KERNEL);
973 if (!epf_test)
974 return -ENOMEM;
975
976 epf->header = &test_header;
977 epf_test->epf = epf;
978
979 INIT_DELAYED_WORK(&epf_test->cmd_handler, pci_epf_test_cmd_handler);
980
981 epf->event_ops = &pci_epf_test_event_ops;
982
983 epf_set_drvdata(epf, epf_test);
984 return 0;
985 }
986
987 static const struct pci_epf_ops ops = {
988 .unbind = pci_epf_test_unbind,
989 .bind = pci_epf_test_bind,
990 };
991
992 static struct pci_epf_driver test_driver = {
993 .driver.name = "pci_epf_test",
994 .probe = pci_epf_test_probe,
995 .id_table = pci_epf_test_ids,
996 .ops = &ops,
997 .owner = THIS_MODULE,
998 };
999
1000 static int __init pci_epf_test_init(void)
1001 {
1002 int ret;
1003
1004 kpcitest_workqueue = alloc_workqueue("kpcitest",
1005 WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
1006 if (!kpcitest_workqueue) {
1007 pr_err("Failed to allocate the kpcitest work queue\n");
1008 return -ENOMEM;
1009 }
1010
1011 ret = pci_epf_register_driver(&test_driver);
1012 if (ret) {
1013 destroy_workqueue(kpcitest_workqueue);
1014 pr_err("Failed to register pci epf test driver --> %d\n", ret);
1015 return ret;
1016 }
1017
1018 return 0;
1019 }
1020 module_init(pci_epf_test_init);
1021
1022 static void __exit pci_epf_test_exit(void)
1023 {
1024 if (kpcitest_workqueue)
1025 destroy_workqueue(kpcitest_workqueue);
1026 pci_epf_unregister_driver(&test_driver);
1027 }
1028 module_exit(pci_epf_test_exit);
1029
1030 MODULE_DESCRIPTION("PCI EPF TEST DRIVER");
1031 MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>");
1032 MODULE_LICENSE("GPL v2");
Kernel DRM miscellaneous fixes and cross-tree changes