?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/evalenti/linux...
[linux/fpc-iii.git] / drivers / ntb / hw / amd / ntb_hw_amd.c
blob6ccba0d862df704877db1589fff036d40c10cf36
1 /*
2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
4 *
5 * GPL LICENSE SUMMARY
6 *
7 * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as
11 * published by the Free Software Foundation.
12 *
13 * BSD LICENSE
14 *
15 * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
16 *
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions
19 * are met:
20 *
21 * * Redistributions of source code must retain the above copyright
22 * notice, this list of conditions and the following disclaimer.
23 * * Redistributions in binary form must reproduce the above copy
24 * notice, this list of conditions and the following disclaimer in
25 * the documentation and/or other materials provided with the
26 * distribution.
27 * * Neither the name of AMD Corporation nor the names of its
28 * contributors may be used to endorse or promote products derived
29 * from this software without specific prior written permission.
30 *
31 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
32 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
33 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
34 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
35 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
36 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
37 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
38 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
39 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
41 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42 *
43 * AMD PCIe NTB Linux driver
44 *
45 * Contact Information:
46 * Xiangliang Yu <Xiangliang.Yu@amd.com>
47 */
48
49 #include <linux/debugfs.h>
50 #include <linux/delay.h>
51 #include <linux/init.h>
52 #include <linux/interrupt.h>
53 #include <linux/module.h>
54 #include <linux/acpi.h>
55 #include <linux/pci.h>
56 #include <linux/random.h>
57 #include <linux/slab.h>
58 #include <linux/ntb.h>
59
60 #include "ntb_hw_amd.h"
61
62 #define NTB_NAME "ntb_hw_amd"
63 #define NTB_DESC "AMD(R) PCI-E Non-Transparent Bridge Driver"
64 #define NTB_VER "1.0"
65
66 MODULE_DESCRIPTION(NTB_DESC);
67 MODULE_VERSION(NTB_VER);
68 MODULE_LICENSE("Dual BSD/GPL");
69 MODULE_AUTHOR("AMD Inc.");
70
71 static const struct file_operations amd_ntb_debugfs_info;
72 static struct dentry *debugfs_dir;
73
74 static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int idx)
75 {
76 if (idx < 0 || idx > ndev->mw_count)
77 return -EINVAL;
78
79 return 1 << idx;
80 }
81
82 static int amd_ntb_mw_count(struct ntb_dev *ntb)
83 {
84 return ntb_ndev(ntb)->mw_count;
85 }
86
87 static int amd_ntb_mw_get_range(struct ntb_dev *ntb, int idx,
88 phys_addr_t *base,
89 resource_size_t *size,
90 resource_size_t *align,
91 resource_size_t *align_size)
92 {
93 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
94 int bar;
95
96 bar = ndev_mw_to_bar(ndev, idx);
97 if (bar < 0)
98 return bar;
99
100 if (base)
101 *base = pci_resource_start(ndev->ntb.pdev, bar);
102
103 if (size)
104 *size = pci_resource_len(ndev->ntb.pdev, bar);
105
106 if (align)
107 *align = SZ_4K;
108
109 if (align_size)
110 *align_size = 1;
111
112 return 0;
113 }
114
115 static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int idx,
116 dma_addr_t addr, resource_size_t size)
117 {
118 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
119 unsigned long xlat_reg, limit_reg = 0;
120 resource_size_t mw_size;
121 void __iomem *mmio, *peer_mmio;
122 u64 base_addr, limit, reg_val;
123 int bar;
124
125 bar = ndev_mw_to_bar(ndev, idx);
126 if (bar < 0)
127 return bar;
128
129 mw_size = pci_resource_len(ndev->ntb.pdev, bar);
130
131 /* make sure the range fits in the usable mw size */
132 if (size > mw_size)
133 return -EINVAL;
134
135 mmio = ndev->self_mmio;
136 peer_mmio = ndev->peer_mmio;
137
138 base_addr = pci_resource_start(ndev->ntb.pdev, bar);
139
140 if (bar != 1) {
141 xlat_reg = AMD_BAR23XLAT_OFFSET + ((bar - 2) << 3);
142 limit_reg = AMD_BAR23LMT_OFFSET + ((bar - 2) << 3);
143
144 /* Set the limit if supported */
145 limit = base_addr + size;
146
147 /* set and verify setting the translation address */
148 write64(addr, peer_mmio + xlat_reg);
149 reg_val = read64(peer_mmio + xlat_reg);
150 if (reg_val != addr) {
151 write64(0, peer_mmio + xlat_reg);
152 return -EIO;
153 }
154
155 /* set and verify setting the limit */
156 write64(limit, mmio + limit_reg);
157 reg_val = read64(mmio + limit_reg);
158 if (reg_val != limit) {
159 write64(base_addr, mmio + limit_reg);
160 write64(0, peer_mmio + xlat_reg);
161 return -EIO;
162 }
163 } else {
164 xlat_reg = AMD_BAR1XLAT_OFFSET;
165 limit_reg = AMD_BAR1LMT_OFFSET;
166
167 /* split bar addr range must all be 32 bit */
168 if (addr & (~0ull << 32))
169 return -EINVAL;
170 if ((addr + size) & (~0ull << 32))
171 return -EINVAL;
172
173 /* Set the limit if supported */
174 limit = base_addr + size;
175
176 /* set and verify setting the translation address */
177 write64(addr, peer_mmio + xlat_reg);
178 reg_val = read64(peer_mmio + xlat_reg);
179 if (reg_val != addr) {
180 write64(0, peer_mmio + xlat_reg);
181 return -EIO;
182 }
183
184 /* set and verify setting the limit */
185 writel(limit, mmio + limit_reg);
186 reg_val = readl(mmio + limit_reg);
187 if (reg_val != limit) {
188 writel(base_addr, mmio + limit_reg);
189 writel(0, peer_mmio + xlat_reg);
190 return -EIO;
191 }
192 }
193
194 return 0;
195 }
196
197 static int amd_link_is_up(struct amd_ntb_dev *ndev)
198 {
199 if (!ndev->peer_sta)
200 return NTB_LNK_STA_ACTIVE(ndev->cntl_sta);
201
202 /* If peer_sta is reset or D0 event, the ISR has
203 * started a timer to check link status of hardware.
204 * So here just clear status bit. And if peer_sta is
205 * D3 or PME_TO, D0/reset event will be happened when
206 * system wakeup/poweron, so do nothing here.
207 */
208 if (ndev->peer_sta & AMD_PEER_RESET_EVENT)
209 ndev->peer_sta &= ~AMD_PEER_RESET_EVENT;
210 else if (ndev->peer_sta & AMD_PEER_D0_EVENT)
211 ndev->peer_sta = 0;
212
213 return 0;
214 }
215
216 static int amd_ntb_link_is_up(struct ntb_dev *ntb,
217 enum ntb_speed *speed,
218 enum ntb_width *width)
219 {
220 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
221 int ret = 0;
222
223 if (amd_link_is_up(ndev)) {
224 if (speed)
225 *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
226 if (width)
227 *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
228
229 dev_dbg(ndev_dev(ndev), "link is up.\n");
230
231 ret = 1;
232 } else {
233 if (speed)
234 *speed = NTB_SPEED_NONE;
235 if (width)
236 *width = NTB_WIDTH_NONE;
237
238 dev_dbg(ndev_dev(ndev), "link is down.\n");
239 }
240
241 return ret;
242 }
243
244 static int amd_ntb_link_enable(struct ntb_dev *ntb,
245 enum ntb_speed max_speed,
246 enum ntb_width max_width)
247 {
248 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
249 void __iomem *mmio = ndev->self_mmio;
250 u32 ntb_ctl;
251
252 /* Enable event interrupt */
253 ndev->int_mask &= ~AMD_EVENT_INTMASK;
254 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
255
256 if (ndev->ntb.topo == NTB_TOPO_SEC)
257 return -EINVAL;
258 dev_dbg(ndev_dev(ndev), "Enabling Link.\n");
259
260 ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
261 ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL);
262 writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
263
264 return 0;
265 }
266
267 static int amd_ntb_link_disable(struct ntb_dev *ntb)
268 {
269 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
270 void __iomem *mmio = ndev->self_mmio;
271 u32 ntb_ctl;
272
273 /* Disable event interrupt */
274 ndev->int_mask |= AMD_EVENT_INTMASK;
275 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
276
277 if (ndev->ntb.topo == NTB_TOPO_SEC)
278 return -EINVAL;
279 dev_dbg(ndev_dev(ndev), "Enabling Link.\n");
280
281 ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
282 ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL);
283 writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
284
285 return 0;
286 }
287
288 static u64 amd_ntb_db_valid_mask(struct ntb_dev *ntb)
289 {
290 return ntb_ndev(ntb)->db_valid_mask;
291 }
292
293 static int amd_ntb_db_vector_count(struct ntb_dev *ntb)
294 {
295 return ntb_ndev(ntb)->db_count;
296 }
297
298 static u64 amd_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
299 {
300 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
301
302 if (db_vector < 0 || db_vector > ndev->db_count)
303 return 0;
304
305 return ntb_ndev(ntb)->db_valid_mask & (1 << db_vector);
306 }
307
308 static u64 amd_ntb_db_read(struct ntb_dev *ntb)
309 {
310 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
311 void __iomem *mmio = ndev->self_mmio;
312
313 return (u64)readw(mmio + AMD_DBSTAT_OFFSET);
314 }
315
316 static int amd_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
317 {
318 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
319 void __iomem *mmio = ndev->self_mmio;
320
321 writew((u16)db_bits, mmio + AMD_DBSTAT_OFFSET);
322
323 return 0;
324 }
325
326 static int amd_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
327 {
328 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
329 void __iomem *mmio = ndev->self_mmio;
330 unsigned long flags;
331
332 if (db_bits & ~ndev->db_valid_mask)
333 return -EINVAL;
334
335 spin_lock_irqsave(&ndev->db_mask_lock, flags);
336 ndev->db_mask |= db_bits;
337 writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
338 spin_unlock_irqrestore(&ndev->db_mask_lock, flags);
339
340 return 0;
341 }
342
343 static int amd_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
344 {
345 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
346 void __iomem *mmio = ndev->self_mmio;
347 unsigned long flags;
348
349 if (db_bits & ~ndev->db_valid_mask)
350 return -EINVAL;
351
352 spin_lock_irqsave(&ndev->db_mask_lock, flags);
353 ndev->db_mask &= ~db_bits;
354 writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
355 spin_unlock_irqrestore(&ndev->db_mask_lock, flags);
356
357 return 0;
358 }
359
360 static int amd_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
361 {
362 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
363 void __iomem *mmio = ndev->self_mmio;
364
365 writew((u16)db_bits, mmio + AMD_DBREQ_OFFSET);
366
367 return 0;
368 }
369
370 static int amd_ntb_spad_count(struct ntb_dev *ntb)
371 {
372 return ntb_ndev(ntb)->spad_count;
373 }
374
375 static u32 amd_ntb_spad_read(struct ntb_dev *ntb, int idx)
376 {
377 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
378 void __iomem *mmio = ndev->self_mmio;
379 u32 offset;
380
381 if (idx < 0 || idx >= ndev->spad_count)
382 return 0;
383
384 offset = ndev->self_spad + (idx << 2);
385 return readl(mmio + AMD_SPAD_OFFSET + offset);
386 }
387
388 static int amd_ntb_spad_write(struct ntb_dev *ntb,
389 int idx, u32 val)
390 {
391 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
392 void __iomem *mmio = ndev->self_mmio;
393 u32 offset;
394
395 if (idx < 0 || idx >= ndev->spad_count)
396 return -EINVAL;
397
398 offset = ndev->self_spad + (idx << 2);
399 writel(val, mmio + AMD_SPAD_OFFSET + offset);
400
401 return 0;
402 }
403
404 static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int idx)
405 {
406 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
407 void __iomem *mmio = ndev->self_mmio;
408 u32 offset;
409
410 if (idx < 0 || idx >= ndev->spad_count)
411 return -EINVAL;
412
413 offset = ndev->peer_spad + (idx << 2);
414 return readl(mmio + AMD_SPAD_OFFSET + offset);
415 }
416
417 static int amd_ntb_peer_spad_write(struct ntb_dev *ntb,
418 int idx, u32 val)
419 {
420 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
421 void __iomem *mmio = ndev->self_mmio;
422 u32 offset;
423
424 if (idx < 0 || idx >= ndev->spad_count)
425 return -EINVAL;
426
427 offset = ndev->peer_spad + (idx << 2);
428 writel(val, mmio + AMD_SPAD_OFFSET + offset);
429
430 return 0;
431 }
432
433 static const struct ntb_dev_ops amd_ntb_ops = {
434 .mw_count = amd_ntb_mw_count,
435 .mw_get_range = amd_ntb_mw_get_range,
436 .mw_set_trans = amd_ntb_mw_set_trans,
437 .link_is_up = amd_ntb_link_is_up,
438 .link_enable = amd_ntb_link_enable,
439 .link_disable = amd_ntb_link_disable,
440 .db_valid_mask = amd_ntb_db_valid_mask,
441 .db_vector_count = amd_ntb_db_vector_count,
442 .db_vector_mask = amd_ntb_db_vector_mask,
443 .db_read = amd_ntb_db_read,
444 .db_clear = amd_ntb_db_clear,
445 .db_set_mask = amd_ntb_db_set_mask,
446 .db_clear_mask = amd_ntb_db_clear_mask,
447 .peer_db_set = amd_ntb_peer_db_set,
448 .spad_count = amd_ntb_spad_count,
449 .spad_read = amd_ntb_spad_read,
450 .spad_write = amd_ntb_spad_write,
451 .peer_spad_read = amd_ntb_peer_spad_read,
452 .peer_spad_write = amd_ntb_peer_spad_write,
453 };
454
455 static void amd_ack_smu(struct amd_ntb_dev *ndev, u32 bit)
456 {
457 void __iomem *mmio = ndev->self_mmio;
458 int reg;
459
460 reg = readl(mmio + AMD_SMUACK_OFFSET);
461 reg |= bit;
462 writel(reg, mmio + AMD_SMUACK_OFFSET);
463
464 ndev->peer_sta |= bit;
465 }
466
467 static void amd_handle_event(struct amd_ntb_dev *ndev, int vec)
468 {
469 void __iomem *mmio = ndev->self_mmio;
470 u32 status;
471
472 status = readl(mmio + AMD_INTSTAT_OFFSET);
473 if (!(status & AMD_EVENT_INTMASK))
474 return;
475
476 dev_dbg(ndev_dev(ndev), "status = 0x%x and vec = %d\n", status, vec);
477
478 status &= AMD_EVENT_INTMASK;
479 switch (status) {
480 case AMD_PEER_FLUSH_EVENT:
481 dev_info(ndev_dev(ndev), "Flush is done.\n");
482 break;
483 case AMD_PEER_RESET_EVENT:
484 amd_ack_smu(ndev, AMD_PEER_RESET_EVENT);
485
486 /* link down first */
487 ntb_link_event(&ndev->ntb);
488 /* polling peer status */
489 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
490
491 break;
492 case AMD_PEER_D3_EVENT:
493 case AMD_PEER_PMETO_EVENT:
494 amd_ack_smu(ndev, status);
495
496 /* link down */
497 ntb_link_event(&ndev->ntb);
498
499 break;
500 case AMD_PEER_D0_EVENT:
501 mmio = ndev->peer_mmio;
502 status = readl(mmio + AMD_PMESTAT_OFFSET);
503 /* check if this is WAKEUP event */
504 if (status & 0x1)
505 dev_info(ndev_dev(ndev), "Wakeup is done.\n");
506
507 amd_ack_smu(ndev, AMD_PEER_D0_EVENT);
508
509 /* start a timer to poll link status */
510 schedule_delayed_work(&ndev->hb_timer,
511 AMD_LINK_HB_TIMEOUT);
512 break;
513 default:
514 dev_info(ndev_dev(ndev), "event status = 0x%x.\n", status);
515 break;
516 }
517 }
518
519 static irqreturn_t ndev_interrupt(struct amd_ntb_dev *ndev, int vec)
520 {
521 dev_dbg(ndev_dev(ndev), "vec %d\n", vec);
522
523 if (vec > (AMD_DB_CNT - 1) || (ndev->msix_vec_count == 1))
524 amd_handle_event(ndev, vec);
525
526 if (vec < AMD_DB_CNT)
527 ntb_db_event(&ndev->ntb, vec);
528
529 return IRQ_HANDLED;
530 }
531
532 static irqreturn_t ndev_vec_isr(int irq, void *dev)
533 {
534 struct amd_ntb_vec *nvec = dev;
535
536 return ndev_interrupt(nvec->ndev, nvec->num);
537 }
538
539 static irqreturn_t ndev_irq_isr(int irq, void *dev)
540 {
541 struct amd_ntb_dev *ndev = dev;
542
543 return ndev_interrupt(ndev, irq - ndev_pdev(ndev)->irq);
544 }
545
546 static int ndev_init_isr(struct amd_ntb_dev *ndev,
547 int msix_min, int msix_max)
548 {
549 struct pci_dev *pdev;
550 int rc, i, msix_count, node;
551
552 pdev = ndev_pdev(ndev);
553
554 node = dev_to_node(&pdev->dev);
555
556 ndev->db_mask = ndev->db_valid_mask;
557
558 /* Try to set up msix irq */
559 ndev->vec = kzalloc_node(msix_max * sizeof(*ndev->vec),
560 GFP_KERNEL, node);
561 if (!ndev->vec)
562 goto err_msix_vec_alloc;
563
564 ndev->msix = kzalloc_node(msix_max * sizeof(*ndev->msix),
565 GFP_KERNEL, node);
566 if (!ndev->msix)
567 goto err_msix_alloc;
568
569 for (i = 0; i < msix_max; ++i)
570 ndev->msix[i].entry = i;
571
572 msix_count = pci_enable_msix_range(pdev, ndev->msix,
573 msix_min, msix_max);
574 if (msix_count < 0)
575 goto err_msix_enable;
576
577 /* NOTE: Disable MSIX if msix count is less than 16 because of
578 * hardware limitation.
579 */
580 if (msix_count < msix_min) {
581 pci_disable_msix(pdev);
582 goto err_msix_enable;
583 }
584
585 for (i = 0; i < msix_count; ++i) {
586 ndev->vec[i].ndev = ndev;
587 ndev->vec[i].num = i;
588 rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
589 "ndev_vec_isr", &ndev->vec[i]);
590 if (rc)
591 goto err_msix_request;
592 }
593
594 dev_dbg(ndev_dev(ndev), "Using msix interrupts\n");
595 ndev->db_count = msix_min;
596 ndev->msix_vec_count = msix_max;
597 return 0;
598
599 err_msix_request:
600 while (i-- > 0)
601 free_irq(ndev->msix[i].vector, ndev);
602 pci_disable_msix(pdev);
603 err_msix_enable:
604 kfree(ndev->msix);
605 err_msix_alloc:
606 kfree(ndev->vec);
607 err_msix_vec_alloc:
608 ndev->msix = NULL;
609 ndev->vec = NULL;
610
611 /* Try to set up msi irq */
612 rc = pci_enable_msi(pdev);
613 if (rc)
614 goto err_msi_enable;
615
616 rc = request_irq(pdev->irq, ndev_irq_isr, 0,
617 "ndev_irq_isr", ndev);
618 if (rc)
619 goto err_msi_request;
620
621 dev_dbg(ndev_dev(ndev), "Using msi interrupts\n");
622 ndev->db_count = 1;
623 ndev->msix_vec_count = 1;
624 return 0;
625
626 err_msi_request:
627 pci_disable_msi(pdev);
628 err_msi_enable:
629
630 /* Try to set up intx irq */
631 pci_intx(pdev, 1);
632
633 rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
634 "ndev_irq_isr", ndev);
635 if (rc)
636 goto err_intx_request;
637
638 dev_dbg(ndev_dev(ndev), "Using intx interrupts\n");
639 ndev->db_count = 1;
640 ndev->msix_vec_count = 1;
641 return 0;
642
643 err_intx_request:
644 return rc;
645 }
646
647 static void ndev_deinit_isr(struct amd_ntb_dev *ndev)
648 {
649 struct pci_dev *pdev;
650 void __iomem *mmio = ndev->self_mmio;
651 int i;
652
653 pdev = ndev_pdev(ndev);
654
655 /* Mask all doorbell interrupts */
656 ndev->db_mask = ndev->db_valid_mask;
657 writel(ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
658
659 if (ndev->msix) {
660 i = ndev->msix_vec_count;
661 while (i--)
662 free_irq(ndev->msix[i].vector, &ndev->vec[i]);
663 pci_disable_msix(pdev);
664 kfree(ndev->msix);
665 kfree(ndev->vec);
666 } else {
667 free_irq(pdev->irq, ndev);
668 if (pci_dev_msi_enabled(pdev))
669 pci_disable_msi(pdev);
670 else
671 pci_intx(pdev, 0);
672 }
673 }
674
675 static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
676 size_t count, loff_t *offp)
677 {
678 struct amd_ntb_dev *ndev;
679 void __iomem *mmio;
680 char *buf;
681 size_t buf_size;
682 ssize_t ret, off;
683 union { u64 v64; u32 v32; u16 v16; } u;
684
685 ndev = filp->private_data;
686 mmio = ndev->self_mmio;
687
688 buf_size = min(count, 0x800ul);
689
690 buf = kmalloc(buf_size, GFP_KERNEL);
691 if (!buf)
692 return -ENOMEM;
693
694 off = 0;
695
696 off += scnprintf(buf + off, buf_size - off,
697 "NTB Device Information:\n");
698
699 off += scnprintf(buf + off, buf_size - off,
700 "Connection Topology -\t%s\n",
701 ntb_topo_string(ndev->ntb.topo));
702
703 off += scnprintf(buf + off, buf_size - off,
704 "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
705
706 if (!amd_link_is_up(ndev)) {
707 off += scnprintf(buf + off, buf_size - off,
708 "Link Status -\t\tDown\n");
709 } else {
710 off += scnprintf(buf + off, buf_size - off,
711 "Link Status -\t\tUp\n");
712 off += scnprintf(buf + off, buf_size - off,
713 "Link Speed -\t\tPCI-E Gen %u\n",
714 NTB_LNK_STA_SPEED(ndev->lnk_sta));
715 off += scnprintf(buf + off, buf_size - off,
716 "Link Width -\t\tx%u\n",
717 NTB_LNK_STA_WIDTH(ndev->lnk_sta));
718 }
719
720 off += scnprintf(buf + off, buf_size - off,
721 "Memory Window Count -\t%u\n", ndev->mw_count);
722 off += scnprintf(buf + off, buf_size - off,
723 "Scratchpad Count -\t%u\n", ndev->spad_count);
724 off += scnprintf(buf + off, buf_size - off,
725 "Doorbell Count -\t%u\n", ndev->db_count);
726 off += scnprintf(buf + off, buf_size - off,
727 "MSIX Vector Count -\t%u\n", ndev->msix_vec_count);
728
729 off += scnprintf(buf + off, buf_size - off,
730 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
731
732 u.v32 = readl(ndev->self_mmio + AMD_DBMASK_OFFSET);
733 off += scnprintf(buf + off, buf_size - off,
734 "Doorbell Mask -\t\t\t%#06x\n", u.v32);
735
736 u.v32 = readl(mmio + AMD_DBSTAT_OFFSET);
737 off += scnprintf(buf + off, buf_size - off,
738 "Doorbell Bell -\t\t\t%#06x\n", u.v32);
739
740 off += scnprintf(buf + off, buf_size - off,
741 "\nNTB Incoming XLAT:\n");
742
743 u.v64 = read64(mmio + AMD_BAR1XLAT_OFFSET);
744 off += scnprintf(buf + off, buf_size - off,
745 "XLAT1 -\t\t%#018llx\n", u.v64);
746
747 u.v64 = read64(ndev->self_mmio + AMD_BAR23XLAT_OFFSET);
748 off += scnprintf(buf + off, buf_size - off,
749 "XLAT23 -\t\t%#018llx\n", u.v64);
750
751 u.v64 = read64(ndev->self_mmio + AMD_BAR45XLAT_OFFSET);
752 off += scnprintf(buf + off, buf_size - off,
753 "XLAT45 -\t\t%#018llx\n", u.v64);
754
755 u.v32 = readl(mmio + AMD_BAR1LMT_OFFSET);
756 off += scnprintf(buf + off, buf_size - off,
757 "LMT1 -\t\t\t%#06x\n", u.v32);
758
759 u.v64 = read64(ndev->self_mmio + AMD_BAR23LMT_OFFSET);
760 off += scnprintf(buf + off, buf_size - off,
761 "LMT23 -\t\t\t%#018llx\n", u.v64);
762
763 u.v64 = read64(ndev->self_mmio + AMD_BAR45LMT_OFFSET);
764 off += scnprintf(buf + off, buf_size - off,
765 "LMT45 -\t\t\t%#018llx\n", u.v64);
766
767 ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
768 kfree(buf);
769 return ret;
770 }
771
772 static void ndev_init_debugfs(struct amd_ntb_dev *ndev)
773 {
774 if (!debugfs_dir) {
775 ndev->debugfs_dir = NULL;
776 ndev->debugfs_info = NULL;
777 } else {
778 ndev->debugfs_dir =
779 debugfs_create_dir(ndev_name(ndev), debugfs_dir);
780 if (!ndev->debugfs_dir)
781 ndev->debugfs_info = NULL;
782 else
783 ndev->debugfs_info =
784 debugfs_create_file("info", S_IRUSR,
785 ndev->debugfs_dir, ndev,
786 &amd_ntb_debugfs_info);
787 }
788 }
789
790 static void ndev_deinit_debugfs(struct amd_ntb_dev *ndev)
791 {
792 debugfs_remove_recursive(ndev->debugfs_dir);
793 }
794
795 static inline void ndev_init_struct(struct amd_ntb_dev *ndev,
796 struct pci_dev *pdev)
797 {
798 ndev->ntb.pdev = pdev;
799 ndev->ntb.topo = NTB_TOPO_NONE;
800 ndev->ntb.ops = &amd_ntb_ops;
801 ndev->int_mask = AMD_EVENT_INTMASK;
802 spin_lock_init(&ndev->db_mask_lock);
803 }
804
805 static int amd_poll_link(struct amd_ntb_dev *ndev)
806 {
807 void __iomem *mmio = ndev->peer_mmio;
808 u32 reg, stat;
809 int rc;
810
811 reg = readl(mmio + AMD_SIDEINFO_OFFSET);
812 reg &= NTB_LIN_STA_ACTIVE_BIT;
813
814 dev_dbg(ndev_dev(ndev), "%s: reg_val = 0x%x.\n", __func__, reg);
815
816 if (reg == ndev->cntl_sta)
817 return 0;
818
819 ndev->cntl_sta = reg;
820
821 rc = pci_read_config_dword(ndev->ntb.pdev,
822 AMD_LINK_STATUS_OFFSET, &stat);
823 if (rc)
824 return 0;
825 ndev->lnk_sta = stat;
826
827 return 1;
828 }
829
830 static void amd_link_hb(struct work_struct *work)
831 {
832 struct amd_ntb_dev *ndev = hb_ndev(work);
833
834 if (amd_poll_link(ndev))
835 ntb_link_event(&ndev->ntb);
836
837 if (!amd_link_is_up(ndev))
838 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
839 }
840
841 static int amd_init_isr(struct amd_ntb_dev *ndev)
842 {
843 return ndev_init_isr(ndev, AMD_DB_CNT, AMD_MSIX_VECTOR_CNT);
844 }
845
846 static void amd_init_side_info(struct amd_ntb_dev *ndev)
847 {
848 void __iomem *mmio = ndev->self_mmio;
849 unsigned int reg;
850
851 reg = readl(mmio + AMD_SIDEINFO_OFFSET);
852 if (!(reg & AMD_SIDE_READY)) {
853 reg |= AMD_SIDE_READY;
854 writel(reg, mmio + AMD_SIDEINFO_OFFSET);
855 }
856 }
857
858 static void amd_deinit_side_info(struct amd_ntb_dev *ndev)
859 {
860 void __iomem *mmio = ndev->self_mmio;
861 unsigned int reg;
862
863 reg = readl(mmio + AMD_SIDEINFO_OFFSET);
864 if (reg & AMD_SIDE_READY) {
865 reg &= ~AMD_SIDE_READY;
866 writel(reg, mmio + AMD_SIDEINFO_OFFSET);
867 readl(mmio + AMD_SIDEINFO_OFFSET);
868 }
869 }
870
871 static int amd_init_ntb(struct amd_ntb_dev *ndev)
872 {
873 void __iomem *mmio = ndev->self_mmio;
874
875 ndev->mw_count = AMD_MW_CNT;
876 ndev->spad_count = AMD_SPADS_CNT;
877 ndev->db_count = AMD_DB_CNT;
878
879 switch (ndev->ntb.topo) {
880 case NTB_TOPO_PRI:
881 case NTB_TOPO_SEC:
882 ndev->spad_count >>= 1;
883 if (ndev->ntb.topo == NTB_TOPO_PRI) {
884 ndev->self_spad = 0;
885 ndev->peer_spad = 0x20;
886 } else {
887 ndev->self_spad = 0x20;
888 ndev->peer_spad = 0;
889 }
890
891 INIT_DELAYED_WORK(&ndev->hb_timer, amd_link_hb);
892 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
893
894 break;
895 default:
896 dev_err(ndev_dev(ndev), "AMD NTB does not support B2B mode.\n");
897 return -EINVAL;
898 }
899
900 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
901
902 /* Mask event interrupts */
903 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
904
905 return 0;
906 }
907
908 static enum ntb_topo amd_get_topo(struct amd_ntb_dev *ndev)
909 {
910 void __iomem *mmio = ndev->self_mmio;
911 u32 info;
912
913 info = readl(mmio + AMD_SIDEINFO_OFFSET);
914 if (info & AMD_SIDE_MASK)
915 return NTB_TOPO_SEC;
916 else
917 return NTB_TOPO_PRI;
918 }
919
920 static int amd_init_dev(struct amd_ntb_dev *ndev)
921 {
922 struct pci_dev *pdev;
923 int rc = 0;
924
925 pdev = ndev_pdev(ndev);
926
927 ndev->ntb.topo = amd_get_topo(ndev);
928 dev_dbg(ndev_dev(ndev), "AMD NTB topo is %s\n",
929 ntb_topo_string(ndev->ntb.topo));
930
931 rc = amd_init_ntb(ndev);
932 if (rc)
933 return rc;
934
935 rc = amd_init_isr(ndev);
936 if (rc) {
937 dev_err(ndev_dev(ndev), "fail to init isr.\n");
938 return rc;
939 }
940
941 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
942
943 return 0;
944 }
945
946 static void amd_deinit_dev(struct amd_ntb_dev *ndev)
947 {
948 cancel_delayed_work_sync(&ndev->hb_timer);
949
950 ndev_deinit_isr(ndev);
951 }
952
953 static int amd_ntb_init_pci(struct amd_ntb_dev *ndev,
954 struct pci_dev *pdev)
955 {
956 int rc;
957
958 pci_set_drvdata(pdev, ndev);
959
960 rc = pci_enable_device(pdev);
961 if (rc)
962 goto err_pci_enable;
963
964 rc = pci_request_regions(pdev, NTB_NAME);
965 if (rc)
966 goto err_pci_regions;
967
968 pci_set_master(pdev);
969
970 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
971 if (rc) {
972 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
973 if (rc)
974 goto err_dma_mask;
975 dev_warn(ndev_dev(ndev), "Cannot DMA highmem\n");
976 }
977
978 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
979 if (rc) {
980 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
981 if (rc)
982 goto err_dma_mask;
983 dev_warn(ndev_dev(ndev), "Cannot DMA consistent highmem\n");
984 }
985
986 ndev->self_mmio = pci_iomap(pdev, 0, 0);
987 if (!ndev->self_mmio) {
988 rc = -EIO;
989 goto err_dma_mask;
990 }
991 ndev->peer_mmio = ndev->self_mmio + AMD_PEER_OFFSET;
992
993 return 0;
994
995 err_dma_mask:
996 pci_clear_master(pdev);
997 err_pci_regions:
998 pci_disable_device(pdev);
999 err_pci_enable:
1000 pci_set_drvdata(pdev, NULL);
1001 return rc;
1002 }
1003
1004 static void amd_ntb_deinit_pci(struct amd_ntb_dev *ndev)
1005 {
1006 struct pci_dev *pdev = ndev_pdev(ndev);
1007
1008 pci_iounmap(pdev, ndev->self_mmio);
1009
1010 pci_clear_master(pdev);
1011 pci_release_regions(pdev);
1012 pci_disable_device(pdev);
1013 pci_set_drvdata(pdev, NULL);
1014 }
1015
1016 static int amd_ntb_pci_probe(struct pci_dev *pdev,
1017 const struct pci_device_id *id)
1018 {
1019 struct amd_ntb_dev *ndev;
1020 int rc, node;
1021
1022 node = dev_to_node(&pdev->dev);
1023
1024 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
1025 if (!ndev) {
1026 rc = -ENOMEM;
1027 goto err_ndev;
1028 }
1029
1030 ndev_init_struct(ndev, pdev);
1031
1032 rc = amd_ntb_init_pci(ndev, pdev);
1033 if (rc)
1034 goto err_init_pci;
1035
1036 rc = amd_init_dev(ndev);
1037 if (rc)
1038 goto err_init_dev;
1039
1040 /* write side info */
1041 amd_init_side_info(ndev);
1042
1043 amd_poll_link(ndev);
1044
1045 ndev_init_debugfs(ndev);
1046
1047 rc = ntb_register_device(&ndev->ntb);
1048 if (rc)
1049 goto err_register;
1050
1051 dev_info(&pdev->dev, "NTB device registered.\n");
1052
1053 return 0;
1054
1055 err_register:
1056 ndev_deinit_debugfs(ndev);
1057 amd_deinit_dev(ndev);
1058 err_init_dev:
1059 amd_ntb_deinit_pci(ndev);
1060 err_init_pci:
1061 kfree(ndev);
1062 err_ndev:
1063 return rc;
1064 }
1065
1066 static void amd_ntb_pci_remove(struct pci_dev *pdev)
1067 {
1068 struct amd_ntb_dev *ndev = pci_get_drvdata(pdev);
1069
1070 ntb_unregister_device(&ndev->ntb);
1071 ndev_deinit_debugfs(ndev);
1072 amd_deinit_side_info(ndev);
1073 amd_deinit_dev(ndev);
1074 amd_ntb_deinit_pci(ndev);
1075 kfree(ndev);
1076 }
1077
1078 static const struct file_operations amd_ntb_debugfs_info = {
1079 .owner = THIS_MODULE,
1080 .open = simple_open,
1081 .read = ndev_debugfs_read,
1082 };
1083
1084 static const struct pci_device_id amd_ntb_pci_tbl[] = {
1085 {PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_NTB)},
1086 {0}
1087 };
1088 MODULE_DEVICE_TABLE(pci, amd_ntb_pci_tbl);
1089
1090 static struct pci_driver amd_ntb_pci_driver = {
1091 .name = KBUILD_MODNAME,
1092 .id_table = amd_ntb_pci_tbl,
1093 .probe = amd_ntb_pci_probe,
1094 .remove = amd_ntb_pci_remove,
1095 };
1096
1097 static int __init amd_ntb_pci_driver_init(void)
1098 {
1099 pr_info("%s %s\n", NTB_DESC, NTB_VER);
1100
1101 if (debugfs_initialized())
1102 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
1103
1104 return pci_register_driver(&amd_ntb_pci_driver);
1105 }
1106 module_init(amd_ntb_pci_driver_init);
1107
1108 static void __exit amd_ntb_pci_driver_exit(void)
1109 {
1110 pci_unregister_driver(&amd_ntb_pci_driver);
1111 debugfs_remove_recursive(debugfs_dir);
1112 }
1113 module_exit(amd_ntb_pci_driver_exit);
Linux with added FPC-III support