2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
7 * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
8 * Copyright (C) 2016 T-Platforms. All Rights Reserved.
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of version 2 of the GNU General Public License as
12 * published by the Free Software Foundation.
16 * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
17 * Copyright (C) 2016 T-Platforms. All Rights Reserved.
19 * Redistribution and use in source and binary forms, with or without
20 * modification, are permitted provided that the following conditions
23 * * Redistributions of source code must retain the above copyright
24 * notice, this list of conditions and the following disclaimer.
25 * * Redistributions in binary form must reproduce the above copy
26 * notice, this list of conditions and the following disclaimer in
27 * the documentation and/or other materials provided with the
29 * * Neither the name of AMD Corporation nor the names of its
30 * contributors may be used to endorse or promote products derived
31 * from this software without specific prior written permission.
33 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
34 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
35 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
36 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
37 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
38 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
39 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
40 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
41 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
42 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
43 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45 * AMD PCIe NTB Linux driver
47 * Contact Information:
48 * Xiangliang Yu <Xiangliang.Yu@amd.com>
51 #include <linux/debugfs.h>
52 #include <linux/delay.h>
53 #include <linux/init.h>
54 #include <linux/interrupt.h>
55 #include <linux/module.h>
56 #include <linux/acpi.h>
57 #include <linux/pci.h>
58 #include <linux/random.h>
59 #include <linux/slab.h>
60 #include <linux/ntb.h>
62 #include "ntb_hw_amd.h"
64 #define NTB_NAME "ntb_hw_amd"
65 #define NTB_DESC "AMD(R) PCI-E Non-Transparent Bridge Driver"
68 MODULE_DESCRIPTION(NTB_DESC
);
69 MODULE_VERSION(NTB_VER
);
70 MODULE_LICENSE("Dual BSD/GPL");
71 MODULE_AUTHOR("AMD Inc.");
73 static const struct file_operations amd_ntb_debugfs_info
;
74 static struct dentry
*debugfs_dir
;
76 static int ndev_mw_to_bar(struct amd_ntb_dev
*ndev
, int idx
)
78 if (idx
< 0 || idx
> ndev
->mw_count
)
84 static int amd_ntb_mw_count(struct ntb_dev
*ntb
, int pidx
)
86 if (pidx
!= NTB_DEF_PEER_IDX
)
89 return ntb_ndev(ntb
)->mw_count
;
92 static int amd_ntb_mw_get_align(struct ntb_dev
*ntb
, int pidx
, int idx
,
93 resource_size_t
*addr_align
,
94 resource_size_t
*size_align
,
95 resource_size_t
*size_max
)
97 struct amd_ntb_dev
*ndev
= ntb_ndev(ntb
);
100 if (pidx
!= NTB_DEF_PEER_IDX
)
103 bar
= ndev_mw_to_bar(ndev
, idx
);
114 *size_max
= pci_resource_len(ndev
->ntb
.pdev
, bar
);
119 static int amd_ntb_mw_set_trans(struct ntb_dev
*ntb
, int pidx
, int idx
,
120 dma_addr_t addr
, resource_size_t size
)
122 struct amd_ntb_dev
*ndev
= ntb_ndev(ntb
);
123 unsigned long xlat_reg
, limit_reg
= 0;
124 resource_size_t mw_size
;
125 void __iomem
*mmio
, *peer_mmio
;
126 u64 base_addr
, limit
, reg_val
;
129 if (pidx
!= NTB_DEF_PEER_IDX
)
132 bar
= ndev_mw_to_bar(ndev
, idx
);
136 mw_size
= pci_resource_len(ntb
->pdev
, bar
);
138 /* make sure the range fits in the usable mw size */
142 mmio
= ndev
->self_mmio
;
143 peer_mmio
= ndev
->peer_mmio
;
145 base_addr
= pci_resource_start(ntb
->pdev
, bar
);
148 xlat_reg
= AMD_BAR23XLAT_OFFSET
+ ((bar
- 2) << 2);
149 limit_reg
= AMD_BAR23LMT_OFFSET
+ ((bar
- 2) << 2);
151 /* Set the limit if supported */
154 /* set and verify setting the translation address */
155 write64(addr
, peer_mmio
+ xlat_reg
);
156 reg_val
= read64(peer_mmio
+ xlat_reg
);
157 if (reg_val
!= addr
) {
158 write64(0, peer_mmio
+ xlat_reg
);
162 /* set and verify setting the limit */
163 write64(limit
, mmio
+ limit_reg
);
164 reg_val
= read64(mmio
+ limit_reg
);
165 if (reg_val
!= limit
) {
166 write64(base_addr
, mmio
+ limit_reg
);
167 write64(0, peer_mmio
+ xlat_reg
);
171 xlat_reg
= AMD_BAR1XLAT_OFFSET
;
172 limit_reg
= AMD_BAR1LMT_OFFSET
;
174 /* Set the limit if supported */
177 /* set and verify setting the translation address */
178 write64(addr
, peer_mmio
+ xlat_reg
);
179 reg_val
= read64(peer_mmio
+ xlat_reg
);
180 if (reg_val
!= addr
) {
181 write64(0, peer_mmio
+ xlat_reg
);
185 /* set and verify setting the limit */
186 writel(limit
, mmio
+ limit_reg
);
187 reg_val
= readl(mmio
+ limit_reg
);
188 if (reg_val
!= limit
) {
189 writel(base_addr
, mmio
+ limit_reg
);
190 writel(0, peer_mmio
+ xlat_reg
);
198 static int amd_link_is_up(struct amd_ntb_dev
*ndev
)
201 return NTB_LNK_STA_ACTIVE(ndev
->cntl_sta
);
203 if (ndev
->peer_sta
& AMD_LINK_UP_EVENT
) {
208 /* If peer_sta is reset or D0 event, the ISR has
209 * started a timer to check link status of hardware.
210 * So here just clear status bit. And if peer_sta is
211 * D3 or PME_TO, D0/reset event will be happened when
212 * system wakeup/poweron, so do nothing here.
214 if (ndev
->peer_sta
& AMD_PEER_RESET_EVENT
)
215 ndev
->peer_sta
&= ~AMD_PEER_RESET_EVENT
;
216 else if (ndev
->peer_sta
& (AMD_PEER_D0_EVENT
| AMD_LINK_DOWN_EVENT
))
222 static u64
amd_ntb_link_is_up(struct ntb_dev
*ntb
,
223 enum ntb_speed
*speed
,
224 enum ntb_width
*width
)
226 struct amd_ntb_dev
*ndev
= ntb_ndev(ntb
);
229 if (amd_link_is_up(ndev
)) {
231 *speed
= NTB_LNK_STA_SPEED(ndev
->lnk_sta
);
233 *width
= NTB_LNK_STA_WIDTH(ndev
->lnk_sta
);
235 dev_dbg(&ntb
->pdev
->dev
, "link is up.\n");
240 *speed
= NTB_SPEED_NONE
;
242 *width
= NTB_WIDTH_NONE
;
244 dev_dbg(&ntb
->pdev
->dev
, "link is down.\n");
250 static int amd_ntb_link_enable(struct ntb_dev
*ntb
,
251 enum ntb_speed max_speed
,
252 enum ntb_width max_width
)
254 struct amd_ntb_dev
*ndev
= ntb_ndev(ntb
);
255 void __iomem
*mmio
= ndev
->self_mmio
;
258 /* Enable event interrupt */
259 ndev
->int_mask
&= ~AMD_EVENT_INTMASK
;
260 writel(ndev
->int_mask
, mmio
+ AMD_INTMASK_OFFSET
);
262 if (ndev
->ntb
.topo
== NTB_TOPO_SEC
)
264 dev_dbg(&ntb
->pdev
->dev
, "Enabling Link.\n");
266 ntb_ctl
= readl(mmio
+ AMD_CNTL_OFFSET
);
267 ntb_ctl
|= (PMM_REG_CTL
| SMM_REG_CTL
);
268 writel(ntb_ctl
, mmio
+ AMD_CNTL_OFFSET
);
273 static int amd_ntb_link_disable(struct ntb_dev
*ntb
)
275 struct amd_ntb_dev
*ndev
= ntb_ndev(ntb
);
276 void __iomem
*mmio
= ndev
->self_mmio
;
279 /* Disable event interrupt */
280 ndev
->int_mask
|= AMD_EVENT_INTMASK
;
281 writel(ndev
->int_mask
, mmio
+ AMD_INTMASK_OFFSET
);
283 if (ndev
->ntb
.topo
== NTB_TOPO_SEC
)
285 dev_dbg(&ntb
->pdev
->dev
, "Enabling Link.\n");
287 ntb_ctl
= readl(mmio
+ AMD_CNTL_OFFSET
);
288 ntb_ctl
&= ~(PMM_REG_CTL
| SMM_REG_CTL
);
289 writel(ntb_ctl
, mmio
+ AMD_CNTL_OFFSET
);
294 static int amd_ntb_peer_mw_count(struct ntb_dev
*ntb
)
296 /* The same as for inbound MWs */
297 return ntb_ndev(ntb
)->mw_count
;
300 static int amd_ntb_peer_mw_get_addr(struct ntb_dev
*ntb
, int idx
,
301 phys_addr_t
*base
, resource_size_t
*size
)
303 struct amd_ntb_dev
*ndev
= ntb_ndev(ntb
);
306 bar
= ndev_mw_to_bar(ndev
, idx
);
311 *base
= pci_resource_start(ndev
->ntb
.pdev
, bar
);
314 *size
= pci_resource_len(ndev
->ntb
.pdev
, bar
);
319 static u64
amd_ntb_db_valid_mask(struct ntb_dev
*ntb
)
321 return ntb_ndev(ntb
)->db_valid_mask
;
324 static int amd_ntb_db_vector_count(struct ntb_dev
*ntb
)
326 return ntb_ndev(ntb
)->db_count
;
329 static u64
amd_ntb_db_vector_mask(struct ntb_dev
*ntb
, int db_vector
)
331 struct amd_ntb_dev
*ndev
= ntb_ndev(ntb
);
333 if (db_vector
< 0 || db_vector
> ndev
->db_count
)
336 return ntb_ndev(ntb
)->db_valid_mask
& (1 << db_vector
);
339 static u64
amd_ntb_db_read(struct ntb_dev
*ntb
)
341 struct amd_ntb_dev
*ndev
= ntb_ndev(ntb
);
342 void __iomem
*mmio
= ndev
->self_mmio
;
344 return (u64
)readw(mmio
+ AMD_DBSTAT_OFFSET
);
347 static int amd_ntb_db_clear(struct ntb_dev
*ntb
, u64 db_bits
)
349 struct amd_ntb_dev
*ndev
= ntb_ndev(ntb
);
350 void __iomem
*mmio
= ndev
->self_mmio
;
352 writew((u16
)db_bits
, mmio
+ AMD_DBSTAT_OFFSET
);
357 static int amd_ntb_db_set_mask(struct ntb_dev
*ntb
, u64 db_bits
)
359 struct amd_ntb_dev
*ndev
= ntb_ndev(ntb
);
360 void __iomem
*mmio
= ndev
->self_mmio
;
363 if (db_bits
& ~ndev
->db_valid_mask
)
366 spin_lock_irqsave(&ndev
->db_mask_lock
, flags
);
367 ndev
->db_mask
|= db_bits
;
368 writew((u16
)ndev
->db_mask
, mmio
+ AMD_DBMASK_OFFSET
);
369 spin_unlock_irqrestore(&ndev
->db_mask_lock
, flags
);
374 static int amd_ntb_db_clear_mask(struct ntb_dev
*ntb
, u64 db_bits
)
376 struct amd_ntb_dev
*ndev
= ntb_ndev(ntb
);
377 void __iomem
*mmio
= ndev
->self_mmio
;
380 if (db_bits
& ~ndev
->db_valid_mask
)
383 spin_lock_irqsave(&ndev
->db_mask_lock
, flags
);
384 ndev
->db_mask
&= ~db_bits
;
385 writew((u16
)ndev
->db_mask
, mmio
+ AMD_DBMASK_OFFSET
);
386 spin_unlock_irqrestore(&ndev
->db_mask_lock
, flags
);
391 static int amd_ntb_peer_db_set(struct ntb_dev
*ntb
, u64 db_bits
)
393 struct amd_ntb_dev
*ndev
= ntb_ndev(ntb
);
394 void __iomem
*mmio
= ndev
->self_mmio
;
396 writew((u16
)db_bits
, mmio
+ AMD_DBREQ_OFFSET
);
401 static int amd_ntb_spad_count(struct ntb_dev
*ntb
)
403 return ntb_ndev(ntb
)->spad_count
;
406 static u32
amd_ntb_spad_read(struct ntb_dev
*ntb
, int idx
)
408 struct amd_ntb_dev
*ndev
= ntb_ndev(ntb
);
409 void __iomem
*mmio
= ndev
->self_mmio
;
412 if (idx
< 0 || idx
>= ndev
->spad_count
)
415 offset
= ndev
->self_spad
+ (idx
<< 2);
416 return readl(mmio
+ AMD_SPAD_OFFSET
+ offset
);
419 static int amd_ntb_spad_write(struct ntb_dev
*ntb
,
422 struct amd_ntb_dev
*ndev
= ntb_ndev(ntb
);
423 void __iomem
*mmio
= ndev
->self_mmio
;
426 if (idx
< 0 || idx
>= ndev
->spad_count
)
429 offset
= ndev
->self_spad
+ (idx
<< 2);
430 writel(val
, mmio
+ AMD_SPAD_OFFSET
+ offset
);
435 static u32
amd_ntb_peer_spad_read(struct ntb_dev
*ntb
, int pidx
, int sidx
)
437 struct amd_ntb_dev
*ndev
= ntb_ndev(ntb
);
438 void __iomem
*mmio
= ndev
->self_mmio
;
441 if (sidx
< 0 || sidx
>= ndev
->spad_count
)
444 offset
= ndev
->peer_spad
+ (sidx
<< 2);
445 return readl(mmio
+ AMD_SPAD_OFFSET
+ offset
);
448 static int amd_ntb_peer_spad_write(struct ntb_dev
*ntb
, int pidx
,
451 struct amd_ntb_dev
*ndev
= ntb_ndev(ntb
);
452 void __iomem
*mmio
= ndev
->self_mmio
;
455 if (sidx
< 0 || sidx
>= ndev
->spad_count
)
458 offset
= ndev
->peer_spad
+ (sidx
<< 2);
459 writel(val
, mmio
+ AMD_SPAD_OFFSET
+ offset
);
464 static const struct ntb_dev_ops amd_ntb_ops
= {
465 .mw_count
= amd_ntb_mw_count
,
466 .mw_get_align
= amd_ntb_mw_get_align
,
467 .mw_set_trans
= amd_ntb_mw_set_trans
,
468 .peer_mw_count
= amd_ntb_peer_mw_count
,
469 .peer_mw_get_addr
= amd_ntb_peer_mw_get_addr
,
470 .link_is_up
= amd_ntb_link_is_up
,
471 .link_enable
= amd_ntb_link_enable
,
472 .link_disable
= amd_ntb_link_disable
,
473 .db_valid_mask
= amd_ntb_db_valid_mask
,
474 .db_vector_count
= amd_ntb_db_vector_count
,
475 .db_vector_mask
= amd_ntb_db_vector_mask
,
476 .db_read
= amd_ntb_db_read
,
477 .db_clear
= amd_ntb_db_clear
,
478 .db_set_mask
= amd_ntb_db_set_mask
,
479 .db_clear_mask
= amd_ntb_db_clear_mask
,
480 .peer_db_set
= amd_ntb_peer_db_set
,
481 .spad_count
= amd_ntb_spad_count
,
482 .spad_read
= amd_ntb_spad_read
,
483 .spad_write
= amd_ntb_spad_write
,
484 .peer_spad_read
= amd_ntb_peer_spad_read
,
485 .peer_spad_write
= amd_ntb_peer_spad_write
,
488 static void amd_ack_smu(struct amd_ntb_dev
*ndev
, u32 bit
)
490 void __iomem
*mmio
= ndev
->self_mmio
;
493 reg
= readl(mmio
+ AMD_SMUACK_OFFSET
);
495 writel(reg
, mmio
+ AMD_SMUACK_OFFSET
);
497 ndev
->peer_sta
|= bit
;
500 static void amd_handle_event(struct amd_ntb_dev
*ndev
, int vec
)
502 void __iomem
*mmio
= ndev
->self_mmio
;
503 struct device
*dev
= &ndev
->ntb
.pdev
->dev
;
506 status
= readl(mmio
+ AMD_INTSTAT_OFFSET
);
507 if (!(status
& AMD_EVENT_INTMASK
))
510 dev_dbg(dev
, "status = 0x%x and vec = %d\n", status
, vec
);
512 status
&= AMD_EVENT_INTMASK
;
514 case AMD_PEER_FLUSH_EVENT
:
515 dev_info(dev
, "Flush is done.\n");
517 case AMD_PEER_RESET_EVENT
:
518 amd_ack_smu(ndev
, AMD_PEER_RESET_EVENT
);
520 /* link down first */
521 ntb_link_event(&ndev
->ntb
);
522 /* polling peer status */
523 schedule_delayed_work(&ndev
->hb_timer
, AMD_LINK_HB_TIMEOUT
);
526 case AMD_PEER_D3_EVENT
:
527 case AMD_PEER_PMETO_EVENT
:
528 case AMD_LINK_UP_EVENT
:
529 case AMD_LINK_DOWN_EVENT
:
530 amd_ack_smu(ndev
, status
);
533 ntb_link_event(&ndev
->ntb
);
536 case AMD_PEER_D0_EVENT
:
537 mmio
= ndev
->peer_mmio
;
538 status
= readl(mmio
+ AMD_PMESTAT_OFFSET
);
539 /* check if this is WAKEUP event */
541 dev_info(dev
, "Wakeup is done.\n");
543 amd_ack_smu(ndev
, AMD_PEER_D0_EVENT
);
545 /* start a timer to poll link status */
546 schedule_delayed_work(&ndev
->hb_timer
,
547 AMD_LINK_HB_TIMEOUT
);
550 dev_info(dev
, "event status = 0x%x.\n", status
);
555 static irqreturn_t
ndev_interrupt(struct amd_ntb_dev
*ndev
, int vec
)
557 dev_dbg(&ndev
->ntb
.pdev
->dev
, "vec %d\n", vec
);
559 if (vec
> (AMD_DB_CNT
- 1) || (ndev
->msix_vec_count
== 1))
560 amd_handle_event(ndev
, vec
);
562 if (vec
< AMD_DB_CNT
)
563 ntb_db_event(&ndev
->ntb
, vec
);
568 static irqreturn_t
ndev_vec_isr(int irq
, void *dev
)
570 struct amd_ntb_vec
*nvec
= dev
;
572 return ndev_interrupt(nvec
->ndev
, nvec
->num
);
575 static irqreturn_t
ndev_irq_isr(int irq
, void *dev
)
577 struct amd_ntb_dev
*ndev
= dev
;
579 return ndev_interrupt(ndev
, irq
- ndev
->ntb
.pdev
->irq
);
582 static int ndev_init_isr(struct amd_ntb_dev
*ndev
,
583 int msix_min
, int msix_max
)
585 struct pci_dev
*pdev
;
586 int rc
, i
, msix_count
, node
;
588 pdev
= ndev
->ntb
.pdev
;
590 node
= dev_to_node(&pdev
->dev
);
592 ndev
->db_mask
= ndev
->db_valid_mask
;
594 /* Try to set up msix irq */
595 ndev
->vec
= kcalloc_node(msix_max
, sizeof(*ndev
->vec
),
598 goto err_msix_vec_alloc
;
600 ndev
->msix
= kcalloc_node(msix_max
, sizeof(*ndev
->msix
),
605 for (i
= 0; i
< msix_max
; ++i
)
606 ndev
->msix
[i
].entry
= i
;
608 msix_count
= pci_enable_msix_range(pdev
, ndev
->msix
,
611 goto err_msix_enable
;
613 /* NOTE: Disable MSIX if msix count is less than 16 because of
614 * hardware limitation.
616 if (msix_count
< msix_min
) {
617 pci_disable_msix(pdev
);
618 goto err_msix_enable
;
621 for (i
= 0; i
< msix_count
; ++i
) {
622 ndev
->vec
[i
].ndev
= ndev
;
623 ndev
->vec
[i
].num
= i
;
624 rc
= request_irq(ndev
->msix
[i
].vector
, ndev_vec_isr
, 0,
625 "ndev_vec_isr", &ndev
->vec
[i
]);
627 goto err_msix_request
;
630 dev_dbg(&pdev
->dev
, "Using msix interrupts\n");
631 ndev
->db_count
= msix_min
;
632 ndev
->msix_vec_count
= msix_max
;
637 free_irq(ndev
->msix
[i
].vector
, &ndev
->vec
[i
]);
638 pci_disable_msix(pdev
);
647 /* Try to set up msi irq */
648 rc
= pci_enable_msi(pdev
);
652 rc
= request_irq(pdev
->irq
, ndev_irq_isr
, 0,
653 "ndev_irq_isr", ndev
);
655 goto err_msi_request
;
657 dev_dbg(&pdev
->dev
, "Using msi interrupts\n");
659 ndev
->msix_vec_count
= 1;
663 pci_disable_msi(pdev
);
666 /* Try to set up intx irq */
669 rc
= request_irq(pdev
->irq
, ndev_irq_isr
, IRQF_SHARED
,
670 "ndev_irq_isr", ndev
);
672 goto err_intx_request
;
674 dev_dbg(&pdev
->dev
, "Using intx interrupts\n");
676 ndev
->msix_vec_count
= 1;
683 static void ndev_deinit_isr(struct amd_ntb_dev
*ndev
)
685 struct pci_dev
*pdev
;
686 void __iomem
*mmio
= ndev
->self_mmio
;
689 pdev
= ndev
->ntb
.pdev
;
691 /* Mask all doorbell interrupts */
692 ndev
->db_mask
= ndev
->db_valid_mask
;
693 writel(ndev
->db_mask
, mmio
+ AMD_DBMASK_OFFSET
);
696 i
= ndev
->msix_vec_count
;
698 free_irq(ndev
->msix
[i
].vector
, &ndev
->vec
[i
]);
699 pci_disable_msix(pdev
);
703 free_irq(pdev
->irq
, ndev
);
704 if (pci_dev_msi_enabled(pdev
))
705 pci_disable_msi(pdev
);
711 static ssize_t
ndev_debugfs_read(struct file
*filp
, char __user
*ubuf
,
712 size_t count
, loff_t
*offp
)
714 struct amd_ntb_dev
*ndev
;
719 union { u64 v64
; u32 v32
; u16 v16
; } u
;
721 ndev
= filp
->private_data
;
722 mmio
= ndev
->self_mmio
;
724 buf_size
= min(count
, 0x800ul
);
726 buf
= kmalloc(buf_size
, GFP_KERNEL
);
732 off
+= scnprintf(buf
+ off
, buf_size
- off
,
733 "NTB Device Information:\n");
735 off
+= scnprintf(buf
+ off
, buf_size
- off
,
736 "Connection Topology -\t%s\n",
737 ntb_topo_string(ndev
->ntb
.topo
));
739 off
+= scnprintf(buf
+ off
, buf_size
- off
,
740 "LNK STA -\t\t%#06x\n", ndev
->lnk_sta
);
742 if (!amd_link_is_up(ndev
)) {
743 off
+= scnprintf(buf
+ off
, buf_size
- off
,
744 "Link Status -\t\tDown\n");
746 off
+= scnprintf(buf
+ off
, buf_size
- off
,
747 "Link Status -\t\tUp\n");
748 off
+= scnprintf(buf
+ off
, buf_size
- off
,
749 "Link Speed -\t\tPCI-E Gen %u\n",
750 NTB_LNK_STA_SPEED(ndev
->lnk_sta
));
751 off
+= scnprintf(buf
+ off
, buf_size
- off
,
752 "Link Width -\t\tx%u\n",
753 NTB_LNK_STA_WIDTH(ndev
->lnk_sta
));
756 off
+= scnprintf(buf
+ off
, buf_size
- off
,
757 "Memory Window Count -\t%u\n", ndev
->mw_count
);
758 off
+= scnprintf(buf
+ off
, buf_size
- off
,
759 "Scratchpad Count -\t%u\n", ndev
->spad_count
);
760 off
+= scnprintf(buf
+ off
, buf_size
- off
,
761 "Doorbell Count -\t%u\n", ndev
->db_count
);
762 off
+= scnprintf(buf
+ off
, buf_size
- off
,
763 "MSIX Vector Count -\t%u\n", ndev
->msix_vec_count
);
765 off
+= scnprintf(buf
+ off
, buf_size
- off
,
766 "Doorbell Valid Mask -\t%#llx\n", ndev
->db_valid_mask
);
768 u
.v32
= readl(ndev
->self_mmio
+ AMD_DBMASK_OFFSET
);
769 off
+= scnprintf(buf
+ off
, buf_size
- off
,
770 "Doorbell Mask -\t\t\t%#06x\n", u
.v32
);
772 u
.v32
= readl(mmio
+ AMD_DBSTAT_OFFSET
);
773 off
+= scnprintf(buf
+ off
, buf_size
- off
,
774 "Doorbell Bell -\t\t\t%#06x\n", u
.v32
);
776 off
+= scnprintf(buf
+ off
, buf_size
- off
,
777 "\nNTB Incoming XLAT:\n");
779 u
.v64
= read64(mmio
+ AMD_BAR1XLAT_OFFSET
);
780 off
+= scnprintf(buf
+ off
, buf_size
- off
,
781 "XLAT1 -\t\t%#018llx\n", u
.v64
);
783 u
.v64
= read64(ndev
->self_mmio
+ AMD_BAR23XLAT_OFFSET
);
784 off
+= scnprintf(buf
+ off
, buf_size
- off
,
785 "XLAT23 -\t\t%#018llx\n", u
.v64
);
787 u
.v64
= read64(ndev
->self_mmio
+ AMD_BAR45XLAT_OFFSET
);
788 off
+= scnprintf(buf
+ off
, buf_size
- off
,
789 "XLAT45 -\t\t%#018llx\n", u
.v64
);
791 u
.v32
= readl(mmio
+ AMD_BAR1LMT_OFFSET
);
792 off
+= scnprintf(buf
+ off
, buf_size
- off
,
793 "LMT1 -\t\t\t%#06x\n", u
.v32
);
795 u
.v64
= read64(ndev
->self_mmio
+ AMD_BAR23LMT_OFFSET
);
796 off
+= scnprintf(buf
+ off
, buf_size
- off
,
797 "LMT23 -\t\t\t%#018llx\n", u
.v64
);
799 u
.v64
= read64(ndev
->self_mmio
+ AMD_BAR45LMT_OFFSET
);
800 off
+= scnprintf(buf
+ off
, buf_size
- off
,
801 "LMT45 -\t\t\t%#018llx\n", u
.v64
);
803 ret
= simple_read_from_buffer(ubuf
, count
, offp
, buf
, off
);
808 static void ndev_init_debugfs(struct amd_ntb_dev
*ndev
)
811 ndev
->debugfs_dir
= NULL
;
812 ndev
->debugfs_info
= NULL
;
815 debugfs_create_dir(pci_name(ndev
->ntb
.pdev
),
817 if (!ndev
->debugfs_dir
)
818 ndev
->debugfs_info
= NULL
;
821 debugfs_create_file("info", S_IRUSR
,
822 ndev
->debugfs_dir
, ndev
,
823 &amd_ntb_debugfs_info
);
827 static void ndev_deinit_debugfs(struct amd_ntb_dev
*ndev
)
829 debugfs_remove_recursive(ndev
->debugfs_dir
);
832 static inline void ndev_init_struct(struct amd_ntb_dev
*ndev
,
833 struct pci_dev
*pdev
)
835 ndev
->ntb
.pdev
= pdev
;
836 ndev
->ntb
.topo
= NTB_TOPO_NONE
;
837 ndev
->ntb
.ops
= &amd_ntb_ops
;
838 ndev
->int_mask
= AMD_EVENT_INTMASK
;
839 spin_lock_init(&ndev
->db_mask_lock
);
842 static int amd_poll_link(struct amd_ntb_dev
*ndev
)
844 void __iomem
*mmio
= ndev
->peer_mmio
;
848 reg
= readl(mmio
+ AMD_SIDEINFO_OFFSET
);
849 reg
&= NTB_LIN_STA_ACTIVE_BIT
;
851 dev_dbg(&ndev
->ntb
.pdev
->dev
, "%s: reg_val = 0x%x.\n", __func__
, reg
);
853 if (reg
== ndev
->cntl_sta
)
856 ndev
->cntl_sta
= reg
;
858 rc
= pci_read_config_dword(ndev
->ntb
.pdev
,
859 AMD_LINK_STATUS_OFFSET
, &stat
);
862 ndev
->lnk_sta
= stat
;
867 static void amd_link_hb(struct work_struct
*work
)
869 struct amd_ntb_dev
*ndev
= hb_ndev(work
);
871 if (amd_poll_link(ndev
))
872 ntb_link_event(&ndev
->ntb
);
874 if (!amd_link_is_up(ndev
))
875 schedule_delayed_work(&ndev
->hb_timer
, AMD_LINK_HB_TIMEOUT
);
878 static int amd_init_isr(struct amd_ntb_dev
*ndev
)
880 return ndev_init_isr(ndev
, AMD_DB_CNT
, AMD_MSIX_VECTOR_CNT
);
883 static void amd_init_side_info(struct amd_ntb_dev
*ndev
)
885 void __iomem
*mmio
= ndev
->self_mmio
;
888 reg
= readl(mmio
+ AMD_SIDEINFO_OFFSET
);
889 if (!(reg
& AMD_SIDE_READY
)) {
890 reg
|= AMD_SIDE_READY
;
891 writel(reg
, mmio
+ AMD_SIDEINFO_OFFSET
);
895 static void amd_deinit_side_info(struct amd_ntb_dev
*ndev
)
897 void __iomem
*mmio
= ndev
->self_mmio
;
900 reg
= readl(mmio
+ AMD_SIDEINFO_OFFSET
);
901 if (reg
& AMD_SIDE_READY
) {
902 reg
&= ~AMD_SIDE_READY
;
903 writel(reg
, mmio
+ AMD_SIDEINFO_OFFSET
);
904 readl(mmio
+ AMD_SIDEINFO_OFFSET
);
908 static int amd_init_ntb(struct amd_ntb_dev
*ndev
)
910 void __iomem
*mmio
= ndev
->self_mmio
;
912 ndev
->mw_count
= AMD_MW_CNT
;
913 ndev
->spad_count
= AMD_SPADS_CNT
;
914 ndev
->db_count
= AMD_DB_CNT
;
916 switch (ndev
->ntb
.topo
) {
919 ndev
->spad_count
>>= 1;
920 if (ndev
->ntb
.topo
== NTB_TOPO_PRI
) {
922 ndev
->peer_spad
= 0x20;
924 ndev
->self_spad
= 0x20;
928 INIT_DELAYED_WORK(&ndev
->hb_timer
, amd_link_hb
);
929 schedule_delayed_work(&ndev
->hb_timer
, AMD_LINK_HB_TIMEOUT
);
933 dev_err(&ndev
->ntb
.pdev
->dev
,
934 "AMD NTB does not support B2B mode.\n");
938 ndev
->db_valid_mask
= BIT_ULL(ndev
->db_count
) - 1;
940 /* Mask event interrupts */
941 writel(ndev
->int_mask
, mmio
+ AMD_INTMASK_OFFSET
);
946 static enum ntb_topo
amd_get_topo(struct amd_ntb_dev
*ndev
)
948 void __iomem
*mmio
= ndev
->self_mmio
;
951 info
= readl(mmio
+ AMD_SIDEINFO_OFFSET
);
952 if (info
& AMD_SIDE_MASK
)
958 static int amd_init_dev(struct amd_ntb_dev
*ndev
)
960 struct pci_dev
*pdev
;
963 pdev
= ndev
->ntb
.pdev
;
965 ndev
->ntb
.topo
= amd_get_topo(ndev
);
966 dev_dbg(&pdev
->dev
, "AMD NTB topo is %s\n",
967 ntb_topo_string(ndev
->ntb
.topo
));
969 rc
= amd_init_ntb(ndev
);
973 rc
= amd_init_isr(ndev
);
975 dev_err(&pdev
->dev
, "fail to init isr.\n");
979 ndev
->db_valid_mask
= BIT_ULL(ndev
->db_count
) - 1;
984 static void amd_deinit_dev(struct amd_ntb_dev
*ndev
)
986 cancel_delayed_work_sync(&ndev
->hb_timer
);
988 ndev_deinit_isr(ndev
);
991 static int amd_ntb_init_pci(struct amd_ntb_dev
*ndev
,
992 struct pci_dev
*pdev
)
996 pci_set_drvdata(pdev
, ndev
);
998 rc
= pci_enable_device(pdev
);
1000 goto err_pci_enable
;
1002 rc
= pci_request_regions(pdev
, NTB_NAME
);
1004 goto err_pci_regions
;
1006 pci_set_master(pdev
);
1008 rc
= pci_set_dma_mask(pdev
, DMA_BIT_MASK(64));
1010 rc
= pci_set_dma_mask(pdev
, DMA_BIT_MASK(32));
1013 dev_warn(&pdev
->dev
, "Cannot DMA highmem\n");
1016 rc
= pci_set_consistent_dma_mask(pdev
, DMA_BIT_MASK(64));
1018 rc
= pci_set_consistent_dma_mask(pdev
, DMA_BIT_MASK(32));
1021 dev_warn(&pdev
->dev
, "Cannot DMA consistent highmem\n");
1023 rc
= dma_coerce_mask_and_coherent(&ndev
->ntb
.dev
,
1024 dma_get_mask(&pdev
->dev
));
1028 ndev
->self_mmio
= pci_iomap(pdev
, 0, 0);
1029 if (!ndev
->self_mmio
) {
1033 ndev
->peer_mmio
= ndev
->self_mmio
+ AMD_PEER_OFFSET
;
1038 pci_clear_master(pdev
);
1040 pci_disable_device(pdev
);
1042 pci_set_drvdata(pdev
, NULL
);
1046 static void amd_ntb_deinit_pci(struct amd_ntb_dev
*ndev
)
1048 struct pci_dev
*pdev
= ndev
->ntb
.pdev
;
1050 pci_iounmap(pdev
, ndev
->self_mmio
);
1052 pci_clear_master(pdev
);
1053 pci_release_regions(pdev
);
1054 pci_disable_device(pdev
);
1055 pci_set_drvdata(pdev
, NULL
);
1058 static int amd_ntb_pci_probe(struct pci_dev
*pdev
,
1059 const struct pci_device_id
*id
)
1061 struct amd_ntb_dev
*ndev
;
1064 node
= dev_to_node(&pdev
->dev
);
1066 ndev
= kzalloc_node(sizeof(*ndev
), GFP_KERNEL
, node
);
1072 ndev_init_struct(ndev
, pdev
);
1074 rc
= amd_ntb_init_pci(ndev
, pdev
);
1078 rc
= amd_init_dev(ndev
);
1082 /* write side info */
1083 amd_init_side_info(ndev
);
1085 amd_poll_link(ndev
);
1087 ndev_init_debugfs(ndev
);
1089 rc
= ntb_register_device(&ndev
->ntb
);
1093 dev_info(&pdev
->dev
, "NTB device registered.\n");
1098 ndev_deinit_debugfs(ndev
);
1099 amd_deinit_dev(ndev
);
1101 amd_ntb_deinit_pci(ndev
);
1108 static void amd_ntb_pci_remove(struct pci_dev
*pdev
)
1110 struct amd_ntb_dev
*ndev
= pci_get_drvdata(pdev
);
1112 ntb_unregister_device(&ndev
->ntb
);
1113 ndev_deinit_debugfs(ndev
);
1114 amd_deinit_side_info(ndev
);
1115 amd_deinit_dev(ndev
);
1116 amd_ntb_deinit_pci(ndev
);
1120 static const struct file_operations amd_ntb_debugfs_info
= {
1121 .owner
= THIS_MODULE
,
1122 .open
= simple_open
,
1123 .read
= ndev_debugfs_read
,
1126 static const struct pci_device_id amd_ntb_pci_tbl
[] = {
1127 {PCI_VDEVICE(AMD
, PCI_DEVICE_ID_AMD_NTB
)},
1130 MODULE_DEVICE_TABLE(pci
, amd_ntb_pci_tbl
);
1132 static struct pci_driver amd_ntb_pci_driver
= {
1133 .name
= KBUILD_MODNAME
,
1134 .id_table
= amd_ntb_pci_tbl
,
1135 .probe
= amd_ntb_pci_probe
,
1136 .remove
= amd_ntb_pci_remove
,
1139 static int __init
amd_ntb_pci_driver_init(void)
1141 pr_info("%s %s\n", NTB_DESC
, NTB_VER
);
1143 if (debugfs_initialized())
1144 debugfs_dir
= debugfs_create_dir(KBUILD_MODNAME
, NULL
);
1146 return pci_register_driver(&amd_ntb_pci_driver
);
1148 module_init(amd_ntb_pci_driver_init
);
1150 static void __exit
amd_ntb_pci_driver_exit(void)
1152 pci_unregister_driver(&amd_ntb_pci_driver
);
1153 debugfs_remove_recursive(debugfs_dir
);
1155 module_exit(amd_ntb_pci_driver_exit
);