arch/mips/pci/msi-xlp.c

   1 /*
   2  * Copyright (c) 2003-2012 Broadcom Corporation
   3  * All Rights Reserved
   4  *
   5  * This software is available to you under a choice of one of two
   6  * licenses.  You may choose to be licensed under the terms of the GNU
   7  * General Public License (GPL) Version 2, available from the file
   8  * COPYING in the main directory of this source tree, or the Broadcom
   9  * license below:
  10  *
  11  * Redistribution and use in source and binary forms, with or without
  12  * modification, are permitted provided that the following conditions
  13  * are met:
  14  *
  15  * 1. Redistributions of source code must retain the above copyright
  16  *    notice, this list of conditions and the following disclaimer.
  17  * 2. Redistributions in binary form must reproduce the above copyright
  18  *    notice, this list of conditions and the following disclaimer in
  19  *    the documentation and/or other materials provided with the
  20  *    distribution.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR
  23  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  24  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  25  * ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE
  26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  27  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  28  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  29  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  30  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
  31  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
  32  * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  33  */
  34
  35 #include <linux/types.h>
  36 #include <linux/pci.h>
  37 #include <linux/kernel.h>
  38 #include <linux/init.h>
  39 #include <linux/msi.h>
  40 #include <linux/mm.h>
  41 #include <linux/irq.h>
  42 #include <linux/irqdesc.h>
  43 #include <linux/console.h>
  44
  45 #include <asm/io.h>
  46
  47 #include <asm/netlogic/interrupt.h>
  48 #include <asm/netlogic/haldefs.h>
  49 #include <asm/netlogic/common.h>
  50 #include <asm/netlogic/mips-extns.h>
  51
  52 #include <asm/netlogic/xlp-hal/iomap.h>
  53 #include <asm/netlogic/xlp-hal/xlp.h>
  54 #include <asm/netlogic/xlp-hal/pic.h>
  55 #include <asm/netlogic/xlp-hal/pcibus.h>
  56 #include <asm/netlogic/xlp-hal/bridge.h>
  57
  58 #define XLP_MSIVEC_PER_LINK     32
  59 #define XLP_MSIXVEC_TOTAL       32
  60 #define XLP_MSIXVEC_PER_LINK    8
  61
  62 /* 128 MSI irqs per node, mapped starting at NLM_MSI_VEC_BASE */
  63 static inline int nlm_link_msiirq(int link, int msivec)
  64 {
  65         return NLM_MSI_VEC_BASE + link * XLP_MSIVEC_PER_LINK + msivec;
  66 }
  67
  68 static inline int nlm_irq_msivec(int irq)
  69 {
  70         return irq % XLP_MSIVEC_PER_LINK;
  71 }
  72
  73 static inline int nlm_irq_msilink(int irq)
  74 {
  75         return (irq % (XLP_MSIVEC_PER_LINK * PCIE_NLINKS)) /
  76                                                 XLP_MSIVEC_PER_LINK;
  77 }
  78
  79 /*
  80  * Only 32 MSI-X vectors are possible because there are only 32 PIC
  81  * interrupts for MSI. We split them statically and use 8 MSI-X vectors
  82  * per link - this keeps the allocation and lookup simple.
  83  */
  84 static inline int nlm_link_msixirq(int link, int bit)
  85 {
  86         return NLM_MSIX_VEC_BASE + link * XLP_MSIXVEC_PER_LINK + bit;
  87 }
  88
  89 static inline int nlm_irq_msixvec(int irq)
  90 {
  91         return irq % XLP_MSIXVEC_TOTAL;  /* works when given xirq */
  92 }
  93
  94 static inline int nlm_irq_msixlink(int irq)
  95 {
  96         return nlm_irq_msixvec(irq) / XLP_MSIXVEC_PER_LINK;
  97 }
  98
  99 /*
 100  * Per link MSI and MSI-X information, set as IRQ handler data for
 101  * MSI and MSI-X interrupts.
 102  */
 103 struct xlp_msi_data {
 104         struct nlm_soc_info *node;
 105         uint64_t        lnkbase;
 106         uint32_t        msi_enabled_mask;
 107         uint32_t        msi_alloc_mask;
 108         uint32_t        msix_alloc_mask;
 109         spinlock_t      msi_lock;
 110 };
 111
 112 /*
 113  * MSI Chip definitions
 114  *
 115  * On XLP, there is a PIC interrupt associated with each PCIe link on the
 116  * chip (which appears as a PCI bridge to us). This gives us 32 MSI irqa
 117  * per link and 128 overall.
 118  *
 119  * When a device connected to the link raises a MSI interrupt, we get a
 120  * link interrupt and we then have to look at PCIE_MSI_STATUS register at
 121  * the bridge to map it to the IRQ
 122  */
 123 static void xlp_msi_enable(struct irq_data *d)
 124 {
 125         struct xlp_msi_data *md = irq_data_get_irq_handler_data(d);
 126         unsigned long flags;
 127         int vec;
 128
 129         vec = nlm_irq_msivec(d->irq);
 130         spin_lock_irqsave(&md->msi_lock, flags);
 131         md->msi_enabled_mask |= 1u << vec;
 132         nlm_write_reg(md->lnkbase, PCIE_MSI_EN, md->msi_enabled_mask);
 133         spin_unlock_irqrestore(&md->msi_lock, flags);
 134 }
 135
 136 static void xlp_msi_disable(struct irq_data *d)
 137 {
 138         struct xlp_msi_data *md = irq_data_get_irq_handler_data(d);
 139         unsigned long flags;
 140         int vec;
 141
 142         vec = nlm_irq_msivec(d->irq);
 143         spin_lock_irqsave(&md->msi_lock, flags);
 144         md->msi_enabled_mask &= ~(1u << vec);
 145         nlm_write_reg(md->lnkbase, PCIE_MSI_EN, md->msi_enabled_mask);
 146         spin_unlock_irqrestore(&md->msi_lock, flags);
 147 }
 148
 149 static void xlp_msi_mask_ack(struct irq_data *d)
 150 {
 151         struct xlp_msi_data *md = irq_data_get_irq_handler_data(d);
 152         int link, vec;
 153
 154         link = nlm_irq_msilink(d->irq);
 155         vec = nlm_irq_msivec(d->irq);
 156         xlp_msi_disable(d);
 157
 158         /* Ack MSI on bridge */
 159         nlm_write_reg(md->lnkbase, PCIE_MSI_STATUS, 1u << vec);
 160
 161         /* Ack at eirr and PIC */
 162         ack_c0_eirr(PIC_PCIE_LINK_MSI_IRQ(link));
 163         nlm_pic_ack(md->node->picbase, PIC_IRT_PCIE_LINK_INDEX(link));
 164 }
 165
 166 static struct irq_chip xlp_msi_chip = {
 167         .name           = "XLP-MSI",
 168         .irq_enable     = xlp_msi_enable,
 169         .irq_disable    = xlp_msi_disable,
 170         .irq_mask_ack   = xlp_msi_mask_ack,
 171         .irq_unmask     = xlp_msi_enable,
 172 };
 173
 174 /*
 175  * The MSI-X interrupt handling is different from MSI, there are 32
 176  * MSI-X interrupts generated by the PIC and each of these correspond
 177  * to a MSI-X vector (0-31) that can be assigned.
 178  *
 179  * We divide the MSI-X vectors to 8 per link and do a per-link
 180  * allocation
 181  *
 182  * Enable and disable done using standard MSI functions.
 183  */
 184 static void xlp_msix_mask_ack(struct irq_data *d)
 185 {
 186         struct xlp_msi_data *md = irq_data_get_irq_handler_data(d);
 187         int link, msixvec;
 188
 189         msixvec = nlm_irq_msixvec(d->irq);
 190         link = nlm_irq_msixlink(d->irq);
 191         mask_msi_irq(d);
 192
 193         /* Ack MSI on bridge */
 194         nlm_write_reg(md->lnkbase, PCIE_MSIX_STATUS, 1u << msixvec);
 195
 196         /* Ack at eirr and PIC */
 197         ack_c0_eirr(PIC_PCIE_MSIX_IRQ(link));
 198         nlm_pic_ack(md->node->picbase, PIC_IRT_PCIE_MSIX_INDEX(msixvec));
 199 }
 200
 201 static struct irq_chip xlp_msix_chip = {
 202         .name           = "XLP-MSIX",
 203         .irq_enable     = unmask_msi_irq,
 204         .irq_disable    = mask_msi_irq,
 205         .irq_mask_ack   = xlp_msix_mask_ack,
 206         .irq_unmask     = unmask_msi_irq,
 207 };
 208
 209 void destroy_irq(unsigned int irq)
 210 {
 211             /* nothing to do yet */
 212 }
 213
 214 void arch_teardown_msi_irq(unsigned int irq)
 215 {
 216         destroy_irq(irq);
 217 }
 218
 219 /*
 220  * Setup a PCIe link for MSI.  By default, the links are in
 221  * legacy interrupt mode.  We will switch them to MSI mode
 222  * at the first MSI request.
 223  */
 224 static void xlp_config_link_msi(uint64_t lnkbase, int lirq, uint64_t msiaddr)
 225 {
 226         u32 val;
 227
 228         val = nlm_read_reg(lnkbase, PCIE_INT_EN0);
 229         if ((val & 0x200) == 0) {
 230                 val |= 0x200;           /* MSI Interrupt enable */
 231                 nlm_write_reg(lnkbase, PCIE_INT_EN0, val);
 232         }
 233
 234         val = nlm_read_reg(lnkbase, 0x1);       /* CMD */
 235         if ((val & 0x0400) == 0) {
 236                 val |= 0x0400;
 237                 nlm_write_reg(lnkbase, 0x1, val);
 238         }
 239
 240         /* Update IRQ in the PCI irq reg */
 241         val = nlm_read_pci_reg(lnkbase, 0xf);
 242         val &= ~0x1fu;
 243         val |= (1 << 8) | lirq;
 244         nlm_write_pci_reg(lnkbase, 0xf, val);
 245
 246         /* MSI addr */
 247         nlm_write_reg(lnkbase, PCIE_BRIDGE_MSI_ADDRH, msiaddr >> 32);
 248         nlm_write_reg(lnkbase, PCIE_BRIDGE_MSI_ADDRL, msiaddr & 0xffffffff);
 249
 250         /* MSI cap for bridge */
 251         val = nlm_read_reg(lnkbase, PCIE_BRIDGE_MSI_CAP);
 252         if ((val & (1 << 16)) == 0) {
 253                 val |= 0xb << 16;               /* mmc32, msi enable */
 254                 nlm_write_reg(lnkbase, PCIE_BRIDGE_MSI_CAP, val);
 255         }
 256 }
 257
 258 /*
 259  * Allocate a MSI vector on a link
 260  */
 261 static int xlp_setup_msi(uint64_t lnkbase, int node, int link,
 262         struct msi_desc *desc)
 263 {
 264         struct xlp_msi_data *md;
 265         struct msi_msg msg;
 266         unsigned long flags;
 267         int msivec, irt, lirq, xirq, ret;
 268         uint64_t msiaddr;
 269
 270         /* Get MSI data for the link */
 271         lirq = PIC_PCIE_LINK_MSI_IRQ(link);
 272         xirq = nlm_irq_to_xirq(node, nlm_link_msiirq(link, 0));
 273         md = irq_get_handler_data(xirq);
 274         msiaddr = MSI_LINK_ADDR(node, link);
 275
 276         spin_lock_irqsave(&md->msi_lock, flags);
 277         if (md->msi_alloc_mask == 0) {
 278                 /* switch the link IRQ to MSI range */
 279                 xlp_config_link_msi(lnkbase, lirq, msiaddr);
 280                 irt = PIC_IRT_PCIE_LINK_INDEX(link);
 281                 nlm_setup_pic_irq(node, lirq, lirq, irt);
 282                 nlm_pic_init_irt(nlm_get_node(node)->picbase, irt, lirq,
 283                                  node * nlm_threads_per_node(), 1 /*en */);
 284         }
 285
 286         /* allocate a MSI vec, and tell the bridge about it */
 287         msivec = fls(md->msi_alloc_mask);
 288         if (msivec == XLP_MSIVEC_PER_LINK) {
 289                 spin_unlock_irqrestore(&md->msi_lock, flags);
 290                 return -ENOMEM;
 291         }
 292         md->msi_alloc_mask |= (1u << msivec);
 293         spin_unlock_irqrestore(&md->msi_lock, flags);
 294
 295         msg.address_hi = msiaddr >> 32;
 296         msg.address_lo = msiaddr & 0xffffffff;
 297         msg.data = 0xc00 | msivec;
 298
 299         xirq = xirq + msivec;           /* msi mapped to global irq space */
 300         ret = irq_set_msi_desc(xirq, desc);
 301         if (ret < 0) {
 302                 destroy_irq(xirq);
 303                 return ret;
 304         }
 305
 306         write_msi_msg(xirq, &msg);
 307         return 0;
 308 }
 309
 310 /*
 311  * Switch a link to MSI-X mode
 312  */
 313 static void xlp_config_link_msix(uint64_t lnkbase, int lirq, uint64_t msixaddr)
 314 {
 315         u32 val;
 316
 317         val = nlm_read_reg(lnkbase, 0x2C);
 318         if ((val & 0x80000000U) == 0) {
 319                 val |= 0x80000000U;
 320                 nlm_write_reg(lnkbase, 0x2C, val);
 321         }
 322         val = nlm_read_reg(lnkbase, PCIE_INT_EN0);
 323         if ((val & 0x200) == 0) {
 324                 val |= 0x200;           /* MSI Interrupt enable */
 325                 nlm_write_reg(lnkbase, PCIE_INT_EN0, val);
 326         }
 327
 328         val = nlm_read_reg(lnkbase, 0x1);       /* CMD */
 329         if ((val & 0x0400) == 0) {
 330                 val |= 0x0400;
 331                 nlm_write_reg(lnkbase, 0x1, val);
 332         }
 333
 334         /* Update IRQ in the PCI irq reg */
 335         val = nlm_read_pci_reg(lnkbase, 0xf);
 336         val &= ~0x1fu;
 337         val |= (1 << 8) | lirq;
 338         nlm_write_pci_reg(lnkbase, 0xf, val);
 339
 340         /* MSI-X addresses */
 341         nlm_write_reg(lnkbase, PCIE_BRIDGE_MSIX_ADDR_BASE, msixaddr >> 8);
 342         nlm_write_reg(lnkbase, PCIE_BRIDGE_MSIX_ADDR_LIMIT,
 343                                         (msixaddr + MSI_ADDR_SZ) >> 8);
 344 }
 345
 346 /*
 347  *  Allocate a MSI-X vector
 348  */
 349 static int xlp_setup_msix(uint64_t lnkbase, int node, int link,
 350         struct msi_desc *desc)
 351 {
 352         struct xlp_msi_data *md;
 353         struct msi_msg msg;
 354         unsigned long flags;
 355         int t, msixvec, lirq, xirq, ret;
 356         uint64_t msixaddr;
 357
 358         /* Get MSI data for the link */
 359         lirq = PIC_PCIE_MSIX_IRQ(link);
 360         xirq = nlm_irq_to_xirq(node, nlm_link_msixirq(link, 0));
 361         md = irq_get_handler_data(xirq);
 362         msixaddr = MSIX_LINK_ADDR(node, link);
 363
 364         spin_lock_irqsave(&md->msi_lock, flags);
 365         /* switch the PCIe link to MSI-X mode at the first alloc */
 366         if (md->msix_alloc_mask == 0)
 367                 xlp_config_link_msix(lnkbase, lirq, msixaddr);
 368
 369         /* allocate a MSI-X vec, and tell the bridge about it */
 370         t = fls(md->msix_alloc_mask);
 371         if (t == XLP_MSIXVEC_PER_LINK) {
 372                 spin_unlock_irqrestore(&md->msi_lock, flags);
 373                 return -ENOMEM;
 374         }
 375         md->msix_alloc_mask |= (1u << t);
 376         spin_unlock_irqrestore(&md->msi_lock, flags);
 377
 378         xirq += t;
 379         msixvec = nlm_irq_msixvec(xirq);
 380         msg.address_hi = msixaddr >> 32;
 381         msg.address_lo = msixaddr & 0xffffffff;
 382         msg.data = 0xc00 | msixvec;
 383
 384         ret = irq_set_msi_desc(xirq, desc);
 385         if (ret < 0) {
 386                 destroy_irq(xirq);
 387                 return ret;
 388         }
 389
 390         write_msi_msg(xirq, &msg);
 391         return 0;
 392 }
 393
 394 int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc)
 395 {
 396         struct pci_dev *lnkdev;
 397         uint64_t lnkbase;
 398         int node, link, slot;
 399
 400         lnkdev = xlp_get_pcie_link(dev);
 401         if (lnkdev == NULL) {
 402                 dev_err(&dev->dev, "Could not find bridge\n");
 403                 return 1;
 404         }
 405         slot = PCI_SLOT(lnkdev->devfn);
 406         link = PCI_FUNC(lnkdev->devfn);
 407         node = slot / 8;
 408         lnkbase = nlm_get_pcie_base(node, link);
 409
 410         if (desc->msi_attrib.is_msix)
 411                 return xlp_setup_msix(lnkbase, node, link, desc);
 412         else
 413                 return xlp_setup_msi(lnkbase, node, link, desc);
 414 }
 415
 416 void __init xlp_init_node_msi_irqs(int node, int link)
 417 {
 418         struct nlm_soc_info *nodep;
 419         struct xlp_msi_data *md;
 420         int irq, i, irt, msixvec;
 421
 422         pr_info("[%d %d] Init node PCI IRT\n", node, link);
 423         nodep = nlm_get_node(node);
 424
 425         /* Alloc an MSI block for the link */
 426         md = kzalloc(sizeof(*md), GFP_KERNEL);
 427         spin_lock_init(&md->msi_lock);
 428         md->msi_enabled_mask = 0;
 429         md->msi_alloc_mask = 0;
 430         md->msix_alloc_mask = 0;
 431         md->node = nodep;
 432         md->lnkbase = nlm_get_pcie_base(node, link);
 433
 434         /* extended space for MSI interrupts */
 435         irq = nlm_irq_to_xirq(node, nlm_link_msiirq(link, 0));
 436         for (i = irq; i < irq + XLP_MSIVEC_PER_LINK; i++) {
 437                 irq_set_chip_and_handler(i, &xlp_msi_chip, handle_level_irq);
 438                 irq_set_handler_data(i, md);
 439         }
 440
 441         for (i = 0; i < XLP_MSIXVEC_PER_LINK; i++) {
 442                 /* Initialize MSI-X irts to generate one interrupt per link */
 443                 msixvec = link * XLP_MSIXVEC_PER_LINK + i;
 444                 irt = PIC_IRT_PCIE_MSIX_INDEX(msixvec);
 445                 nlm_pic_init_irt(nodep->picbase, irt, PIC_PCIE_MSIX_IRQ(link),
 446                         node * nlm_threads_per_node(), 1 /* enable */);
 447
 448                 /* Initialize MSI-X extended irq space for the link  */
 449                 irq = nlm_irq_to_xirq(node, nlm_link_msixirq(link, i));
 450                 irq_set_chip_and_handler(irq, &xlp_msix_chip, handle_level_irq);
 451                 irq_set_handler_data(irq, md);
 452         }
 453
 454 }
 455
 456 void nlm_dispatch_msi(int node, int lirq)
 457 {
 458         struct xlp_msi_data *md;
 459         int link, i, irqbase;
 460         u32 status;
 461
 462         link = lirq - PIC_PCIE_LINK_MSI_IRQ_BASE;
 463         irqbase = nlm_irq_to_xirq(node, nlm_link_msiirq(link, 0));
 464         md = irq_get_handler_data(irqbase);
 465         status = nlm_read_reg(md->lnkbase, PCIE_MSI_STATUS) &
 466                                                 md->msi_enabled_mask;
 467         while (status) {
 468                 i = __ffs(status);
 469                 do_IRQ(irqbase + i);
 470                 status &= status - 1;
 471         }
 472 }
 473
 474 void nlm_dispatch_msix(int node, int lirq)
 475 {
 476         struct xlp_msi_data *md;
 477         int link, i, irqbase;
 478         u32 status;
 479
 480         link = lirq - PIC_PCIE_MSIX_IRQ_BASE;
 481         irqbase = nlm_irq_to_xirq(node, nlm_link_msixirq(link, 0));
 482         md = irq_get_handler_data(irqbase);
 483         status = nlm_read_reg(md->lnkbase, PCIE_MSIX_STATUS);
 484
 485         /* narrow it down to the MSI-x vectors for our link */
 486         status = (status >> (link * XLP_MSIXVEC_PER_LINK)) &
 487                         ((1 << XLP_MSIXVEC_PER_LINK) - 1);
 488
 489         while (status) {
 490                 i = __ffs(status);
 491                 do_IRQ(irqbase + i);
 492                 status &= status - 1;
 493         }
 494 }