mm: fix exec activate_mm vs TLB shootdown and lazy tlb switching race

[linux/fpc-iii.git] / arch / tile / kernel / pci_gx.c

/*
 * Copyright 2012 Tilera Corporation. All Rights Reserved.
 *
 *   This program is free software; you can redistribute it and/or
 *   modify it under the terms of the GNU General Public License
 *   as published by the Free Software Foundation, version 2.
 *
 *   This program is distributed in the hope that it will be useful, but
 *   WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
 *   NON INFRINGEMENT.  See the GNU General Public License for
 *   more details.
 */

#include <linux/kernel.h>
#include <linux/mmzone.h>
#include <linux/pci.h>
#include <linux/delay.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/capability.h>
#include <linux/sched.h>
#include <linux/errno.h>
#include <linux/irq.h>
#include <linux/msi.h>
#include <linux/io.h>
#include <linux/uaccess.h>
#include <linux/ctype.h>

#include <asm/processor.h>
#include <asm/sections.h>
#include <asm/byteorder.h>

#include <gxio/iorpc_globals.h>
#include <gxio/kiorpc.h>
#include <gxio/trio.h>
#include <gxio/iorpc_trio.h>
#include <hv/drv_trio_intf.h>

#include <arch/sim.h>

/*
 * This file contains the routines to search for PCI buses,
 * enumerate the buses, and configure any attached devices.
 */

#define DEBUG_PCI_CFG   0

#if DEBUG_PCI_CFG
#define TRACE_CFG_WR(size, val, bus, dev, func, offset) \
        pr_info("CFG WR %d-byte VAL %#x to bus %d dev %d func %d addr %u\n", \
                size, val, bus, dev, func, offset & 0xFFF);
#define TRACE_CFG_RD(size, val, bus, dev, func, offset) \
        pr_info("CFG RD %d-byte VAL %#x from bus %d dev %d func %d addr %u\n", \
                size, val, bus, dev, func, offset & 0xFFF);
#else
#define TRACE_CFG_WR(...)
#define TRACE_CFG_RD(...)
#endif

static int pci_probe = 1;

/* Information on the PCIe RC ports configuration. */
static int pcie_rc[TILEGX_NUM_TRIO][TILEGX_TRIO_PCIES];

/*
 * On some platforms with one or more Gx endpoint ports, we need to
 * delay the PCIe RC port probe for a few seconds to work around
 * a HW PCIe link-training bug. The exact delay is specified with
 * a kernel boot argument in the form of "pcie_rc_delay=T,P,S",
 * where T is the TRIO instance number, P is the port number and S is
 * the delay in seconds. If the argument is specified, but the delay is
 * not provided, the value will be DEFAULT_RC_DELAY.
 */
static int rc_delay[TILEGX_NUM_TRIO][TILEGX_TRIO_PCIES];

/* Default number of seconds that the PCIe RC port probe can be delayed. */
#define DEFAULT_RC_DELAY        10

/* The PCI I/O space size in each PCI domain. */
#define IO_SPACE_SIZE           0x10000

/* Provide shorter versions of some very long constant names. */
#define AUTO_CONFIG_RC  \
        TRIO_PCIE_INTFC_PORT_CONFIG__STRAP_STATE_VAL_AUTO_CONFIG_RC
#define AUTO_CONFIG_RC_G1       \
        TRIO_PCIE_INTFC_PORT_CONFIG__STRAP_STATE_VAL_AUTO_CONFIG_RC_G1
#define AUTO_CONFIG_EP  \
        TRIO_PCIE_INTFC_PORT_CONFIG__STRAP_STATE_VAL_AUTO_CONFIG_ENDPOINT
#define AUTO_CONFIG_EP_G1       \
        TRIO_PCIE_INTFC_PORT_CONFIG__STRAP_STATE_VAL_AUTO_CONFIG_ENDPOINT_G1

/* Array of the PCIe ports configuration info obtained from the BIB. */
struct pcie_trio_ports_property pcie_ports[TILEGX_NUM_TRIO];

/* Number of configured TRIO instances. */
int num_trio_shims;

/* All drivers share the TRIO contexts defined here. */
gxio_trio_context_t trio_contexts[TILEGX_NUM_TRIO];

/* Pointer to an array of PCIe RC controllers. */
struct pci_controller pci_controllers[TILEGX_NUM_TRIO * TILEGX_TRIO_PCIES];
int num_rc_controllers;

static struct pci_ops tile_cfg_ops;

/* Mask of CPUs that should receive PCIe interrupts. */
static struct cpumask intr_cpus_map;

/*
 * Pick a CPU to receive and handle the PCIe interrupts, based on the IRQ #.
 * For now, we simply send interrupts to non-dataplane CPUs.
 * We may implement methods to allow user to specify the target CPUs,
 * e.g. via boot arguments.
 */
static int tile_irq_cpu(int irq)
{
        unsigned int count;
        int i = 0;
        int cpu;

        count = cpumask_weight(&intr_cpus_map);
        if (unlikely(count == 0)) {
                pr_warn("intr_cpus_map empty, interrupts will be delivered to dataplane tiles\n");
                return irq % (smp_height * smp_width);
        }

        count = irq % count;
        for_each_cpu(cpu, &intr_cpus_map) {
                if (i++ == count)
                        break;
        }
        return cpu;
}

/* Open a file descriptor to the TRIO shim. */
static int tile_pcie_open(int trio_index)
{
        gxio_trio_context_t *context = &trio_contexts[trio_index];
        int ret;
        int mac;

        /* This opens a file descriptor to the TRIO shim. */
        ret = gxio_trio_init(context, trio_index);
        if (ret < 0)
                goto gxio_trio_init_failure;

        /* Allocate an ASID for the kernel. */
        ret = gxio_trio_alloc_asids(context, 1, 0, 0);
        if (ret < 0) {
                pr_err("PCI: ASID alloc failure on TRIO %d, give up\n",
                        trio_index);
                goto asid_alloc_failure;
        }

        context->asid = ret;

#ifdef USE_SHARED_PCIE_CONFIG_REGION
        /*
         * Alloc a PIO region for config access, shared by all MACs per TRIO.
         * This shouldn't fail since the kernel is supposed to the first
         * client of the TRIO's PIO regions.
         */
        ret = gxio_trio_alloc_pio_regions(context, 1, 0, 0);
        if (ret < 0) {
                pr_err("PCI: CFG PIO alloc failure on TRIO %d, give up\n",
                        trio_index);
                goto pio_alloc_failure;
        }

        context->pio_cfg_index = ret;

        /*
         * For PIO CFG, the bus_address_hi parameter is 0. The mac parameter
         * is also 0 because it is specified in PIO_REGION_SETUP_CFG_ADDR.
         */
        ret = gxio_trio_init_pio_region_aux(context, context->pio_cfg_index,
                0, 0, HV_TRIO_PIO_FLAG_CONFIG_SPACE);
        if (ret < 0) {
                pr_err("PCI: CFG PIO init failure on TRIO %d, give up\n",
                        trio_index);
                goto pio_alloc_failure;
        }
#endif

        /* Get the properties of the PCIe ports on this TRIO instance. */
        ret = gxio_trio_get_port_property(context, &pcie_ports[trio_index]);
        if (ret < 0) {
                pr_err("PCI: PCIE_GET_PORT_PROPERTY failure, error %d, on TRIO %d\n",
                       ret, trio_index);
                goto get_port_property_failure;
        }

        context->mmio_base_mac =
                iorpc_ioremap(context->fd, 0, HV_TRIO_CONFIG_IOREMAP_SIZE);
        if (context->mmio_base_mac == NULL) {
                pr_err("PCI: TRIO config space mapping failure, error %d, on TRIO %d\n",
                       ret, trio_index);
                ret = -ENOMEM;

                goto trio_mmio_mapping_failure;
        }

        /* Check the port strap state which will override the BIB setting. */
        for (mac = 0; mac < TILEGX_TRIO_PCIES; mac++) {
                TRIO_PCIE_INTFC_PORT_CONFIG_t port_config;
                unsigned int reg_offset;

                /* Ignore ports that are not specified in the BIB. */
                if (!pcie_ports[trio_index].ports[mac].allow_rc &&
                    !pcie_ports[trio_index].ports[mac].allow_ep)
                        continue;

                reg_offset =
                        (TRIO_PCIE_INTFC_PORT_CONFIG <<
                                TRIO_CFG_REGION_ADDR__REG_SHIFT) |
                        (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_INTERFACE <<
                                TRIO_CFG_REGION_ADDR__INTFC_SHIFT) |
                        (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT);

                port_config.word =
                        __gxio_mmio_read(context->mmio_base_mac + reg_offset);

                if (port_config.strap_state != AUTO_CONFIG_RC &&
                    port_config.strap_state != AUTO_CONFIG_RC_G1) {
                        /*
                         * If this is really intended to be an EP port, record
                         * it so that the endpoint driver will know about it.
                         */
                        if (port_config.strap_state == AUTO_CONFIG_EP ||
                            port_config.strap_state == AUTO_CONFIG_EP_G1)
                                pcie_ports[trio_index].ports[mac].allow_ep = 1;
                }
        }

        return ret;

trio_mmio_mapping_failure:
get_port_property_failure:
asid_alloc_failure:
#ifdef USE_SHARED_PCIE_CONFIG_REGION
pio_alloc_failure:
#endif
        hv_dev_close(context->fd);
gxio_trio_init_failure:
        context->fd = -1;

        return ret;
}

static int __init tile_trio_init(void)
{
        int i;

        /* We loop over all the TRIO shims. */
        for (i = 0; i < TILEGX_NUM_TRIO; i++) {
                if (tile_pcie_open(i) < 0)
                        continue;
                num_trio_shims++;
        }

        return 0;
}
postcore_initcall(tile_trio_init);

static void tilegx_legacy_irq_ack(struct irq_data *d)
{
        __insn_mtspr(SPR_IPI_EVENT_RESET_K, 1UL << d->irq);
}

static void tilegx_legacy_irq_mask(struct irq_data *d)
{
        __insn_mtspr(SPR_IPI_MASK_SET_K, 1UL << d->irq);
}

static void tilegx_legacy_irq_unmask(struct irq_data *d)
{
        __insn_mtspr(SPR_IPI_MASK_RESET_K, 1UL << d->irq);
}

static struct irq_chip tilegx_legacy_irq_chip = {
        .name                   = "tilegx_legacy_irq",
        .irq_ack                = tilegx_legacy_irq_ack,
        .irq_mask               = tilegx_legacy_irq_mask,
        .irq_unmask             = tilegx_legacy_irq_unmask,

        /* TBD: support set_affinity. */
};

/*
 * This is a wrapper function of the kernel level-trigger interrupt
 * handler handle_level_irq() for PCI legacy interrupts. The TRIO
 * is configured such that only INTx Assert interrupts are proxied
 * to Linux which just calls handle_level_irq() after clearing the
 * MAC INTx Assert status bit associated with this interrupt.
 */
static void trio_handle_level_irq(struct irq_desc *desc)
{
        struct pci_controller *controller = irq_desc_get_handler_data(desc);
        gxio_trio_context_t *trio_context = controller->trio;
        uint64_t intx = (uint64_t)irq_desc_get_chip_data(desc);
        int mac = controller->mac;
        unsigned int reg_offset;
        uint64_t level_mask;

        handle_level_irq(desc);

        /*
         * Clear the INTx Level status, otherwise future interrupts are
         * not sent.
         */
        reg_offset = (TRIO_PCIE_INTFC_MAC_INT_STS <<
                TRIO_CFG_REGION_ADDR__REG_SHIFT) |
                (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_INTERFACE <<
                TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) |
                (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT);

        level_mask = TRIO_PCIE_INTFC_MAC_INT_STS__INT_LEVEL_MASK << intx;

        __gxio_mmio_write(trio_context->mmio_base_mac + reg_offset, level_mask);
}

/*
 * Create kernel irqs and set up the handlers for the legacy interrupts.
 * Also some minimum initialization for the MSI support.
 */
static int tile_init_irqs(struct pci_controller *controller)
{
        int i;
        int j;
        int irq;
        int result;

        cpumask_copy(&intr_cpus_map, cpu_online_mask);


        for (i = 0; i < 4; i++) {
                gxio_trio_context_t *context = controller->trio;
                int cpu;

                /* Ask the kernel to allocate an IRQ. */
                irq = irq_alloc_hwirq(-1);
                if (!irq) {
                        pr_err("PCI: no free irq vectors, failed for %d\n", i);
                        goto free_irqs;
                }
                controller->irq_intx_table[i] = irq;

                /* Distribute the 4 IRQs to different tiles. */
                cpu = tile_irq_cpu(irq);

                /* Configure the TRIO intr binding for this IRQ. */
                result = gxio_trio_config_legacy_intr(context, cpu_x(cpu),
                                                      cpu_y(cpu), KERNEL_PL,
                                                      irq, controller->mac, i);
                if (result < 0) {
                        pr_err("PCI: MAC intx config failed for %d\n", i);

                        goto free_irqs;
                }

                /* Register the IRQ handler with the kernel. */
                irq_set_chip_and_handler(irq, &tilegx_legacy_irq_chip,
                                        trio_handle_level_irq);
                irq_set_chip_data(irq, (void *)(uint64_t)i);
                irq_set_handler_data(irq, controller);
        }

        return 0;

free_irqs:
        for (j = 0; j < i; j++)
                irq_free_hwirq(controller->irq_intx_table[j]);

        return -1;
}

/*
 * Return 1 if the port is strapped to operate in RC mode.
 */
static int
strapped_for_rc(gxio_trio_context_t *trio_context, int mac)
{
        TRIO_PCIE_INTFC_PORT_CONFIG_t port_config;
        unsigned int reg_offset;

        /* Check the port configuration. */
        reg_offset =
                (TRIO_PCIE_INTFC_PORT_CONFIG <<
                        TRIO_CFG_REGION_ADDR__REG_SHIFT) |
                (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_INTERFACE <<
                        TRIO_CFG_REGION_ADDR__INTFC_SHIFT) |
                (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT);
        port_config.word =
                __gxio_mmio_read(trio_context->mmio_base_mac + reg_offset);

        if (port_config.strap_state == AUTO_CONFIG_RC ||
            port_config.strap_state == AUTO_CONFIG_RC_G1)
                return 1;
        else
                return 0;
}

/*
 * Find valid controllers and fill in pci_controller structs for each
 * of them.
 *
 * Return the number of controllers discovered.
 */
int __init tile_pci_init(void)
{
        int ctl_index = 0;
        int i, j;

        if (!pci_probe) {
                pr_info("PCI: disabled by boot argument\n");
                return 0;
        }

        pr_info("PCI: Searching for controllers...\n");

        if (num_trio_shims == 0 || sim_is_simulator())
                return 0;

        /*
         * Now determine which PCIe ports are configured to operate in RC
         * mode. There is a difference in the port configuration capability
         * between the Gx36 and Gx72 devices.
         *
         * The Gx36 has configuration capability for each of the 3 PCIe
         * interfaces (disable, auto endpoint, auto RC, etc.).
         * On the Gx72, you can only select one of the 3 PCIe interfaces per
         * TRIO to train automatically. Further, the allowable training modes
         * are reduced to four options (auto endpoint, auto RC, stream x1,
         * stream x4).
         *
         * For Gx36 ports, it must be allowed to be in RC mode by the
         * Board Information Block, and the hardware strapping pins must be
         * set to RC mode.
         *
         * For Gx72 ports, the port will operate in RC mode if either of the
         * following is true:
         * 1. It is allowed to be in RC mode by the Board Information Block,
         *    and the BIB doesn't allow the EP mode.
         * 2. It is allowed to be in either the RC or the EP mode by the BIB,
         *    and the hardware strapping pin is set to RC mode.
         */
        for (i = 0; i < TILEGX_NUM_TRIO; i++) {
                gxio_trio_context_t *context = &trio_contexts[i];

                if (context->fd < 0)
                        continue;

                for (j = 0; j < TILEGX_TRIO_PCIES; j++) {
                        int is_rc = 0;

                        if (pcie_ports[i].is_gx72 &&
                            pcie_ports[i].ports[j].allow_rc) {
                                if (!pcie_ports[i].ports[j].allow_ep ||
                                    strapped_for_rc(context, j))
                                        is_rc = 1;
                        } else if (pcie_ports[i].ports[j].allow_rc &&
                                   strapped_for_rc(context, j)) {
                                is_rc = 1;
                        }
                        if (is_rc) {
                                pcie_rc[i][j] = 1;
                                num_rc_controllers++;
                        }
                }
        }

        /* Return if no PCIe ports are configured to operate in RC mode. */
        if (num_rc_controllers == 0)
                return 0;

        /* Set the TRIO pointer and MAC index for each PCIe RC port. */
        for (i = 0; i < TILEGX_NUM_TRIO; i++) {
                for (j = 0; j < TILEGX_TRIO_PCIES; j++) {
                        if (pcie_rc[i][j]) {
                                pci_controllers[ctl_index].trio =
                                        &trio_contexts[i];
                                pci_controllers[ctl_index].mac = j;
                                pci_controllers[ctl_index].trio_index = i;
                                ctl_index++;
                                if (ctl_index == num_rc_controllers)
                                        goto out;
                        }
                }
        }

out:
        /* Configure each PCIe RC port. */
        for (i = 0; i < num_rc_controllers; i++) {

                /* Configure the PCIe MAC to run in RC mode. */
                struct pci_controller *controller = &pci_controllers[i];

                controller->index = i;
                controller->ops = &tile_cfg_ops;

                controller->io_space.start = PCIBIOS_MIN_IO +
                        (i * IO_SPACE_SIZE);
                controller->io_space.end = controller->io_space.start +
                        IO_SPACE_SIZE - 1;
                BUG_ON(controller->io_space.end > IO_SPACE_LIMIT);
                controller->io_space.flags = IORESOURCE_IO;
                snprintf(controller->io_space_name,
                         sizeof(controller->io_space_name),
                         "PCI I/O domain %d", i);
                controller->io_space.name = controller->io_space_name;

                /*
                 * The PCI memory resource is located above the PA space.
                 * For every host bridge, the BAR window or the MMIO aperture
                 * is in range [3GB, 4GB - 1] of a 4GB space beyond the
                 * PA space.
                 */
                controller->mem_offset = TILE_PCI_MEM_START +
                        (i * TILE_PCI_BAR_WINDOW_TOP);
                controller->mem_space.start = controller->mem_offset +
                        TILE_PCI_BAR_WINDOW_TOP - TILE_PCI_BAR_WINDOW_SIZE;
                controller->mem_space.end = controller->mem_offset +
                        TILE_PCI_BAR_WINDOW_TOP - 1;
                controller->mem_space.flags = IORESOURCE_MEM;
                snprintf(controller->mem_space_name,
                         sizeof(controller->mem_space_name),
                         "PCI mem domain %d", i);
                controller->mem_space.name = controller->mem_space_name;
        }

        return num_rc_controllers;
}

/*
 * (pin - 1) converts from the PCI standard's [1:4] convention to
 * a normal [0:3] range.
 */
static int tile_map_irq(const struct pci_dev *dev, u8 device, u8 pin)
{
        struct pci_controller *controller =
                (struct pci_controller *)dev->sysdata;
        return controller->irq_intx_table[pin - 1];
}

static void fixup_read_and_payload_sizes(struct pci_controller *controller)
{
        gxio_trio_context_t *trio_context = controller->trio;
        struct pci_bus *root_bus = controller->root_bus;
        TRIO_PCIE_RC_DEVICE_CONTROL_t dev_control;
        TRIO_PCIE_RC_DEVICE_CAP_t rc_dev_cap;
        unsigned int reg_offset;
        struct pci_bus *child;
        int mac;
        int err;

        mac = controller->mac;

        /* Set our max read request size to be 4KB. */
        reg_offset =
                (TRIO_PCIE_RC_DEVICE_CONTROL <<
                        TRIO_CFG_REGION_ADDR__REG_SHIFT) |
                (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_STANDARD <<
                        TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) |
                (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT);

        dev_control.word = __gxio_mmio_read32(trio_context->mmio_base_mac +
                                              reg_offset);
        dev_control.max_read_req_sz = 5;
        __gxio_mmio_write32(trio_context->mmio_base_mac + reg_offset,
                            dev_control.word);

        /*
         * Set the max payload size supported by this Gx PCIe MAC.
         * Though Gx PCIe supports Max Payload Size of up to 1024 bytes,
         * experiments have shown that setting MPS to 256 yields the
         * best performance.
         */
        reg_offset =
                (TRIO_PCIE_RC_DEVICE_CAP <<
                        TRIO_CFG_REGION_ADDR__REG_SHIFT) |
                (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_STANDARD <<
                        TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) |
                (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT);

        rc_dev_cap.word = __gxio_mmio_read32(trio_context->mmio_base_mac +
                                             reg_offset);
        rc_dev_cap.mps_sup = 1;
        __gxio_mmio_write32(trio_context->mmio_base_mac + reg_offset,
                            rc_dev_cap.word);

        /* Configure PCI Express MPS setting. */
        list_for_each_entry(child, &root_bus->children, node)
                pcie_bus_configure_settings(child);

        /*
         * Set the mac_config register in trio based on the MPS/MRS of the link.
         */
        reg_offset =
                (TRIO_PCIE_RC_DEVICE_CONTROL <<
                        TRIO_CFG_REGION_ADDR__REG_SHIFT) |
                (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_STANDARD <<
                        TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) |
                (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT);

        dev_control.word = __gxio_mmio_read32(trio_context->mmio_base_mac +
                                                reg_offset);

        err = gxio_trio_set_mps_mrs(trio_context,
                                    dev_control.max_payload_size,
                                    dev_control.max_read_req_sz,
                                    mac);
        if (err < 0) {
                pr_err("PCI: PCIE_CONFIGURE_MAC_MPS_MRS failure, MAC %d on TRIO %d\n",
                       mac, controller->trio_index);
        }
}

static int setup_pcie_rc_delay(char *str)
{
        unsigned long delay = 0;
        unsigned long trio_index;
        unsigned long mac;

        if (str == NULL || !isdigit(*str))
                return -EINVAL;
        trio_index = simple_strtoul(str, (char **)&str, 10);
        if (trio_index >= TILEGX_NUM_TRIO)
                return -EINVAL;

        if (*str != ',')
                return -EINVAL;

        str++;
        if (!isdigit(*str))
                return -EINVAL;
        mac = simple_strtoul(str, (char **)&str, 10);
        if (mac >= TILEGX_TRIO_PCIES)
                return -EINVAL;

        if (*str != '\0') {
                if (*str != ',')
                        return -EINVAL;

                str++;
                if (!isdigit(*str))
                        return -EINVAL;
                delay = simple_strtoul(str, (char **)&str, 10);
        }

        rc_delay[trio_index][mac] = delay ? : DEFAULT_RC_DELAY;
        return 0;
}
early_param("pcie_rc_delay", setup_pcie_rc_delay);

/* PCI initialization entry point, called by subsys_initcall. */
int __init pcibios_init(void)
{
        resource_size_t offset;
        LIST_HEAD(resources);
        int next_busno;
        struct pci_host_bridge *bridge;
        int i;

        tile_pci_init();

        if (num_rc_controllers == 0)
                return 0;

        /*
         * Delay a bit in case devices aren't ready.  Some devices are
         * known to require at least 20ms here, but we use a more
         * conservative value.
         */
        msleep(250);

        /* Scan all of the recorded PCI controllers.  */
        for (next_busno = 0, i = 0; i < num_rc_controllers; i++) {
                struct pci_controller *controller = &pci_controllers[i];
                gxio_trio_context_t *trio_context = controller->trio;
                TRIO_PCIE_INTFC_PORT_STATUS_t port_status;
                TRIO_PCIE_INTFC_TX_FIFO_CTL_t tx_fifo_ctl;
                struct pci_bus *bus;
                unsigned int reg_offset;
                unsigned int class_code_revision;
                int trio_index;
                int mac;
                int ret;

                if (trio_context->fd < 0)
                        continue;

                trio_index = controller->trio_index;
                mac = controller->mac;

                /*
                 * Check for PCIe link-up status to decide if we need
                 * to force the link to come up.
                 */
                reg_offset =
                        (TRIO_PCIE_INTFC_PORT_STATUS <<
                                TRIO_CFG_REGION_ADDR__REG_SHIFT) |
                        (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_INTERFACE <<
                                TRIO_CFG_REGION_ADDR__INTFC_SHIFT) |
                        (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT);

                port_status.word =
                        __gxio_mmio_read(trio_context->mmio_base_mac +
                                         reg_offset);
                if (!port_status.dl_up) {
                        if (rc_delay[trio_index][mac]) {
                                pr_info("Delaying PCIe RC TRIO init %d sec on MAC %d on TRIO %d\n",
                                        rc_delay[trio_index][mac], mac,
                                        trio_index);
                                msleep(rc_delay[trio_index][mac] * 1000);
                        }
                        ret = gxio_trio_force_rc_link_up(trio_context, mac);
                        if (ret < 0)
                                pr_err("PCI: PCIE_FORCE_LINK_UP failure, MAC %d on TRIO %d\n",
                                       mac, trio_index);
                }

                pr_info("PCI: Found PCI controller #%d on TRIO %d MAC %d\n",
                        i, trio_index, controller->mac);

                /* Delay the bus probe if needed. */
                if (rc_delay[trio_index][mac]) {
                        pr_info("Delaying PCIe RC bus enumerating %d sec on MAC %d on TRIO %d\n",
                                rc_delay[trio_index][mac], mac, trio_index);
                        msleep(rc_delay[trio_index][mac] * 1000);
                } else {
                        /*
                         * Wait a bit here because some EP devices
                         * take longer to come up.
                         */
                        msleep(1000);
                }

                /* Check for PCIe link-up status again. */
                port_status.word =
                        __gxio_mmio_read(trio_context->mmio_base_mac +
                                         reg_offset);
                if (!port_status.dl_up) {
                        if (pcie_ports[trio_index].ports[mac].removable) {
                                pr_info("PCI: link is down, MAC %d on TRIO %d\n",
                                        mac, trio_index);
                                pr_info("This is expected if no PCIe card is connected to this link\n");
                        } else
                                pr_err("PCI: link is down, MAC %d on TRIO %d\n",
                                       mac, trio_index);
                        continue;
                }

                /*
                 * Ensure that the link can come out of L1 power down state.
                 * Strictly speaking, this is needed only in the case of
                 * heavy RC-initiated DMAs.
                 */
                reg_offset =
                        (TRIO_PCIE_INTFC_TX_FIFO_CTL <<
                                TRIO_CFG_REGION_ADDR__REG_SHIFT) |
                        (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_INTERFACE <<
                                TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) |
                        (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT);
                tx_fifo_ctl.word =
                        __gxio_mmio_read(trio_context->mmio_base_mac +
                                         reg_offset);
                tx_fifo_ctl.min_p_credits = 0;
                __gxio_mmio_write(trio_context->mmio_base_mac + reg_offset,
                                  tx_fifo_ctl.word);

                /*
                 * Change the device ID so that Linux bus crawl doesn't confuse
                 * the internal bridge with any Tilera endpoints.
                 */
                reg_offset =
                        (TRIO_PCIE_RC_DEVICE_ID_VEN_ID <<
                                TRIO_CFG_REGION_ADDR__REG_SHIFT) |
                        (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_STANDARD <<
                                TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) |
                        (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT);

                __gxio_mmio_write32(trio_context->mmio_base_mac + reg_offset,
                                    (TILERA_GX36_RC_DEV_ID <<
                                    TRIO_PCIE_RC_DEVICE_ID_VEN_ID__DEV_ID_SHIFT) |
                                    TILERA_VENDOR_ID);

                /* Set the internal P2P bridge class code. */
                reg_offset =
                        (TRIO_PCIE_RC_REVISION_ID <<
                                TRIO_CFG_REGION_ADDR__REG_SHIFT) |
                        (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_STANDARD <<
                                TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) |
                        (mac << TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT);

                class_code_revision =
                        __gxio_mmio_read32(trio_context->mmio_base_mac +
                                           reg_offset);
                class_code_revision = (class_code_revision & 0xff) |
                        (PCI_CLASS_BRIDGE_PCI << 16);

                __gxio_mmio_write32(trio_context->mmio_base_mac +
                                    reg_offset, class_code_revision);

#ifdef USE_SHARED_PCIE_CONFIG_REGION

                /* Map in the MMIO space for the PIO region. */
                offset = HV_TRIO_PIO_OFFSET(trio_context->pio_cfg_index) |
                        (((unsigned long long)mac) <<
                        TRIO_TILE_PIO_REGION_SETUP_CFG_ADDR__MAC_SHIFT);

#else

                /* Alloc a PIO region for PCI config access per MAC. */
                ret = gxio_trio_alloc_pio_regions(trio_context, 1, 0, 0);
                if (ret < 0) {
                        pr_err("PCI: PCI CFG PIO alloc failure for mac %d on TRIO %d, give up\n",
                               mac, trio_index);

                        continue;
                }

                trio_context->pio_cfg_index[mac] = ret;

                /* For PIO CFG, the bus_address_hi parameter is 0. */
                ret = gxio_trio_init_pio_region_aux(trio_context,
                        trio_context->pio_cfg_index[mac],
                        mac, 0, HV_TRIO_PIO_FLAG_CONFIG_SPACE);
                if (ret < 0) {
                        pr_err("PCI: PCI CFG PIO init failure for mac %d on TRIO %d, give up\n",
                               mac, trio_index);

                        continue;
                }

                offset = HV_TRIO_PIO_OFFSET(trio_context->pio_cfg_index[mac]) |
                        (((unsigned long long)mac) <<
                        TRIO_TILE_PIO_REGION_SETUP_CFG_ADDR__MAC_SHIFT);

#endif

                /*
                 * To save VMALLOC space, we take advantage of the fact that
                 * bit 29 in the PIO CFG address format is reserved 0. With
                 * TRIO_TILE_PIO_REGION_SETUP_CFG_ADDR__MAC_SHIFT being 30,
                 * this cuts VMALLOC space usage from 1GB to 512MB per mac.
                 */
                trio_context->mmio_base_pio_cfg[mac] =
                        iorpc_ioremap(trio_context->fd, offset, (1UL <<
                        (TRIO_TILE_PIO_REGION_SETUP_CFG_ADDR__MAC_SHIFT - 1)));
                if (trio_context->mmio_base_pio_cfg[mac] == NULL) {
                        pr_err("PCI: PIO map failure for mac %d on TRIO %d\n",
                               mac, trio_index);

                        continue;
                }

                /* Initialize the PCIe interrupts. */
                if (tile_init_irqs(controller)) {
                        pr_err("PCI: IRQs init failure for mac %d on TRIO %d\n",
                                mac, trio_index);

                        continue;
                }

                /*
                 * The PCI memory resource is located above the PA space.
                 * The memory range for the PCI root bus should not overlap
                 * with the physical RAM.
                 */
                pci_add_resource_offset(&resources, &controller->mem_space,
                                        controller->mem_offset);
                pci_add_resource(&resources, &controller->io_space);
                controller->first_busno = next_busno;

                bridge = pci_alloc_host_bridge(0);
                if (!bridge)
                        break;

                list_splice_init(&resources, &bridge->windows);
                bridge->dev.parent = NULL;
                bridge->sysdata = controller;
                bridge->busnr = next_busno;
                bridge->ops = controller->ops;
                bridge->swizzle_irq = pci_common_swizzle;
                bridge->map_irq = tile_map_irq;

                pci_scan_root_bus_bridge(bridge);
                bus = bridge->bus;
                controller->root_bus = bus;
                next_busno = bus->busn_res.end + 1;
        }

        /*
         * This comes from the generic Linux PCI driver.
         *
         * It allocates all of the resources (I/O memory, etc)
         * associated with the devices read in above.
         */
        pci_assign_unassigned_resources();

        /* Record the I/O resources in the PCI controller structure. */
        for (i = 0; i < num_rc_controllers; i++) {
                struct pci_controller *controller = &pci_controllers[i];
                gxio_trio_context_t *trio_context = controller->trio;
                struct pci_bus *root_bus = pci_controllers[i].root_bus;
                int ret;
                int j;

                /*
                 * Skip controllers that are not properly initialized or
                 * have down links.
                 */
                if (root_bus == NULL)
                        continue;

                /* Configure the max_payload_size values for this domain. */
                fixup_read_and_payload_sizes(controller);

                /* Alloc a PIO region for PCI memory access for each RC port. */
                ret = gxio_trio_alloc_pio_regions(trio_context, 1, 0, 0);
                if (ret < 0) {
                        pr_err("PCI: MEM PIO alloc failure on TRIO %d mac %d, give up\n",
                               controller->trio_index, controller->mac);

                        continue;
                }

                controller->pio_mem_index = ret;

                /*
                 * For PIO MEM, the bus_address_hi parameter is hard-coded 0
                 * because we always assign 32-bit PCI bus BAR ranges.
                 */
                ret = gxio_trio_init_pio_region_aux(trio_context,
                                                    controller->pio_mem_index,
                                                    controller->mac,
                                                    0,
                                                    0);
                if (ret < 0) {
                        pr_err("PCI: MEM PIO init failure on TRIO %d mac %d, give up\n",
                               controller->trio_index, controller->mac);

                        continue;
                }

#ifdef CONFIG_TILE_PCI_IO
                /*
                 * Alloc a PIO region for PCI I/O space access for each RC port.
                 */
                ret = gxio_trio_alloc_pio_regions(trio_context, 1, 0, 0);
                if (ret < 0) {
                        pr_err("PCI: I/O PIO alloc failure on TRIO %d mac %d, give up\n",
                               controller->trio_index, controller->mac);

                        continue;
                }

                controller->pio_io_index = ret;

                /*
                 * For PIO IO, the bus_address_hi parameter is hard-coded 0
                 * because PCI I/O address space is 32-bit.
                 */
                ret = gxio_trio_init_pio_region_aux(trio_context,
                                                    controller->pio_io_index,
                                                    controller->mac,
                                                    0,
                                                    HV_TRIO_PIO_FLAG_IO_SPACE);
                if (ret < 0) {
                        pr_err("PCI: I/O PIO init failure on TRIO %d mac %d, give up\n",
                               controller->trio_index, controller->mac);

                        continue;
                }
#endif

                /*
                 * Configure a Mem-Map region for each memory controller so
                 * that Linux can map all of its PA space to the PCI bus.
                 * Use the IOMMU to handle hash-for-home memory.
                 */
                for_each_online_node(j) {
                        unsigned long start_pfn = node_start_pfn[j];
                        unsigned long end_pfn = node_end_pfn[j];
                        unsigned long nr_pages = end_pfn - start_pfn;

                        ret = gxio_trio_alloc_memory_maps(trio_context, 1, 0,
                                                          0);
                        if (ret < 0) {
                                pr_err("PCI: Mem-Map alloc failure on TRIO %d mac %d for MC %d, give up\n",
                                       controller->trio_index, controller->mac,
                                       j);

                                goto alloc_mem_map_failed;
                        }

                        controller->mem_maps[j] = ret;

                        /*
                         * Initialize the Mem-Map and the I/O MMU so that all
                         * the physical memory can be accessed by the endpoint
                         * devices. The base bus address is set to the base CPA
                         * of this memory controller plus an offset (see pci.h).
                         * The region's base VA is set to the base CPA. The
                         * I/O MMU table essentially translates the CPA to
                         * the real PA. Implicitly, for node 0, we create
                         * a separate Mem-Map region that serves as the inbound
                         * window for legacy 32-bit devices. This is a direct
                         * map of the low 4GB CPA space.
                         */
                        ret = gxio_trio_init_memory_map_mmu_aux(trio_context,
                                controller->mem_maps[j],
                                start_pfn << PAGE_SHIFT,
                                nr_pages << PAGE_SHIFT,
                                trio_context->asid,
                                controller->mac,
                                (start_pfn << PAGE_SHIFT) +
                                TILE_PCI_MEM_MAP_BASE_OFFSET,
                                j,
                                GXIO_TRIO_ORDER_MODE_UNORDERED);
                        if (ret < 0) {
                                pr_err("PCI: Mem-Map init failure on TRIO %d mac %d for MC %d, give up\n",
                                       controller->trio_index, controller->mac,
                                       j);

                                goto alloc_mem_map_failed;
                        }
                        continue;

alloc_mem_map_failed:
                        break;
                }

                pci_bus_add_devices(root_bus);
        }

        return 0;
}
subsys_initcall(pcibios_init);

/* Process any "pci=" kernel boot arguments. */
char *__init pcibios_setup(char *str)
{
        if (!strcmp(str, "off")) {
                pci_probe = 0;
                return NULL;
        }
        return str;
}

/*
 * Called for each device after PCI setup is done.
 * We initialize the PCI device capabilities conservatively, assuming that
 * all devices can only address the 32-bit DMA space. The exception here is
 * that the device dma_offset is set to the value that matches the 64-bit
 * capable devices. This is OK because dma_offset is not used by legacy
 * dma_ops, nor by the hybrid dma_ops's streaming DMAs, which are 64-bit ops.
 * This implementation matches the kernel design of setting PCI devices'
 * coherent_dma_mask to 0xffffffffull by default, allowing the device drivers
 * to skip calling pci_set_consistent_dma_mask(DMA_BIT_MASK(32)).
 */
static void pcibios_fixup_final(struct pci_dev *pdev)
{
        set_dma_ops(&pdev->dev, gx_legacy_pci_dma_map_ops);
        set_dma_offset(&pdev->dev, TILE_PCI_MEM_MAP_BASE_OFFSET);
        pdev->dev.archdata.max_direct_dma_addr =
                TILE_PCI_MAX_DIRECT_DMA_ADDRESS;
        pdev->dev.coherent_dma_mask = TILE_PCI_MAX_DIRECT_DMA_ADDRESS;
}
DECLARE_PCI_FIXUP_FINAL(PCI_ANY_ID, PCI_ANY_ID, pcibios_fixup_final);

/* Map a PCI MMIO bus address into VA space. */
void __iomem *ioremap(resource_size_t phys_addr, unsigned long size)
{
        struct pci_controller *controller = NULL;
        resource_size_t bar_start;
        resource_size_t bar_end;
        resource_size_t offset;
        resource_size_t start;
        resource_size_t end;
        int trio_fd;
        int i;

        start = phys_addr;
        end = phys_addr + size - 1;

        /*
         * By searching phys_addr in each controller's mem_space, we can
         * determine the controller that should accept the PCI memory access.
         */
        for (i = 0; i < num_rc_controllers; i++) {
                /*
                 * Skip controllers that are not properly initialized or
                 * have down links.
                 */
                if (pci_controllers[i].root_bus == NULL)
                        continue;

                bar_start = pci_controllers[i].mem_space.start;
                bar_end = pci_controllers[i].mem_space.end;

                if ((start >= bar_start) && (end <= bar_end)) {
                        controller = &pci_controllers[i];
                        break;
                }
        }

        if (controller == NULL)
                return NULL;

        trio_fd = controller->trio->fd;

        /* Convert the resource start to the bus address offset. */
        start = phys_addr - controller->mem_offset;

        offset = HV_TRIO_PIO_OFFSET(controller->pio_mem_index) + start;

        /* We need to keep the PCI bus address's in-page offset in the VA. */
        return iorpc_ioremap(trio_fd, offset, size) +
                (start & (PAGE_SIZE - 1));
}
EXPORT_SYMBOL(ioremap);

#ifdef CONFIG_TILE_PCI_IO
/* Map a PCI I/O address into VA space. */
void __iomem *ioport_map(unsigned long port, unsigned int size)
{
        struct pci_controller *controller = NULL;
        resource_size_t bar_start;
        resource_size_t bar_end;
        resource_size_t offset;
        resource_size_t start;
        resource_size_t end;
        int trio_fd;
        int i;

        start = port;
        end = port + size - 1;

        /*
         * By searching the port in each controller's io_space, we can
         * determine the controller that should accept the PCI I/O access.
         */
        for (i = 0; i < num_rc_controllers; i++) {
                /*
                 * Skip controllers that are not properly initialized or
                 * have down links.
                 */
                if (pci_controllers[i].root_bus == NULL)
                        continue;

                bar_start = pci_controllers[i].io_space.start;
                bar_end = pci_controllers[i].io_space.end;

                if ((start >= bar_start) && (end <= bar_end)) {
                        controller = &pci_controllers[i];
                        break;
                }
        }

        if (controller == NULL)
                return NULL;

        trio_fd = controller->trio->fd;

        /* Convert the resource start to the bus address offset. */
        port -= controller->io_space.start;

        offset = HV_TRIO_PIO_OFFSET(controller->pio_io_index) + port;

        /* We need to keep the PCI bus address's in-page offset in the VA. */
        return iorpc_ioremap(trio_fd, offset, size) + (port & (PAGE_SIZE - 1));
}
EXPORT_SYMBOL(ioport_map);

void ioport_unmap(void __iomem *addr)
{
        iounmap(addr);
}
EXPORT_SYMBOL(ioport_unmap);
#endif

void pci_iounmap(struct pci_dev *dev, void __iomem *addr)
{
        iounmap(addr);
}
EXPORT_SYMBOL(pci_iounmap);

/****************************************************************
 *
 * Tile PCI config space read/write routines
 *
 ****************************************************************/

/*
 * These are the normal read and write ops
 * These are expanded with macros from  pci_bus_read_config_byte() etc.
 *
 * devfn is the combined PCI device & function.
 *
 * offset is in bytes, from the start of config space for the
 * specified bus & device.
 */
static int tile_cfg_read(struct pci_bus *bus, unsigned int devfn, int offset,
                         int size, u32 *val)
{
        struct pci_controller *controller = bus->sysdata;
        gxio_trio_context_t *trio_context = controller->trio;
        int busnum = bus->number & 0xff;
        int device = PCI_SLOT(devfn);
        int function = PCI_FUNC(devfn);
        int config_type = 1;
        TRIO_TILE_PIO_REGION_SETUP_CFG_ADDR_t cfg_addr;
        void *mmio_addr;

        /*
         * Map all accesses to the local device on root bus into the
         * MMIO space of the MAC. Accesses to the downstream devices
         * go to the PIO space.
         */
        if (pci_is_root_bus(bus)) {
                if (device == 0) {
                        /*
                         * This is the internal downstream P2P bridge,
                         * access directly.
                         */
                        unsigned int reg_offset;

                        reg_offset = ((offset & 0xFFF) <<
                                TRIO_CFG_REGION_ADDR__REG_SHIFT) |
                                (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_PROTECTED
                                << TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) |
                                (controller->mac <<
                                        TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT);

                        mmio_addr = trio_context->mmio_base_mac + reg_offset;

                        goto valid_device;

                } else {
                        /*
                         * We fake an empty device for (device > 0),
                         * since there is only one device on bus 0.
                         */
                        goto invalid_device;
                }
        }

        /*
         * Accesses to the directly attached device have to be
         * sent as type-0 configs.
         */
        if (busnum == (controller->first_busno + 1)) {
                /*
                 * There is only one device off of our built-in P2P bridge.
                 */
                if (device != 0)
                        goto invalid_device;

                config_type = 0;
        }

        cfg_addr.word = 0;
        cfg_addr.reg_addr = (offset & 0xFFF);
        cfg_addr.fn = function;
        cfg_addr.dev = device;
        cfg_addr.bus = busnum;
        cfg_addr.type = config_type;

        /*
         * Note that we don't set the mac field in cfg_addr because the
         * mapping is per port.
         */
        mmio_addr = trio_context->mmio_base_pio_cfg[controller->mac] +
                cfg_addr.word;

valid_device:

        switch (size) {
        case 4:
                *val = __gxio_mmio_read32(mmio_addr);
                break;

        case 2:
                *val = __gxio_mmio_read16(mmio_addr);
                break;

        case 1:
                *val = __gxio_mmio_read8(mmio_addr);
                break;

        default:
                return PCIBIOS_FUNC_NOT_SUPPORTED;
        }

        TRACE_CFG_RD(size, *val, busnum, device, function, offset);

        return 0;

invalid_device:

        switch (size) {
        case 4:
                *val = 0xFFFFFFFF;
                break;

        case 2:
                *val = 0xFFFF;
                break;

        case 1:
                *val = 0xFF;
                break;

        default:
                return PCIBIOS_FUNC_NOT_SUPPORTED;
        }

        return 0;
}


/*
 * See tile_cfg_read() for relevant comments.
 * Note that "val" is the value to write, not a pointer to that value.
 */
static int tile_cfg_write(struct pci_bus *bus, unsigned int devfn, int offset,
                          int size, u32 val)
{
        struct pci_controller *controller = bus->sysdata;
        gxio_trio_context_t *trio_context = controller->trio;
        int busnum = bus->number & 0xff;
        int device = PCI_SLOT(devfn);
        int function = PCI_FUNC(devfn);
        int config_type = 1;
        TRIO_TILE_PIO_REGION_SETUP_CFG_ADDR_t cfg_addr;
        void *mmio_addr;
        u32 val_32 = (u32)val;
        u16 val_16 = (u16)val;
        u8 val_8 = (u8)val;

        /*
         * Map all accesses to the local device on root bus into the
         * MMIO space of the MAC. Accesses to the downstream devices
         * go to the PIO space.
         */
        if (pci_is_root_bus(bus)) {
                if (device == 0) {
                        /*
                         * This is the internal downstream P2P bridge,
                         * access directly.
                         */
                        unsigned int reg_offset;

                        reg_offset = ((offset & 0xFFF) <<
                                TRIO_CFG_REGION_ADDR__REG_SHIFT) |
                                (TRIO_CFG_REGION_ADDR__INTFC_VAL_MAC_PROTECTED
                                << TRIO_CFG_REGION_ADDR__INTFC_SHIFT ) |
                                (controller->mac <<
                                        TRIO_CFG_REGION_ADDR__MAC_SEL_SHIFT);

                        mmio_addr = trio_context->mmio_base_mac + reg_offset;

                        goto valid_device;

                } else {
                        /*
                         * We fake an empty device for (device > 0),
                         * since there is only one device on bus 0.
                         */
                        goto invalid_device;
                }
        }

        /*
         * Accesses to the directly attached device have to be
         * sent as type-0 configs.
         */
        if (busnum == (controller->first_busno + 1)) {
                /*
                 * There is only one device off of our built-in P2P bridge.
                 */
                if (device != 0)
                        goto invalid_device;

                config_type = 0;
        }

        cfg_addr.word = 0;
        cfg_addr.reg_addr = (offset & 0xFFF);
        cfg_addr.fn = function;
        cfg_addr.dev = device;
        cfg_addr.bus = busnum;
        cfg_addr.type = config_type;

        /*
         * Note that we don't set the mac field in cfg_addr because the
         * mapping is per port.
         */
        mmio_addr = trio_context->mmio_base_pio_cfg[controller->mac] +
                        cfg_addr.word;

valid_device:

        switch (size) {
        case 4:
                __gxio_mmio_write32(mmio_addr, val_32);
                TRACE_CFG_WR(size, val_32, busnum, device, function, offset);
                break;

        case 2:
                __gxio_mmio_write16(mmio_addr, val_16);
                TRACE_CFG_WR(size, val_16, busnum, device, function, offset);
                break;

        case 1:
                __gxio_mmio_write8(mmio_addr, val_8);
                TRACE_CFG_WR(size, val_8, busnum, device, function, offset);
                break;

        default:
                return PCIBIOS_FUNC_NOT_SUPPORTED;
        }

invalid_device:

        return 0;
}


static struct pci_ops tile_cfg_ops = {
        .read =         tile_cfg_read,
        .write =        tile_cfg_write,
};


/* MSI support starts here. */
static unsigned int tilegx_msi_startup(struct irq_data *d)
{
        if (irq_data_get_msi_desc(d))
                pci_msi_unmask_irq(d);

        return 0;
}

static void tilegx_msi_ack(struct irq_data *d)
{
        __insn_mtspr(SPR_IPI_EVENT_RESET_K, 1UL << d->irq);
}

static void tilegx_msi_mask(struct irq_data *d)
{
        pci_msi_mask_irq(d);
        __insn_mtspr(SPR_IPI_MASK_SET_K, 1UL << d->irq);
}

static void tilegx_msi_unmask(struct irq_data *d)
{
        __insn_mtspr(SPR_IPI_MASK_RESET_K, 1UL << d->irq);
        pci_msi_unmask_irq(d);
}

static struct irq_chip tilegx_msi_chip = {
        .name                   = "tilegx_msi",
        .irq_startup            = tilegx_msi_startup,
        .irq_ack                = tilegx_msi_ack,
        .irq_mask               = tilegx_msi_mask,
        .irq_unmask             = tilegx_msi_unmask,

        /* TBD: support set_affinity. */
};

int arch_setup_msi_irq(struct pci_dev *pdev, struct msi_desc *desc)
{
        struct pci_controller *controller;
        gxio_trio_context_t *trio_context;
        struct msi_msg msg;
        int default_irq;
        uint64_t mem_map_base;
        uint64_t mem_map_limit;
        u64 msi_addr;
        int mem_map;
        int cpu;
        int irq;
        int ret;

        irq = irq_alloc_hwirq(-1);
        if (!irq)
                return -ENOSPC;

        /*
         * Since we use a 64-bit Mem-Map to accept the MSI write, we fail
         * devices that are not capable of generating a 64-bit message address.
         * These devices will fall back to using the legacy interrupts.
         * Most PCIe endpoint devices do support 64-bit message addressing.
         */
        if (desc->msi_attrib.is_64 == 0) {
                dev_info(&pdev->dev, "64-bit MSI message address not supported, falling back to legacy interrupts\n");

                ret = -ENOMEM;
                goto is_64_failure;
        }

        default_irq = desc->msi_attrib.default_irq;
        controller = irq_get_handler_data(default_irq);

        BUG_ON(!controller);

        trio_context = controller->trio;

        /*
         * Allocate a scatter-queue that will accept the MSI write and
         * trigger the TILE-side interrupts. We use the scatter-queue regions
         * before the mem map regions, because the latter are needed by more
         * applications.
         */
        mem_map = gxio_trio_alloc_scatter_queues(trio_context, 1, 0, 0);
        if (mem_map >= 0) {
                TRIO_MAP_SQ_DOORBELL_FMT_t doorbell_template = {{
                        .pop = 0,
                        .doorbell = 1,
                }};

                mem_map += TRIO_NUM_MAP_MEM_REGIONS;
                mem_map_base = MEM_MAP_INTR_REGIONS_BASE +
                        mem_map * MEM_MAP_INTR_REGION_SIZE;
                mem_map_limit = mem_map_base + MEM_MAP_INTR_REGION_SIZE - 1;

                msi_addr = mem_map_base + MEM_MAP_INTR_REGION_SIZE - 8;
                msg.data = (unsigned int)doorbell_template.word;
        } else {
                /* SQ regions are out, allocate from map mem regions. */
                mem_map = gxio_trio_alloc_memory_maps(trio_context, 1, 0, 0);
                if (mem_map < 0) {
                        dev_info(&pdev->dev, "%s Mem-Map alloc failure - failed to initialize MSI interrupts - falling back to legacy interrupts\n",
                                 desc->msi_attrib.is_msix ? "MSI-X" : "MSI");
                        ret = -ENOMEM;
                        goto msi_mem_map_alloc_failure;
                }

                mem_map_base = MEM_MAP_INTR_REGIONS_BASE +
                        mem_map * MEM_MAP_INTR_REGION_SIZE;
                mem_map_limit = mem_map_base + MEM_MAP_INTR_REGION_SIZE - 1;

                msi_addr = mem_map_base + TRIO_MAP_MEM_REG_INT3 -
                        TRIO_MAP_MEM_REG_INT0;

                msg.data = mem_map;
        }

        /* We try to distribute different IRQs to different tiles. */
        cpu = tile_irq_cpu(irq);

        /*
         * Now call up to the HV to configure the MSI interrupt and
         * set up the IPI binding.
         */
        ret = gxio_trio_config_msi_intr(trio_context, cpu_x(cpu), cpu_y(cpu),
                                        KERNEL_PL, irq, controller->mac,
                                        mem_map, mem_map_base, mem_map_limit,
                                        trio_context->asid);
        if (ret < 0) {
                dev_info(&pdev->dev, "HV MSI config failed\n");

                goto hv_msi_config_failure;
        }

        irq_set_msi_desc(irq, desc);

        msg.address_hi = msi_addr >> 32;
        msg.address_lo = msi_addr & 0xffffffff;

        pci_write_msi_msg(irq, &msg);
        irq_set_chip_and_handler(irq, &tilegx_msi_chip, handle_level_irq);
        irq_set_handler_data(irq, controller);

        return 0;

hv_msi_config_failure:
        /* Free mem-map */
msi_mem_map_alloc_failure:
is_64_failure:
        irq_free_hwirq(irq);
        return ret;
}

void arch_teardown_msi_irq(unsigned int irq)
{
        irq_free_hwirq(irq);
}