drivers/pci/pcie/aer/aerdrv_core.c

   1 /*
   2  * drivers/pci/pcie/aer/aerdrv_core.c
   3  *
   4  * This file is subject to the terms and conditions of the GNU General Public
   5  * License.  See the file "COPYING" in the main directory of this archive
   6  * for more details.
   7  *
   8  * This file implements the core part of PCI-Express AER. When an pci-express
   9  * error is delivered, an error message will be collected and printed to
  10  * console, then, an error recovery procedure will be executed by following
  11  * the pci error recovery rules.
  12  *
  13  * Copyright (C) 2006 Intel Corp.
  14  *      Tom Long Nguyen (tom.l.nguyen@intel.com)
  15  *      Zhang Yanmin (yanmin.zhang@intel.com)
  16  *
  17  */
  18
  19 #include <linux/module.h>
  20 #include <linux/pci.h>
  21 #include <linux/kernel.h>
  22 #include <linux/errno.h>
  23 #include <linux/pm.h>
  24 #include <linux/suspend.h>
  25 #include <linux/delay.h>
  26 #include <linux/slab.h>
  27 #include "aerdrv.h"
  28
  29 static int forceload;
  30 static int nosourceid;
  31 module_param(forceload, bool, 0);
  32 module_param(nosourceid, bool, 0);
  33
  34 int pci_enable_pcie_error_reporting(struct pci_dev *dev)
  35 {
  36         u16 reg16 = 0;
  37         int pos;
  38
  39         if (pcie_aer_get_firmware_first(dev))
  40                 return -EIO;
  41
  42         pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
  43         if (!pos)
  44                 return -EIO;
  45
  46         pos = pci_pcie_cap(dev);
  47         if (!pos)
  48                 return -EIO;
  49
  50         pci_read_config_word(dev, pos + PCI_EXP_DEVCTL, &reg16);
  51         reg16 |= (PCI_EXP_DEVCTL_CERE |
  52                 PCI_EXP_DEVCTL_NFERE |
  53                 PCI_EXP_DEVCTL_FERE |
  54                 PCI_EXP_DEVCTL_URRE);
  55         pci_write_config_word(dev, pos + PCI_EXP_DEVCTL, reg16);
  56
  57         return 0;
  58 }
  59 EXPORT_SYMBOL_GPL(pci_enable_pcie_error_reporting);
  60
  61 int pci_disable_pcie_error_reporting(struct pci_dev *dev)
  62 {
  63         u16 reg16 = 0;
  64         int pos;
  65
  66         if (pcie_aer_get_firmware_first(dev))
  67                 return -EIO;
  68
  69         pos = pci_pcie_cap(dev);
  70         if (!pos)
  71                 return -EIO;
  72
  73         pci_read_config_word(dev, pos + PCI_EXP_DEVCTL, &reg16);
  74         reg16 &= ~(PCI_EXP_DEVCTL_CERE |
  75                 PCI_EXP_DEVCTL_NFERE |
  76                 PCI_EXP_DEVCTL_FERE |
  77                 PCI_EXP_DEVCTL_URRE);
  78         pci_write_config_word(dev, pos + PCI_EXP_DEVCTL, reg16);
  79
  80         return 0;
  81 }
  82 EXPORT_SYMBOL_GPL(pci_disable_pcie_error_reporting);
  83
  84 int pci_cleanup_aer_uncorrect_error_status(struct pci_dev *dev)
  85 {
  86         int pos;
  87         u32 status;
  88
  89         pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
  90         if (!pos)
  91                 return -EIO;
  92
  93         pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
  94         if (status)
  95                 pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
  96
  97         return 0;
  98 }
  99 EXPORT_SYMBOL_GPL(pci_cleanup_aer_uncorrect_error_status);
 100
 101 /**
 102  * add_error_device - list device to be handled
 103  * @e_info: pointer to error info
 104  * @dev: pointer to pci_dev to be added
 105  */
 106 static int add_error_device(struct aer_err_info *e_info, struct pci_dev *dev)
 107 {
 108         if (e_info->error_dev_num < AER_MAX_MULTI_ERR_DEVICES) {
 109                 e_info->dev[e_info->error_dev_num] = dev;
 110                 e_info->error_dev_num++;
 111                 return 0;
 112         }
 113         return -ENOSPC;
 114 }
 115
 116 #define PCI_BUS(x)      (((x) >> 8) & 0xff)
 117
 118 /**
 119  * is_error_source - check whether the device is source of reported error
 120  * @dev: pointer to pci_dev to be checked
 121  * @e_info: pointer to reported error info
 122  */
 123 static bool is_error_source(struct pci_dev *dev, struct aer_err_info *e_info)
 124 {
 125         int pos;
 126         u32 status, mask;
 127         u16 reg16;
 128
 129         /*
 130          * When bus id is equal to 0, it might be a bad id
 131          * reported by root port.
 132          */
 133         if (!nosourceid && (PCI_BUS(e_info->id) != 0)) {
 134                 /* Device ID match? */
 135                 if (e_info->id == ((dev->bus->number << 8) | dev->devfn))
 136                         return true;
 137
 138                 /* Continue id comparing if there is no multiple error */
 139                 if (!e_info->multi_error_valid)
 140                         return false;
 141         }
 142
 143         /*
 144          * When either
 145          *      1) nosourceid==y;
 146          *      2) bus id is equal to 0. Some ports might lose the bus
 147          *              id of error source id;
 148          *      3) There are multiple errors and prior id comparing fails;
 149          * We check AER status registers to find possible reporter.
 150          */
 151         if (atomic_read(&dev->enable_cnt) == 0)
 152                 return false;
 153         pos = pci_pcie_cap(dev);
 154         if (!pos)
 155                 return false;
 156
 157         /* Check if AER is enabled */
 158         pci_read_config_word(dev, pos + PCI_EXP_DEVCTL, &reg16);
 159         if (!(reg16 & (
 160                 PCI_EXP_DEVCTL_CERE |
 161                 PCI_EXP_DEVCTL_NFERE |
 162                 PCI_EXP_DEVCTL_FERE |
 163                 PCI_EXP_DEVCTL_URRE)))
 164                 return false;
 165         pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
 166         if (!pos)
 167                 return false;
 168
 169         /* Check if error is recorded */
 170         if (e_info->severity == AER_CORRECTABLE) {
 171                 pci_read_config_dword(dev, pos + PCI_ERR_COR_STATUS, &status);
 172                 pci_read_config_dword(dev, pos + PCI_ERR_COR_MASK, &mask);
 173         } else {
 174                 pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
 175                 pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_MASK, &mask);
 176         }
 177         if (status & ~mask)
 178                 return true;
 179
 180         return false;
 181 }
 182
 183 static int find_device_iter(struct pci_dev *dev, void *data)
 184 {
 185         struct aer_err_info *e_info = (struct aer_err_info *)data;
 186
 187         if (is_error_source(dev, e_info)) {
 188                 /* List this device */
 189                 if (add_error_device(e_info, dev)) {
 190                         /* We cannot handle more... Stop iteration */
 191                         /* TODO: Should print error message here? */
 192                         return 1;
 193                 }
 194
 195                 /* If there is only a single error, stop iteration */
 196                 if (!e_info->multi_error_valid)
 197                         return 1;
 198         }
 199         return 0;
 200 }
 201
 202 /**
 203  * find_source_device - search through device hierarchy for source device
 204  * @parent: pointer to Root Port pci_dev data structure
 205  * @e_info: including detailed error information such like id
 206  *
 207  * Return true if found.
 208  *
 209  * Invoked by DPC when error is detected at the Root Port.
 210  * Caller of this function must set id, severity, and multi_error_valid of
 211  * struct aer_err_info pointed by @e_info properly.  This function must fill
 212  * e_info->error_dev_num and e_info->dev[], based on the given information.
 213  */
 214 static bool find_source_device(struct pci_dev *parent,
 215                 struct aer_err_info *e_info)
 216 {
 217         struct pci_dev *dev = parent;
 218         int result;
 219
 220         /* Must reset in this function */
 221         e_info->error_dev_num = 0;
 222
 223         /* Is Root Port an agent that sends error message? */
 224         result = find_device_iter(dev, e_info);
 225         if (result)
 226                 return true;
 227
 228         pci_walk_bus(parent->subordinate, find_device_iter, e_info);
 229
 230         if (!e_info->error_dev_num) {
 231                 dev_printk(KERN_DEBUG, &parent->dev,
 232                                 "can't find device of ID%04x\n",
 233                                 e_info->id);
 234                 return false;
 235         }
 236         return true;
 237 }
 238
 239 static int report_error_detected(struct pci_dev *dev, void *data)
 240 {
 241         pci_ers_result_t vote;
 242         struct pci_error_handlers *err_handler;
 243         struct aer_broadcast_data *result_data;
 244         result_data = (struct aer_broadcast_data *) data;
 245
 246         dev->error_state = result_data->state;
 247
 248         if (!dev->driver ||
 249                 !dev->driver->err_handler ||
 250                 !dev->driver->err_handler->error_detected) {
 251                 if (result_data->state == pci_channel_io_frozen &&
 252                         !(dev->hdr_type & PCI_HEADER_TYPE_BRIDGE)) {
 253                         /*
 254                          * In case of fatal recovery, if one of down-
 255                          * stream device has no driver. We might be
 256                          * unable to recover because a later insmod
 257                          * of a driver for this device is unaware of
 258                          * its hw state.
 259                          */
 260                         dev_printk(KERN_DEBUG, &dev->dev, "device has %s\n",
 261                                    dev->driver ?
 262                                    "no AER-aware driver" : "no driver");
 263                 }
 264                 return 0;
 265         }
 266
 267         err_handler = dev->driver->err_handler;
 268         vote = err_handler->error_detected(dev, result_data->state);
 269         result_data->result = merge_result(result_data->result, vote);
 270         return 0;
 271 }
 272
 273 static int report_mmio_enabled(struct pci_dev *dev, void *data)
 274 {
 275         pci_ers_result_t vote;
 276         struct pci_error_handlers *err_handler;
 277         struct aer_broadcast_data *result_data;
 278         result_data = (struct aer_broadcast_data *) data;
 279
 280         if (!dev->driver ||
 281                 !dev->driver->err_handler ||
 282                 !dev->driver->err_handler->mmio_enabled)
 283                 return 0;
 284
 285         err_handler = dev->driver->err_handler;
 286         vote = err_handler->mmio_enabled(dev);
 287         result_data->result = merge_result(result_data->result, vote);
 288         return 0;
 289 }
 290
 291 static int report_slot_reset(struct pci_dev *dev, void *data)
 292 {
 293         pci_ers_result_t vote;
 294         struct pci_error_handlers *err_handler;
 295         struct aer_broadcast_data *result_data;
 296         result_data = (struct aer_broadcast_data *) data;
 297
 298         if (!dev->driver ||
 299                 !dev->driver->err_handler ||
 300                 !dev->driver->err_handler->slot_reset)
 301                 return 0;
 302
 303         err_handler = dev->driver->err_handler;
 304         vote = err_handler->slot_reset(dev);
 305         result_data->result = merge_result(result_data->result, vote);
 306         return 0;
 307 }
 308
 309 static int report_resume(struct pci_dev *dev, void *data)
 310 {
 311         struct pci_error_handlers *err_handler;
 312
 313         dev->error_state = pci_channel_io_normal;
 314
 315         if (!dev->driver ||
 316                 !dev->driver->err_handler ||
 317                 !dev->driver->err_handler->resume)
 318                 return 0;
 319
 320         err_handler = dev->driver->err_handler;
 321         err_handler->resume(dev);
 322         return 0;
 323 }
 324
 325 /**
 326  * broadcast_error_message - handle message broadcast to downstream drivers
 327  * @dev: pointer to from where in a hierarchy message is broadcasted down
 328  * @state: error state
 329  * @error_mesg: message to print
 330  * @cb: callback to be broadcasted
 331  *
 332  * Invoked during error recovery process. Once being invoked, the content
 333  * of error severity will be broadcasted to all downstream drivers in a
 334  * hierarchy in question.
 335  */
 336 static pci_ers_result_t broadcast_error_message(struct pci_dev *dev,
 337         enum pci_channel_state state,
 338         char *error_mesg,
 339         int (*cb)(struct pci_dev *, void *))
 340 {
 341         struct aer_broadcast_data result_data;
 342
 343         dev_printk(KERN_DEBUG, &dev->dev, "broadcast %s message\n", error_mesg);
 344         result_data.state = state;
 345         if (cb == report_error_detected)
 346                 result_data.result = PCI_ERS_RESULT_CAN_RECOVER;
 347         else
 348                 result_data.result = PCI_ERS_RESULT_RECOVERED;
 349
 350         if (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE) {
 351                 /*
 352                  * If the error is reported by a bridge, we think this error
 353                  * is related to the downstream link of the bridge, so we
 354                  * do error recovery on all subordinates of the bridge instead
 355                  * of the bridge and clear the error status of the bridge.
 356                  */
 357                 if (cb == report_error_detected)
 358                         dev->error_state = state;
 359                 pci_walk_bus(dev->subordinate, cb, &result_data);
 360                 if (cb == report_resume) {
 361                         pci_cleanup_aer_uncorrect_error_status(dev);
 362                         dev->error_state = pci_channel_io_normal;
 363                 }
 364         } else {
 365                 /*
 366                  * If the error is reported by an end point, we think this
 367                  * error is related to the upstream link of the end point.
 368                  */
 369                 pci_walk_bus(dev->bus, cb, &result_data);
 370         }
 371
 372         return result_data.result;
 373 }
 374
 375 /**
 376  * aer_do_secondary_bus_reset - perform secondary bus reset
 377  * @dev: pointer to bridge's pci_dev data structure
 378  *
 379  * Invoked when performing link reset at Root Port or Downstream Port.
 380  */
 381 void aer_do_secondary_bus_reset(struct pci_dev *dev)
 382 {
 383         u16 p2p_ctrl;
 384
 385         /* Assert Secondary Bus Reset */
 386         pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &p2p_ctrl);
 387         p2p_ctrl |= PCI_BRIDGE_CTL_BUS_RESET;
 388         pci_write_config_word(dev, PCI_BRIDGE_CONTROL, p2p_ctrl);
 389
 390         /*
 391          * we should send hot reset message for 2ms to allow it time to
 392          * propagate to all downstream ports
 393          */
 394         msleep(2);
 395
 396         /* De-assert Secondary Bus Reset */
 397         p2p_ctrl &= ~PCI_BRIDGE_CTL_BUS_RESET;
 398         pci_write_config_word(dev, PCI_BRIDGE_CONTROL, p2p_ctrl);
 399
 400         /*
 401          * System software must wait for at least 100ms from the end
 402          * of a reset of one or more device before it is permitted
 403          * to issue Configuration Requests to those devices.
 404          */
 405         msleep(200);
 406 }
 407
 408 /**
 409  * default_downstream_reset_link - default reset function for Downstream Port
 410  * @dev: pointer to downstream port's pci_dev data structure
 411  *
 412  * Invoked when performing link reset at Downstream Port w/ no aer driver.
 413  */
 414 static pci_ers_result_t default_downstream_reset_link(struct pci_dev *dev)
 415 {
 416         aer_do_secondary_bus_reset(dev);
 417         dev_printk(KERN_DEBUG, &dev->dev,
 418                 "Downstream Port link has been reset\n");
 419         return PCI_ERS_RESULT_RECOVERED;
 420 }
 421
 422 static int find_aer_service_iter(struct device *device, void *data)
 423 {
 424         struct pcie_port_service_driver *service_driver, **drv;
 425
 426         drv = (struct pcie_port_service_driver **) data;
 427
 428         if (device->bus == &pcie_port_bus_type && device->driver) {
 429                 service_driver = to_service_driver(device->driver);
 430                 if (service_driver->service == PCIE_PORT_SERVICE_AER) {
 431                         *drv = service_driver;
 432                         return 1;
 433                 }
 434         }
 435
 436         return 0;
 437 }
 438
 439 static struct pcie_port_service_driver *find_aer_service(struct pci_dev *dev)
 440 {
 441         struct pcie_port_service_driver *drv = NULL;
 442
 443         device_for_each_child(&dev->dev, &drv, find_aer_service_iter);
 444
 445         return drv;
 446 }
 447
 448 static pci_ers_result_t reset_link(struct pcie_device *aerdev,
 449                 struct pci_dev *dev)
 450 {
 451         struct pci_dev *udev;
 452         pci_ers_result_t status;
 453         struct pcie_port_service_driver *driver;
 454
 455         if (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE) {
 456                 /* Reset this port for all subordinates */
 457                 udev = dev;
 458         } else {
 459                 /* Reset the upstream component (likely downstream port) */
 460                 udev = dev->bus->self;
 461         }
 462
 463         /* Use the aer driver of the component firstly */
 464         driver = find_aer_service(udev);
 465
 466         if (driver && driver->reset_link) {
 467                 status = driver->reset_link(udev);
 468         } else if (udev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM) {
 469                 status = default_downstream_reset_link(udev);
 470         } else {
 471                 dev_printk(KERN_DEBUG, &dev->dev,
 472                         "no link-reset support at upstream device %s\n",
 473                         pci_name(udev));
 474                 return PCI_ERS_RESULT_DISCONNECT;
 475         }
 476
 477         if (status != PCI_ERS_RESULT_RECOVERED) {
 478                 dev_printk(KERN_DEBUG, &dev->dev,
 479                         "link reset at upstream device %s failed\n",
 480                         pci_name(udev));
 481                 return PCI_ERS_RESULT_DISCONNECT;
 482         }
 483
 484         return status;
 485 }
 486
 487 /**
 488  * do_recovery - handle nonfatal/fatal error recovery process
 489  * @aerdev: pointer to a pcie_device data structure of root port
 490  * @dev: pointer to a pci_dev data structure of agent detecting an error
 491  * @severity: error severity type
 492  *
 493  * Invoked when an error is nonfatal/fatal. Once being invoked, broadcast
 494  * error detected message to all downstream drivers within a hierarchy in
 495  * question and return the returned code.
 496  */
 497 static void do_recovery(struct pcie_device *aerdev, struct pci_dev *dev,
 498                 int severity)
 499 {
 500         pci_ers_result_t status, result = PCI_ERS_RESULT_RECOVERED;
 501         enum pci_channel_state state;
 502
 503         if (severity == AER_FATAL)
 504                 state = pci_channel_io_frozen;
 505         else
 506                 state = pci_channel_io_normal;
 507
 508         status = broadcast_error_message(dev,
 509                         state,
 510                         "error_detected",
 511                         report_error_detected);
 512
 513         if (severity == AER_FATAL) {
 514                 result = reset_link(aerdev, dev);
 515                 if (result != PCI_ERS_RESULT_RECOVERED)
 516                         goto failed;
 517         }
 518
 519         if (status == PCI_ERS_RESULT_CAN_RECOVER)
 520                 status = broadcast_error_message(dev,
 521                                 state,
 522                                 "mmio_enabled",
 523                                 report_mmio_enabled);
 524
 525         if (status == PCI_ERS_RESULT_NEED_RESET) {
 526                 /*
 527                  * TODO: Should call platform-specific
 528                  * functions to reset slot before calling
 529                  * drivers' slot_reset callbacks?
 530                  */
 531                 status = broadcast_error_message(dev,
 532                                 state,
 533                                 "slot_reset",
 534                                 report_slot_reset);
 535         }
 536
 537         if (status != PCI_ERS_RESULT_RECOVERED)
 538                 goto failed;
 539
 540         broadcast_error_message(dev,
 541                                 state,
 542                                 "resume",
 543                                 report_resume);
 544
 545         dev_printk(KERN_DEBUG, &dev->dev,
 546                 "AER driver successfully recovered\n");
 547         return;
 548
 549 failed:
 550         /* TODO: Should kernel panic here? */
 551         dev_printk(KERN_DEBUG, &dev->dev,
 552                 "AER driver didn't recover\n");
 553 }
 554
 555 /**
 556  * handle_error_source - handle logging error into an event log
 557  * @aerdev: pointer to pcie_device data structure of the root port
 558  * @dev: pointer to pci_dev data structure of error source device
 559  * @info: comprehensive error information
 560  *
 561  * Invoked when an error being detected by Root Port.
 562  */
 563 static void handle_error_source(struct pcie_device *aerdev,
 564         struct pci_dev *dev,
 565         struct aer_err_info *info)
 566 {
 567         int pos;
 568
 569         if (info->severity == AER_CORRECTABLE) {
 570                 /*
 571                  * Correctable error does not need software intevention.
 572                  * No need to go through error recovery process.
 573                  */
 574                 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
 575                 if (pos)
 576                         pci_write_config_dword(dev, pos + PCI_ERR_COR_STATUS,
 577                                         info->status);
 578         } else
 579                 do_recovery(aerdev, dev, info->severity);
 580 }
 581
 582 /**
 583  * get_device_error_info - read error status from dev and store it to info
 584  * @dev: pointer to the device expected to have a error record
 585  * @info: pointer to structure to store the error record
 586  *
 587  * Return 1 on success, 0 on error.
 588  *
 589  * Note that @info is reused among all error devices. Clear fields properly.
 590  */
 591 static int get_device_error_info(struct pci_dev *dev, struct aer_err_info *info)
 592 {
 593         int pos, temp;
 594
 595         /* Must reset in this function */
 596         info->status = 0;
 597         info->tlp_header_valid = 0;
 598
 599         pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
 600
 601         /* The device might not support AER */
 602         if (!pos)
 603                 return 1;
 604
 605         if (info->severity == AER_CORRECTABLE) {
 606                 pci_read_config_dword(dev, pos + PCI_ERR_COR_STATUS,
 607                         &info->status);
 608                 pci_read_config_dword(dev, pos + PCI_ERR_COR_MASK,
 609                         &info->mask);
 610                 if (!(info->status & ~info->mask))
 611                         return 0;
 612         } else if (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE ||
 613                 info->severity == AER_NONFATAL) {
 614
 615                 /* Link is still healthy for IO reads */
 616                 pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS,
 617                         &info->status);
 618                 pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_MASK,
 619                         &info->mask);
 620                 if (!(info->status & ~info->mask))
 621                         return 0;
 622
 623                 /* Get First Error Pointer */
 624                 pci_read_config_dword(dev, pos + PCI_ERR_CAP, &temp);
 625                 info->first_error = PCI_ERR_CAP_FEP(temp);
 626
 627                 if (info->status & AER_LOG_TLP_MASKS) {
 628                         info->tlp_header_valid = 1;
 629                         pci_read_config_dword(dev,
 630                                 pos + PCI_ERR_HEADER_LOG, &info->tlp.dw0);
 631                         pci_read_config_dword(dev,
 632                                 pos + PCI_ERR_HEADER_LOG + 4, &info->tlp.dw1);
 633                         pci_read_config_dword(dev,
 634                                 pos + PCI_ERR_HEADER_LOG + 8, &info->tlp.dw2);
 635                         pci_read_config_dword(dev,
 636                                 pos + PCI_ERR_HEADER_LOG + 12, &info->tlp.dw3);
 637                 }
 638         }
 639
 640         return 1;
 641 }
 642
 643 static inline void aer_process_err_devices(struct pcie_device *p_device,
 644                         struct aer_err_info *e_info)
 645 {
 646         int i;
 647
 648         /* Report all before handle them, not to lost records by reset etc. */
 649         for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) {
 650                 if (get_device_error_info(e_info->dev[i], e_info))
 651                         aer_print_error(e_info->dev[i], e_info);
 652         }
 653         for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) {
 654                 if (get_device_error_info(e_info->dev[i], e_info))
 655                         handle_error_source(p_device, e_info->dev[i], e_info);
 656         }
 657 }
 658
 659 /**
 660  * aer_isr_one_error - consume an error detected by root port
 661  * @p_device: pointer to error root port service device
 662  * @e_src: pointer to an error source
 663  */
 664 static void aer_isr_one_error(struct pcie_device *p_device,
 665                 struct aer_err_source *e_src)
 666 {
 667         struct aer_err_info *e_info;
 668
 669         /* struct aer_err_info might be big, so we allocate it with slab */
 670         e_info = kmalloc(sizeof(struct aer_err_info), GFP_KERNEL);
 671         if (!e_info) {
 672                 dev_printk(KERN_DEBUG, &p_device->port->dev,
 673                         "Can't allocate mem when processing AER errors\n");
 674                 return;
 675         }
 676
 677         /*
 678          * There is a possibility that both correctable error and
 679          * uncorrectable error being logged. Report correctable error first.
 680          */
 681         if (e_src->status & PCI_ERR_ROOT_COR_RCV) {
 682                 e_info->id = ERR_COR_ID(e_src->id);
 683                 e_info->severity = AER_CORRECTABLE;
 684
 685                 if (e_src->status & PCI_ERR_ROOT_MULTI_COR_RCV)
 686                         e_info->multi_error_valid = 1;
 687                 else
 688                         e_info->multi_error_valid = 0;
 689
 690                 aer_print_port_info(p_device->port, e_info);
 691
 692                 if (find_source_device(p_device->port, e_info))
 693                         aer_process_err_devices(p_device, e_info);
 694         }
 695
 696         if (e_src->status & PCI_ERR_ROOT_UNCOR_RCV) {
 697                 e_info->id = ERR_UNCOR_ID(e_src->id);
 698
 699                 if (e_src->status & PCI_ERR_ROOT_FATAL_RCV)
 700                         e_info->severity = AER_FATAL;
 701                 else
 702                         e_info->severity = AER_NONFATAL;
 703
 704                 if (e_src->status & PCI_ERR_ROOT_MULTI_UNCOR_RCV)
 705                         e_info->multi_error_valid = 1;
 706                 else
 707                         e_info->multi_error_valid = 0;
 708
 709                 aer_print_port_info(p_device->port, e_info);
 710
 711                 if (find_source_device(p_device->port, e_info))
 712                         aer_process_err_devices(p_device, e_info);
 713         }
 714
 715         kfree(e_info);
 716 }
 717
 718 /**
 719  * get_e_source - retrieve an error source
 720  * @rpc: pointer to the root port which holds an error
 721  * @e_src: pointer to store retrieved error source
 722  *
 723  * Return 1 if an error source is retrieved, otherwise 0.
 724  *
 725  * Invoked by DPC handler to consume an error.
 726  */
 727 static int get_e_source(struct aer_rpc *rpc, struct aer_err_source *e_src)
 728 {
 729         unsigned long flags;
 730
 731         /* Lock access to Root error producer/consumer index */
 732         spin_lock_irqsave(&rpc->e_lock, flags);
 733         if (rpc->prod_idx == rpc->cons_idx) {
 734                 spin_unlock_irqrestore(&rpc->e_lock, flags);
 735                 return 0;
 736         }
 737
 738         *e_src = rpc->e_sources[rpc->cons_idx];
 739         rpc->cons_idx++;
 740         if (rpc->cons_idx == AER_ERROR_SOURCES_MAX)
 741                 rpc->cons_idx = 0;
 742         spin_unlock_irqrestore(&rpc->e_lock, flags);
 743
 744         return 1;
 745 }
 746
 747 /**
 748  * aer_isr - consume errors detected by root port
 749  * @work: definition of this work item
 750  *
 751  * Invoked, as DPC, when root port records new detected error
 752  */
 753 void aer_isr(struct work_struct *work)
 754 {
 755         struct aer_rpc *rpc = container_of(work, struct aer_rpc, dpc_handler);
 756         struct pcie_device *p_device = rpc->rpd;
 757         struct aer_err_source uninitialized_var(e_src);
 758
 759         mutex_lock(&rpc->rpc_mutex);
 760         while (get_e_source(rpc, &e_src))
 761                 aer_isr_one_error(p_device, &e_src);
 762         mutex_unlock(&rpc->rpc_mutex);
 763
 764         wake_up(&rpc->wait_release);
 765 }
 766
 767 /**
 768  * aer_init - provide AER initialization
 769  * @dev: pointer to AER pcie device
 770  *
 771  * Invoked when AER service driver is loaded.
 772  */
 773 int aer_init(struct pcie_device *dev)
 774 {
 775         if (forceload) {
 776                 dev_printk(KERN_DEBUG, &dev->device,
 777                            "aerdrv forceload requested.\n");
 778                 pcie_aer_force_firmware_first(dev->port, 0);
 779         }
 780         return 0;
 781 }