?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
workqueue: Make worker_attach/detach_pool() update worker->pool
[linux/fpc-iii.git] / drivers / edac / thunderx_edac.c
blob4803c6468bab4d18eafe65514344c2de02dd909c
1 /*
2 * Cavium ThunderX memory controller kernel module
3 *
4 * This file is subject to the terms and conditions of the GNU General Public
5 * License. See the file "COPYING" in the main directory of this archive
6 * for more details.
7 *
8 * Copyright Cavium, Inc. (C) 2015-2017. All rights reserved.
9 *
10 */
11
12 #include <linux/module.h>
13 #include <linux/pci.h>
14 #include <linux/edac.h>
15 #include <linux/interrupt.h>
16 #include <linux/string.h>
17 #include <linux/stop_machine.h>
18 #include <linux/delay.h>
19 #include <linux/sizes.h>
20 #include <linux/atomic.h>
21 #include <linux/bitfield.h>
22 #include <linux/circ_buf.h>
23
24 #include <asm/page.h>
25
26 #include "edac_module.h"
27
28 #define phys_to_pfn(phys) (PFN_DOWN(phys))
29
30 #define THUNDERX_NODE GENMASK(45, 44)
31
32 enum {
33 ERR_CORRECTED = 1,
34 ERR_UNCORRECTED = 2,
35 ERR_UNKNOWN = 3,
36 };
37
38 #define MAX_SYNDROME_REGS 4
39
40 struct error_syndrome {
41 u64 reg[MAX_SYNDROME_REGS];
42 };
43
44 struct error_descr {
45 int type;
46 u64 mask;
47 char *descr;
48 };
49
50 static void decode_register(char *str, size_t size,
51 const struct error_descr *descr,
52 const uint64_t reg)
53 {
54 int ret = 0;
55
56 while (descr->type && descr->mask && descr->descr) {
57 if (reg & descr->mask) {
58 ret = snprintf(str, size, "\n\t%s, %s",
59 descr->type == ERR_CORRECTED ?
60 "Corrected" : "Uncorrected",
61 descr->descr);
62 str += ret;
63 size -= ret;
64 }
65 descr++;
66 }
67 }
68
69 static unsigned long get_bits(unsigned long data, int pos, int width)
70 {
71 return (data >> pos) & ((1 << width) - 1);
72 }
73
74 #define L2C_CTL 0x87E080800000
75 #define L2C_CTL_DISIDXALIAS BIT(0)
76
77 #define PCI_DEVICE_ID_THUNDER_LMC 0xa022
78
79 #define LMC_FADR 0x20
80 #define LMC_FADR_FDIMM(x) ((x >> 37) & 0x1)
81 #define LMC_FADR_FBUNK(x) ((x >> 36) & 0x1)
82 #define LMC_FADR_FBANK(x) ((x >> 32) & 0xf)
83 #define LMC_FADR_FROW(x) ((x >> 14) & 0xffff)
84 #define LMC_FADR_FCOL(x) ((x >> 0) & 0x1fff)
85
86 #define LMC_NXM_FADR 0x28
87 #define LMC_ECC_SYND 0x38
88
89 #define LMC_ECC_PARITY_TEST 0x108
90
91 #define LMC_INT_W1S 0x150
92
93 #define LMC_INT_ENA_W1C 0x158
94 #define LMC_INT_ENA_W1S 0x160
95
96 #define LMC_CONFIG 0x188
97
98 #define LMC_CONFIG_BG2 BIT(62)
99 #define LMC_CONFIG_RANK_ENA BIT(42)
100 #define LMC_CONFIG_PBANK_LSB(x) (((x) >> 5) & 0xF)
101 #define LMC_CONFIG_ROW_LSB(x) (((x) >> 2) & 0x7)
102
103 #define LMC_CONTROL 0x190
104 #define LMC_CONTROL_XOR_BANK BIT(16)
105
106 #define LMC_INT 0x1F0
107
108 #define LMC_INT_DDR_ERR BIT(11)
109 #define LMC_INT_DED_ERR (0xFUL << 5)
110 #define LMC_INT_SEC_ERR (0xFUL << 1)
111 #define LMC_INT_NXM_WR_MASK BIT(0)
112
113 #define LMC_DDR_PLL_CTL 0x258
114 #define LMC_DDR_PLL_CTL_DDR4 BIT(29)
115
116 #define LMC_FADR_SCRAMBLED 0x330
117
118 #define LMC_INT_UE (LMC_INT_DDR_ERR | LMC_INT_DED_ERR | \
119 LMC_INT_NXM_WR_MASK)
120
121 #define LMC_INT_CE (LMC_INT_SEC_ERR)
122
123 static const struct error_descr lmc_errors[] = {
124 {
125 .type = ERR_CORRECTED,
126 .mask = LMC_INT_SEC_ERR,
127 .descr = "Single-bit ECC error",
128 },
129 {
130 .type = ERR_UNCORRECTED,
131 .mask = LMC_INT_DDR_ERR,
132 .descr = "DDR chip error",
133 },
134 {
135 .type = ERR_UNCORRECTED,
136 .mask = LMC_INT_DED_ERR,
137 .descr = "Double-bit ECC error",
138 },
139 {
140 .type = ERR_UNCORRECTED,
141 .mask = LMC_INT_NXM_WR_MASK,
142 .descr = "Non-existent memory write",
143 },
144 {0, 0, NULL},
145 };
146
147 #define LMC_INT_EN_DDR_ERROR_ALERT_ENA BIT(5)
148 #define LMC_INT_EN_DLCRAM_DED_ERR BIT(4)
149 #define LMC_INT_EN_DLCRAM_SEC_ERR BIT(3)
150 #define LMC_INT_INTR_DED_ENA BIT(2)
151 #define LMC_INT_INTR_SEC_ENA BIT(1)
152 #define LMC_INT_INTR_NXM_WR_ENA BIT(0)
153
154 #define LMC_INT_ENA_ALL GENMASK(5, 0)
155
156 #define LMC_DDR_PLL_CTL 0x258
157 #define LMC_DDR_PLL_CTL_DDR4 BIT(29)
158
159 #define LMC_CONTROL 0x190
160 #define LMC_CONTROL_RDIMM BIT(0)
161
162 #define LMC_SCRAM_FADR 0x330
163
164 #define LMC_CHAR_MASK0 0x228
165 #define LMC_CHAR_MASK2 0x238
166
167 #define RING_ENTRIES 8
168
169 struct debugfs_entry {
170 const char *name;
171 umode_t mode;
172 const struct file_operations fops;
173 };
174
175 struct lmc_err_ctx {
176 u64 reg_int;
177 u64 reg_fadr;
178 u64 reg_nxm_fadr;
179 u64 reg_scram_fadr;
180 u64 reg_ecc_synd;
181 };
182
183 struct thunderx_lmc {
184 void __iomem *regs;
185 struct pci_dev *pdev;
186 struct msix_entry msix_ent;
187
188 atomic_t ecc_int;
189
190 u64 mask0;
191 u64 mask2;
192 u64 parity_test;
193 u64 node;
194
195 int xbits;
196 int bank_width;
197 int pbank_lsb;
198 int dimm_lsb;
199 int rank_lsb;
200 int bank_lsb;
201 int row_lsb;
202 int col_hi_lsb;
203
204 int xor_bank;
205 int l2c_alias;
206
207 struct page *mem;
208
209 struct lmc_err_ctx err_ctx[RING_ENTRIES];
210 unsigned long ring_head;
211 unsigned long ring_tail;
212 };
213
214 #define ring_pos(pos, size) ((pos) & (size - 1))
215
216 #define DEBUGFS_STRUCT(_name, _mode, _write, _read) \
217 static struct debugfs_entry debugfs_##_name = { \
218 .name = __stringify(_name), \
219 .mode = VERIFY_OCTAL_PERMISSIONS(_mode), \
220 .fops = { \
221 .open = simple_open, \
222 .write = _write, \
223 .read = _read, \
224 .llseek = generic_file_llseek, \
225 }, \
226 }
227
228 #define DEBUGFS_FIELD_ATTR(_type, _field) \
229 static ssize_t thunderx_##_type##_##_field##_read(struct file *file, \
230 char __user *data, \
231 size_t count, loff_t *ppos) \
232 { \
233 struct thunderx_##_type *pdata = file->private_data; \
234 char buf[20]; \
235 \
236 snprintf(buf, count, "0x%016llx", pdata->_field); \
237 return simple_read_from_buffer(data, count, ppos, \
238 buf, sizeof(buf)); \
239 } \
240 \
241 static ssize_t thunderx_##_type##_##_field##_write(struct file *file, \
242 const char __user *data, \
243 size_t count, loff_t *ppos) \
244 { \
245 struct thunderx_##_type *pdata = file->private_data; \
246 int res; \
247 \
248 res = kstrtoull_from_user(data, count, 0, &pdata->_field); \
249 \
250 return res ? res : count; \
251 } \
252 \
253 DEBUGFS_STRUCT(_field, 0600, \
254 thunderx_##_type##_##_field##_write, \
255 thunderx_##_type##_##_field##_read) \
256
257 #define DEBUGFS_REG_ATTR(_type, _name, _reg) \
258 static ssize_t thunderx_##_type##_##_name##_read(struct file *file, \
259 char __user *data, \
260 size_t count, loff_t *ppos) \
261 { \
262 struct thunderx_##_type *pdata = file->private_data; \
263 char buf[20]; \
264 \
265 sprintf(buf, "0x%016llx", readq(pdata->regs + _reg)); \
266 return simple_read_from_buffer(data, count, ppos, \
267 buf, sizeof(buf)); \
268 } \
269 \
270 static ssize_t thunderx_##_type##_##_name##_write(struct file *file, \
271 const char __user *data, \
272 size_t count, loff_t *ppos) \
273 { \
274 struct thunderx_##_type *pdata = file->private_data; \
275 u64 val; \
276 int res; \
277 \
278 res = kstrtoull_from_user(data, count, 0, &val); \
279 \
280 if (!res) { \
281 writeq(val, pdata->regs + _reg); \
282 res = count; \
283 } \
284 \
285 return res; \
286 } \
287 \
288 DEBUGFS_STRUCT(_name, 0600, \
289 thunderx_##_type##_##_name##_write, \
290 thunderx_##_type##_##_name##_read)
291
292 #define LMC_DEBUGFS_ENT(_field) DEBUGFS_FIELD_ATTR(lmc, _field)
293
294 /*
295 * To get an ECC error injected, the following steps are needed:
296 * - Setup the ECC injection by writing the appropriate parameters:
297 * echo <bit mask value> > /sys/kernel/debug/<device number>/ecc_mask0
298 * echo <bit mask value> > /sys/kernel/debug/<device number>/ecc_mask2
299 * echo 0x802 > /sys/kernel/debug/<device number>/ecc_parity_test
300 * - Do the actual injection:
301 * echo 1 > /sys/kernel/debug/<device number>/inject_ecc
302 */
303 static ssize_t thunderx_lmc_inject_int_write(struct file *file,
304 const char __user *data,
305 size_t count, loff_t *ppos)
306 {
307 struct thunderx_lmc *lmc = file->private_data;
308 u64 val;
309 int res;
310
311 res = kstrtoull_from_user(data, count, 0, &val);
312
313 if (!res) {
314 /* Trigger the interrupt */
315 writeq(val, lmc->regs + LMC_INT_W1S);
316 res = count;
317 }
318
319 return res;
320 }
321
322 static ssize_t thunderx_lmc_int_read(struct file *file,
323 char __user *data,
324 size_t count, loff_t *ppos)
325 {
326 struct thunderx_lmc *lmc = file->private_data;
327 char buf[20];
328 u64 lmc_int = readq(lmc->regs + LMC_INT);
329
330 snprintf(buf, sizeof(buf), "0x%016llx", lmc_int);
331 return simple_read_from_buffer(data, count, ppos, buf, sizeof(buf));
332 }
333
334 #define TEST_PATTERN 0xa5
335
336 static int inject_ecc_fn(void *arg)
337 {
338 struct thunderx_lmc *lmc = arg;
339 uintptr_t addr, phys;
340 unsigned int cline_size = cache_line_size();
341 const unsigned int lines = PAGE_SIZE / cline_size;
342 unsigned int i, cl_idx;
343
344 addr = (uintptr_t)page_address(lmc->mem);
345 phys = (uintptr_t)page_to_phys(lmc->mem);
346
347 cl_idx = (phys & 0x7f) >> 4;
348 lmc->parity_test &= ~(7ULL << 8);
349 lmc->parity_test |= (cl_idx << 8);
350
351 writeq(lmc->mask0, lmc->regs + LMC_CHAR_MASK0);
352 writeq(lmc->mask2, lmc->regs + LMC_CHAR_MASK2);
353 writeq(lmc->parity_test, lmc->regs + LMC_ECC_PARITY_TEST);
354
355 readq(lmc->regs + LMC_CHAR_MASK0);
356 readq(lmc->regs + LMC_CHAR_MASK2);
357 readq(lmc->regs + LMC_ECC_PARITY_TEST);
358
359 for (i = 0; i < lines; i++) {
360 memset((void *)addr, TEST_PATTERN, cline_size);
361 barrier();
362
363 /*
364 * Flush L1 cachelines to the PoC (L2).
365 * This will cause cacheline eviction to the L2.
366 */
367 asm volatile("dc civac, %0\n"
368 "dsb sy\n"
369 : : "r"(addr + i * cline_size));
370 }
371
372 for (i = 0; i < lines; i++) {
373 /*
374 * Flush L2 cachelines to the DRAM.
375 * This will cause cacheline eviction to the DRAM
376 * and ECC corruption according to the masks set.
377 */
378 __asm__ volatile("sys #0,c11,C1,#2, %0\n"
379 : : "r"(phys + i * cline_size));
380 }
381
382 for (i = 0; i < lines; i++) {
383 /*
384 * Invalidate L2 cachelines.
385 * The subsequent load will cause cacheline fetch
386 * from the DRAM and an error interrupt
387 */
388 __asm__ volatile("sys #0,c11,C1,#1, %0"
389 : : "r"(phys + i * cline_size));
390 }
391
392 for (i = 0; i < lines; i++) {
393 /*
394 * Invalidate L1 cachelines.
395 * The subsequent load will cause cacheline fetch
396 * from the L2 and/or DRAM
397 */
398 asm volatile("dc ivac, %0\n"
399 "dsb sy\n"
400 : : "r"(addr + i * cline_size));
401 }
402
403 return 0;
404 }
405
406 static ssize_t thunderx_lmc_inject_ecc_write(struct file *file,
407 const char __user *data,
408 size_t count, loff_t *ppos)
409 {
410 struct thunderx_lmc *lmc = file->private_data;
411
412 unsigned int cline_size = cache_line_size();
413
414 u8 tmp[cline_size];
415 void __iomem *addr;
416 unsigned int offs, timeout = 100000;
417
418 atomic_set(&lmc->ecc_int, 0);
419
420 lmc->mem = alloc_pages_node(lmc->node, GFP_KERNEL, 0);
421
422 if (!lmc->mem)
423 return -ENOMEM;
424
425 addr = page_address(lmc->mem);
426
427 while (!atomic_read(&lmc->ecc_int) && timeout--) {
428 stop_machine(inject_ecc_fn, lmc, NULL);
429
430 for (offs = 0; offs < PAGE_SIZE; offs += sizeof(tmp)) {
431 /*
432 * Do a load from the previously rigged location
433 * This should generate an error interrupt.
434 */
435 memcpy(tmp, addr + offs, cline_size);
436 asm volatile("dsb ld\n");
437 }
438 }
439
440 __free_pages(lmc->mem, 0);
441
442 return count;
443 }
444
445 LMC_DEBUGFS_ENT(mask0);
446 LMC_DEBUGFS_ENT(mask2);
447 LMC_DEBUGFS_ENT(parity_test);
448
449 DEBUGFS_STRUCT(inject_int, 0200, thunderx_lmc_inject_int_write, NULL);
450 DEBUGFS_STRUCT(inject_ecc, 0200, thunderx_lmc_inject_ecc_write, NULL);
451 DEBUGFS_STRUCT(int_w1c, 0400, NULL, thunderx_lmc_int_read);
452
453 struct debugfs_entry *lmc_dfs_ents[] = {
454 &debugfs_mask0,
455 &debugfs_mask2,
456 &debugfs_parity_test,
457 &debugfs_inject_ecc,
458 &debugfs_inject_int,
459 &debugfs_int_w1c,
460 };
461
462 static int thunderx_create_debugfs_nodes(struct dentry *parent,
463 struct debugfs_entry *attrs[],
464 void *data,
465 size_t num)
466 {
467 int i;
468 struct dentry *ent;
469
470 if (!IS_ENABLED(CONFIG_EDAC_DEBUG))
471 return 0;
472
473 if (!parent)
474 return -ENOENT;
475
476 for (i = 0; i < num; i++) {
477 ent = edac_debugfs_create_file(attrs[i]->name, attrs[i]->mode,
478 parent, data, &attrs[i]->fops);
479
480 if (!ent)
481 break;
482 }
483
484 return i;
485 }
486
487 static phys_addr_t thunderx_faddr_to_phys(u64 faddr, struct thunderx_lmc *lmc)
488 {
489 phys_addr_t addr = 0;
490 int bank, xbits;
491
492 addr |= lmc->node << 40;
493 addr |= LMC_FADR_FDIMM(faddr) << lmc->dimm_lsb;
494 addr |= LMC_FADR_FBUNK(faddr) << lmc->rank_lsb;
495 addr |= LMC_FADR_FROW(faddr) << lmc->row_lsb;
496 addr |= (LMC_FADR_FCOL(faddr) >> 4) << lmc->col_hi_lsb;
497
498 bank = LMC_FADR_FBANK(faddr) << lmc->bank_lsb;
499
500 if (lmc->xor_bank)
501 bank ^= get_bits(addr, 12 + lmc->xbits, lmc->bank_width);
502
503 addr |= bank << lmc->bank_lsb;
504
505 xbits = PCI_FUNC(lmc->pdev->devfn);
506
507 if (lmc->l2c_alias)
508 xbits ^= get_bits(addr, 20, lmc->xbits) ^
509 get_bits(addr, 12, lmc->xbits);
510
511 addr |= xbits << 7;
512
513 return addr;
514 }
515
516 static unsigned int thunderx_get_num_lmcs(unsigned int node)
517 {
518 unsigned int number = 0;
519 struct pci_dev *pdev = NULL;
520
521 do {
522 pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
523 PCI_DEVICE_ID_THUNDER_LMC,
524 pdev);
525 if (pdev) {
526 #ifdef CONFIG_NUMA
527 if (pdev->dev.numa_node == node)
528 number++;
529 #else
530 number++;
531 #endif
532 }
533 } while (pdev);
534
535 return number;
536 }
537
538 #define LMC_MESSAGE_SIZE 120
539 #define LMC_OTHER_SIZE (50 * ARRAY_SIZE(lmc_errors))
540
541 static irqreturn_t thunderx_lmc_err_isr(int irq, void *dev_id)
542 {
543 struct mem_ctl_info *mci = dev_id;
544 struct thunderx_lmc *lmc = mci->pvt_info;
545
546 unsigned long head = ring_pos(lmc->ring_head, ARRAY_SIZE(lmc->err_ctx));
547 struct lmc_err_ctx *ctx = &lmc->err_ctx[head];
548
549 writeq(0, lmc->regs + LMC_CHAR_MASK0);
550 writeq(0, lmc->regs + LMC_CHAR_MASK2);
551 writeq(0x2, lmc->regs + LMC_ECC_PARITY_TEST);
552
553 ctx->reg_int = readq(lmc->regs + LMC_INT);
554 ctx->reg_fadr = readq(lmc->regs + LMC_FADR);
555 ctx->reg_nxm_fadr = readq(lmc->regs + LMC_NXM_FADR);
556 ctx->reg_scram_fadr = readq(lmc->regs + LMC_SCRAM_FADR);
557 ctx->reg_ecc_synd = readq(lmc->regs + LMC_ECC_SYND);
558
559 lmc->ring_head++;
560
561 atomic_set(&lmc->ecc_int, 1);
562
563 /* Clear the interrupt */
564 writeq(ctx->reg_int, lmc->regs + LMC_INT);
565
566 return IRQ_WAKE_THREAD;
567 }
568
569 static irqreturn_t thunderx_lmc_threaded_isr(int irq, void *dev_id)
570 {
571 struct mem_ctl_info *mci = dev_id;
572 struct thunderx_lmc *lmc = mci->pvt_info;
573 phys_addr_t phys_addr;
574
575 unsigned long tail;
576 struct lmc_err_ctx *ctx;
577
578 irqreturn_t ret = IRQ_NONE;
579
580 char *msg;
581 char *other;
582
583 msg = kmalloc(LMC_MESSAGE_SIZE, GFP_KERNEL);
584 other = kmalloc(LMC_OTHER_SIZE, GFP_KERNEL);
585
586 if (!msg || !other)
587 goto err_free;
588
589 while (CIRC_CNT(lmc->ring_head, lmc->ring_tail,
590 ARRAY_SIZE(lmc->err_ctx))) {
591 tail = ring_pos(lmc->ring_tail, ARRAY_SIZE(lmc->err_ctx));
592
593 ctx = &lmc->err_ctx[tail];
594
595 dev_dbg(&lmc->pdev->dev, "LMC_INT: %016llx\n",
596 ctx->reg_int);
597 dev_dbg(&lmc->pdev->dev, "LMC_FADR: %016llx\n",
598 ctx->reg_fadr);
599 dev_dbg(&lmc->pdev->dev, "LMC_NXM_FADR: %016llx\n",
600 ctx->reg_nxm_fadr);
601 dev_dbg(&lmc->pdev->dev, "LMC_SCRAM_FADR: %016llx\n",
602 ctx->reg_scram_fadr);
603 dev_dbg(&lmc->pdev->dev, "LMC_ECC_SYND: %016llx\n",
604 ctx->reg_ecc_synd);
605
606 snprintf(msg, LMC_MESSAGE_SIZE,
607 "DIMM %lld rank %lld bank %lld row %lld col %lld",
608 LMC_FADR_FDIMM(ctx->reg_scram_fadr),
609 LMC_FADR_FBUNK(ctx->reg_scram_fadr),
610 LMC_FADR_FBANK(ctx->reg_scram_fadr),
611 LMC_FADR_FROW(ctx->reg_scram_fadr),
612 LMC_FADR_FCOL(ctx->reg_scram_fadr));
613
614 decode_register(other, LMC_OTHER_SIZE, lmc_errors,
615 ctx->reg_int);
616
617 phys_addr = thunderx_faddr_to_phys(ctx->reg_fadr, lmc);
618
619 if (ctx->reg_int & LMC_INT_UE)
620 edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci, 1,
621 phys_to_pfn(phys_addr),
622 offset_in_page(phys_addr),
623 0, -1, -1, -1, msg, other);
624 else if (ctx->reg_int & LMC_INT_CE)
625 edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, 1,
626 phys_to_pfn(phys_addr),
627 offset_in_page(phys_addr),
628 0, -1, -1, -1, msg, other);
629
630 lmc->ring_tail++;
631 }
632
633 ret = IRQ_HANDLED;
634
635 err_free:
636 kfree(msg);
637 kfree(other);
638
639 return ret;
640 }
641
642 static const struct pci_device_id thunderx_lmc_pci_tbl[] = {
643 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_LMC) },
644 { 0, },
645 };
646
647 static inline int pci_dev_to_mc_idx(struct pci_dev *pdev)
648 {
649 int node = dev_to_node(&pdev->dev);
650 int ret = PCI_FUNC(pdev->devfn);
651
652 ret += max(node, 0) << 3;
653
654 return ret;
655 }
656
657 static int thunderx_lmc_probe(struct pci_dev *pdev,
658 const struct pci_device_id *id)
659 {
660 struct thunderx_lmc *lmc;
661 struct edac_mc_layer layer;
662 struct mem_ctl_info *mci;
663 u64 lmc_control, lmc_ddr_pll_ctl, lmc_config;
664 int ret;
665 u64 lmc_int;
666 void *l2c_ioaddr;
667
668 layer.type = EDAC_MC_LAYER_SLOT;
669 layer.size = 2;
670 layer.is_virt_csrow = false;
671
672 ret = pcim_enable_device(pdev);
673 if (ret) {
674 dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
675 return ret;
676 }
677
678 ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_lmc");
679 if (ret) {
680 dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
681 return ret;
682 }
683
684 mci = edac_mc_alloc(pci_dev_to_mc_idx(pdev), 1, &layer,
685 sizeof(struct thunderx_lmc));
686 if (!mci)
687 return -ENOMEM;
688
689 mci->pdev = &pdev->dev;
690 lmc = mci->pvt_info;
691
692 pci_set_drvdata(pdev, mci);
693
694 lmc->regs = pcim_iomap_table(pdev)[0];
695
696 lmc_control = readq(lmc->regs + LMC_CONTROL);
697 lmc_ddr_pll_ctl = readq(lmc->regs + LMC_DDR_PLL_CTL);
698 lmc_config = readq(lmc->regs + LMC_CONFIG);
699
700 if (lmc_control & LMC_CONTROL_RDIMM) {
701 mci->mtype_cap = FIELD_GET(LMC_DDR_PLL_CTL_DDR4,
702 lmc_ddr_pll_ctl) ?
703 MEM_RDDR4 : MEM_RDDR3;
704 } else {
705 mci->mtype_cap = FIELD_GET(LMC_DDR_PLL_CTL_DDR4,
706 lmc_ddr_pll_ctl) ?
707 MEM_DDR4 : MEM_DDR3;
708 }
709
710 mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED;
711 mci->edac_cap = EDAC_FLAG_SECDED;
712
713 mci->mod_name = "thunderx-lmc";
714 mci->ctl_name = "thunderx-lmc";
715 mci->dev_name = dev_name(&pdev->dev);
716 mci->scrub_mode = SCRUB_NONE;
717
718 lmc->pdev = pdev;
719 lmc->msix_ent.entry = 0;
720
721 lmc->ring_head = 0;
722 lmc->ring_tail = 0;
723
724 ret = pci_enable_msix_exact(pdev, &lmc->msix_ent, 1);
725 if (ret) {
726 dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
727 goto err_free;
728 }
729
730 ret = devm_request_threaded_irq(&pdev->dev, lmc->msix_ent.vector,
731 thunderx_lmc_err_isr,
732 thunderx_lmc_threaded_isr, 0,
733 "[EDAC] ThunderX LMC", mci);
734 if (ret) {
735 dev_err(&pdev->dev, "Cannot set ISR: %d\n", ret);
736 goto err_free;
737 }
738
739 lmc->node = FIELD_GET(THUNDERX_NODE, pci_resource_start(pdev, 0));
740
741 lmc->xbits = thunderx_get_num_lmcs(lmc->node) >> 1;
742 lmc->bank_width = (FIELD_GET(LMC_DDR_PLL_CTL_DDR4, lmc_ddr_pll_ctl) &&
743 FIELD_GET(LMC_CONFIG_BG2, lmc_config)) ? 4 : 3;
744
745 lmc->pbank_lsb = (lmc_config >> 5) & 0xf;
746 lmc->dimm_lsb = 28 + lmc->pbank_lsb + lmc->xbits;
747 lmc->rank_lsb = lmc->dimm_lsb;
748 lmc->rank_lsb -= FIELD_GET(LMC_CONFIG_RANK_ENA, lmc_config) ? 1 : 0;
749 lmc->bank_lsb = 7 + lmc->xbits;
750 lmc->row_lsb = 14 + LMC_CONFIG_ROW_LSB(lmc_config) + lmc->xbits;
751
752 lmc->col_hi_lsb = lmc->bank_lsb + lmc->bank_width;
753
754 lmc->xor_bank = lmc_control & LMC_CONTROL_XOR_BANK;
755
756 l2c_ioaddr = ioremap(L2C_CTL | FIELD_PREP(THUNDERX_NODE, lmc->node), PAGE_SIZE);
757 if (!l2c_ioaddr) {
758 dev_err(&pdev->dev, "Cannot map L2C_CTL\n");
759 ret = -ENOMEM;
760 goto err_free;
761 }
762
763 lmc->l2c_alias = !(readq(l2c_ioaddr) & L2C_CTL_DISIDXALIAS);
764
765 iounmap(l2c_ioaddr);
766
767 ret = edac_mc_add_mc(mci);
768 if (ret) {
769 dev_err(&pdev->dev, "Cannot add the MC: %d\n", ret);
770 goto err_free;
771 }
772
773 lmc_int = readq(lmc->regs + LMC_INT);
774 writeq(lmc_int, lmc->regs + LMC_INT);
775
776 writeq(LMC_INT_ENA_ALL, lmc->regs + LMC_INT_ENA_W1S);
777
778 if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
779 ret = thunderx_create_debugfs_nodes(mci->debugfs,
780 lmc_dfs_ents,
781 lmc,
782 ARRAY_SIZE(lmc_dfs_ents));
783
784 if (ret != ARRAY_SIZE(lmc_dfs_ents)) {
785 dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
786 ret, ret >= 0 ? " created" : "");
787 }
788 }
789
790 return 0;
791
792 err_free:
793 pci_set_drvdata(pdev, NULL);
794 edac_mc_free(mci);
795
796 return ret;
797 }
798
799 static void thunderx_lmc_remove(struct pci_dev *pdev)
800 {
801 struct mem_ctl_info *mci = pci_get_drvdata(pdev);
802 struct thunderx_lmc *lmc = mci->pvt_info;
803
804 writeq(LMC_INT_ENA_ALL, lmc->regs + LMC_INT_ENA_W1C);
805
806 edac_mc_del_mc(&pdev->dev);
807 edac_mc_free(mci);
808 }
809
810 MODULE_DEVICE_TABLE(pci, thunderx_lmc_pci_tbl);
811
812 static struct pci_driver thunderx_lmc_driver = {
813 .name = "thunderx_lmc_edac",
814 .probe = thunderx_lmc_probe,
815 .remove = thunderx_lmc_remove,
816 .id_table = thunderx_lmc_pci_tbl,
817 };
818
819 /*---------------------- OCX driver ---------------------------------*/
820
821 #define PCI_DEVICE_ID_THUNDER_OCX 0xa013
822
823 #define OCX_LINK_INTS 3
824 #define OCX_INTS (OCX_LINK_INTS + 1)
825 #define OCX_RX_LANES 24
826 #define OCX_RX_LANE_STATS 15
827
828 #define OCX_COM_INT 0x100
829 #define OCX_COM_INT_W1S 0x108
830 #define OCX_COM_INT_ENA_W1S 0x110
831 #define OCX_COM_INT_ENA_W1C 0x118
832
833 #define OCX_COM_IO_BADID BIT(54)
834 #define OCX_COM_MEM_BADID BIT(53)
835 #define OCX_COM_COPR_BADID BIT(52)
836 #define OCX_COM_WIN_REQ_BADID BIT(51)
837 #define OCX_COM_WIN_REQ_TOUT BIT(50)
838 #define OCX_COM_RX_LANE GENMASK(23, 0)
839
840 #define OCX_COM_INT_CE (OCX_COM_IO_BADID | \
841 OCX_COM_MEM_BADID | \
842 OCX_COM_COPR_BADID | \
843 OCX_COM_WIN_REQ_BADID | \
844 OCX_COM_WIN_REQ_TOUT)
845
846 static const struct error_descr ocx_com_errors[] = {
847 {
848 .type = ERR_CORRECTED,
849 .mask = OCX_COM_IO_BADID,
850 .descr = "Invalid IO transaction node ID",
851 },
852 {
853 .type = ERR_CORRECTED,
854 .mask = OCX_COM_MEM_BADID,
855 .descr = "Invalid memory transaction node ID",
856 },
857 {
858 .type = ERR_CORRECTED,
859 .mask = OCX_COM_COPR_BADID,
860 .descr = "Invalid coprocessor transaction node ID",
861 },
862 {
863 .type = ERR_CORRECTED,
864 .mask = OCX_COM_WIN_REQ_BADID,
865 .descr = "Invalid SLI transaction node ID",
866 },
867 {
868 .type = ERR_CORRECTED,
869 .mask = OCX_COM_WIN_REQ_TOUT,
870 .descr = "Window/core request timeout",
871 },
872 {0, 0, NULL},
873 };
874
875 #define OCX_COM_LINKX_INT(x) (0x120 + (x) * 8)
876 #define OCX_COM_LINKX_INT_W1S(x) (0x140 + (x) * 8)
877 #define OCX_COM_LINKX_INT_ENA_W1S(x) (0x160 + (x) * 8)
878 #define OCX_COM_LINKX_INT_ENA_W1C(x) (0x180 + (x) * 8)
879
880 #define OCX_COM_LINK_BAD_WORD BIT(13)
881 #define OCX_COM_LINK_ALIGN_FAIL BIT(12)
882 #define OCX_COM_LINK_ALIGN_DONE BIT(11)
883 #define OCX_COM_LINK_UP BIT(10)
884 #define OCX_COM_LINK_STOP BIT(9)
885 #define OCX_COM_LINK_BLK_ERR BIT(8)
886 #define OCX_COM_LINK_REINIT BIT(7)
887 #define OCX_COM_LINK_LNK_DATA BIT(6)
888 #define OCX_COM_LINK_RXFIFO_DBE BIT(5)
889 #define OCX_COM_LINK_RXFIFO_SBE BIT(4)
890 #define OCX_COM_LINK_TXFIFO_DBE BIT(3)
891 #define OCX_COM_LINK_TXFIFO_SBE BIT(2)
892 #define OCX_COM_LINK_REPLAY_DBE BIT(1)
893 #define OCX_COM_LINK_REPLAY_SBE BIT(0)
894
895 static const struct error_descr ocx_com_link_errors[] = {
896 {
897 .type = ERR_CORRECTED,
898 .mask = OCX_COM_LINK_REPLAY_SBE,
899 .descr = "Replay buffer single-bit error",
900 },
901 {
902 .type = ERR_CORRECTED,
903 .mask = OCX_COM_LINK_TXFIFO_SBE,
904 .descr = "TX FIFO single-bit error",
905 },
906 {
907 .type = ERR_CORRECTED,
908 .mask = OCX_COM_LINK_RXFIFO_SBE,
909 .descr = "RX FIFO single-bit error",
910 },
911 {
912 .type = ERR_CORRECTED,
913 .mask = OCX_COM_LINK_BLK_ERR,
914 .descr = "Block code error",
915 },
916 {
917 .type = ERR_CORRECTED,
918 .mask = OCX_COM_LINK_ALIGN_FAIL,
919 .descr = "Link alignment failure",
920 },
921 {
922 .type = ERR_CORRECTED,
923 .mask = OCX_COM_LINK_BAD_WORD,
924 .descr = "Bad code word",
925 },
926 {
927 .type = ERR_UNCORRECTED,
928 .mask = OCX_COM_LINK_REPLAY_DBE,
929 .descr = "Replay buffer double-bit error",
930 },
931 {
932 .type = ERR_UNCORRECTED,
933 .mask = OCX_COM_LINK_TXFIFO_DBE,
934 .descr = "TX FIFO double-bit error",
935 },
936 {
937 .type = ERR_UNCORRECTED,
938 .mask = OCX_COM_LINK_RXFIFO_DBE,
939 .descr = "RX FIFO double-bit error",
940 },
941 {
942 .type = ERR_UNCORRECTED,
943 .mask = OCX_COM_LINK_STOP,
944 .descr = "Link stopped",
945 },
946 {0, 0, NULL},
947 };
948
949 #define OCX_COM_LINK_INT_UE (OCX_COM_LINK_REPLAY_DBE | \
950 OCX_COM_LINK_TXFIFO_DBE | \
951 OCX_COM_LINK_RXFIFO_DBE | \
952 OCX_COM_LINK_STOP)
953
954 #define OCX_COM_LINK_INT_CE (OCX_COM_LINK_REPLAY_SBE | \
955 OCX_COM_LINK_TXFIFO_SBE | \
956 OCX_COM_LINK_RXFIFO_SBE | \
957 OCX_COM_LINK_BLK_ERR | \
958 OCX_COM_LINK_ALIGN_FAIL | \
959 OCX_COM_LINK_BAD_WORD)
960
961 #define OCX_LNE_INT(x) (0x8018 + (x) * 0x100)
962 #define OCX_LNE_INT_EN(x) (0x8020 + (x) * 0x100)
963 #define OCX_LNE_BAD_CNT(x) (0x8028 + (x) * 0x100)
964 #define OCX_LNE_CFG(x) (0x8000 + (x) * 0x100)
965 #define OCX_LNE_STAT(x, y) (0x8040 + (x) * 0x100 + (y) * 8)
966
967 #define OCX_LNE_CFG_RX_BDRY_LOCK_DIS BIT(8)
968 #define OCX_LNE_CFG_RX_STAT_WRAP_DIS BIT(2)
969 #define OCX_LNE_CFG_RX_STAT_RDCLR BIT(1)
970 #define OCX_LNE_CFG_RX_STAT_ENA BIT(0)
971
972
973 #define OCX_LANE_BAD_64B67B BIT(8)
974 #define OCX_LANE_DSKEW_FIFO_OVFL BIT(5)
975 #define OCX_LANE_SCRM_SYNC_LOSS BIT(4)
976 #define OCX_LANE_UKWN_CNTL_WORD BIT(3)
977 #define OCX_LANE_CRC32_ERR BIT(2)
978 #define OCX_LANE_BDRY_SYNC_LOSS BIT(1)
979 #define OCX_LANE_SERDES_LOCK_LOSS BIT(0)
980
981 #define OCX_COM_LANE_INT_UE (0)
982 #define OCX_COM_LANE_INT_CE (OCX_LANE_SERDES_LOCK_LOSS | \
983 OCX_LANE_BDRY_SYNC_LOSS | \
984 OCX_LANE_CRC32_ERR | \
985 OCX_LANE_UKWN_CNTL_WORD | \
986 OCX_LANE_SCRM_SYNC_LOSS | \
987 OCX_LANE_DSKEW_FIFO_OVFL | \
988 OCX_LANE_BAD_64B67B)
989
990 static const struct error_descr ocx_lane_errors[] = {
991 {
992 .type = ERR_CORRECTED,
993 .mask = OCX_LANE_SERDES_LOCK_LOSS,
994 .descr = "RX SerDes lock lost",
995 },
996 {
997 .type = ERR_CORRECTED,
998 .mask = OCX_LANE_BDRY_SYNC_LOSS,
999 .descr = "RX word boundary lost",
1000 },
1001 {
1002 .type = ERR_CORRECTED,
1003 .mask = OCX_LANE_CRC32_ERR,
1004 .descr = "CRC32 error",
1005 },
1006 {
1007 .type = ERR_CORRECTED,
1008 .mask = OCX_LANE_UKWN_CNTL_WORD,
1009 .descr = "Unknown control word",
1010 },
1011 {
1012 .type = ERR_CORRECTED,
1013 .mask = OCX_LANE_SCRM_SYNC_LOSS,
1014 .descr = "Scrambler synchronization lost",
1015 },
1016 {
1017 .type = ERR_CORRECTED,
1018 .mask = OCX_LANE_DSKEW_FIFO_OVFL,
1019 .descr = "RX deskew FIFO overflow",
1020 },
1021 {
1022 .type = ERR_CORRECTED,
1023 .mask = OCX_LANE_BAD_64B67B,
1024 .descr = "Bad 64B/67B codeword",
1025 },
1026 {0, 0, NULL},
1027 };
1028
1029 #define OCX_LNE_INT_ENA_ALL (GENMASK(9, 8) | GENMASK(6, 0))
1030 #define OCX_COM_INT_ENA_ALL (GENMASK(54, 50) | GENMASK(23, 0))
1031 #define OCX_COM_LINKX_INT_ENA_ALL (GENMASK(13, 12) | \
1032 GENMASK(9, 7) | GENMASK(5, 0))
1033
1034 #define OCX_TLKX_ECC_CTL(x) (0x10018 + (x) * 0x2000)
1035 #define OCX_RLKX_ECC_CTL(x) (0x18018 + (x) * 0x2000)
1036
1037 struct ocx_com_err_ctx {
1038 u64 reg_com_int;
1039 u64 reg_lane_int[OCX_RX_LANES];
1040 u64 reg_lane_stat11[OCX_RX_LANES];
1041 };
1042
1043 struct ocx_link_err_ctx {
1044 u64 reg_com_link_int;
1045 int link;
1046 };
1047
1048 struct thunderx_ocx {
1049 void __iomem *regs;
1050 int com_link;
1051 struct pci_dev *pdev;
1052 struct edac_device_ctl_info *edac_dev;
1053
1054 struct dentry *debugfs;
1055 struct msix_entry msix_ent[OCX_INTS];
1056
1057 struct ocx_com_err_ctx com_err_ctx[RING_ENTRIES];
1058 struct ocx_link_err_ctx link_err_ctx[RING_ENTRIES];
1059
1060 unsigned long com_ring_head;
1061 unsigned long com_ring_tail;
1062
1063 unsigned long link_ring_head;
1064 unsigned long link_ring_tail;
1065 };
1066
1067 #define OCX_MESSAGE_SIZE SZ_1K
1068 #define OCX_OTHER_SIZE (50 * ARRAY_SIZE(ocx_com_link_errors))
1069
1070 /* This handler is threaded */
1071 static irqreturn_t thunderx_ocx_com_isr(int irq, void *irq_id)
1072 {
1073 struct msix_entry *msix = irq_id;
1074 struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1075 msix_ent[msix->entry]);
1076
1077 int lane;
1078 unsigned long head = ring_pos(ocx->com_ring_head,
1079 ARRAY_SIZE(ocx->com_err_ctx));
1080 struct ocx_com_err_ctx *ctx = &ocx->com_err_ctx[head];
1081
1082 ctx->reg_com_int = readq(ocx->regs + OCX_COM_INT);
1083
1084 for (lane = 0; lane < OCX_RX_LANES; lane++) {
1085 ctx->reg_lane_int[lane] =
1086 readq(ocx->regs + OCX_LNE_INT(lane));
1087 ctx->reg_lane_stat11[lane] =
1088 readq(ocx->regs + OCX_LNE_STAT(lane, 11));
1089
1090 writeq(ctx->reg_lane_int[lane], ocx->regs + OCX_LNE_INT(lane));
1091 }
1092
1093 writeq(ctx->reg_com_int, ocx->regs + OCX_COM_INT);
1094
1095 ocx->com_ring_head++;
1096
1097 return IRQ_WAKE_THREAD;
1098 }
1099
1100 static irqreturn_t thunderx_ocx_com_threaded_isr(int irq, void *irq_id)
1101 {
1102 struct msix_entry *msix = irq_id;
1103 struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1104 msix_ent[msix->entry]);
1105
1106 irqreturn_t ret = IRQ_NONE;
1107
1108 unsigned long tail;
1109 struct ocx_com_err_ctx *ctx;
1110 int lane;
1111 char *msg;
1112 char *other;
1113
1114 msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
1115 other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
1116
1117 if (!msg || !other)
1118 goto err_free;
1119
1120 while (CIRC_CNT(ocx->com_ring_head, ocx->com_ring_tail,
1121 ARRAY_SIZE(ocx->com_err_ctx))) {
1122 tail = ring_pos(ocx->com_ring_tail,
1123 ARRAY_SIZE(ocx->com_err_ctx));
1124 ctx = &ocx->com_err_ctx[tail];
1125
1126 snprintf(msg, OCX_MESSAGE_SIZE, "%s: OCX_COM_INT: %016llx",
1127 ocx->edac_dev->ctl_name, ctx->reg_com_int);
1128
1129 decode_register(other, OCX_OTHER_SIZE,
1130 ocx_com_errors, ctx->reg_com_int);
1131
1132 strncat(msg, other, OCX_MESSAGE_SIZE);
1133
1134 for (lane = 0; lane < OCX_RX_LANES; lane++)
1135 if (ctx->reg_com_int & BIT(lane)) {
1136 snprintf(other, OCX_OTHER_SIZE,
1137 "\n\tOCX_LNE_INT[%02d]: %016llx OCX_LNE_STAT11[%02d]: %016llx",
1138 lane, ctx->reg_lane_int[lane],
1139 lane, ctx->reg_lane_stat11[lane]);
1140
1141 strncat(msg, other, OCX_MESSAGE_SIZE);
1142
1143 decode_register(other, OCX_OTHER_SIZE,
1144 ocx_lane_errors,
1145 ctx->reg_lane_int[lane]);
1146 strncat(msg, other, OCX_MESSAGE_SIZE);
1147 }
1148
1149 if (ctx->reg_com_int & OCX_COM_INT_CE)
1150 edac_device_handle_ce(ocx->edac_dev, 0, 0, msg);
1151
1152 ocx->com_ring_tail++;
1153 }
1154
1155 ret = IRQ_HANDLED;
1156
1157 err_free:
1158 kfree(other);
1159 kfree(msg);
1160
1161 return ret;
1162 }
1163
1164 static irqreturn_t thunderx_ocx_lnk_isr(int irq, void *irq_id)
1165 {
1166 struct msix_entry *msix = irq_id;
1167 struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1168 msix_ent[msix->entry]);
1169 unsigned long head = ring_pos(ocx->link_ring_head,
1170 ARRAY_SIZE(ocx->link_err_ctx));
1171 struct ocx_link_err_ctx *ctx = &ocx->link_err_ctx[head];
1172
1173 ctx->link = msix->entry;
1174 ctx->reg_com_link_int = readq(ocx->regs + OCX_COM_LINKX_INT(ctx->link));
1175
1176 writeq(ctx->reg_com_link_int, ocx->regs + OCX_COM_LINKX_INT(ctx->link));
1177
1178 ocx->link_ring_head++;
1179
1180 return IRQ_WAKE_THREAD;
1181 }
1182
1183 static irqreturn_t thunderx_ocx_lnk_threaded_isr(int irq, void *irq_id)
1184 {
1185 struct msix_entry *msix = irq_id;
1186 struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1187 msix_ent[msix->entry]);
1188 irqreturn_t ret = IRQ_NONE;
1189 unsigned long tail;
1190 struct ocx_link_err_ctx *ctx;
1191
1192 char *msg;
1193 char *other;
1194
1195 msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
1196 other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
1197
1198 if (!msg || !other)
1199 goto err_free;
1200
1201 while (CIRC_CNT(ocx->link_ring_head, ocx->link_ring_tail,
1202 ARRAY_SIZE(ocx->link_err_ctx))) {
1203 tail = ring_pos(ocx->link_ring_head,
1204 ARRAY_SIZE(ocx->link_err_ctx));
1205
1206 ctx = &ocx->link_err_ctx[tail];
1207
1208 snprintf(msg, OCX_MESSAGE_SIZE,
1209 "%s: OCX_COM_LINK_INT[%d]: %016llx",
1210 ocx->edac_dev->ctl_name,
1211 ctx->link, ctx->reg_com_link_int);
1212
1213 decode_register(other, OCX_OTHER_SIZE,
1214 ocx_com_link_errors, ctx->reg_com_link_int);
1215
1216 strncat(msg, other, OCX_MESSAGE_SIZE);
1217
1218 if (ctx->reg_com_link_int & OCX_COM_LINK_INT_UE)
1219 edac_device_handle_ue(ocx->edac_dev, 0, 0, msg);
1220 else if (ctx->reg_com_link_int & OCX_COM_LINK_INT_CE)
1221 edac_device_handle_ce(ocx->edac_dev, 0, 0, msg);
1222
1223 ocx->link_ring_tail++;
1224 }
1225
1226 ret = IRQ_HANDLED;
1227 err_free:
1228 kfree(other);
1229 kfree(msg);
1230
1231 return ret;
1232 }
1233
1234 #define OCX_DEBUGFS_ATTR(_name, _reg) DEBUGFS_REG_ATTR(ocx, _name, _reg)
1235
1236 OCX_DEBUGFS_ATTR(tlk0_ecc_ctl, OCX_TLKX_ECC_CTL(0));
1237 OCX_DEBUGFS_ATTR(tlk1_ecc_ctl, OCX_TLKX_ECC_CTL(1));
1238 OCX_DEBUGFS_ATTR(tlk2_ecc_ctl, OCX_TLKX_ECC_CTL(2));
1239
1240 OCX_DEBUGFS_ATTR(rlk0_ecc_ctl, OCX_RLKX_ECC_CTL(0));
1241 OCX_DEBUGFS_ATTR(rlk1_ecc_ctl, OCX_RLKX_ECC_CTL(1));
1242 OCX_DEBUGFS_ATTR(rlk2_ecc_ctl, OCX_RLKX_ECC_CTL(2));
1243
1244 OCX_DEBUGFS_ATTR(com_link0_int, OCX_COM_LINKX_INT_W1S(0));
1245 OCX_DEBUGFS_ATTR(com_link1_int, OCX_COM_LINKX_INT_W1S(1));
1246 OCX_DEBUGFS_ATTR(com_link2_int, OCX_COM_LINKX_INT_W1S(2));
1247
1248 OCX_DEBUGFS_ATTR(lne00_badcnt, OCX_LNE_BAD_CNT(0));
1249 OCX_DEBUGFS_ATTR(lne01_badcnt, OCX_LNE_BAD_CNT(1));
1250 OCX_DEBUGFS_ATTR(lne02_badcnt, OCX_LNE_BAD_CNT(2));
1251 OCX_DEBUGFS_ATTR(lne03_badcnt, OCX_LNE_BAD_CNT(3));
1252 OCX_DEBUGFS_ATTR(lne04_badcnt, OCX_LNE_BAD_CNT(4));
1253 OCX_DEBUGFS_ATTR(lne05_badcnt, OCX_LNE_BAD_CNT(5));
1254 OCX_DEBUGFS_ATTR(lne06_badcnt, OCX_LNE_BAD_CNT(6));
1255 OCX_DEBUGFS_ATTR(lne07_badcnt, OCX_LNE_BAD_CNT(7));
1256
1257 OCX_DEBUGFS_ATTR(lne08_badcnt, OCX_LNE_BAD_CNT(8));
1258 OCX_DEBUGFS_ATTR(lne09_badcnt, OCX_LNE_BAD_CNT(9));
1259 OCX_DEBUGFS_ATTR(lne10_badcnt, OCX_LNE_BAD_CNT(10));
1260 OCX_DEBUGFS_ATTR(lne11_badcnt, OCX_LNE_BAD_CNT(11));
1261 OCX_DEBUGFS_ATTR(lne12_badcnt, OCX_LNE_BAD_CNT(12));
1262 OCX_DEBUGFS_ATTR(lne13_badcnt, OCX_LNE_BAD_CNT(13));
1263 OCX_DEBUGFS_ATTR(lne14_badcnt, OCX_LNE_BAD_CNT(14));
1264 OCX_DEBUGFS_ATTR(lne15_badcnt, OCX_LNE_BAD_CNT(15));
1265
1266 OCX_DEBUGFS_ATTR(lne16_badcnt, OCX_LNE_BAD_CNT(16));
1267 OCX_DEBUGFS_ATTR(lne17_badcnt, OCX_LNE_BAD_CNT(17));
1268 OCX_DEBUGFS_ATTR(lne18_badcnt, OCX_LNE_BAD_CNT(18));
1269 OCX_DEBUGFS_ATTR(lne19_badcnt, OCX_LNE_BAD_CNT(19));
1270 OCX_DEBUGFS_ATTR(lne20_badcnt, OCX_LNE_BAD_CNT(20));
1271 OCX_DEBUGFS_ATTR(lne21_badcnt, OCX_LNE_BAD_CNT(21));
1272 OCX_DEBUGFS_ATTR(lne22_badcnt, OCX_LNE_BAD_CNT(22));
1273 OCX_DEBUGFS_ATTR(lne23_badcnt, OCX_LNE_BAD_CNT(23));
1274
1275 OCX_DEBUGFS_ATTR(com_int, OCX_COM_INT_W1S);
1276
1277 struct debugfs_entry *ocx_dfs_ents[] = {
1278 &debugfs_tlk0_ecc_ctl,
1279 &debugfs_tlk1_ecc_ctl,
1280 &debugfs_tlk2_ecc_ctl,
1281
1282 &debugfs_rlk0_ecc_ctl,
1283 &debugfs_rlk1_ecc_ctl,
1284 &debugfs_rlk2_ecc_ctl,
1285
1286 &debugfs_com_link0_int,
1287 &debugfs_com_link1_int,
1288 &debugfs_com_link2_int,
1289
1290 &debugfs_lne00_badcnt,
1291 &debugfs_lne01_badcnt,
1292 &debugfs_lne02_badcnt,
1293 &debugfs_lne03_badcnt,
1294 &debugfs_lne04_badcnt,
1295 &debugfs_lne05_badcnt,
1296 &debugfs_lne06_badcnt,
1297 &debugfs_lne07_badcnt,
1298 &debugfs_lne08_badcnt,
1299 &debugfs_lne09_badcnt,
1300 &debugfs_lne10_badcnt,
1301 &debugfs_lne11_badcnt,
1302 &debugfs_lne12_badcnt,
1303 &debugfs_lne13_badcnt,
1304 &debugfs_lne14_badcnt,
1305 &debugfs_lne15_badcnt,
1306 &debugfs_lne16_badcnt,
1307 &debugfs_lne17_badcnt,
1308 &debugfs_lne18_badcnt,
1309 &debugfs_lne19_badcnt,
1310 &debugfs_lne20_badcnt,
1311 &debugfs_lne21_badcnt,
1312 &debugfs_lne22_badcnt,
1313 &debugfs_lne23_badcnt,
1314
1315 &debugfs_com_int,
1316 };
1317
1318 static const struct pci_device_id thunderx_ocx_pci_tbl[] = {
1319 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_OCX) },
1320 { 0, },
1321 };
1322
1323 static void thunderx_ocx_clearstats(struct thunderx_ocx *ocx)
1324 {
1325 int lane, stat, cfg;
1326
1327 for (lane = 0; lane < OCX_RX_LANES; lane++) {
1328 cfg = readq(ocx->regs + OCX_LNE_CFG(lane));
1329 cfg |= OCX_LNE_CFG_RX_STAT_RDCLR;
1330 cfg &= ~OCX_LNE_CFG_RX_STAT_ENA;
1331 writeq(cfg, ocx->regs + OCX_LNE_CFG(lane));
1332
1333 for (stat = 0; stat < OCX_RX_LANE_STATS; stat++)
1334 readq(ocx->regs + OCX_LNE_STAT(lane, stat));
1335 }
1336 }
1337
1338 static int thunderx_ocx_probe(struct pci_dev *pdev,
1339 const struct pci_device_id *id)
1340 {
1341 struct thunderx_ocx *ocx;
1342 struct edac_device_ctl_info *edac_dev;
1343 char name[32];
1344 int idx;
1345 int i;
1346 int ret;
1347 u64 reg;
1348
1349 ret = pcim_enable_device(pdev);
1350 if (ret) {
1351 dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
1352 return ret;
1353 }
1354
1355 ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_ocx");
1356 if (ret) {
1357 dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
1358 return ret;
1359 }
1360
1361 idx = edac_device_alloc_index();
1362 snprintf(name, sizeof(name), "OCX%d", idx);
1363 edac_dev = edac_device_alloc_ctl_info(sizeof(struct thunderx_ocx),
1364 name, 1, "CCPI", 1,
1365 0, NULL, 0, idx);
1366 if (!edac_dev) {
1367 dev_err(&pdev->dev, "Cannot allocate EDAC device: %d\n", ret);
1368 return -ENOMEM;
1369 }
1370 ocx = edac_dev->pvt_info;
1371 ocx->edac_dev = edac_dev;
1372 ocx->com_ring_head = 0;
1373 ocx->com_ring_tail = 0;
1374 ocx->link_ring_head = 0;
1375 ocx->link_ring_tail = 0;
1376
1377 ocx->regs = pcim_iomap_table(pdev)[0];
1378 if (!ocx->regs) {
1379 dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
1380 ret = -ENODEV;
1381 goto err_free;
1382 }
1383
1384 ocx->pdev = pdev;
1385
1386 for (i = 0; i < OCX_INTS; i++) {
1387 ocx->msix_ent[i].entry = i;
1388 ocx->msix_ent[i].vector = 0;
1389 }
1390
1391 ret = pci_enable_msix_exact(pdev, ocx->msix_ent, OCX_INTS);
1392 if (ret) {
1393 dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
1394 goto err_free;
1395 }
1396
1397 for (i = 0; i < OCX_INTS; i++) {
1398 ret = devm_request_threaded_irq(&pdev->dev,
1399 ocx->msix_ent[i].vector,
1400 (i == 3) ?
1401 thunderx_ocx_com_isr :
1402 thunderx_ocx_lnk_isr,
1403 (i == 3) ?
1404 thunderx_ocx_com_threaded_isr :
1405 thunderx_ocx_lnk_threaded_isr,
1406 0, "[EDAC] ThunderX OCX",
1407 &ocx->msix_ent[i]);
1408 if (ret)
1409 goto err_free;
1410 }
1411
1412 edac_dev->dev = &pdev->dev;
1413 edac_dev->dev_name = dev_name(&pdev->dev);
1414 edac_dev->mod_name = "thunderx-ocx";
1415 edac_dev->ctl_name = "thunderx-ocx";
1416
1417 ret = edac_device_add_device(edac_dev);
1418 if (ret) {
1419 dev_err(&pdev->dev, "Cannot add EDAC device: %d\n", ret);
1420 goto err_free;
1421 }
1422
1423 if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
1424 ocx->debugfs = edac_debugfs_create_dir(pdev->dev.kobj.name);
1425
1426 ret = thunderx_create_debugfs_nodes(ocx->debugfs,
1427 ocx_dfs_ents,
1428 ocx,
1429 ARRAY_SIZE(ocx_dfs_ents));
1430 if (ret != ARRAY_SIZE(ocx_dfs_ents)) {
1431 dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
1432 ret, ret >= 0 ? " created" : "");
1433 }
1434 }
1435
1436 pci_set_drvdata(pdev, edac_dev);
1437
1438 thunderx_ocx_clearstats(ocx);
1439
1440 for (i = 0; i < OCX_RX_LANES; i++) {
1441 writeq(OCX_LNE_INT_ENA_ALL,
1442 ocx->regs + OCX_LNE_INT_EN(i));
1443
1444 reg = readq(ocx->regs + OCX_LNE_INT(i));
1445 writeq(reg, ocx->regs + OCX_LNE_INT(i));
1446
1447 }
1448
1449 for (i = 0; i < OCX_LINK_INTS; i++) {
1450 reg = readq(ocx->regs + OCX_COM_LINKX_INT(i));
1451 writeq(reg, ocx->regs + OCX_COM_LINKX_INT(i));
1452
1453 writeq(OCX_COM_LINKX_INT_ENA_ALL,
1454 ocx->regs + OCX_COM_LINKX_INT_ENA_W1S(i));
1455 }
1456
1457 reg = readq(ocx->regs + OCX_COM_INT);
1458 writeq(reg, ocx->regs + OCX_COM_INT);
1459
1460 writeq(OCX_COM_INT_ENA_ALL, ocx->regs + OCX_COM_INT_ENA_W1S);
1461
1462 return 0;
1463 err_free:
1464 edac_device_free_ctl_info(edac_dev);
1465
1466 return ret;
1467 }
1468
1469 static void thunderx_ocx_remove(struct pci_dev *pdev)
1470 {
1471 struct edac_device_ctl_info *edac_dev = pci_get_drvdata(pdev);
1472 struct thunderx_ocx *ocx = edac_dev->pvt_info;
1473 int i;
1474
1475 writeq(OCX_COM_INT_ENA_ALL, ocx->regs + OCX_COM_INT_ENA_W1C);
1476
1477 for (i = 0; i < OCX_INTS; i++) {
1478 writeq(OCX_COM_LINKX_INT_ENA_ALL,
1479 ocx->regs + OCX_COM_LINKX_INT_ENA_W1C(i));
1480 }
1481
1482 edac_debugfs_remove_recursive(ocx->debugfs);
1483
1484 edac_device_del_device(&pdev->dev);
1485 edac_device_free_ctl_info(edac_dev);
1486 }
1487
1488 MODULE_DEVICE_TABLE(pci, thunderx_ocx_pci_tbl);
1489
1490 static struct pci_driver thunderx_ocx_driver = {
1491 .name = "thunderx_ocx_edac",
1492 .probe = thunderx_ocx_probe,
1493 .remove = thunderx_ocx_remove,
1494 .id_table = thunderx_ocx_pci_tbl,
1495 };
1496
1497 /*---------------------- L2C driver ---------------------------------*/
1498
1499 #define PCI_DEVICE_ID_THUNDER_L2C_TAD 0xa02e
1500 #define PCI_DEVICE_ID_THUNDER_L2C_CBC 0xa02f
1501 #define PCI_DEVICE_ID_THUNDER_L2C_MCI 0xa030
1502
1503 #define L2C_TAD_INT_W1C 0x40000
1504 #define L2C_TAD_INT_W1S 0x40008
1505
1506 #define L2C_TAD_INT_ENA_W1C 0x40020
1507 #define L2C_TAD_INT_ENA_W1S 0x40028
1508
1509
1510 #define L2C_TAD_INT_L2DDBE BIT(1)
1511 #define L2C_TAD_INT_SBFSBE BIT(2)
1512 #define L2C_TAD_INT_SBFDBE BIT(3)
1513 #define L2C_TAD_INT_FBFSBE BIT(4)
1514 #define L2C_TAD_INT_FBFDBE BIT(5)
1515 #define L2C_TAD_INT_TAGDBE BIT(9)
1516 #define L2C_TAD_INT_RDDISLMC BIT(15)
1517 #define L2C_TAD_INT_WRDISLMC BIT(16)
1518 #define L2C_TAD_INT_LFBTO BIT(17)
1519 #define L2C_TAD_INT_GSYNCTO BIT(18)
1520 #define L2C_TAD_INT_RTGSBE BIT(32)
1521 #define L2C_TAD_INT_RTGDBE BIT(33)
1522 #define L2C_TAD_INT_RDDISOCI BIT(34)
1523 #define L2C_TAD_INT_WRDISOCI BIT(35)
1524
1525 #define L2C_TAD_INT_ECC (L2C_TAD_INT_L2DDBE | \
1526 L2C_TAD_INT_SBFSBE | L2C_TAD_INT_SBFDBE | \
1527 L2C_TAD_INT_FBFSBE | L2C_TAD_INT_FBFDBE)
1528
1529 #define L2C_TAD_INT_CE (L2C_TAD_INT_SBFSBE | \
1530 L2C_TAD_INT_FBFSBE)
1531
1532 #define L2C_TAD_INT_UE (L2C_TAD_INT_L2DDBE | \
1533 L2C_TAD_INT_SBFDBE | \
1534 L2C_TAD_INT_FBFDBE | \
1535 L2C_TAD_INT_TAGDBE | \
1536 L2C_TAD_INT_RTGDBE | \
1537 L2C_TAD_INT_WRDISOCI | \
1538 L2C_TAD_INT_RDDISOCI | \
1539 L2C_TAD_INT_WRDISLMC | \
1540 L2C_TAD_INT_RDDISLMC | \
1541 L2C_TAD_INT_LFBTO | \
1542 L2C_TAD_INT_GSYNCTO)
1543
1544 static const struct error_descr l2_tad_errors[] = {
1545 {
1546 .type = ERR_CORRECTED,
1547 .mask = L2C_TAD_INT_SBFSBE,
1548 .descr = "SBF single-bit error",
1549 },
1550 {
1551 .type = ERR_CORRECTED,
1552 .mask = L2C_TAD_INT_FBFSBE,
1553 .descr = "FBF single-bit error",
1554 },
1555 {
1556 .type = ERR_UNCORRECTED,
1557 .mask = L2C_TAD_INT_L2DDBE,
1558 .descr = "L2D double-bit error",
1559 },
1560 {
1561 .type = ERR_UNCORRECTED,
1562 .mask = L2C_TAD_INT_SBFDBE,
1563 .descr = "SBF double-bit error",
1564 },
1565 {
1566 .type = ERR_UNCORRECTED,
1567 .mask = L2C_TAD_INT_FBFDBE,
1568 .descr = "FBF double-bit error",
1569 },
1570 {
1571 .type = ERR_UNCORRECTED,
1572 .mask = L2C_TAD_INT_TAGDBE,
1573 .descr = "TAG double-bit error",
1574 },
1575 {
1576 .type = ERR_UNCORRECTED,
1577 .mask = L2C_TAD_INT_RTGDBE,
1578 .descr = "RTG double-bit error",
1579 },
1580 {
1581 .type = ERR_UNCORRECTED,
1582 .mask = L2C_TAD_INT_WRDISOCI,
1583 .descr = "Write to a disabled CCPI",
1584 },
1585 {
1586 .type = ERR_UNCORRECTED,
1587 .mask = L2C_TAD_INT_RDDISOCI,
1588 .descr = "Read from a disabled CCPI",
1589 },
1590 {
1591 .type = ERR_UNCORRECTED,
1592 .mask = L2C_TAD_INT_WRDISLMC,
1593 .descr = "Write to a disabled LMC",
1594 },
1595 {
1596 .type = ERR_UNCORRECTED,
1597 .mask = L2C_TAD_INT_RDDISLMC,
1598 .descr = "Read from a disabled LMC",
1599 },
1600 {
1601 .type = ERR_UNCORRECTED,
1602 .mask = L2C_TAD_INT_LFBTO,
1603 .descr = "LFB entry timeout",
1604 },
1605 {
1606 .type = ERR_UNCORRECTED,
1607 .mask = L2C_TAD_INT_GSYNCTO,
1608 .descr = "Global sync CCPI timeout",
1609 },
1610 {0, 0, NULL},
1611 };
1612
1613 #define L2C_TAD_INT_TAG (L2C_TAD_INT_TAGDBE)
1614
1615 #define L2C_TAD_INT_RTG (L2C_TAD_INT_RTGDBE)
1616
1617 #define L2C_TAD_INT_DISLMC (L2C_TAD_INT_WRDISLMC | L2C_TAD_INT_RDDISLMC)
1618
1619 #define L2C_TAD_INT_DISOCI (L2C_TAD_INT_WRDISOCI | L2C_TAD_INT_RDDISOCI)
1620
1621 #define L2C_TAD_INT_ENA_ALL (L2C_TAD_INT_ECC | L2C_TAD_INT_TAG | \
1622 L2C_TAD_INT_RTG | \
1623 L2C_TAD_INT_DISLMC | L2C_TAD_INT_DISOCI | \
1624 L2C_TAD_INT_LFBTO)
1625
1626 #define L2C_TAD_TIMETWO 0x50000
1627 #define L2C_TAD_TIMEOUT 0x50100
1628 #define L2C_TAD_ERR 0x60000
1629 #define L2C_TAD_TQD_ERR 0x60100
1630 #define L2C_TAD_TTG_ERR 0x60200
1631
1632
1633 #define L2C_CBC_INT_W1C 0x60000
1634
1635 #define L2C_CBC_INT_RSDSBE BIT(0)
1636 #define L2C_CBC_INT_RSDDBE BIT(1)
1637
1638 #define L2C_CBC_INT_RSD (L2C_CBC_INT_RSDSBE | L2C_CBC_INT_RSDDBE)
1639
1640 #define L2C_CBC_INT_MIBSBE BIT(4)
1641 #define L2C_CBC_INT_MIBDBE BIT(5)
1642
1643 #define L2C_CBC_INT_MIB (L2C_CBC_INT_MIBSBE | L2C_CBC_INT_MIBDBE)
1644
1645 #define L2C_CBC_INT_IORDDISOCI BIT(6)
1646 #define L2C_CBC_INT_IOWRDISOCI BIT(7)
1647
1648 #define L2C_CBC_INT_IODISOCI (L2C_CBC_INT_IORDDISOCI | \
1649 L2C_CBC_INT_IOWRDISOCI)
1650
1651 #define L2C_CBC_INT_CE (L2C_CBC_INT_RSDSBE | L2C_CBC_INT_MIBSBE)
1652 #define L2C_CBC_INT_UE (L2C_CBC_INT_RSDDBE | L2C_CBC_INT_MIBDBE)
1653
1654
1655 static const struct error_descr l2_cbc_errors[] = {
1656 {
1657 .type = ERR_CORRECTED,
1658 .mask = L2C_CBC_INT_RSDSBE,
1659 .descr = "RSD single-bit error",
1660 },
1661 {
1662 .type = ERR_CORRECTED,
1663 .mask = L2C_CBC_INT_MIBSBE,
1664 .descr = "MIB single-bit error",
1665 },
1666 {
1667 .type = ERR_UNCORRECTED,
1668 .mask = L2C_CBC_INT_RSDDBE,
1669 .descr = "RSD double-bit error",
1670 },
1671 {
1672 .type = ERR_UNCORRECTED,
1673 .mask = L2C_CBC_INT_MIBDBE,
1674 .descr = "MIB double-bit error",
1675 },
1676 {
1677 .type = ERR_UNCORRECTED,
1678 .mask = L2C_CBC_INT_IORDDISOCI,
1679 .descr = "Read from a disabled CCPI",
1680 },
1681 {
1682 .type = ERR_UNCORRECTED,
1683 .mask = L2C_CBC_INT_IOWRDISOCI,
1684 .descr = "Write to a disabled CCPI",
1685 },
1686 {0, 0, NULL},
1687 };
1688
1689 #define L2C_CBC_INT_W1S 0x60008
1690 #define L2C_CBC_INT_ENA_W1C 0x60020
1691
1692 #define L2C_CBC_INT_ENA_ALL (L2C_CBC_INT_RSD | L2C_CBC_INT_MIB | \
1693 L2C_CBC_INT_IODISOCI)
1694
1695 #define L2C_CBC_INT_ENA_W1S 0x60028
1696
1697 #define L2C_CBC_IODISOCIERR 0x80008
1698 #define L2C_CBC_IOCERR 0x80010
1699 #define L2C_CBC_RSDERR 0x80018
1700 #define L2C_CBC_MIBERR 0x80020
1701
1702
1703 #define L2C_MCI_INT_W1C 0x0
1704
1705 #define L2C_MCI_INT_VBFSBE BIT(0)
1706 #define L2C_MCI_INT_VBFDBE BIT(1)
1707
1708 static const struct error_descr l2_mci_errors[] = {
1709 {
1710 .type = ERR_CORRECTED,
1711 .mask = L2C_MCI_INT_VBFSBE,
1712 .descr = "VBF single-bit error",
1713 },
1714 {
1715 .type = ERR_UNCORRECTED,
1716 .mask = L2C_MCI_INT_VBFDBE,
1717 .descr = "VBF double-bit error",
1718 },
1719 {0, 0, NULL},
1720 };
1721
1722 #define L2C_MCI_INT_W1S 0x8
1723 #define L2C_MCI_INT_ENA_W1C 0x20
1724
1725 #define L2C_MCI_INT_ENA_ALL (L2C_MCI_INT_VBFSBE | L2C_MCI_INT_VBFDBE)
1726
1727 #define L2C_MCI_INT_ENA_W1S 0x28
1728
1729 #define L2C_MCI_ERR 0x10000
1730
1731 #define L2C_MESSAGE_SIZE SZ_1K
1732 #define L2C_OTHER_SIZE (50 * ARRAY_SIZE(l2_tad_errors))
1733
1734 struct l2c_err_ctx {
1735 char *reg_ext_name;
1736 u64 reg_int;
1737 u64 reg_ext;
1738 };
1739
1740 struct thunderx_l2c {
1741 void __iomem *regs;
1742 struct pci_dev *pdev;
1743 struct edac_device_ctl_info *edac_dev;
1744
1745 struct dentry *debugfs;
1746
1747 int index;
1748
1749 struct msix_entry msix_ent;
1750
1751 struct l2c_err_ctx err_ctx[RING_ENTRIES];
1752 unsigned long ring_head;
1753 unsigned long ring_tail;
1754 };
1755
1756 static irqreturn_t thunderx_l2c_tad_isr(int irq, void *irq_id)
1757 {
1758 struct msix_entry *msix = irq_id;
1759 struct thunderx_l2c *tad = container_of(msix, struct thunderx_l2c,
1760 msix_ent);
1761
1762 unsigned long head = ring_pos(tad->ring_head, ARRAY_SIZE(tad->err_ctx));
1763 struct l2c_err_ctx *ctx = &tad->err_ctx[head];
1764
1765 ctx->reg_int = readq(tad->regs + L2C_TAD_INT_W1C);
1766
1767 if (ctx->reg_int & L2C_TAD_INT_ECC) {
1768 ctx->reg_ext_name = "TQD_ERR";
1769 ctx->reg_ext = readq(tad->regs + L2C_TAD_TQD_ERR);
1770 } else if (ctx->reg_int & L2C_TAD_INT_TAG) {
1771 ctx->reg_ext_name = "TTG_ERR";
1772 ctx->reg_ext = readq(tad->regs + L2C_TAD_TTG_ERR);
1773 } else if (ctx->reg_int & L2C_TAD_INT_LFBTO) {
1774 ctx->reg_ext_name = "TIMEOUT";
1775 ctx->reg_ext = readq(tad->regs + L2C_TAD_TIMEOUT);
1776 } else if (ctx->reg_int & L2C_TAD_INT_DISOCI) {
1777 ctx->reg_ext_name = "ERR";
1778 ctx->reg_ext = readq(tad->regs + L2C_TAD_ERR);
1779 }
1780
1781 writeq(ctx->reg_int, tad->regs + L2C_TAD_INT_W1C);
1782
1783 tad->ring_head++;
1784
1785 return IRQ_WAKE_THREAD;
1786 }
1787
1788 static irqreturn_t thunderx_l2c_cbc_isr(int irq, void *irq_id)
1789 {
1790 struct msix_entry *msix = irq_id;
1791 struct thunderx_l2c *cbc = container_of(msix, struct thunderx_l2c,
1792 msix_ent);
1793
1794 unsigned long head = ring_pos(cbc->ring_head, ARRAY_SIZE(cbc->err_ctx));
1795 struct l2c_err_ctx *ctx = &cbc->err_ctx[head];
1796
1797 ctx->reg_int = readq(cbc->regs + L2C_CBC_INT_W1C);
1798
1799 if (ctx->reg_int & L2C_CBC_INT_RSD) {
1800 ctx->reg_ext_name = "RSDERR";
1801 ctx->reg_ext = readq(cbc->regs + L2C_CBC_RSDERR);
1802 } else if (ctx->reg_int & L2C_CBC_INT_MIB) {
1803 ctx->reg_ext_name = "MIBERR";
1804 ctx->reg_ext = readq(cbc->regs + L2C_CBC_MIBERR);
1805 } else if (ctx->reg_int & L2C_CBC_INT_IODISOCI) {
1806 ctx->reg_ext_name = "IODISOCIERR";
1807 ctx->reg_ext = readq(cbc->regs + L2C_CBC_IODISOCIERR);
1808 }
1809
1810 writeq(ctx->reg_int, cbc->regs + L2C_CBC_INT_W1C);
1811
1812 cbc->ring_head++;
1813
1814 return IRQ_WAKE_THREAD;
1815 }
1816
1817 static irqreturn_t thunderx_l2c_mci_isr(int irq, void *irq_id)
1818 {
1819 struct msix_entry *msix = irq_id;
1820 struct thunderx_l2c *mci = container_of(msix, struct thunderx_l2c,
1821 msix_ent);
1822
1823 unsigned long head = ring_pos(mci->ring_head, ARRAY_SIZE(mci->err_ctx));
1824 struct l2c_err_ctx *ctx = &mci->err_ctx[head];
1825
1826 ctx->reg_int = readq(mci->regs + L2C_MCI_INT_W1C);
1827 ctx->reg_ext = readq(mci->regs + L2C_MCI_ERR);
1828
1829 writeq(ctx->reg_int, mci->regs + L2C_MCI_INT_W1C);
1830
1831 ctx->reg_ext_name = "ERR";
1832
1833 mci->ring_head++;
1834
1835 return IRQ_WAKE_THREAD;
1836 }
1837
1838 static irqreturn_t thunderx_l2c_threaded_isr(int irq, void *irq_id)
1839 {
1840 struct msix_entry *msix = irq_id;
1841 struct thunderx_l2c *l2c = container_of(msix, struct thunderx_l2c,
1842 msix_ent);
1843
1844 unsigned long tail = ring_pos(l2c->ring_tail, ARRAY_SIZE(l2c->err_ctx));
1845 struct l2c_err_ctx *ctx = &l2c->err_ctx[tail];
1846 irqreturn_t ret = IRQ_NONE;
1847
1848 u64 mask_ue, mask_ce;
1849 const struct error_descr *l2_errors;
1850 char *reg_int_name;
1851
1852 char *msg;
1853 char *other;
1854
1855 msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
1856 other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
1857
1858 if (!msg || !other)
1859 goto err_free;
1860
1861 switch (l2c->pdev->device) {
1862 case PCI_DEVICE_ID_THUNDER_L2C_TAD:
1863 reg_int_name = "L2C_TAD_INT";
1864 mask_ue = L2C_TAD_INT_UE;
1865 mask_ce = L2C_TAD_INT_CE;
1866 l2_errors = l2_tad_errors;
1867 break;
1868 case PCI_DEVICE_ID_THUNDER_L2C_CBC:
1869 reg_int_name = "L2C_CBC_INT";
1870 mask_ue = L2C_CBC_INT_UE;
1871 mask_ce = L2C_CBC_INT_CE;
1872 l2_errors = l2_cbc_errors;
1873 break;
1874 case PCI_DEVICE_ID_THUNDER_L2C_MCI:
1875 reg_int_name = "L2C_MCI_INT";
1876 mask_ue = L2C_MCI_INT_VBFDBE;
1877 mask_ce = L2C_MCI_INT_VBFSBE;
1878 l2_errors = l2_mci_errors;
1879 break;
1880 default:
1881 dev_err(&l2c->pdev->dev, "Unsupported device: %04x\n",
1882 l2c->pdev->device);
1883 return IRQ_NONE;
1884 }
1885
1886 while (CIRC_CNT(l2c->ring_head, l2c->ring_tail,
1887 ARRAY_SIZE(l2c->err_ctx))) {
1888 snprintf(msg, L2C_MESSAGE_SIZE,
1889 "%s: %s: %016llx, %s: %016llx",
1890 l2c->edac_dev->ctl_name, reg_int_name, ctx->reg_int,
1891 ctx->reg_ext_name, ctx->reg_ext);
1892
1893 decode_register(other, L2C_OTHER_SIZE, l2_errors, ctx->reg_int);
1894
1895 strncat(msg, other, L2C_MESSAGE_SIZE);
1896
1897 if (ctx->reg_int & mask_ue)
1898 edac_device_handle_ue(l2c->edac_dev, 0, 0, msg);
1899 else if (ctx->reg_int & mask_ce)
1900 edac_device_handle_ce(l2c->edac_dev, 0, 0, msg);
1901
1902 l2c->ring_tail++;
1903 }
1904
1905 return IRQ_HANDLED;
1906
1907 err_free:
1908 kfree(other);
1909 kfree(msg);
1910
1911 return ret;
1912 }
1913
1914 #define L2C_DEBUGFS_ATTR(_name, _reg) DEBUGFS_REG_ATTR(l2c, _name, _reg)
1915
1916 L2C_DEBUGFS_ATTR(tad_int, L2C_TAD_INT_W1S);
1917
1918 struct debugfs_entry *l2c_tad_dfs_ents[] = {
1919 &debugfs_tad_int,
1920 };
1921
1922 L2C_DEBUGFS_ATTR(cbc_int, L2C_CBC_INT_W1S);
1923
1924 struct debugfs_entry *l2c_cbc_dfs_ents[] = {
1925 &debugfs_cbc_int,
1926 };
1927
1928 L2C_DEBUGFS_ATTR(mci_int, L2C_MCI_INT_W1S);
1929
1930 struct debugfs_entry *l2c_mci_dfs_ents[] = {
1931 &debugfs_mci_int,
1932 };
1933
1934 static const struct pci_device_id thunderx_l2c_pci_tbl[] = {
1935 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_TAD), },
1936 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_CBC), },
1937 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_MCI), },
1938 { 0, },
1939 };
1940
1941 static int thunderx_l2c_probe(struct pci_dev *pdev,
1942 const struct pci_device_id *id)
1943 {
1944 struct thunderx_l2c *l2c;
1945 struct edac_device_ctl_info *edac_dev;
1946 struct debugfs_entry **l2c_devattr;
1947 size_t dfs_entries;
1948 irqreturn_t (*thunderx_l2c_isr)(int, void *) = NULL;
1949 char name[32];
1950 const char *fmt;
1951 u64 reg_en_offs, reg_en_mask;
1952 int idx;
1953 int ret;
1954
1955 ret = pcim_enable_device(pdev);
1956 if (ret) {
1957 dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
1958 return ret;
1959 }
1960
1961 ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_l2c");
1962 if (ret) {
1963 dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
1964 return ret;
1965 }
1966
1967 switch (pdev->device) {
1968 case PCI_DEVICE_ID_THUNDER_L2C_TAD:
1969 thunderx_l2c_isr = thunderx_l2c_tad_isr;
1970 l2c_devattr = l2c_tad_dfs_ents;
1971 dfs_entries = ARRAY_SIZE(l2c_tad_dfs_ents);
1972 fmt = "L2C-TAD%d";
1973 reg_en_offs = L2C_TAD_INT_ENA_W1S;
1974 reg_en_mask = L2C_TAD_INT_ENA_ALL;
1975 break;
1976 case PCI_DEVICE_ID_THUNDER_L2C_CBC:
1977 thunderx_l2c_isr = thunderx_l2c_cbc_isr;
1978 l2c_devattr = l2c_cbc_dfs_ents;
1979 dfs_entries = ARRAY_SIZE(l2c_cbc_dfs_ents);
1980 fmt = "L2C-CBC%d";
1981 reg_en_offs = L2C_CBC_INT_ENA_W1S;
1982 reg_en_mask = L2C_CBC_INT_ENA_ALL;
1983 break;
1984 case PCI_DEVICE_ID_THUNDER_L2C_MCI:
1985 thunderx_l2c_isr = thunderx_l2c_mci_isr;
1986 l2c_devattr = l2c_mci_dfs_ents;
1987 dfs_entries = ARRAY_SIZE(l2c_mci_dfs_ents);
1988 fmt = "L2C-MCI%d";
1989 reg_en_offs = L2C_MCI_INT_ENA_W1S;
1990 reg_en_mask = L2C_MCI_INT_ENA_ALL;
1991 break;
1992 default:
1993 //Should never ever get here
1994 dev_err(&pdev->dev, "Unsupported PCI device: %04x\n",
1995 pdev->device);
1996 return -EINVAL;
1997 }
1998
1999 idx = edac_device_alloc_index();
2000 snprintf(name, sizeof(name), fmt, idx);
2001
2002 edac_dev = edac_device_alloc_ctl_info(sizeof(struct thunderx_l2c),
2003 name, 1, "L2C", 1, 0,
2004 NULL, 0, idx);
2005 if (!edac_dev) {
2006 dev_err(&pdev->dev, "Cannot allocate EDAC device\n");
2007 return -ENOMEM;
2008 }
2009
2010 l2c = edac_dev->pvt_info;
2011 l2c->edac_dev = edac_dev;
2012
2013 l2c->regs = pcim_iomap_table(pdev)[0];
2014 if (!l2c->regs) {
2015 dev_err(&pdev->dev, "Cannot map PCI resources\n");
2016 ret = -ENODEV;
2017 goto err_free;
2018 }
2019
2020 l2c->pdev = pdev;
2021
2022 l2c->ring_head = 0;
2023 l2c->ring_tail = 0;
2024
2025 l2c->msix_ent.entry = 0;
2026 l2c->msix_ent.vector = 0;
2027
2028 ret = pci_enable_msix_exact(pdev, &l2c->msix_ent, 1);
2029 if (ret) {
2030 dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
2031 goto err_free;
2032 }
2033
2034 ret = devm_request_threaded_irq(&pdev->dev, l2c->msix_ent.vector,
2035 thunderx_l2c_isr,
2036 thunderx_l2c_threaded_isr,
2037 0, "[EDAC] ThunderX L2C",
2038 &l2c->msix_ent);
2039 if (ret)
2040 goto err_free;
2041
2042 edac_dev->dev = &pdev->dev;
2043 edac_dev->dev_name = dev_name(&pdev->dev);
2044 edac_dev->mod_name = "thunderx-l2c";
2045 edac_dev->ctl_name = "thunderx-l2c";
2046
2047 ret = edac_device_add_device(edac_dev);
2048 if (ret) {
2049 dev_err(&pdev->dev, "Cannot add EDAC device: %d\n", ret);
2050 goto err_free;
2051 }
2052
2053 if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
2054 l2c->debugfs = edac_debugfs_create_dir(pdev->dev.kobj.name);
2055
2056 ret = thunderx_create_debugfs_nodes(l2c->debugfs, l2c_devattr,
2057 l2c, dfs_entries);
2058
2059 if (ret != dfs_entries) {
2060 dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
2061 ret, ret >= 0 ? " created" : "");
2062 }
2063 }
2064
2065 pci_set_drvdata(pdev, edac_dev);
2066
2067 writeq(reg_en_mask, l2c->regs + reg_en_offs);
2068
2069 return 0;
2070
2071 err_free:
2072 edac_device_free_ctl_info(edac_dev);
2073
2074 return ret;
2075 }
2076
2077 static void thunderx_l2c_remove(struct pci_dev *pdev)
2078 {
2079 struct edac_device_ctl_info *edac_dev = pci_get_drvdata(pdev);
2080 struct thunderx_l2c *l2c = edac_dev->pvt_info;
2081
2082 switch (pdev->device) {
2083 case PCI_DEVICE_ID_THUNDER_L2C_TAD:
2084 writeq(L2C_TAD_INT_ENA_ALL, l2c->regs + L2C_TAD_INT_ENA_W1C);
2085 break;
2086 case PCI_DEVICE_ID_THUNDER_L2C_CBC:
2087 writeq(L2C_CBC_INT_ENA_ALL, l2c->regs + L2C_CBC_INT_ENA_W1C);
2088 break;
2089 case PCI_DEVICE_ID_THUNDER_L2C_MCI:
2090 writeq(L2C_MCI_INT_ENA_ALL, l2c->regs + L2C_MCI_INT_ENA_W1C);
2091 break;
2092 }
2093
2094 edac_debugfs_remove_recursive(l2c->debugfs);
2095
2096 edac_device_del_device(&pdev->dev);
2097 edac_device_free_ctl_info(edac_dev);
2098 }
2099
2100 MODULE_DEVICE_TABLE(pci, thunderx_l2c_pci_tbl);
2101
2102 static struct pci_driver thunderx_l2c_driver = {
2103 .name = "thunderx_l2c_edac",
2104 .probe = thunderx_l2c_probe,
2105 .remove = thunderx_l2c_remove,
2106 .id_table = thunderx_l2c_pci_tbl,
2107 };
2108
2109 static int __init thunderx_edac_init(void)
2110 {
2111 int rc = 0;
2112
2113 rc = pci_register_driver(&thunderx_lmc_driver);
2114 if (rc)
2115 return rc;
2116
2117 rc = pci_register_driver(&thunderx_ocx_driver);
2118 if (rc)
2119 goto err_lmc;
2120
2121 rc = pci_register_driver(&thunderx_l2c_driver);
2122 if (rc)
2123 goto err_ocx;
2124
2125 return rc;
2126 err_ocx:
2127 pci_unregister_driver(&thunderx_ocx_driver);
2128 err_lmc:
2129 pci_unregister_driver(&thunderx_lmc_driver);
2130
2131 return rc;
2132 }
2133
2134 static void __exit thunderx_edac_exit(void)
2135 {
2136 pci_unregister_driver(&thunderx_l2c_driver);
2137 pci_unregister_driver(&thunderx_ocx_driver);
2138 pci_unregister_driver(&thunderx_lmc_driver);
2139
2140 }
2141
2142 module_init(thunderx_edac_init);
2143 module_exit(thunderx_edac_exit);
2144
2145 MODULE_LICENSE("GPL v2");
2146 MODULE_AUTHOR("Cavium, Inc.");
2147 MODULE_DESCRIPTION("EDAC Driver for Cavium ThunderX");
Linux with added FPC-III support