1 // SPDX-License-Identifier: GPL-2.0-only
3 * Driver for Pondicherry2 memory controller.
5 * Copyright (c) 2016, Intel Corporation.
7 * [Derived from sb_edac.c]
9 * Translation of system physical addresses to DIMM addresses
10 * is a two stage process:
12 * First the Pondicherry 2 memory controller handles slice and channel interleaving
13 * in "sys2pmi()". This is (almost) completley common between platforms.
15 * Then a platform specific dunit (DIMM unit) completes the process to provide DIMM,
16 * rank, bank, row and column using the appropriate "dunit_ops" functions/parameters.
19 #include <linux/module.h>
20 #include <linux/init.h>
21 #include <linux/pci.h>
22 #include <linux/pci_ids.h>
23 #include <linux/slab.h>
24 #include <linux/delay.h>
25 #include <linux/edac.h>
26 #include <linux/mmzone.h>
27 #include <linux/smp.h>
28 #include <linux/bitmap.h>
29 #include <linux/math64.h>
30 #include <linux/mod_devicetable.h>
31 #include <asm/cpu_device_id.h>
32 #include <asm/intel-family.h>
33 #include <asm/processor.h>
37 #include "edac_module.h"
38 #include "pnd2_edac.h"
40 #define EDAC_MOD_STR "pnd2_edac"
42 #define APL_NUM_CHANNELS 4
43 #define DNV_NUM_CHANNELS 2
44 #define DNV_MAX_DIMMS 2 /* Max DIMMs per channel */
48 DNV
, /* All requests go to PMI CH0 on each slice (CH1 disabled) */
61 int dimm_geom
[APL_NUM_CHANNELS
];
66 * System address space is divided into multiple regions with
67 * different interleave rules in each. The as0/as1 regions
68 * have no interleaving at all. The as2 region is interleaved
69 * between two channels. The mot region is magic and may overlap
70 * other regions, with its interleave rules taking precedence.
71 * Addresses not in any of these regions are interleaved across
74 static struct region
{
80 static struct dunit_ops
{
86 int dimms_per_channel
;
87 int (*rd_reg
)(int port
, int off
, int op
, void *data
, size_t sz
, char *name
);
88 int (*get_registers
)(void);
89 int (*check_ecc
)(void);
90 void (*mk_region
)(char *name
, struct region
*rp
, void *asym
);
91 void (*get_dimm_config
)(struct mem_ctl_info
*mci
);
92 int (*pmi2mem
)(struct mem_ctl_info
*mci
, u64 pmiaddr
, u32 pmiidx
,
93 struct dram_addr
*daddr
, char *msg
);
96 static struct mem_ctl_info
*pnd2_mci
;
98 #define PND2_MSG_SIZE 256
101 #define pnd2_printk(level, fmt, arg...) \
102 edac_printk(level, "pnd2", fmt, ##arg)
104 #define pnd2_mc_printk(mci, level, fmt, arg...) \
105 edac_mc_chipset_printk(mci, level, "pnd2", fmt, ##arg)
107 #define MOT_CHAN_INTLV_BIT_1SLC_2CH 12
108 #define MOT_CHAN_INTLV_BIT_2SLC_2CH 13
109 #define SELECTOR_DISABLED (-1)
110 #define _4GB (1ul << 32)
112 #define PMI_ADDRESS_WIDTH 31
113 #define PND_MAX_PHYS_BIT 39
115 #define APL_ASYMSHIFT 28
116 #define DNV_ASYMSHIFT 31
117 #define CH_HASH_MASK_LSB 6
118 #define SLICE_HASH_MASK_LSB 6
119 #define MOT_SLC_INTLV_BIT 12
120 #define LOG2_PMI_ADDR_GRANULARITY 5
123 #define GET_BITFIELD(v, lo, hi) (((v) & GENMASK_ULL(hi, lo)) >> (lo))
124 #define U64_LSHIFT(val, s) ((u64)(val) << (s))
127 * On Apollo Lake we access memory controller registers via a
128 * side-band mailbox style interface in a hidden PCI device
129 * configuration space.
131 static struct pci_bus
*p2sb_bus
;
132 #define P2SB_DEVFN PCI_DEVFN(0xd, 0)
133 #define P2SB_ADDR_OFF 0xd0
134 #define P2SB_DATA_OFF 0xd4
135 #define P2SB_STAT_OFF 0xd8
136 #define P2SB_ROUT_OFF 0xda
137 #define P2SB_EADD_OFF 0xdc
138 #define P2SB_HIDE_OFF 0xe1
142 #define P2SB_READ(size, off, ptr) \
143 pci_bus_read_config_##size(p2sb_bus, P2SB_DEVFN, off, ptr)
144 #define P2SB_WRITE(size, off, val) \
145 pci_bus_write_config_##size(p2sb_bus, P2SB_DEVFN, off, val)
147 static bool p2sb_is_busy(u16
*status
)
149 P2SB_READ(word
, P2SB_STAT_OFF
, status
);
151 return !!(*status
& P2SB_BUSY
);
154 static int _apl_rd_reg(int port
, int off
, int op
, u32
*data
)
156 int retries
= 0xff, ret
;
160 /* Unhide the P2SB device, if it's hidden */
161 P2SB_READ(byte
, P2SB_HIDE_OFF
, &hidden
);
163 P2SB_WRITE(byte
, P2SB_HIDE_OFF
, 0);
165 if (p2sb_is_busy(&status
)) {
170 P2SB_WRITE(dword
, P2SB_ADDR_OFF
, (port
<< 24) | off
);
171 P2SB_WRITE(dword
, P2SB_DATA_OFF
, 0);
172 P2SB_WRITE(dword
, P2SB_EADD_OFF
, 0);
173 P2SB_WRITE(word
, P2SB_ROUT_OFF
, 0);
174 P2SB_WRITE(word
, P2SB_STAT_OFF
, (op
<< 8) | P2SB_BUSY
);
176 while (p2sb_is_busy(&status
)) {
177 if (retries
-- == 0) {
183 P2SB_READ(dword
, P2SB_DATA_OFF
, data
);
184 ret
= (status
>> 1) & 0x3;
186 /* Hide the P2SB device, if it was hidden before */
188 P2SB_WRITE(byte
, P2SB_HIDE_OFF
, hidden
);
193 static int apl_rd_reg(int port
, int off
, int op
, void *data
, size_t sz
, char *name
)
197 edac_dbg(2, "Read %s port=%x off=%x op=%x\n", name
, port
, off
, op
);
200 ret
= _apl_rd_reg(port
, off
+ 4, op
, (u32
*)(data
+ 4));
203 ret
|= _apl_rd_reg(port
, off
, op
, (u32
*)data
);
204 pnd2_printk(KERN_DEBUG
, "%s=%x%08x ret=%d\n", name
,
205 sz
== 8 ? *((u32
*)(data
+ 4)) : 0, *((u32
*)data
), ret
);
212 static u64
get_mem_ctrl_hub_base_addr(void)
214 struct b_cr_mchbar_lo_pci lo
;
215 struct b_cr_mchbar_hi_pci hi
;
216 struct pci_dev
*pdev
;
218 pdev
= pci_get_device(PCI_VENDOR_ID_INTEL
, 0x1980, NULL
);
220 pci_read_config_dword(pdev
, 0x48, (u32
*)&lo
);
221 pci_read_config_dword(pdev
, 0x4c, (u32
*)&hi
);
228 edac_dbg(2, "MMIO via memory controller hub base address is disabled!\n");
232 return U64_LSHIFT(hi
.base
, 32) | U64_LSHIFT(lo
.base
, 15);
235 static u64
get_sideband_reg_base_addr(void)
237 struct pci_dev
*pdev
;
241 pdev
= pci_get_device(PCI_VENDOR_ID_INTEL
, 0x19dd, NULL
);
243 /* Unhide the P2SB device, if it's hidden */
244 pci_read_config_byte(pdev
, 0xe1, &hidden
);
246 pci_write_config_byte(pdev
, 0xe1, 0);
248 pci_read_config_dword(pdev
, 0x10, &lo
);
249 pci_read_config_dword(pdev
, 0x14, &hi
);
252 /* Hide the P2SB device, if it was hidden before */
254 pci_write_config_byte(pdev
, 0xe1, hidden
);
257 return (U64_LSHIFT(hi
, 32) | U64_LSHIFT(lo
, 0));
263 #define DNV_MCHBAR_SIZE 0x8000
264 #define DNV_SB_PORT_SIZE 0x10000
265 static int dnv_rd_reg(int port
, int off
, int op
, void *data
, size_t sz
, char *name
)
267 struct pci_dev
*pdev
;
273 pdev
= pci_get_device(PCI_VENDOR_ID_INTEL
, 0x1980, NULL
);
277 pci_read_config_dword(pdev
, off
, data
);
280 /* MMIO via memory controller hub base address */
281 if (op
== 0 && port
== 0x4c) {
282 addr
= get_mem_ctrl_hub_base_addr();
285 size
= DNV_MCHBAR_SIZE
;
287 /* MMIO via sideband register base address */
288 addr
= get_sideband_reg_base_addr();
291 addr
+= (port
<< 16);
292 size
= DNV_SB_PORT_SIZE
;
295 base
= ioremap((resource_size_t
)addr
, size
);
300 *(u32
*)(data
+ 4) = *(u32
*)(base
+ off
+ 4);
301 *(u32
*)data
= *(u32
*)(base
+ off
);
306 edac_dbg(2, "Read %s=%.8x_%.8x\n", name
,
307 (sz
== 8) ? *(u32
*)(data
+ 4) : 0, *(u32
*)data
);
312 #define RD_REGP(regp, regname, port) \
315 regname##_r_opcode, \
316 regp, sizeof(struct regname), \
319 #define RD_REG(regp, regname) \
320 ops->rd_reg(regname ## _port, \
322 regname##_r_opcode, \
323 regp, sizeof(struct regname), \
326 static u64 top_lm
, top_hm
;
327 static bool two_slices
;
328 static bool two_channels
; /* Both PMI channels in one slice enabled */
330 static u8 sym_chan_mask
;
331 static u8 asym_chan_mask
;
334 static int slice_selector
= -1;
335 static int chan_selector
= -1;
336 static u64 slice_hash_mask
;
337 static u64 chan_hash_mask
;
339 static void mk_region(char *name
, struct region
*rp
, u64 base
, u64 limit
)
344 edac_dbg(2, "Region:%s [%llx, %llx]\n", name
, base
, limit
);
347 static void mk_region_mask(char *name
, struct region
*rp
, u64 base
, u64 mask
)
350 pr_info(FW_BUG
"MOT mask cannot be zero\n");
353 if (mask
!= GENMASK_ULL(PND_MAX_PHYS_BIT
, __ffs(mask
))) {
354 pr_info(FW_BUG
"MOT mask not power of two\n");
358 pr_info(FW_BUG
"MOT region base/mask alignment error\n");
362 rp
->limit
= (base
| ~mask
) & GENMASK_ULL(PND_MAX_PHYS_BIT
, 0);
364 edac_dbg(2, "Region:%s [%llx, %llx]\n", name
, base
, rp
->limit
);
367 static bool in_region(struct region
*rp
, u64 addr
)
372 return rp
->base
<= addr
&& addr
<= rp
->limit
;
375 static int gen_sym_mask(struct b_cr_slice_channel_hash
*p
)
379 if (!p
->slice_0_mem_disabled
)
380 mask
|= p
->sym_slice0_channel_enabled
;
382 if (!p
->slice_1_disabled
)
383 mask
|= p
->sym_slice1_channel_enabled
<< 2;
385 if (p
->ch_1_disabled
|| p
->enable_pmi_dual_data_mode
)
391 static int gen_asym_mask(struct b_cr_slice_channel_hash
*p
,
392 struct b_cr_asym_mem_region0_mchbar
*as0
,
393 struct b_cr_asym_mem_region1_mchbar
*as1
,
394 struct b_cr_asym_2way_mem_region_mchbar
*as2way
)
396 const int intlv
[] = { 0x5, 0xA, 0x3, 0xC };
399 if (as2way
->asym_2way_interleave_enable
)
400 mask
= intlv
[as2way
->asym_2way_intlv_mode
];
401 if (as0
->slice0_asym_enable
)
402 mask
|= (1 << as0
->slice0_asym_channel_select
);
403 if (as1
->slice1_asym_enable
)
404 mask
|= (4 << as1
->slice1_asym_channel_select
);
405 if (p
->slice_0_mem_disabled
)
407 if (p
->slice_1_disabled
)
409 if (p
->ch_1_disabled
|| p
->enable_pmi_dual_data_mode
)
415 static struct b_cr_tolud_pci tolud
;
416 static struct b_cr_touud_lo_pci touud_lo
;
417 static struct b_cr_touud_hi_pci touud_hi
;
418 static struct b_cr_asym_mem_region0_mchbar asym0
;
419 static struct b_cr_asym_mem_region1_mchbar asym1
;
420 static struct b_cr_asym_2way_mem_region_mchbar asym_2way
;
421 static struct b_cr_mot_out_base_mchbar mot_base
;
422 static struct b_cr_mot_out_mask_mchbar mot_mask
;
423 static struct b_cr_slice_channel_hash chash
;
425 /* Apollo Lake dunit */
427 * Validated on board with just two DIMMs in the [0] and [2] positions
428 * in this array. Other port number matches documentation, but caution
431 static const int apl_dports
[APL_NUM_CHANNELS
] = { 0x18, 0x10, 0x11, 0x19 };
432 static struct d_cr_drp0 drp0
[APL_NUM_CHANNELS
];
434 /* Denverton dunit */
435 static const int dnv_dports
[DNV_NUM_CHANNELS
] = { 0x10, 0x12 };
436 static struct d_cr_dsch dsch
;
437 static struct d_cr_ecc_ctrl ecc_ctrl
[DNV_NUM_CHANNELS
];
438 static struct d_cr_drp drp
[DNV_NUM_CHANNELS
];
439 static struct d_cr_dmap dmap
[DNV_NUM_CHANNELS
];
440 static struct d_cr_dmap1 dmap1
[DNV_NUM_CHANNELS
];
441 static struct d_cr_dmap2 dmap2
[DNV_NUM_CHANNELS
];
442 static struct d_cr_dmap3 dmap3
[DNV_NUM_CHANNELS
];
443 static struct d_cr_dmap4 dmap4
[DNV_NUM_CHANNELS
];
444 static struct d_cr_dmap5 dmap5
[DNV_NUM_CHANNELS
];
446 static void apl_mk_region(char *name
, struct region
*rp
, void *asym
)
448 struct b_cr_asym_mem_region0_mchbar
*a
= asym
;
451 U64_LSHIFT(a
->slice0_asym_base
, APL_ASYMSHIFT
),
452 U64_LSHIFT(a
->slice0_asym_limit
, APL_ASYMSHIFT
) +
453 GENMASK_ULL(APL_ASYMSHIFT
- 1, 0));
456 static void dnv_mk_region(char *name
, struct region
*rp
, void *asym
)
458 struct b_cr_asym_mem_region_denverton
*a
= asym
;
461 U64_LSHIFT(a
->slice_asym_base
, DNV_ASYMSHIFT
),
462 U64_LSHIFT(a
->slice_asym_limit
, DNV_ASYMSHIFT
) +
463 GENMASK_ULL(DNV_ASYMSHIFT
- 1, 0));
466 static int apl_get_registers(void)
471 if (RD_REG(&asym_2way
, b_cr_asym_2way_mem_region_mchbar
))
475 * RD_REGP() will fail for unpopulated or non-existent
476 * DIMM slots. Return success if we find at least one DIMM.
478 for (i
= 0; i
< APL_NUM_CHANNELS
; i
++)
479 if (!RD_REGP(&drp0
[i
], d_cr_drp0
, apl_dports
[i
]))
485 static int dnv_get_registers(void)
489 if (RD_REG(&dsch
, d_cr_dsch
))
492 for (i
= 0; i
< DNV_NUM_CHANNELS
; i
++)
493 if (RD_REGP(&ecc_ctrl
[i
], d_cr_ecc_ctrl
, dnv_dports
[i
]) ||
494 RD_REGP(&drp
[i
], d_cr_drp
, dnv_dports
[i
]) ||
495 RD_REGP(&dmap
[i
], d_cr_dmap
, dnv_dports
[i
]) ||
496 RD_REGP(&dmap1
[i
], d_cr_dmap1
, dnv_dports
[i
]) ||
497 RD_REGP(&dmap2
[i
], d_cr_dmap2
, dnv_dports
[i
]) ||
498 RD_REGP(&dmap3
[i
], d_cr_dmap3
, dnv_dports
[i
]) ||
499 RD_REGP(&dmap4
[i
], d_cr_dmap4
, dnv_dports
[i
]) ||
500 RD_REGP(&dmap5
[i
], d_cr_dmap5
, dnv_dports
[i
]))
507 * Read all the h/w config registers once here (they don't
508 * change at run time. Figure out which address ranges have
509 * which interleave characteristics.
511 static int get_registers(void)
513 const int intlv
[] = { 10, 11, 12, 12 };
515 if (RD_REG(&tolud
, b_cr_tolud_pci
) ||
516 RD_REG(&touud_lo
, b_cr_touud_lo_pci
) ||
517 RD_REG(&touud_hi
, b_cr_touud_hi_pci
) ||
518 RD_REG(&asym0
, b_cr_asym_mem_region0_mchbar
) ||
519 RD_REG(&asym1
, b_cr_asym_mem_region1_mchbar
) ||
520 RD_REG(&mot_base
, b_cr_mot_out_base_mchbar
) ||
521 RD_REG(&mot_mask
, b_cr_mot_out_mask_mchbar
) ||
522 RD_REG(&chash
, b_cr_slice_channel_hash
))
525 if (ops
->get_registers())
528 if (ops
->type
== DNV
) {
529 /* PMI channel idx (always 0) for asymmetric region */
530 asym0
.slice0_asym_channel_select
= 0;
531 asym1
.slice1_asym_channel_select
= 0;
532 /* PMI channel bitmap (always 1) for symmetric region */
533 chash
.sym_slice0_channel_enabled
= 0x1;
534 chash
.sym_slice1_channel_enabled
= 0x1;
537 if (asym0
.slice0_asym_enable
)
538 ops
->mk_region("as0", &as0
, &asym0
);
540 if (asym1
.slice1_asym_enable
)
541 ops
->mk_region("as1", &as1
, &asym1
);
543 if (asym_2way
.asym_2way_interleave_enable
) {
544 mk_region("as2way", &as2
,
545 U64_LSHIFT(asym_2way
.asym_2way_base
, APL_ASYMSHIFT
),
546 U64_LSHIFT(asym_2way
.asym_2way_limit
, APL_ASYMSHIFT
) +
547 GENMASK_ULL(APL_ASYMSHIFT
- 1, 0));
550 if (mot_base
.imr_en
) {
551 mk_region_mask("mot", &mot
,
552 U64_LSHIFT(mot_base
.mot_out_base
, MOT_SHIFT
),
553 U64_LSHIFT(mot_mask
.mot_out_mask
, MOT_SHIFT
));
556 top_lm
= U64_LSHIFT(tolud
.tolud
, 20);
557 top_hm
= U64_LSHIFT(touud_hi
.touud
, 32) | U64_LSHIFT(touud_lo
.touud
, 20);
559 two_slices
= !chash
.slice_1_disabled
&&
560 !chash
.slice_0_mem_disabled
&&
561 (chash
.sym_slice0_channel_enabled
!= 0) &&
562 (chash
.sym_slice1_channel_enabled
!= 0);
563 two_channels
= !chash
.ch_1_disabled
&&
564 !chash
.enable_pmi_dual_data_mode
&&
565 ((chash
.sym_slice0_channel_enabled
== 3) ||
566 (chash
.sym_slice1_channel_enabled
== 3));
568 sym_chan_mask
= gen_sym_mask(&chash
);
569 asym_chan_mask
= gen_asym_mask(&chash
, &asym0
, &asym1
, &asym_2way
);
570 chan_mask
= sym_chan_mask
| asym_chan_mask
;
572 if (two_slices
&& !two_channels
) {
576 slice_selector
= intlv
[chash
.interleave_mode
];
577 } else if (!two_slices
&& two_channels
) {
581 chan_selector
= intlv
[chash
.interleave_mode
];
582 } else if (two_slices
&& two_channels
) {
583 if (chash
.hvm_mode
) {
587 slice_selector
= intlv
[chash
.interleave_mode
];
588 chan_selector
= intlv
[chash
.interleave_mode
] + 1;
594 slice_hash_mask
= chash
.slice_hash_mask
<< SLICE_HASH_MASK_LSB
;
596 slice_hash_mask
|= BIT_ULL(slice_selector
);
601 chan_hash_mask
= chash
.ch_hash_mask
<< CH_HASH_MASK_LSB
;
603 chan_hash_mask
|= BIT_ULL(chan_selector
);
609 /* Get a contiguous memory address (remove the MMIO gap) */
610 static u64
remove_mmio_gap(u64 sys
)
612 return (sys
< _4GB
) ? sys
: sys
- (_4GB
- top_lm
);
615 /* Squeeze out one address bit, shift upper part down to fill gap */
616 static void remove_addr_bit(u64
*addr
, int bitidx
)
623 mask
= (1ull << bitidx
) - 1;
624 *addr
= ((*addr
>> 1) & ~mask
) | (*addr
& mask
);
627 /* XOR all the bits from addr specified in mask */
628 static int hash_by_mask(u64 addr
, u64 mask
)
630 u64 result
= addr
& mask
;
632 result
= (result
>> 32) ^ result
;
633 result
= (result
>> 16) ^ result
;
634 result
= (result
>> 8) ^ result
;
635 result
= (result
>> 4) ^ result
;
636 result
= (result
>> 2) ^ result
;
637 result
= (result
>> 1) ^ result
;
639 return (int)result
& 1;
643 * First stage decode. Take the system address and figure out which
644 * second stage will deal with it based on interleave modes.
646 static int sys2pmi(const u64 addr
, u32
*pmiidx
, u64
*pmiaddr
, char *msg
)
648 u64 contig_addr
, contig_base
, contig_offset
, contig_base_adj
;
649 int mot_intlv_bit
= two_slices
? MOT_CHAN_INTLV_BIT_2SLC_2CH
:
650 MOT_CHAN_INTLV_BIT_1SLC_2CH
;
651 int slice_intlv_bit_rm
= SELECTOR_DISABLED
;
652 int chan_intlv_bit_rm
= SELECTOR_DISABLED
;
653 /* Determine if address is in the MOT region. */
654 bool mot_hit
= in_region(&mot
, addr
);
655 /* Calculate the number of symmetric regions enabled. */
656 int sym_channels
= hweight8(sym_chan_mask
);
659 * The amount we need to shift the asym base can be determined by the
660 * number of enabled symmetric channels.
661 * NOTE: This can only work because symmetric memory is not supposed
662 * to do a 3-way interleave.
664 int sym_chan_shift
= sym_channels
>> 1;
666 /* Give up if address is out of range, or in MMIO gap */
667 if (addr
>= (1ul << PND_MAX_PHYS_BIT
) ||
668 (addr
>= top_lm
&& addr
< _4GB
) || addr
>= top_hm
) {
669 snprintf(msg
, PND2_MSG_SIZE
, "Error address 0x%llx is not DRAM", addr
);
673 /* Get a contiguous memory address (remove the MMIO gap) */
674 contig_addr
= remove_mmio_gap(addr
);
676 if (in_region(&as0
, addr
)) {
677 *pmiidx
= asym0
.slice0_asym_channel_select
;
679 contig_base
= remove_mmio_gap(as0
.base
);
680 contig_offset
= contig_addr
- contig_base
;
681 contig_base_adj
= (contig_base
>> sym_chan_shift
) *
682 ((chash
.sym_slice0_channel_enabled
>> (*pmiidx
& 1)) & 1);
683 contig_addr
= contig_offset
+ ((sym_channels
> 0) ? contig_base_adj
: 0ull);
684 } else if (in_region(&as1
, addr
)) {
685 *pmiidx
= 2u + asym1
.slice1_asym_channel_select
;
687 contig_base
= remove_mmio_gap(as1
.base
);
688 contig_offset
= contig_addr
- contig_base
;
689 contig_base_adj
= (contig_base
>> sym_chan_shift
) *
690 ((chash
.sym_slice1_channel_enabled
>> (*pmiidx
& 1)) & 1);
691 contig_addr
= contig_offset
+ ((sym_channels
> 0) ? contig_base_adj
: 0ull);
692 } else if (in_region(&as2
, addr
) && (asym_2way
.asym_2way_intlv_mode
== 0x3ul
)) {
695 mot_intlv_bit
= MOT_CHAN_INTLV_BIT_1SLC_2CH
;
696 *pmiidx
= (asym_2way
.asym_2way_intlv_mode
& 1) << 1;
697 channel1
= mot_hit
? ((bool)((addr
>> mot_intlv_bit
) & 1)) :
698 hash_by_mask(contig_addr
, chan_hash_mask
);
699 *pmiidx
|= (u32
)channel1
;
701 contig_base
= remove_mmio_gap(as2
.base
);
702 chan_intlv_bit_rm
= mot_hit
? mot_intlv_bit
: chan_selector
;
703 contig_offset
= contig_addr
- contig_base
;
704 remove_addr_bit(&contig_offset
, chan_intlv_bit_rm
);
705 contig_addr
= (contig_base
>> sym_chan_shift
) + contig_offset
;
707 /* Otherwise we're in normal, boring symmetric mode. */
714 slice_intlv_bit_rm
= MOT_SLC_INTLV_BIT
;
715 slice1
= (addr
>> MOT_SLC_INTLV_BIT
) & 1;
717 slice_intlv_bit_rm
= slice_selector
;
718 slice1
= hash_by_mask(addr
, slice_hash_mask
);
721 *pmiidx
= (u32
)slice1
<< 1;
727 mot_intlv_bit
= two_slices
? MOT_CHAN_INTLV_BIT_2SLC_2CH
:
728 MOT_CHAN_INTLV_BIT_1SLC_2CH
;
731 chan_intlv_bit_rm
= mot_intlv_bit
;
732 channel1
= (addr
>> mot_intlv_bit
) & 1;
734 chan_intlv_bit_rm
= chan_selector
;
735 channel1
= hash_by_mask(contig_addr
, chan_hash_mask
);
738 *pmiidx
|= (u32
)channel1
;
742 /* Remove the chan_selector bit first */
743 remove_addr_bit(&contig_addr
, chan_intlv_bit_rm
);
744 /* Remove the slice bit (we remove it second because it must be lower */
745 remove_addr_bit(&contig_addr
, slice_intlv_bit_rm
);
746 *pmiaddr
= contig_addr
;
751 /* Translate PMI address to memory (rank, row, bank, column) */
752 #define C(n) (0x10 | (n)) /* column */
753 #define B(n) (0x20 | (n)) /* bank */
754 #define R(n) (0x40 | (n)) /* row */
755 #define RS (0x80) /* rank */
771 static struct dimm_geometry
{
776 u16 bits
[PMI_ADDRESS_WIDTH
];
779 .addrdec
= AMAP_1KB
, .dden
= DEN_4Gb
, .dwid
= X16
,
780 .rowbits
= 15, .colbits
= 10,
782 C(2), C(3), C(4), C(5), C(6), B(0), B(1), B(2), R(0),
783 R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), R(9),
784 R(10), C(7), C(8), C(9), R(11), RS
, R(12), R(13), R(14),
789 .addrdec
= AMAP_1KB
, .dden
= DEN_4Gb
, .dwid
= X8
,
790 .rowbits
= 16, .colbits
= 10,
792 C(2), C(3), C(4), C(5), C(6), B(0), B(1), B(2), R(0),
793 R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), R(9),
794 R(10), C(7), C(8), C(9), R(11), RS
, R(12), R(13), R(14),
799 .addrdec
= AMAP_1KB
, .dden
= DEN_8Gb
, .dwid
= X16
,
800 .rowbits
= 16, .colbits
= 10,
802 C(2), C(3), C(4), C(5), C(6), B(0), B(1), B(2), R(0),
803 R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), R(9),
804 R(10), C(7), C(8), C(9), R(11), RS
, R(12), R(13), R(14),
809 .addrdec
= AMAP_1KB
, .dden
= DEN_8Gb
, .dwid
= X8
,
810 .rowbits
= 16, .colbits
= 11,
812 C(2), C(3), C(4), C(5), C(6), B(0), B(1), B(2), R(0),
813 R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), R(9),
814 R(10), C(7), C(8), C(9), R(11), RS
, C(11), R(12), R(13),
819 .addrdec
= AMAP_2KB
, .dden
= DEN_4Gb
, .dwid
= X16
,
820 .rowbits
= 15, .colbits
= 10,
822 C(2), C(3), C(4), C(5), C(6), C(7), B(0), B(1), B(2),
823 R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8),
824 R(9), R(10), C(8), C(9), R(11), RS
, R(12), R(13), R(14),
829 .addrdec
= AMAP_2KB
, .dden
= DEN_4Gb
, .dwid
= X8
,
830 .rowbits
= 16, .colbits
= 10,
832 C(2), C(3), C(4), C(5), C(6), C(7), B(0), B(1), B(2),
833 R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8),
834 R(9), R(10), C(8), C(9), R(11), RS
, R(12), R(13), R(14),
839 .addrdec
= AMAP_2KB
, .dden
= DEN_8Gb
, .dwid
= X16
,
840 .rowbits
= 16, .colbits
= 10,
842 C(2), C(3), C(4), C(5), C(6), C(7), B(0), B(1), B(2),
843 R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8),
844 R(9), R(10), C(8), C(9), R(11), RS
, R(12), R(13), R(14),
849 .addrdec
= AMAP_2KB
, .dden
= DEN_8Gb
, .dwid
= X8
,
850 .rowbits
= 16, .colbits
= 11,
852 C(2), C(3), C(4), C(5), C(6), C(7), B(0), B(1), B(2),
853 R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8),
854 R(9), R(10), C(8), C(9), R(11), RS
, C(11), R(12), R(13),
859 .addrdec
= AMAP_4KB
, .dden
= DEN_4Gb
, .dwid
= X16
,
860 .rowbits
= 15, .colbits
= 10,
862 C(2), C(3), C(4), C(5), C(6), C(7), C(8), B(0), B(1),
863 B(2), R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7),
864 R(8), R(9), R(10), C(9), R(11), RS
, R(12), R(13), R(14),
869 .addrdec
= AMAP_4KB
, .dden
= DEN_4Gb
, .dwid
= X8
,
870 .rowbits
= 16, .colbits
= 10,
872 C(2), C(3), C(4), C(5), C(6), C(7), C(8), B(0), B(1),
873 B(2), R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7),
874 R(8), R(9), R(10), C(9), R(11), RS
, R(12), R(13), R(14),
879 .addrdec
= AMAP_4KB
, .dden
= DEN_8Gb
, .dwid
= X16
,
880 .rowbits
= 16, .colbits
= 10,
882 C(2), C(3), C(4), C(5), C(6), C(7), C(8), B(0), B(1),
883 B(2), R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7),
884 R(8), R(9), R(10), C(9), R(11), RS
, R(12), R(13), R(14),
889 .addrdec
= AMAP_4KB
, .dden
= DEN_8Gb
, .dwid
= X8
,
890 .rowbits
= 16, .colbits
= 11,
892 C(2), C(3), C(4), C(5), C(6), C(7), C(8), B(0), B(1),
893 B(2), R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7),
894 R(8), R(9), R(10), C(9), R(11), RS
, C(11), R(12), R(13),
900 static int bank_hash(u64 pmiaddr
, int idx
, int shft
)
906 bhash
^= ((pmiaddr
>> (12 + shft
)) ^ (pmiaddr
>> (9 + shft
))) & 1;
909 bhash
^= (((pmiaddr
>> (10 + shft
)) ^ (pmiaddr
>> (8 + shft
))) & 1) << 1;
910 bhash
^= ((pmiaddr
>> 22) & 1) << 1;
913 bhash
^= (((pmiaddr
>> (13 + shft
)) ^ (pmiaddr
>> (11 + shft
))) & 1) << 2;
920 static int rank_hash(u64 pmiaddr
)
922 return ((pmiaddr
>> 16) ^ (pmiaddr
>> 10)) & 1;
925 /* Second stage decode. Compute rank, bank, row & column. */
926 static int apl_pmi2mem(struct mem_ctl_info
*mci
, u64 pmiaddr
, u32 pmiidx
,
927 struct dram_addr
*daddr
, char *msg
)
929 struct d_cr_drp0
*cr_drp0
= &drp0
[pmiidx
];
930 struct pnd2_pvt
*pvt
= mci
->pvt_info
;
931 int g
= pvt
->dimm_geom
[pmiidx
];
932 struct dimm_geometry
*d
= &dimms
[g
];
933 int column
= 0, bank
= 0, row
= 0, rank
= 0;
934 int i
, idx
, type
, skiprs
= 0;
936 for (i
= 0; i
< PMI_ADDRESS_WIDTH
; i
++) {
937 int bit
= (pmiaddr
>> i
) & 1;
939 if (i
+ skiprs
>= PMI_ADDRESS_WIDTH
) {
940 snprintf(msg
, PND2_MSG_SIZE
, "Bad dimm_geometry[] table\n");
944 type
= d
->bits
[i
+ skiprs
] & ~0xf;
945 idx
= d
->bits
[i
+ skiprs
] & 0xf;
948 * On single rank DIMMs ignore the rank select bit
949 * and shift remainder of "bits[]" down one place.
951 if (type
== RS
&& (cr_drp0
->rken0
+ cr_drp0
->rken1
) == 1) {
953 type
= d
->bits
[i
+ skiprs
] & ~0xf;
954 idx
= d
->bits
[i
+ skiprs
] & 0xf;
959 column
|= (bit
<< idx
);
962 bank
|= (bit
<< idx
);
964 bank
^= bank_hash(pmiaddr
, idx
, d
->addrdec
);
972 rank
^= rank_hash(pmiaddr
);
976 snprintf(msg
, PND2_MSG_SIZE
, "Bad translation\n");
993 /* Pluck bit "in" from pmiaddr and return value shifted to bit "out" */
994 #define dnv_get_bit(pmi, in, out) ((int)(((pmi) >> (in)) & 1u) << (out))
996 static int dnv_pmi2mem(struct mem_ctl_info
*mci
, u64 pmiaddr
, u32 pmiidx
,
997 struct dram_addr
*daddr
, char *msg
)
1000 daddr
->rank
= dnv_get_bit(pmiaddr
, dmap
[pmiidx
].rs0
+ 13, 0);
1002 daddr
->rank
|= dnv_get_bit(pmiaddr
, dmap
[pmiidx
].rs1
+ 13, 1);
1005 * Normally ranks 0,1 are DIMM0, and 2,3 are DIMM1, but we
1006 * flip them if DIMM1 is larger than DIMM0.
1008 daddr
->dimm
= (daddr
->rank
>= 2) ^ drp
[pmiidx
].dimmflip
;
1010 daddr
->bank
= dnv_get_bit(pmiaddr
, dmap
[pmiidx
].ba0
+ 6, 0);
1011 daddr
->bank
|= dnv_get_bit(pmiaddr
, dmap
[pmiidx
].ba1
+ 6, 1);
1012 daddr
->bank
|= dnv_get_bit(pmiaddr
, dmap
[pmiidx
].bg0
+ 6, 2);
1014 daddr
->bank
|= dnv_get_bit(pmiaddr
, dmap
[pmiidx
].bg1
+ 6, 3);
1015 if (dmap1
[pmiidx
].bxor
) {
1017 daddr
->bank
^= dnv_get_bit(pmiaddr
, dmap3
[pmiidx
].row6
+ 6, 0);
1018 daddr
->bank
^= dnv_get_bit(pmiaddr
, dmap3
[pmiidx
].row7
+ 6, 1);
1019 if (dsch
.chan_width
== 0)
1020 /* 64/72 bit dram channel width */
1021 daddr
->bank
^= dnv_get_bit(pmiaddr
, dmap5
[pmiidx
].ca3
+ 6, 2);
1023 /* 32/40 bit dram channel width */
1024 daddr
->bank
^= dnv_get_bit(pmiaddr
, dmap5
[pmiidx
].ca4
+ 6, 2);
1025 daddr
->bank
^= dnv_get_bit(pmiaddr
, dmap2
[pmiidx
].row2
+ 6, 3);
1027 daddr
->bank
^= dnv_get_bit(pmiaddr
, dmap2
[pmiidx
].row2
+ 6, 0);
1028 daddr
->bank
^= dnv_get_bit(pmiaddr
, dmap3
[pmiidx
].row6
+ 6, 1);
1029 if (dsch
.chan_width
== 0)
1030 daddr
->bank
^= dnv_get_bit(pmiaddr
, dmap5
[pmiidx
].ca3
+ 6, 2);
1032 daddr
->bank
^= dnv_get_bit(pmiaddr
, dmap5
[pmiidx
].ca4
+ 6, 2);
1036 daddr
->row
= dnv_get_bit(pmiaddr
, dmap2
[pmiidx
].row0
+ 6, 0);
1037 daddr
->row
|= dnv_get_bit(pmiaddr
, dmap2
[pmiidx
].row1
+ 6, 1);
1038 daddr
->row
|= dnv_get_bit(pmiaddr
, dmap2
[pmiidx
].row2
+ 6, 2);
1039 daddr
->row
|= dnv_get_bit(pmiaddr
, dmap2
[pmiidx
].row3
+ 6, 3);
1040 daddr
->row
|= dnv_get_bit(pmiaddr
, dmap2
[pmiidx
].row4
+ 6, 4);
1041 daddr
->row
|= dnv_get_bit(pmiaddr
, dmap2
[pmiidx
].row5
+ 6, 5);
1042 daddr
->row
|= dnv_get_bit(pmiaddr
, dmap3
[pmiidx
].row6
+ 6, 6);
1043 daddr
->row
|= dnv_get_bit(pmiaddr
, dmap3
[pmiidx
].row7
+ 6, 7);
1044 daddr
->row
|= dnv_get_bit(pmiaddr
, dmap3
[pmiidx
].row8
+ 6, 8);
1045 daddr
->row
|= dnv_get_bit(pmiaddr
, dmap3
[pmiidx
].row9
+ 6, 9);
1046 daddr
->row
|= dnv_get_bit(pmiaddr
, dmap3
[pmiidx
].row10
+ 6, 10);
1047 daddr
->row
|= dnv_get_bit(pmiaddr
, dmap3
[pmiidx
].row11
+ 6, 11);
1048 daddr
->row
|= dnv_get_bit(pmiaddr
, dmap4
[pmiidx
].row12
+ 6, 12);
1049 daddr
->row
|= dnv_get_bit(pmiaddr
, dmap4
[pmiidx
].row13
+ 6, 13);
1050 if (dmap4
[pmiidx
].row14
!= 31)
1051 daddr
->row
|= dnv_get_bit(pmiaddr
, dmap4
[pmiidx
].row14
+ 6, 14);
1052 if (dmap4
[pmiidx
].row15
!= 31)
1053 daddr
->row
|= dnv_get_bit(pmiaddr
, dmap4
[pmiidx
].row15
+ 6, 15);
1054 if (dmap4
[pmiidx
].row16
!= 31)
1055 daddr
->row
|= dnv_get_bit(pmiaddr
, dmap4
[pmiidx
].row16
+ 6, 16);
1056 if (dmap4
[pmiidx
].row17
!= 31)
1057 daddr
->row
|= dnv_get_bit(pmiaddr
, dmap4
[pmiidx
].row17
+ 6, 17);
1059 daddr
->col
= dnv_get_bit(pmiaddr
, dmap5
[pmiidx
].ca3
+ 6, 3);
1060 daddr
->col
|= dnv_get_bit(pmiaddr
, dmap5
[pmiidx
].ca4
+ 6, 4);
1061 daddr
->col
|= dnv_get_bit(pmiaddr
, dmap5
[pmiidx
].ca5
+ 6, 5);
1062 daddr
->col
|= dnv_get_bit(pmiaddr
, dmap5
[pmiidx
].ca6
+ 6, 6);
1063 daddr
->col
|= dnv_get_bit(pmiaddr
, dmap5
[pmiidx
].ca7
+ 6, 7);
1064 daddr
->col
|= dnv_get_bit(pmiaddr
, dmap5
[pmiidx
].ca8
+ 6, 8);
1065 daddr
->col
|= dnv_get_bit(pmiaddr
, dmap5
[pmiidx
].ca9
+ 6, 9);
1066 if (!dsch
.ddr4en
&& dmap1
[pmiidx
].ca11
!= 0x3f)
1067 daddr
->col
|= dnv_get_bit(pmiaddr
, dmap1
[pmiidx
].ca11
+ 13, 11);
1072 static int check_channel(int ch
)
1074 if (drp0
[ch
].dramtype
!= 0) {
1075 pnd2_printk(KERN_INFO
, "Unsupported DIMM in channel %d\n", ch
);
1077 } else if (drp0
[ch
].eccen
== 0) {
1078 pnd2_printk(KERN_INFO
, "ECC disabled on channel %d\n", ch
);
1084 static int apl_check_ecc_active(void)
1088 /* Check dramtype and ECC mode for each present DIMM */
1089 for (i
= 0; i
< APL_NUM_CHANNELS
; i
++)
1090 if (chan_mask
& BIT(i
))
1091 ret
+= check_channel(i
);
1092 return ret
? -EINVAL
: 0;
1095 #define DIMMS_PRESENT(d) ((d)->rken0 + (d)->rken1 + (d)->rken2 + (d)->rken3)
1097 static int check_unit(int ch
)
1099 struct d_cr_drp
*d
= &drp
[ch
];
1101 if (DIMMS_PRESENT(d
) && !ecc_ctrl
[ch
].eccen
) {
1102 pnd2_printk(KERN_INFO
, "ECC disabled on channel %d\n", ch
);
1108 static int dnv_check_ecc_active(void)
1112 for (i
= 0; i
< DNV_NUM_CHANNELS
; i
++)
1113 ret
+= check_unit(i
);
1114 return ret
? -EINVAL
: 0;
1117 static int get_memory_error_data(struct mem_ctl_info
*mci
, u64 addr
,
1118 struct dram_addr
*daddr
, char *msg
)
1124 ret
= sys2pmi(addr
, &pmiidx
, &pmiaddr
, msg
);
1128 pmiaddr
>>= ops
->pmiaddr_shift
;
1129 /* pmi channel idx to dimm channel idx */
1130 pmiidx
>>= ops
->pmiidx_shift
;
1131 daddr
->chan
= pmiidx
;
1133 ret
= ops
->pmi2mem(mci
, pmiaddr
, pmiidx
, daddr
, msg
);
1137 edac_dbg(0, "SysAddr=%llx PmiAddr=%llx Channel=%d DIMM=%d Rank=%d Bank=%d Row=%d Column=%d\n",
1138 addr
, pmiaddr
, daddr
->chan
, daddr
->dimm
, daddr
->rank
, daddr
->bank
, daddr
->row
, daddr
->col
);
1143 static void pnd2_mce_output_error(struct mem_ctl_info
*mci
, const struct mce
*m
,
1144 struct dram_addr
*daddr
)
1146 enum hw_event_mc_err_type tp_event
;
1147 char *optype
, msg
[PND2_MSG_SIZE
];
1148 bool ripv
= m
->mcgstatus
& MCG_STATUS_RIPV
;
1149 bool overflow
= m
->status
& MCI_STATUS_OVER
;
1150 bool uc_err
= m
->status
& MCI_STATUS_UC
;
1151 bool recov
= m
->status
& MCI_STATUS_S
;
1152 u32 core_err_cnt
= GET_BITFIELD(m
->status
, 38, 52);
1153 u32 mscod
= GET_BITFIELD(m
->status
, 16, 31);
1154 u32 errcode
= GET_BITFIELD(m
->status
, 0, 15);
1155 u32 optypenum
= GET_BITFIELD(m
->status
, 4, 6);
1158 tp_event
= uc_err
? (ripv
? HW_EVENT_ERR_UNCORRECTED
: HW_EVENT_ERR_FATAL
) :
1159 HW_EVENT_ERR_CORRECTED
;
1162 * According with Table 15-9 of the Intel Architecture spec vol 3A,
1163 * memory errors should fit in this mask:
1164 * 000f 0000 1mmm cccc (binary)
1166 * f = Correction Report Filtering Bit. If 1, subsequent errors
1170 * If the mask doesn't match, report an error to the parsing logic
1172 if (!((errcode
& 0xef80) == 0x80)) {
1173 optype
= "Can't parse: it is not a mem";
1175 switch (optypenum
) {
1177 optype
= "generic undef request error";
1180 optype
= "memory read error";
1183 optype
= "memory write error";
1186 optype
= "addr/cmd error";
1189 optype
= "memory scrubbing error";
1192 optype
= "reserved";
1197 /* Only decode errors with an valid address (ADDRV) */
1198 if (!(m
->status
& MCI_STATUS_ADDRV
))
1201 rc
= get_memory_error_data(mci
, m
->addr
, daddr
, msg
);
1205 snprintf(msg
, sizeof(msg
),
1206 "%s%s err_code:%04x:%04x channel:%d DIMM:%d rank:%d row:%d bank:%d col:%d",
1207 overflow
? " OVERFLOW" : "", (uc_err
&& recov
) ? " recoverable" : "", mscod
,
1208 errcode
, daddr
->chan
, daddr
->dimm
, daddr
->rank
, daddr
->row
, daddr
->bank
, daddr
->col
);
1210 edac_dbg(0, "%s\n", msg
);
1212 /* Call the helper to output message */
1213 edac_mc_handle_error(tp_event
, mci
, core_err_cnt
, m
->addr
>> PAGE_SHIFT
,
1214 m
->addr
& ~PAGE_MASK
, 0, daddr
->chan
, daddr
->dimm
, -1, optype
, msg
);
1219 edac_mc_handle_error(tp_event
, mci
, core_err_cnt
, 0, 0, 0, -1, -1, -1, msg
, "");
1222 static void apl_get_dimm_config(struct mem_ctl_info
*mci
)
1224 struct pnd2_pvt
*pvt
= mci
->pvt_info
;
1225 struct dimm_info
*dimm
;
1226 struct d_cr_drp0
*d
;
1230 for (i
= 0; i
< APL_NUM_CHANNELS
; i
++) {
1231 if (!(chan_mask
& BIT(i
)))
1234 dimm
= edac_get_dimm(mci
, i
, 0, 0);
1236 edac_dbg(0, "No allocated DIMM for channel %d\n", i
);
1241 for (g
= 0; g
< ARRAY_SIZE(dimms
); g
++)
1242 if (dimms
[g
].addrdec
== d
->addrdec
&&
1243 dimms
[g
].dden
== d
->dden
&&
1244 dimms
[g
].dwid
== d
->dwid
)
1247 if (g
== ARRAY_SIZE(dimms
)) {
1248 edac_dbg(0, "Channel %d: unrecognized DIMM\n", i
);
1252 pvt
->dimm_geom
[i
] = g
;
1253 capacity
= (d
->rken0
+ d
->rken1
) * 8 * (1ul << dimms
[g
].rowbits
) *
1254 (1ul << dimms
[g
].colbits
);
1255 edac_dbg(0, "Channel %d: %lld MByte DIMM\n", i
, capacity
>> (20 - 3));
1256 dimm
->nr_pages
= MiB_TO_PAGES(capacity
>> (20 - 3));
1258 dimm
->dtype
= (d
->dwid
== 0) ? DEV_X8
: DEV_X16
;
1259 dimm
->mtype
= MEM_DDR3
;
1260 dimm
->edac_mode
= EDAC_SECDED
;
1261 snprintf(dimm
->label
, sizeof(dimm
->label
), "Slice#%d_Chan#%d", i
/ 2, i
% 2);
1265 static const int dnv_dtypes
[] = {
1266 DEV_X8
, DEV_X4
, DEV_X16
, DEV_UNKNOWN
1269 static void dnv_get_dimm_config(struct mem_ctl_info
*mci
)
1271 int i
, j
, ranks_of_dimm
[DNV_MAX_DIMMS
], banks
, rowbits
, colbits
, memtype
;
1272 struct dimm_info
*dimm
;
1285 for (i
= 0; i
< DNV_NUM_CHANNELS
; i
++) {
1286 if (dmap4
[i
].row14
== 31)
1288 else if (dmap4
[i
].row15
== 31)
1290 else if (dmap4
[i
].row16
== 31)
1292 else if (dmap4
[i
].row17
== 31)
1297 if (memtype
== MEM_DDR3
) {
1298 if (dmap1
[i
].ca11
!= 0x3f)
1305 /* DIMM0 is present if rank0 and/or rank1 is enabled */
1306 ranks_of_dimm
[0] = d
->rken0
+ d
->rken1
;
1307 /* DIMM1 is present if rank2 and/or rank3 is enabled */
1308 ranks_of_dimm
[1] = d
->rken2
+ d
->rken3
;
1310 for (j
= 0; j
< DNV_MAX_DIMMS
; j
++) {
1311 if (!ranks_of_dimm
[j
])
1314 dimm
= edac_get_dimm(mci
, i
, j
, 0);
1316 edac_dbg(0, "No allocated DIMM for channel %d DIMM %d\n", i
, j
);
1320 capacity
= ranks_of_dimm
[j
] * banks
* (1ul << rowbits
) * (1ul << colbits
);
1321 edac_dbg(0, "Channel %d DIMM %d: %lld MByte DIMM\n", i
, j
, capacity
>> (20 - 3));
1322 dimm
->nr_pages
= MiB_TO_PAGES(capacity
>> (20 - 3));
1324 dimm
->dtype
= dnv_dtypes
[j
? d
->dimmdwid0
: d
->dimmdwid1
];
1325 dimm
->mtype
= memtype
;
1326 dimm
->edac_mode
= EDAC_SECDED
;
1327 snprintf(dimm
->label
, sizeof(dimm
->label
), "Chan#%d_DIMM#%d", i
, j
);
1332 static int pnd2_register_mci(struct mem_ctl_info
**ppmci
)
1334 struct edac_mc_layer layers
[2];
1335 struct mem_ctl_info
*mci
;
1336 struct pnd2_pvt
*pvt
;
1339 rc
= ops
->check_ecc();
1343 /* Allocate a new MC control structure */
1344 layers
[0].type
= EDAC_MC_LAYER_CHANNEL
;
1345 layers
[0].size
= ops
->channels
;
1346 layers
[0].is_virt_csrow
= false;
1347 layers
[1].type
= EDAC_MC_LAYER_SLOT
;
1348 layers
[1].size
= ops
->dimms_per_channel
;
1349 layers
[1].is_virt_csrow
= true;
1350 mci
= edac_mc_alloc(0, ARRAY_SIZE(layers
), layers
, sizeof(*pvt
));
1354 pvt
= mci
->pvt_info
;
1355 memset(pvt
, 0, sizeof(*pvt
));
1357 mci
->mod_name
= EDAC_MOD_STR
;
1358 mci
->dev_name
= ops
->name
;
1359 mci
->ctl_name
= "Pondicherry2";
1361 /* Get dimm basic config and the memory layout */
1362 ops
->get_dimm_config(mci
);
1364 if (edac_mc_add_mc(mci
)) {
1365 edac_dbg(0, "MC: failed edac_mc_add_mc()\n");
1375 static void pnd2_unregister_mci(struct mem_ctl_info
*mci
)
1377 if (unlikely(!mci
|| !mci
->pvt_info
)) {
1378 pnd2_printk(KERN_ERR
, "Couldn't find mci handler\n");
1382 /* Remove MC sysfs nodes */
1383 edac_mc_del_mc(NULL
);
1384 edac_dbg(1, "%s: free mci struct\n", mci
->ctl_name
);
1389 * Callback function registered with core kernel mce code.
1390 * Called once for each logged error.
1392 static int pnd2_mce_check_error(struct notifier_block
*nb
, unsigned long val
, void *data
)
1394 struct mce
*mce
= (struct mce
*)data
;
1395 struct mem_ctl_info
*mci
;
1396 struct dram_addr daddr
;
1400 if (!mci
|| (mce
->kflags
& MCE_HANDLED_CEC
))
1404 * Just let mcelog handle it if the error is
1405 * outside the memory controller. A memory error
1406 * is indicated by bit 7 = 1 and bits = 8-11,13-15 = 0.
1407 * bit 12 has an special meaning.
1409 if ((mce
->status
& 0xefff) >> 7 != 1)
1412 if (mce
->mcgstatus
& MCG_STATUS_MCIP
)
1417 pnd2_mc_printk(mci
, KERN_INFO
, "HANDLING MCE MEMORY ERROR\n");
1418 pnd2_mc_printk(mci
, KERN_INFO
, "CPU %u: Machine Check %s: %llx Bank %u: %llx\n",
1419 mce
->extcpu
, type
, mce
->mcgstatus
, mce
->bank
, mce
->status
);
1420 pnd2_mc_printk(mci
, KERN_INFO
, "TSC %llx ", mce
->tsc
);
1421 pnd2_mc_printk(mci
, KERN_INFO
, "ADDR %llx ", mce
->addr
);
1422 pnd2_mc_printk(mci
, KERN_INFO
, "MISC %llx ", mce
->misc
);
1423 pnd2_mc_printk(mci
, KERN_INFO
, "PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x\n",
1424 mce
->cpuvendor
, mce
->cpuid
, mce
->time
, mce
->socketid
, mce
->apicid
);
1426 pnd2_mce_output_error(mci
, mce
, &daddr
);
1428 /* Advice mcelog that the error were handled */
1429 mce
->kflags
|= MCE_HANDLED_EDAC
;
1433 static struct notifier_block pnd2_mce_dec
= {
1434 .notifier_call
= pnd2_mce_check_error
,
1435 .priority
= MCE_PRIO_EDAC
,
1438 #ifdef CONFIG_EDAC_DEBUG
1440 * Write an address to this file to exercise the address decode
1441 * logic in this driver.
1443 static u64 pnd2_fake_addr
;
1444 #define PND2_BLOB_SIZE 1024
1445 static char pnd2_result
[PND2_BLOB_SIZE
];
1446 static struct dentry
*pnd2_test
;
1447 static struct debugfs_blob_wrapper pnd2_blob
= {
1448 .data
= pnd2_result
,
1452 static int debugfs_u64_set(void *data
, u64 val
)
1454 struct dram_addr daddr
;
1459 /* ADDRV + MemRd + Unknown channel */
1460 m
.status
= MCI_STATUS_ADDRV
+ 0x9f;
1462 pnd2_mce_output_error(pnd2_mci
, &m
, &daddr
);
1463 snprintf(pnd2_blob
.data
, PND2_BLOB_SIZE
,
1464 "SysAddr=%llx Channel=%d DIMM=%d Rank=%d Bank=%d Row=%d Column=%d\n",
1465 m
.addr
, daddr
.chan
, daddr
.dimm
, daddr
.rank
, daddr
.bank
, daddr
.row
, daddr
.col
);
1466 pnd2_blob
.size
= strlen(pnd2_blob
.data
);
1470 DEFINE_DEBUGFS_ATTRIBUTE(fops_u64_wo
, NULL
, debugfs_u64_set
, "%llu\n");
1472 static void setup_pnd2_debug(void)
1474 pnd2_test
= edac_debugfs_create_dir("pnd2_test");
1475 edac_debugfs_create_file("pnd2_debug_addr", 0200, pnd2_test
,
1476 &pnd2_fake_addr
, &fops_u64_wo
);
1477 debugfs_create_blob("pnd2_debug_results", 0400, pnd2_test
, &pnd2_blob
);
1480 static void teardown_pnd2_debug(void)
1482 debugfs_remove_recursive(pnd2_test
);
1485 static void setup_pnd2_debug(void) {}
1486 static void teardown_pnd2_debug(void) {}
1487 #endif /* CONFIG_EDAC_DEBUG */
1490 static int pnd2_probe(void)
1495 rc
= get_registers();
1499 return pnd2_register_mci(&pnd2_mci
);
1502 static void pnd2_remove(void)
1505 pnd2_unregister_mci(pnd2_mci
);
1508 static struct dunit_ops apl_ops
= {
1511 .pmiaddr_shift
= LOG2_PMI_ADDR_GRANULARITY
,
1513 .channels
= APL_NUM_CHANNELS
,
1514 .dimms_per_channel
= 1,
1515 .rd_reg
= apl_rd_reg
,
1516 .get_registers
= apl_get_registers
,
1517 .check_ecc
= apl_check_ecc_active
,
1518 .mk_region
= apl_mk_region
,
1519 .get_dimm_config
= apl_get_dimm_config
,
1520 .pmi2mem
= apl_pmi2mem
,
1523 static struct dunit_ops dnv_ops
= {
1528 .channels
= DNV_NUM_CHANNELS
,
1529 .dimms_per_channel
= 2,
1530 .rd_reg
= dnv_rd_reg
,
1531 .get_registers
= dnv_get_registers
,
1532 .check_ecc
= dnv_check_ecc_active
,
1533 .mk_region
= dnv_mk_region
,
1534 .get_dimm_config
= dnv_get_dimm_config
,
1535 .pmi2mem
= dnv_pmi2mem
,
1538 static const struct x86_cpu_id pnd2_cpuids
[] = {
1539 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT
, &apl_ops
),
1540 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_D
, &dnv_ops
),
1543 MODULE_DEVICE_TABLE(x86cpu
, pnd2_cpuids
);
1545 static int __init
pnd2_init(void)
1547 const struct x86_cpu_id
*id
;
1553 owner
= edac_get_owner();
1554 if (owner
&& strncmp(owner
, EDAC_MOD_STR
, sizeof(EDAC_MOD_STR
)))
1557 id
= x86_match_cpu(pnd2_cpuids
);
1561 ops
= (struct dunit_ops
*)id
->driver_data
;
1563 if (ops
->type
== APL
) {
1564 p2sb_bus
= pci_find_bus(0, 0);
1569 /* Ensure that the OPSTATE is set correctly for POLL or NMI */
1574 pnd2_printk(KERN_ERR
, "Failed to register device with error %d.\n", rc
);
1581 mce_register_decode_chain(&pnd2_mce_dec
);
1587 static void __exit
pnd2_exit(void)
1590 teardown_pnd2_debug();
1591 mce_unregister_decode_chain(&pnd2_mce_dec
);
1595 module_init(pnd2_init
);
1596 module_exit(pnd2_exit
);
1598 module_param(edac_op_state
, int, 0444);
1599 MODULE_PARM_DESC(edac_op_state
, "EDAC Error Reporting state: 0=Poll,1=NMI");
1601 MODULE_LICENSE("GPL v2");
1602 MODULE_AUTHOR("Tony Luck");
1603 MODULE_DESCRIPTION("MC Driver for Intel SoC using Pondicherry memory controller");