?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 4.16.11
[linux/fpc-iii.git] / drivers / edac / amd64_edac.c
blob329cb96f886fd136062707324680327a3083b763
1 #include "amd64_edac.h"
2 #include <asm/amd_nb.h>
3
4 static struct edac_pci_ctl_info *pci_ctl;
5
6 static int report_gart_errors;
7 module_param(report_gart_errors, int, 0644);
8
9 /*
10 * Set by command line parameter. If BIOS has enabled the ECC, this override is
11 * cleared to prevent re-enabling the hardware by this driver.
12 */
13 static int ecc_enable_override;
14 module_param(ecc_enable_override, int, 0644);
15
16 static struct msr __percpu *msrs;
17
18 /* Per-node stuff */
19 static struct ecc_settings **ecc_stngs;
20
21 /*
22 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
23 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
24 * or higher value'.
25 *
26 *FIXME: Produce a better mapping/linearisation.
27 */
28 static const struct scrubrate {
29 u32 scrubval; /* bit pattern for scrub rate */
30 u32 bandwidth; /* bandwidth consumed (bytes/sec) */
31 } scrubrates[] = {
32 { 0x01, 1600000000UL},
33 { 0x02, 800000000UL},
34 { 0x03, 400000000UL},
35 { 0x04, 200000000UL},
36 { 0x05, 100000000UL},
37 { 0x06, 50000000UL},
38 { 0x07, 25000000UL},
39 { 0x08, 12284069UL},
40 { 0x09, 6274509UL},
41 { 0x0A, 3121951UL},
42 { 0x0B, 1560975UL},
43 { 0x0C, 781440UL},
44 { 0x0D, 390720UL},
45 { 0x0E, 195300UL},
46 { 0x0F, 97650UL},
47 { 0x10, 48854UL},
48 { 0x11, 24427UL},
49 { 0x12, 12213UL},
50 { 0x13, 6101UL},
51 { 0x14, 3051UL},
52 { 0x15, 1523UL},
53 { 0x16, 761UL},
54 { 0x00, 0UL}, /* scrubbing off */
55 };
56
57 int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
58 u32 *val, const char *func)
59 {
60 int err = 0;
61
62 err = pci_read_config_dword(pdev, offset, val);
63 if (err)
64 amd64_warn("%s: error reading F%dx%03x.\n",
65 func, PCI_FUNC(pdev->devfn), offset);
66
67 return err;
68 }
69
70 int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
71 u32 val, const char *func)
72 {
73 int err = 0;
74
75 err = pci_write_config_dword(pdev, offset, val);
76 if (err)
77 amd64_warn("%s: error writing to F%dx%03x.\n",
78 func, PCI_FUNC(pdev->devfn), offset);
79
80 return err;
81 }
82
83 /*
84 * Select DCT to which PCI cfg accesses are routed
85 */
86 static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct)
87 {
88 u32 reg = 0;
89
90 amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, &reg);
91 reg &= (pvt->model == 0x30) ? ~3 : ~1;
92 reg |= dct;
93 amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
94 }
95
96 /*
97 *
98 * Depending on the family, F2 DCT reads need special handling:
99 *
100 * K8: has a single DCT only and no address offsets >= 0x100
101 *
102 * F10h: each DCT has its own set of regs
103 * DCT0 -> F2x040..
104 * DCT1 -> F2x140..
105 *
106 * F16h: has only 1 DCT
107 *
108 * F15h: we select which DCT we access using F1x10C[DctCfgSel]
109 */
110 static inline int amd64_read_dct_pci_cfg(struct amd64_pvt *pvt, u8 dct,
111 int offset, u32 *val)
112 {
113 switch (pvt->fam) {
114 case 0xf:
115 if (dct || offset >= 0x100)
116 return -EINVAL;
117 break;
118
119 case 0x10:
120 if (dct) {
121 /*
122 * Note: If ganging is enabled, barring the regs
123 * F2x[1,0]98 and F2x[1,0]9C; reads reads to F2x1xx
124 * return 0. (cf. Section 2.8.1 F10h BKDG)
125 */
126 if (dct_ganging_enabled(pvt))
127 return 0;
128
129 offset += 0x100;
130 }
131 break;
132
133 case 0x15:
134 /*
135 * F15h: F2x1xx addresses do not map explicitly to DCT1.
136 * We should select which DCT we access using F1x10C[DctCfgSel]
137 */
138 dct = (dct && pvt->model == 0x30) ? 3 : dct;
139 f15h_select_dct(pvt, dct);
140 break;
141
142 case 0x16:
143 if (dct)
144 return -EINVAL;
145 break;
146
147 default:
148 break;
149 }
150 return amd64_read_pci_cfg(pvt->F2, offset, val);
151 }
152
153 /*
154 * Memory scrubber control interface. For K8, memory scrubbing is handled by
155 * hardware and can involve L2 cache, dcache as well as the main memory. With
156 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
157 * functionality.
158 *
159 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
160 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
161 * bytes/sec for the setting.
162 *
163 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
164 * other archs, we might not have access to the caches directly.
165 */
166
167 static inline void __f17h_set_scrubval(struct amd64_pvt *pvt, u32 scrubval)
168 {
169 /*
170 * Fam17h supports scrub values between 0x5 and 0x14. Also, the values
171 * are shifted down by 0x5, so scrubval 0x5 is written to the register
172 * as 0x0, scrubval 0x6 as 0x1, etc.
173 */
174 if (scrubval >= 0x5 && scrubval <= 0x14) {
175 scrubval -= 0x5;
176 pci_write_bits32(pvt->F6, F17H_SCR_LIMIT_ADDR, scrubval, 0xF);
177 pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 1, 0x1);
178 } else {
179 pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 0, 0x1);
180 }
181 }
182 /*
183 * Scan the scrub rate mapping table for a close or matching bandwidth value to
184 * issue. If requested is too big, then use last maximum value found.
185 */
186 static int __set_scrub_rate(struct amd64_pvt *pvt, u32 new_bw, u32 min_rate)
187 {
188 u32 scrubval;
189 int i;
190
191 /*
192 * map the configured rate (new_bw) to a value specific to the AMD64
193 * memory controller and apply to register. Search for the first
194 * bandwidth entry that is greater or equal than the setting requested
195 * and program that. If at last entry, turn off DRAM scrubbing.
196 *
197 * If no suitable bandwidth is found, turn off DRAM scrubbing entirely
198 * by falling back to the last element in scrubrates[].
199 */
200 for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) {
201 /*
202 * skip scrub rates which aren't recommended
203 * (see F10 BKDG, F3x58)
204 */
205 if (scrubrates[i].scrubval < min_rate)
206 continue;
207
208 if (scrubrates[i].bandwidth <= new_bw)
209 break;
210 }
211
212 scrubval = scrubrates[i].scrubval;
213
214 if (pvt->fam == 0x17) {
215 __f17h_set_scrubval(pvt, scrubval);
216 } else if (pvt->fam == 0x15 && pvt->model == 0x60) {
217 f15h_select_dct(pvt, 0);
218 pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
219 f15h_select_dct(pvt, 1);
220 pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
221 } else {
222 pci_write_bits32(pvt->F3, SCRCTRL, scrubval, 0x001F);
223 }
224
225 if (scrubval)
226 return scrubrates[i].bandwidth;
227
228 return 0;
229 }
230
231 static int set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
232 {
233 struct amd64_pvt *pvt = mci->pvt_info;
234 u32 min_scrubrate = 0x5;
235
236 if (pvt->fam == 0xf)
237 min_scrubrate = 0x0;
238
239 if (pvt->fam == 0x15) {
240 /* Erratum #505 */
241 if (pvt->model < 0x10)
242 f15h_select_dct(pvt, 0);
243
244 if (pvt->model == 0x60)
245 min_scrubrate = 0x6;
246 }
247 return __set_scrub_rate(pvt, bw, min_scrubrate);
248 }
249
250 static int get_scrub_rate(struct mem_ctl_info *mci)
251 {
252 struct amd64_pvt *pvt = mci->pvt_info;
253 int i, retval = -EINVAL;
254 u32 scrubval = 0;
255
256 switch (pvt->fam) {
257 case 0x15:
258 /* Erratum #505 */
259 if (pvt->model < 0x10)
260 f15h_select_dct(pvt, 0);
261
262 if (pvt->model == 0x60)
263 amd64_read_pci_cfg(pvt->F2, F15H_M60H_SCRCTRL, &scrubval);
264 break;
265
266 case 0x17:
267 amd64_read_pci_cfg(pvt->F6, F17H_SCR_BASE_ADDR, &scrubval);
268 if (scrubval & BIT(0)) {
269 amd64_read_pci_cfg(pvt->F6, F17H_SCR_LIMIT_ADDR, &scrubval);
270 scrubval &= 0xF;
271 scrubval += 0x5;
272 } else {
273 scrubval = 0;
274 }
275 break;
276
277 default:
278 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
279 break;
280 }
281
282 scrubval = scrubval & 0x001F;
283
284 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
285 if (scrubrates[i].scrubval == scrubval) {
286 retval = scrubrates[i].bandwidth;
287 break;
288 }
289 }
290 return retval;
291 }
292
293 /*
294 * returns true if the SysAddr given by sys_addr matches the
295 * DRAM base/limit associated with node_id
296 */
297 static bool base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, u8 nid)
298 {
299 u64 addr;
300
301 /* The K8 treats this as a 40-bit value. However, bits 63-40 will be
302 * all ones if the most significant implemented address bit is 1.
303 * Here we discard bits 63-40. See section 3.4.2 of AMD publication
304 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
305 * Application Programming.
306 */
307 addr = sys_addr & 0x000000ffffffffffull;
308
309 return ((addr >= get_dram_base(pvt, nid)) &&
310 (addr <= get_dram_limit(pvt, nid)));
311 }
312
313 /*
314 * Attempt to map a SysAddr to a node. On success, return a pointer to the
315 * mem_ctl_info structure for the node that the SysAddr maps to.
316 *
317 * On failure, return NULL.
318 */
319 static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
320 u64 sys_addr)
321 {
322 struct amd64_pvt *pvt;
323 u8 node_id;
324 u32 intlv_en, bits;
325
326 /*
327 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
328 * 3.4.4.2) registers to map the SysAddr to a node ID.
329 */
330 pvt = mci->pvt_info;
331
332 /*
333 * The value of this field should be the same for all DRAM Base
334 * registers. Therefore we arbitrarily choose to read it from the
335 * register for node 0.
336 */
337 intlv_en = dram_intlv_en(pvt, 0);
338
339 if (intlv_en == 0) {
340 for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
341 if (base_limit_match(pvt, sys_addr, node_id))
342 goto found;
343 }
344 goto err_no_match;
345 }
346
347 if (unlikely((intlv_en != 0x01) &&
348 (intlv_en != 0x03) &&
349 (intlv_en != 0x07))) {
350 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
351 return NULL;
352 }
353
354 bits = (((u32) sys_addr) >> 12) & intlv_en;
355
356 for (node_id = 0; ; ) {
357 if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
358 break; /* intlv_sel field matches */
359
360 if (++node_id >= DRAM_RANGES)
361 goto err_no_match;
362 }
363
364 /* sanity test for sys_addr */
365 if (unlikely(!base_limit_match(pvt, sys_addr, node_id))) {
366 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
367 "range for node %d with node interleaving enabled.\n",
368 __func__, sys_addr, node_id);
369 return NULL;
370 }
371
372 found:
373 return edac_mc_find((int)node_id);
374
375 err_no_match:
376 edac_dbg(2, "sys_addr 0x%lx doesn't match any node\n",
377 (unsigned long)sys_addr);
378
379 return NULL;
380 }
381
382 /*
383 * compute the CS base address of the @csrow on the DRAM controller @dct.
384 * For details see F2x[5C:40] in the processor's BKDG
385 */
386 static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
387 u64 *base, u64 *mask)
388 {
389 u64 csbase, csmask, base_bits, mask_bits;
390 u8 addr_shift;
391
392 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
393 csbase = pvt->csels[dct].csbases[csrow];
394 csmask = pvt->csels[dct].csmasks[csrow];
395 base_bits = GENMASK_ULL(31, 21) | GENMASK_ULL(15, 9);
396 mask_bits = GENMASK_ULL(29, 21) | GENMASK_ULL(15, 9);
397 addr_shift = 4;
398
399 /*
400 * F16h and F15h, models 30h and later need two addr_shift values:
401 * 8 for high and 6 for low (cf. F16h BKDG).
402 */
403 } else if (pvt->fam == 0x16 ||
404 (pvt->fam == 0x15 && pvt->model >= 0x30)) {
405 csbase = pvt->csels[dct].csbases[csrow];
406 csmask = pvt->csels[dct].csmasks[csrow >> 1];
407
408 *base = (csbase & GENMASK_ULL(15, 5)) << 6;
409 *base |= (csbase & GENMASK_ULL(30, 19)) << 8;
410
411 *mask = ~0ULL;
412 /* poke holes for the csmask */
413 *mask &= ~((GENMASK_ULL(15, 5) << 6) |
414 (GENMASK_ULL(30, 19) << 8));
415
416 *mask |= (csmask & GENMASK_ULL(15, 5)) << 6;
417 *mask |= (csmask & GENMASK_ULL(30, 19)) << 8;
418
419 return;
420 } else {
421 csbase = pvt->csels[dct].csbases[csrow];
422 csmask = pvt->csels[dct].csmasks[csrow >> 1];
423 addr_shift = 8;
424
425 if (pvt->fam == 0x15)
426 base_bits = mask_bits =
427 GENMASK_ULL(30,19) | GENMASK_ULL(13,5);
428 else
429 base_bits = mask_bits =
430 GENMASK_ULL(28,19) | GENMASK_ULL(13,5);
431 }
432
433 *base = (csbase & base_bits) << addr_shift;
434
435 *mask = ~0ULL;
436 /* poke holes for the csmask */
437 *mask &= ~(mask_bits << addr_shift);
438 /* OR them in */
439 *mask |= (csmask & mask_bits) << addr_shift;
440 }
441
442 #define for_each_chip_select(i, dct, pvt) \
443 for (i = 0; i < pvt->csels[dct].b_cnt; i++)
444
445 #define chip_select_base(i, dct, pvt) \
446 pvt->csels[dct].csbases[i]
447
448 #define for_each_chip_select_mask(i, dct, pvt) \
449 for (i = 0; i < pvt->csels[dct].m_cnt; i++)
450
451 /*
452 * @input_addr is an InputAddr associated with the node given by mci. Return the
453 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
454 */
455 static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
456 {
457 struct amd64_pvt *pvt;
458 int csrow;
459 u64 base, mask;
460
461 pvt = mci->pvt_info;
462
463 for_each_chip_select(csrow, 0, pvt) {
464 if (!csrow_enabled(csrow, 0, pvt))
465 continue;
466
467 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
468
469 mask = ~mask;
470
471 if ((input_addr & mask) == (base & mask)) {
472 edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)\n",
473 (unsigned long)input_addr, csrow,
474 pvt->mc_node_id);
475
476 return csrow;
477 }
478 }
479 edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)\n",
480 (unsigned long)input_addr, pvt->mc_node_id);
481
482 return -1;
483 }
484
485 /*
486 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
487 * for the node represented by mci. Info is passed back in *hole_base,
488 * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if
489 * info is invalid. Info may be invalid for either of the following reasons:
490 *
491 * - The revision of the node is not E or greater. In this case, the DRAM Hole
492 * Address Register does not exist.
493 *
494 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
495 * indicating that its contents are not valid.
496 *
497 * The values passed back in *hole_base, *hole_offset, and *hole_size are
498 * complete 32-bit values despite the fact that the bitfields in the DHAR
499 * only represent bits 31-24 of the base and offset values.
500 */
501 int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
502 u64 *hole_offset, u64 *hole_size)
503 {
504 struct amd64_pvt *pvt = mci->pvt_info;
505
506 /* only revE and later have the DRAM Hole Address Register */
507 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_E) {
508 edac_dbg(1, " revision %d for node %d does not support DHAR\n",
509 pvt->ext_model, pvt->mc_node_id);
510 return 1;
511 }
512
513 /* valid for Fam10h and above */
514 if (pvt->fam >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
515 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this system\n");
516 return 1;
517 }
518
519 if (!dhar_valid(pvt)) {
520 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this node %d\n",
521 pvt->mc_node_id);
522 return 1;
523 }
524
525 /* This node has Memory Hoisting */
526
527 /* +------------------+--------------------+--------------------+-----
528 * | memory | DRAM hole | relocated |
529 * | [0, (x - 1)] | [x, 0xffffffff] | addresses from |
530 * | | | DRAM hole |
531 * | | | [0x100000000, |
532 * | | | (0x100000000+ |
533 * | | | (0xffffffff-x))] |
534 * +------------------+--------------------+--------------------+-----
535 *
536 * Above is a diagram of physical memory showing the DRAM hole and the
537 * relocated addresses from the DRAM hole. As shown, the DRAM hole
538 * starts at address x (the base address) and extends through address
539 * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the
540 * addresses in the hole so that they start at 0x100000000.
541 */
542
543 *hole_base = dhar_base(pvt);
544 *hole_size = (1ULL << 32) - *hole_base;
545
546 *hole_offset = (pvt->fam > 0xf) ? f10_dhar_offset(pvt)
547 : k8_dhar_offset(pvt);
548
549 edac_dbg(1, " DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
550 pvt->mc_node_id, (unsigned long)*hole_base,
551 (unsigned long)*hole_offset, (unsigned long)*hole_size);
552
553 return 0;
554 }
555 EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info);
556
557 /*
558 * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is
559 * assumed that sys_addr maps to the node given by mci.
560 *
561 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
562 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
563 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
564 * then it is also involved in translating a SysAddr to a DramAddr. Sections
565 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
566 * These parts of the documentation are unclear. I interpret them as follows:
567 *
568 * When node n receives a SysAddr, it processes the SysAddr as follows:
569 *
570 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
571 * Limit registers for node n. If the SysAddr is not within the range
572 * specified by the base and limit values, then node n ignores the Sysaddr
573 * (since it does not map to node n). Otherwise continue to step 2 below.
574 *
575 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
576 * disabled so skip to step 3 below. Otherwise see if the SysAddr is within
577 * the range of relocated addresses (starting at 0x100000000) from the DRAM
578 * hole. If not, skip to step 3 below. Else get the value of the
579 * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
580 * offset defined by this value from the SysAddr.
581 *
582 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
583 * Base register for node n. To obtain the DramAddr, subtract the base
584 * address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
585 */
586 static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
587 {
588 struct amd64_pvt *pvt = mci->pvt_info;
589 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
590 int ret;
591
592 dram_base = get_dram_base(pvt, pvt->mc_node_id);
593
594 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
595 &hole_size);
596 if (!ret) {
597 if ((sys_addr >= (1ULL << 32)) &&
598 (sys_addr < ((1ULL << 32) + hole_size))) {
599 /* use DHAR to translate SysAddr to DramAddr */
600 dram_addr = sys_addr - hole_offset;
601
602 edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
603 (unsigned long)sys_addr,
604 (unsigned long)dram_addr);
605
606 return dram_addr;
607 }
608 }
609
610 /*
611 * Translate the SysAddr to a DramAddr as shown near the start of
612 * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8
613 * only deals with 40-bit values. Therefore we discard bits 63-40 of
614 * sys_addr below. If bit 39 of sys_addr is 1 then the bits we
615 * discard are all 1s. Otherwise the bits we discard are all 0s. See
616 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
617 * Programmer's Manual Volume 1 Application Programming.
618 */
619 dram_addr = (sys_addr & GENMASK_ULL(39, 0)) - dram_base;
620
621 edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
622 (unsigned long)sys_addr, (unsigned long)dram_addr);
623 return dram_addr;
624 }
625
626 /*
627 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
628 * (section 3.4.4.1). Return the number of bits from a SysAddr that are used
629 * for node interleaving.
630 */
631 static int num_node_interleave_bits(unsigned intlv_en)
632 {
633 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
634 int n;
635
636 BUG_ON(intlv_en > 7);
637 n = intlv_shift_table[intlv_en];
638 return n;
639 }
640
641 /* Translate the DramAddr given by @dram_addr to an InputAddr. */
642 static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
643 {
644 struct amd64_pvt *pvt;
645 int intlv_shift;
646 u64 input_addr;
647
648 pvt = mci->pvt_info;
649
650 /*
651 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
652 * concerning translating a DramAddr to an InputAddr.
653 */
654 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
655 input_addr = ((dram_addr >> intlv_shift) & GENMASK_ULL(35, 12)) +
656 (dram_addr & 0xfff);
657
658 edac_dbg(2, " Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
659 intlv_shift, (unsigned long)dram_addr,
660 (unsigned long)input_addr);
661
662 return input_addr;
663 }
664
665 /*
666 * Translate the SysAddr represented by @sys_addr to an InputAddr. It is
667 * assumed that @sys_addr maps to the node given by mci.
668 */
669 static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
670 {
671 u64 input_addr;
672
673 input_addr =
674 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
675
676 edac_dbg(2, "SysAddr 0x%lx translates to InputAddr 0x%lx\n",
677 (unsigned long)sys_addr, (unsigned long)input_addr);
678
679 return input_addr;
680 }
681
682 /* Map the Error address to a PAGE and PAGE OFFSET. */
683 static inline void error_address_to_page_and_offset(u64 error_address,
684 struct err_info *err)
685 {
686 err->page = (u32) (error_address >> PAGE_SHIFT);
687 err->offset = ((u32) error_address) & ~PAGE_MASK;
688 }
689
690 /*
691 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
692 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
693 * of a node that detected an ECC memory error. mci represents the node that
694 * the error address maps to (possibly different from the node that detected
695 * the error). Return the number of the csrow that sys_addr maps to, or -1 on
696 * error.
697 */
698 static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
699 {
700 int csrow;
701
702 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
703
704 if (csrow == -1)
705 amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
706 "address 0x%lx\n", (unsigned long)sys_addr);
707 return csrow;
708 }
709
710 static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
711
712 /*
713 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
714 * are ECC capable.
715 */
716 static unsigned long determine_edac_cap(struct amd64_pvt *pvt)
717 {
718 unsigned long edac_cap = EDAC_FLAG_NONE;
719 u8 bit;
720
721 if (pvt->umc) {
722 u8 i, umc_en_mask = 0, dimm_ecc_en_mask = 0;
723
724 for (i = 0; i < NUM_UMCS; i++) {
725 if (!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT))
726 continue;
727
728 umc_en_mask |= BIT(i);
729
730 /* UMC Configuration bit 12 (DimmEccEn) */
731 if (pvt->umc[i].umc_cfg & BIT(12))
732 dimm_ecc_en_mask |= BIT(i);
733 }
734
735 if (umc_en_mask == dimm_ecc_en_mask)
736 edac_cap = EDAC_FLAG_SECDED;
737 } else {
738 bit = (pvt->fam > 0xf || pvt->ext_model >= K8_REV_F)
739 ? 19
740 : 17;
741
742 if (pvt->dclr0 & BIT(bit))
743 edac_cap = EDAC_FLAG_SECDED;
744 }
745
746 return edac_cap;
747 }
748
749 static void debug_display_dimm_sizes(struct amd64_pvt *, u8);
750
751 static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan)
752 {
753 edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
754
755 if (pvt->dram_type == MEM_LRDDR3) {
756 u32 dcsm = pvt->csels[chan].csmasks[0];
757 /*
758 * It's assumed all LRDIMMs in a DCT are going to be of
759 * same 'type' until proven otherwise. So, use a cs
760 * value of '0' here to get dcsm value.
761 */
762 edac_dbg(1, " LRDIMM %dx rank multiply\n", (dcsm & 0x3));
763 }
764
765 edac_dbg(1, "All DIMMs support ECC:%s\n",
766 (dclr & BIT(19)) ? "yes" : "no");
767
768
769 edac_dbg(1, " PAR/ERR parity: %s\n",
770 (dclr & BIT(8)) ? "enabled" : "disabled");
771
772 if (pvt->fam == 0x10)
773 edac_dbg(1, " DCT 128bit mode width: %s\n",
774 (dclr & BIT(11)) ? "128b" : "64b");
775
776 edac_dbg(1, " x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
777 (dclr & BIT(12)) ? "yes" : "no",
778 (dclr & BIT(13)) ? "yes" : "no",
779 (dclr & BIT(14)) ? "yes" : "no",
780 (dclr & BIT(15)) ? "yes" : "no");
781 }
782
783 static void debug_display_dimm_sizes_df(struct amd64_pvt *pvt, u8 ctrl)
784 {
785 int dimm, size0, size1, cs0, cs1;
786
787 edac_printk(KERN_DEBUG, EDAC_MC, "UMC%d chip selects:\n", ctrl);
788
789 for (dimm = 0; dimm < 4; dimm++) {
790 size0 = 0;
791 cs0 = dimm * 2;
792
793 if (csrow_enabled(cs0, ctrl, pvt))
794 size0 = pvt->ops->dbam_to_cs(pvt, ctrl, 0, cs0);
795
796 size1 = 0;
797 cs1 = dimm * 2 + 1;
798
799 if (csrow_enabled(cs1, ctrl, pvt))
800 size1 = pvt->ops->dbam_to_cs(pvt, ctrl, 0, cs1);
801
802 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
803 cs0, size0,
804 cs1, size1);
805 }
806 }
807
808 static void __dump_misc_regs_df(struct amd64_pvt *pvt)
809 {
810 struct amd64_umc *umc;
811 u32 i, tmp, umc_base;
812
813 for (i = 0; i < NUM_UMCS; i++) {
814 umc_base = get_umc_base(i);
815 umc = &pvt->umc[i];
816
817 edac_dbg(1, "UMC%d DIMM cfg: 0x%x\n", i, umc->dimm_cfg);
818 edac_dbg(1, "UMC%d UMC cfg: 0x%x\n", i, umc->umc_cfg);
819 edac_dbg(1, "UMC%d SDP ctrl: 0x%x\n", i, umc->sdp_ctrl);
820 edac_dbg(1, "UMC%d ECC ctrl: 0x%x\n", i, umc->ecc_ctrl);
821
822 amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ECC_BAD_SYMBOL, &tmp);
823 edac_dbg(1, "UMC%d ECC bad symbol: 0x%x\n", i, tmp);
824
825 amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_UMC_CAP, &tmp);
826 edac_dbg(1, "UMC%d UMC cap: 0x%x\n", i, tmp);
827 edac_dbg(1, "UMC%d UMC cap high: 0x%x\n", i, umc->umc_cap_hi);
828
829 edac_dbg(1, "UMC%d ECC capable: %s, ChipKill ECC capable: %s\n",
830 i, (umc->umc_cap_hi & BIT(30)) ? "yes" : "no",
831 (umc->umc_cap_hi & BIT(31)) ? "yes" : "no");
832 edac_dbg(1, "UMC%d All DIMMs support ECC: %s\n",
833 i, (umc->umc_cfg & BIT(12)) ? "yes" : "no");
834 edac_dbg(1, "UMC%d x4 DIMMs present: %s\n",
835 i, (umc->dimm_cfg & BIT(6)) ? "yes" : "no");
836 edac_dbg(1, "UMC%d x16 DIMMs present: %s\n",
837 i, (umc->dimm_cfg & BIT(7)) ? "yes" : "no");
838
839 if (pvt->dram_type == MEM_LRDDR4) {
840 amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ADDR_CFG, &tmp);
841 edac_dbg(1, "UMC%d LRDIMM %dx rank multiply\n",
842 i, 1 << ((tmp >> 4) & 0x3));
843 }
844
845 debug_display_dimm_sizes_df(pvt, i);
846 }
847
848 edac_dbg(1, "F0x104 (DRAM Hole Address): 0x%08x, base: 0x%08x\n",
849 pvt->dhar, dhar_base(pvt));
850 }
851
852 /* Display and decode various NB registers for debug purposes. */
853 static void __dump_misc_regs(struct amd64_pvt *pvt)
854 {
855 edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
856
857 edac_dbg(1, " NB two channel DRAM capable: %s\n",
858 (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
859
860 edac_dbg(1, " ECC capable: %s, ChipKill ECC capable: %s\n",
861 (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
862 (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
863
864 debug_dump_dramcfg_low(pvt, pvt->dclr0, 0);
865
866 edac_dbg(1, "F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
867
868 edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x\n",
869 pvt->dhar, dhar_base(pvt),
870 (pvt->fam == 0xf) ? k8_dhar_offset(pvt)
871 : f10_dhar_offset(pvt));
872
873 debug_display_dimm_sizes(pvt, 0);
874
875 /* everything below this point is Fam10h and above */
876 if (pvt->fam == 0xf)
877 return;
878
879 debug_display_dimm_sizes(pvt, 1);
880
881 /* Only if NOT ganged does dclr1 have valid info */
882 if (!dct_ganging_enabled(pvt))
883 debug_dump_dramcfg_low(pvt, pvt->dclr1, 1);
884 }
885
886 /* Display and decode various NB registers for debug purposes. */
887 static void dump_misc_regs(struct amd64_pvt *pvt)
888 {
889 if (pvt->umc)
890 __dump_misc_regs_df(pvt);
891 else
892 __dump_misc_regs(pvt);
893
894 edac_dbg(1, " DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
895
896 amd64_info("using %s syndromes.\n",
897 ((pvt->ecc_sym_sz == 8) ? "x8" : "x4"));
898 }
899
900 /*
901 * See BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
902 */
903 static void prep_chip_selects(struct amd64_pvt *pvt)
904 {
905 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
906 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
907 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
908 } else if (pvt->fam == 0x15 && pvt->model == 0x30) {
909 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 4;
910 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 2;
911 } else {
912 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
913 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
914 }
915 }
916
917 /*
918 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
919 */
920 static void read_dct_base_mask(struct amd64_pvt *pvt)
921 {
922 int base_reg0, base_reg1, mask_reg0, mask_reg1, cs;
923
924 prep_chip_selects(pvt);
925
926 if (pvt->umc) {
927 base_reg0 = get_umc_base(0) + UMCCH_BASE_ADDR;
928 base_reg1 = get_umc_base(1) + UMCCH_BASE_ADDR;
929 mask_reg0 = get_umc_base(0) + UMCCH_ADDR_MASK;
930 mask_reg1 = get_umc_base(1) + UMCCH_ADDR_MASK;
931 } else {
932 base_reg0 = DCSB0;
933 base_reg1 = DCSB1;
934 mask_reg0 = DCSM0;
935 mask_reg1 = DCSM1;
936 }
937
938 for_each_chip_select(cs, 0, pvt) {
939 int reg0 = base_reg0 + (cs * 4);
940 int reg1 = base_reg1 + (cs * 4);
941 u32 *base0 = &pvt->csels[0].csbases[cs];
942 u32 *base1 = &pvt->csels[1].csbases[cs];
943
944 if (pvt->umc) {
945 if (!amd_smn_read(pvt->mc_node_id, reg0, base0))
946 edac_dbg(0, " DCSB0[%d]=0x%08x reg: 0x%x\n",
947 cs, *base0, reg0);
948
949 if (!amd_smn_read(pvt->mc_node_id, reg1, base1))
950 edac_dbg(0, " DCSB1[%d]=0x%08x reg: 0x%x\n",
951 cs, *base1, reg1);
952 } else {
953 if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, base0))
954 edac_dbg(0, " DCSB0[%d]=0x%08x reg: F2x%x\n",
955 cs, *base0, reg0);
956
957 if (pvt->fam == 0xf)
958 continue;
959
960 if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, base1))
961 edac_dbg(0, " DCSB1[%d]=0x%08x reg: F2x%x\n",
962 cs, *base1, (pvt->fam == 0x10) ? reg1
963 : reg0);
964 }
965 }
966
967 for_each_chip_select_mask(cs, 0, pvt) {
968 int reg0 = mask_reg0 + (cs * 4);
969 int reg1 = mask_reg1 + (cs * 4);
970 u32 *mask0 = &pvt->csels[0].csmasks[cs];
971 u32 *mask1 = &pvt->csels[1].csmasks[cs];
972
973 if (pvt->umc) {
974 if (!amd_smn_read(pvt->mc_node_id, reg0, mask0))
975 edac_dbg(0, " DCSM0[%d]=0x%08x reg: 0x%x\n",
976 cs, *mask0, reg0);
977
978 if (!amd_smn_read(pvt->mc_node_id, reg1, mask1))
979 edac_dbg(0, " DCSM1[%d]=0x%08x reg: 0x%x\n",
980 cs, *mask1, reg1);
981 } else {
982 if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, mask0))
983 edac_dbg(0, " DCSM0[%d]=0x%08x reg: F2x%x\n",
984 cs, *mask0, reg0);
985
986 if (pvt->fam == 0xf)
987 continue;
988
989 if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, mask1))
990 edac_dbg(0, " DCSM1[%d]=0x%08x reg: F2x%x\n",
991 cs, *mask1, (pvt->fam == 0x10) ? reg1
992 : reg0);
993 }
994 }
995 }
996
997 static void determine_memory_type(struct amd64_pvt *pvt)
998 {
999 u32 dram_ctrl, dcsm;
1000
1001 switch (pvt->fam) {
1002 case 0xf:
1003 if (pvt->ext_model >= K8_REV_F)
1004 goto ddr3;
1005
1006 pvt->dram_type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
1007 return;
1008
1009 case 0x10:
1010 if (pvt->dchr0 & DDR3_MODE)
1011 goto ddr3;
1012
1013 pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
1014 return;
1015
1016 case 0x15:
1017 if (pvt->model < 0x60)
1018 goto ddr3;
1019
1020 /*
1021 * Model 0x60h needs special handling:
1022 *
1023 * We use a Chip Select value of '0' to obtain dcsm.
1024 * Theoretically, it is possible to populate LRDIMMs of different
1025 * 'Rank' value on a DCT. But this is not the common case. So,
1026 * it's reasonable to assume all DIMMs are going to be of same
1027 * 'type' until proven otherwise.
1028 */
1029 amd64_read_dct_pci_cfg(pvt, 0, DRAM_CONTROL, &dram_ctrl);
1030 dcsm = pvt->csels[0].csmasks[0];
1031
1032 if (((dram_ctrl >> 8) & 0x7) == 0x2)
1033 pvt->dram_type = MEM_DDR4;
1034 else if (pvt->dclr0 & BIT(16))
1035 pvt->dram_type = MEM_DDR3;
1036 else if (dcsm & 0x3)
1037 pvt->dram_type = MEM_LRDDR3;
1038 else
1039 pvt->dram_type = MEM_RDDR3;
1040
1041 return;
1042
1043 case 0x16:
1044 goto ddr3;
1045
1046 case 0x17:
1047 if ((pvt->umc[0].dimm_cfg | pvt->umc[1].dimm_cfg) & BIT(5))
1048 pvt->dram_type = MEM_LRDDR4;
1049 else if ((pvt->umc[0].dimm_cfg | pvt->umc[1].dimm_cfg) & BIT(4))
1050 pvt->dram_type = MEM_RDDR4;
1051 else
1052 pvt->dram_type = MEM_DDR4;
1053 return;
1054
1055 default:
1056 WARN(1, KERN_ERR "%s: Family??? 0x%x\n", __func__, pvt->fam);
1057 pvt->dram_type = MEM_EMPTY;
1058 }
1059 return;
1060
1061 ddr3:
1062 pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
1063 }
1064
1065 /* Get the number of DCT channels the memory controller is using. */
1066 static int k8_early_channel_count(struct amd64_pvt *pvt)
1067 {
1068 int flag;
1069
1070 if (pvt->ext_model >= K8_REV_F)
1071 /* RevF (NPT) and later */
1072 flag = pvt->dclr0 & WIDTH_128;
1073 else
1074 /* RevE and earlier */
1075 flag = pvt->dclr0 & REVE_WIDTH_128;
1076
1077 /* not used */
1078 pvt->dclr1 = 0;
1079
1080 return (flag) ? 2 : 1;
1081 }
1082
1083 /* On F10h and later ErrAddr is MC4_ADDR[47:1] */
1084 static u64 get_error_address(struct amd64_pvt *pvt, struct mce *m)
1085 {
1086 u16 mce_nid = amd_get_nb_id(m->extcpu);
1087 struct mem_ctl_info *mci;
1088 u8 start_bit = 1;
1089 u8 end_bit = 47;
1090 u64 addr;
1091
1092 mci = edac_mc_find(mce_nid);
1093 if (!mci)
1094 return 0;
1095
1096 pvt = mci->pvt_info;
1097
1098 if (pvt->fam == 0xf) {
1099 start_bit = 3;
1100 end_bit = 39;
1101 }
1102
1103 addr = m->addr & GENMASK_ULL(end_bit, start_bit);
1104
1105 /*
1106 * Erratum 637 workaround
1107 */
1108 if (pvt->fam == 0x15) {
1109 u64 cc6_base, tmp_addr;
1110 u32 tmp;
1111 u8 intlv_en;
1112
1113 if ((addr & GENMASK_ULL(47, 24)) >> 24 != 0x00fdf7)
1114 return addr;
1115
1116
1117 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp);
1118 intlv_en = tmp >> 21 & 0x7;
1119
1120 /* add [47:27] + 3 trailing bits */
1121 cc6_base = (tmp & GENMASK_ULL(20, 0)) << 3;
1122
1123 /* reverse and add DramIntlvEn */
1124 cc6_base |= intlv_en ^ 0x7;
1125
1126 /* pin at [47:24] */
1127 cc6_base <<= 24;
1128
1129 if (!intlv_en)
1130 return cc6_base | (addr & GENMASK_ULL(23, 0));
1131
1132 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp);
1133
1134 /* faster log2 */
1135 tmp_addr = (addr & GENMASK_ULL(23, 12)) << __fls(intlv_en + 1);
1136
1137 /* OR DramIntlvSel into bits [14:12] */
1138 tmp_addr |= (tmp & GENMASK_ULL(23, 21)) >> 9;
1139
1140 /* add remaining [11:0] bits from original MC4_ADDR */
1141 tmp_addr |= addr & GENMASK_ULL(11, 0);
1142
1143 return cc6_base | tmp_addr;
1144 }
1145
1146 return addr;
1147 }
1148
1149 static struct pci_dev *pci_get_related_function(unsigned int vendor,
1150 unsigned int device,
1151 struct pci_dev *related)
1152 {
1153 struct pci_dev *dev = NULL;
1154
1155 while ((dev = pci_get_device(vendor, device, dev))) {
1156 if (pci_domain_nr(dev->bus) == pci_domain_nr(related->bus) &&
1157 (dev->bus->number == related->bus->number) &&
1158 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
1159 break;
1160 }
1161
1162 return dev;
1163 }
1164
1165 static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
1166 {
1167 struct amd_northbridge *nb;
1168 struct pci_dev *f1 = NULL;
1169 unsigned int pci_func;
1170 int off = range << 3;
1171 u32 llim;
1172
1173 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo);
1174 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
1175
1176 if (pvt->fam == 0xf)
1177 return;
1178
1179 if (!dram_rw(pvt, range))
1180 return;
1181
1182 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi);
1183 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
1184
1185 /* F15h: factor in CC6 save area by reading dst node's limit reg */
1186 if (pvt->fam != 0x15)
1187 return;
1188
1189 nb = node_to_amd_nb(dram_dst_node(pvt, range));
1190 if (WARN_ON(!nb))
1191 return;
1192
1193 if (pvt->model == 0x60)
1194 pci_func = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1;
1195 else if (pvt->model == 0x30)
1196 pci_func = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1;
1197 else
1198 pci_func = PCI_DEVICE_ID_AMD_15H_NB_F1;
1199
1200 f1 = pci_get_related_function(nb->misc->vendor, pci_func, nb->misc);
1201 if (WARN_ON(!f1))
1202 return;
1203
1204 amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim);
1205
1206 pvt->ranges[range].lim.lo &= GENMASK_ULL(15, 0);
1207
1208 /* {[39:27],111b} */
1209 pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16;
1210
1211 pvt->ranges[range].lim.hi &= GENMASK_ULL(7, 0);
1212
1213 /* [47:40] */
1214 pvt->ranges[range].lim.hi |= llim >> 13;
1215
1216 pci_dev_put(f1);
1217 }
1218
1219 static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
1220 struct err_info *err)
1221 {
1222 struct amd64_pvt *pvt = mci->pvt_info;
1223
1224 error_address_to_page_and_offset(sys_addr, err);
1225
1226 /*
1227 * Find out which node the error address belongs to. This may be
1228 * different from the node that detected the error.
1229 */
1230 err->src_mci = find_mc_by_sys_addr(mci, sys_addr);
1231 if (!err->src_mci) {
1232 amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
1233 (unsigned long)sys_addr);
1234 err->err_code = ERR_NODE;
1235 return;
1236 }
1237
1238 /* Now map the sys_addr to a CSROW */
1239 err->csrow = sys_addr_to_csrow(err->src_mci, sys_addr);
1240 if (err->csrow < 0) {
1241 err->err_code = ERR_CSROW;
1242 return;
1243 }
1244
1245 /* CHIPKILL enabled */
1246 if (pvt->nbcfg & NBCFG_CHIPKILL) {
1247 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
1248 if (err->channel < 0) {
1249 /*
1250 * Syndrome didn't map, so we don't know which of the
1251 * 2 DIMMs is in error. So we need to ID 'both' of them
1252 * as suspect.
1253 */
1254 amd64_mc_warn(err->src_mci, "unknown syndrome 0x%04x - "
1255 "possible error reporting race\n",
1256 err->syndrome);
1257 err->err_code = ERR_CHANNEL;
1258 return;
1259 }
1260 } else {
1261 /*
1262 * non-chipkill ecc mode
1263 *
1264 * The k8 documentation is unclear about how to determine the
1265 * channel number when using non-chipkill memory. This method
1266 * was obtained from email communication with someone at AMD.
1267 * (Wish the email was placed in this comment - norsk)
1268 */
1269 err->channel = ((sys_addr & BIT(3)) != 0);
1270 }
1271 }
1272
1273 static int ddr2_cs_size(unsigned i, bool dct_width)
1274 {
1275 unsigned shift = 0;
1276
1277 if (i <= 2)
1278 shift = i;
1279 else if (!(i & 0x1))
1280 shift = i >> 1;
1281 else
1282 shift = (i + 1) >> 1;
1283
1284 return 128 << (shift + !!dct_width);
1285 }
1286
1287 static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1288 unsigned cs_mode, int cs_mask_nr)
1289 {
1290 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1291
1292 if (pvt->ext_model >= K8_REV_F) {
1293 WARN_ON(cs_mode > 11);
1294 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1295 }
1296 else if (pvt->ext_model >= K8_REV_D) {
1297 unsigned diff;
1298 WARN_ON(cs_mode > 10);
1299
1300 /*
1301 * the below calculation, besides trying to win an obfuscated C
1302 * contest, maps cs_mode values to DIMM chip select sizes. The
1303 * mappings are:
1304 *
1305 * cs_mode CS size (mb)
1306 * ======= ============
1307 * 0 32
1308 * 1 64
1309 * 2 128
1310 * 3 128
1311 * 4 256
1312 * 5 512
1313 * 6 256
1314 * 7 512
1315 * 8 1024
1316 * 9 1024
1317 * 10 2048
1318 *
1319 * Basically, it calculates a value with which to shift the
1320 * smallest CS size of 32MB.
1321 *
1322 * ddr[23]_cs_size have a similar purpose.
1323 */
1324 diff = cs_mode/3 + (unsigned)(cs_mode > 5);
1325
1326 return 32 << (cs_mode - diff);
1327 }
1328 else {
1329 WARN_ON(cs_mode > 6);
1330 return 32 << cs_mode;
1331 }
1332 }
1333
1334 /*
1335 * Get the number of DCT channels in use.
1336 *
1337 * Return:
1338 * number of Memory Channels in operation
1339 * Pass back:
1340 * contents of the DCL0_LOW register
1341 */
1342 static int f1x_early_channel_count(struct amd64_pvt *pvt)
1343 {
1344 int i, j, channels = 0;
1345
1346 /* On F10h, if we are in 128 bit mode, then we are using 2 channels */
1347 if (pvt->fam == 0x10 && (pvt->dclr0 & WIDTH_128))
1348 return 2;
1349
1350 /*
1351 * Need to check if in unganged mode: In such, there are 2 channels,
1352 * but they are not in 128 bit mode and thus the above 'dclr0' status
1353 * bit will be OFF.
1354 *
1355 * Need to check DCT0[0] and DCT1[0] to see if only one of them has
1356 * their CSEnable bit on. If so, then SINGLE DIMM case.
1357 */
1358 edac_dbg(0, "Data width is not 128 bits - need more decoding\n");
1359
1360 /*
1361 * Check DRAM Bank Address Mapping values for each DIMM to see if there
1362 * is more than just one DIMM present in unganged mode. Need to check
1363 * both controllers since DIMMs can be placed in either one.
1364 */
1365 for (i = 0; i < 2; i++) {
1366 u32 dbam = (i ? pvt->dbam1 : pvt->dbam0);
1367
1368 for (j = 0; j < 4; j++) {
1369 if (DBAM_DIMM(j, dbam) > 0) {
1370 channels++;
1371 break;
1372 }
1373 }
1374 }
1375
1376 if (channels > 2)
1377 channels = 2;
1378
1379 amd64_info("MCT channel count: %d\n", channels);
1380
1381 return channels;
1382 }
1383
1384 static int f17_early_channel_count(struct amd64_pvt *pvt)
1385 {
1386 int i, channels = 0;
1387
1388 /* SDP Control bit 31 (SdpInit) is clear for unused UMC channels */
1389 for (i = 0; i < NUM_UMCS; i++)
1390 channels += !!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT);
1391
1392 amd64_info("MCT channel count: %d\n", channels);
1393
1394 return channels;
1395 }
1396
1397 static int ddr3_cs_size(unsigned i, bool dct_width)
1398 {
1399 unsigned shift = 0;
1400 int cs_size = 0;
1401
1402 if (i == 0 || i == 3 || i == 4)
1403 cs_size = -1;
1404 else if (i <= 2)
1405 shift = i;
1406 else if (i == 12)
1407 shift = 7;
1408 else if (!(i & 0x1))
1409 shift = i >> 1;
1410 else
1411 shift = (i + 1) >> 1;
1412
1413 if (cs_size != -1)
1414 cs_size = (128 * (1 << !!dct_width)) << shift;
1415
1416 return cs_size;
1417 }
1418
1419 static int ddr3_lrdimm_cs_size(unsigned i, unsigned rank_multiply)
1420 {
1421 unsigned shift = 0;
1422 int cs_size = 0;
1423
1424 if (i < 4 || i == 6)
1425 cs_size = -1;
1426 else if (i == 12)
1427 shift = 7;
1428 else if (!(i & 0x1))
1429 shift = i >> 1;
1430 else
1431 shift = (i + 1) >> 1;
1432
1433 if (cs_size != -1)
1434 cs_size = rank_multiply * (128 << shift);
1435
1436 return cs_size;
1437 }
1438
1439 static int ddr4_cs_size(unsigned i)
1440 {
1441 int cs_size = 0;
1442
1443 if (i == 0)
1444 cs_size = -1;
1445 else if (i == 1)
1446 cs_size = 1024;
1447 else
1448 /* Min cs_size = 1G */
1449 cs_size = 1024 * (1 << (i >> 1));
1450
1451 return cs_size;
1452 }
1453
1454 static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1455 unsigned cs_mode, int cs_mask_nr)
1456 {
1457 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1458
1459 WARN_ON(cs_mode > 11);
1460
1461 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
1462 return ddr3_cs_size(cs_mode, dclr & WIDTH_128);
1463 else
1464 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1465 }
1466
1467 /*
1468 * F15h supports only 64bit DCT interfaces
1469 */
1470 static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1471 unsigned cs_mode, int cs_mask_nr)
1472 {
1473 WARN_ON(cs_mode > 12);
1474
1475 return ddr3_cs_size(cs_mode, false);
1476 }
1477
1478 /* F15h M60h supports DDR4 mapping as well.. */
1479 static int f15_m60h_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1480 unsigned cs_mode, int cs_mask_nr)
1481 {
1482 int cs_size;
1483 u32 dcsm = pvt->csels[dct].csmasks[cs_mask_nr];
1484
1485 WARN_ON(cs_mode > 12);
1486
1487 if (pvt->dram_type == MEM_DDR4) {
1488 if (cs_mode > 9)
1489 return -1;
1490
1491 cs_size = ddr4_cs_size(cs_mode);
1492 } else if (pvt->dram_type == MEM_LRDDR3) {
1493 unsigned rank_multiply = dcsm & 0xf;
1494
1495 if (rank_multiply == 3)
1496 rank_multiply = 4;
1497 cs_size = ddr3_lrdimm_cs_size(cs_mode, rank_multiply);
1498 } else {
1499 /* Minimum cs size is 512mb for F15hM60h*/
1500 if (cs_mode == 0x1)
1501 return -1;
1502
1503 cs_size = ddr3_cs_size(cs_mode, false);
1504 }
1505
1506 return cs_size;
1507 }
1508
1509 /*
1510 * F16h and F15h model 30h have only limited cs_modes.
1511 */
1512 static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1513 unsigned cs_mode, int cs_mask_nr)
1514 {
1515 WARN_ON(cs_mode > 12);
1516
1517 if (cs_mode == 6 || cs_mode == 8 ||
1518 cs_mode == 9 || cs_mode == 12)
1519 return -1;
1520 else
1521 return ddr3_cs_size(cs_mode, false);
1522 }
1523
1524 static int f17_base_addr_to_cs_size(struct amd64_pvt *pvt, u8 umc,
1525 unsigned int cs_mode, int csrow_nr)
1526 {
1527 u32 base_addr = pvt->csels[umc].csbases[csrow_nr];
1528
1529 /* Each mask is used for every two base addresses. */
1530 u32 addr_mask = pvt->csels[umc].csmasks[csrow_nr >> 1];
1531
1532 /* Register [31:1] = Address [39:9]. Size is in kBs here. */
1533 u32 size = ((addr_mask >> 1) - (base_addr >> 1) + 1) >> 1;
1534
1535 edac_dbg(1, "BaseAddr: 0x%x, AddrMask: 0x%x\n", base_addr, addr_mask);
1536
1537 /* Return size in MBs. */
1538 return size >> 10;
1539 }
1540
1541 static void read_dram_ctl_register(struct amd64_pvt *pvt)
1542 {
1543
1544 if (pvt->fam == 0xf)
1545 return;
1546
1547 if (!amd64_read_pci_cfg(pvt->F2, DCT_SEL_LO, &pvt->dct_sel_lo)) {
1548 edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
1549 pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
1550
1551 edac_dbg(0, " DCTs operate in %s mode\n",
1552 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"));
1553
1554 if (!dct_ganging_enabled(pvt))
1555 edac_dbg(0, " Address range split per DCT: %s\n",
1556 (dct_high_range_enabled(pvt) ? "yes" : "no"));
1557
1558 edac_dbg(0, " data interleave for ECC: %s, DRAM cleared since last warm reset: %s\n",
1559 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
1560 (dct_memory_cleared(pvt) ? "yes" : "no"));
1561
1562 edac_dbg(0, " channel interleave: %s, "
1563 "interleave bits selector: 0x%x\n",
1564 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
1565 dct_sel_interleave_addr(pvt));
1566 }
1567
1568 amd64_read_pci_cfg(pvt->F2, DCT_SEL_HI, &pvt->dct_sel_hi);
1569 }
1570
1571 /*
1572 * Determine channel (DCT) based on the interleaving mode (see F15h M30h BKDG,
1573 * 2.10.12 Memory Interleaving Modes).
1574 */
1575 static u8 f15_m30h_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1576 u8 intlv_en, int num_dcts_intlv,
1577 u32 dct_sel)
1578 {
1579 u8 channel = 0;
1580 u8 select;
1581
1582 if (!(intlv_en))
1583 return (u8)(dct_sel);
1584
1585 if (num_dcts_intlv == 2) {
1586 select = (sys_addr >> 8) & 0x3;
1587 channel = select ? 0x3 : 0;
1588 } else if (num_dcts_intlv == 4) {
1589 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1590 switch (intlv_addr) {
1591 case 0x4:
1592 channel = (sys_addr >> 8) & 0x3;
1593 break;
1594 case 0x5:
1595 channel = (sys_addr >> 9) & 0x3;
1596 break;
1597 }
1598 }
1599 return channel;
1600 }
1601
1602 /*
1603 * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
1604 * Interleaving Modes.
1605 */
1606 static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1607 bool hi_range_sel, u8 intlv_en)
1608 {
1609 u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
1610
1611 if (dct_ganging_enabled(pvt))
1612 return 0;
1613
1614 if (hi_range_sel)
1615 return dct_sel_high;
1616
1617 /*
1618 * see F2x110[DctSelIntLvAddr] - channel interleave mode
1619 */
1620 if (dct_interleave_enabled(pvt)) {
1621 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1622
1623 /* return DCT select function: 0=DCT0, 1=DCT1 */
1624 if (!intlv_addr)
1625 return sys_addr >> 6 & 1;
1626
1627 if (intlv_addr & 0x2) {
1628 u8 shift = intlv_addr & 0x1 ? 9 : 6;
1629 u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) & 1;
1630
1631 return ((sys_addr >> shift) & 1) ^ temp;
1632 }
1633
1634 if (intlv_addr & 0x4) {
1635 u8 shift = intlv_addr & 0x1 ? 9 : 8;
1636
1637 return (sys_addr >> shift) & 1;
1638 }
1639
1640 return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
1641 }
1642
1643 if (dct_high_range_enabled(pvt))
1644 return ~dct_sel_high & 1;
1645
1646 return 0;
1647 }
1648
1649 /* Convert the sys_addr to the normalized DCT address */
1650 static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range,
1651 u64 sys_addr, bool hi_rng,
1652 u32 dct_sel_base_addr)
1653 {
1654 u64 chan_off;
1655 u64 dram_base = get_dram_base(pvt, range);
1656 u64 hole_off = f10_dhar_offset(pvt);
1657 u64 dct_sel_base_off = (u64)(pvt->dct_sel_hi & 0xFFFFFC00) << 16;
1658
1659 if (hi_rng) {
1660 /*
1661 * if
1662 * base address of high range is below 4Gb
1663 * (bits [47:27] at [31:11])
1664 * DRAM address space on this DCT is hoisted above 4Gb &&
1665 * sys_addr > 4Gb
1666 *
1667 * remove hole offset from sys_addr
1668 * else
1669 * remove high range offset from sys_addr
1670 */
1671 if ((!(dct_sel_base_addr >> 16) ||
1672 dct_sel_base_addr < dhar_base(pvt)) &&
1673 dhar_valid(pvt) &&
1674 (sys_addr >= BIT_64(32)))
1675 chan_off = hole_off;
1676 else
1677 chan_off = dct_sel_base_off;
1678 } else {
1679 /*
1680 * if
1681 * we have a valid hole &&
1682 * sys_addr > 4Gb
1683 *
1684 * remove hole
1685 * else
1686 * remove dram base to normalize to DCT address
1687 */
1688 if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))
1689 chan_off = hole_off;
1690 else
1691 chan_off = dram_base;
1692 }
1693
1694 return (sys_addr & GENMASK_ULL(47,6)) - (chan_off & GENMASK_ULL(47,23));
1695 }
1696
1697 /*
1698 * checks if the csrow passed in is marked as SPARED, if so returns the new
1699 * spare row
1700 */
1701 static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
1702 {
1703 int tmp_cs;
1704
1705 if (online_spare_swap_done(pvt, dct) &&
1706 csrow == online_spare_bad_dramcs(pvt, dct)) {
1707
1708 for_each_chip_select(tmp_cs, dct, pvt) {
1709 if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
1710 csrow = tmp_cs;
1711 break;
1712 }
1713 }
1714 }
1715 return csrow;
1716 }
1717
1718 /*
1719 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
1720 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
1721 *
1722 * Return:
1723 * -EINVAL: NOT FOUND
1724 * 0..csrow = Chip-Select Row
1725 */
1726 static int f1x_lookup_addr_in_dct(u64 in_addr, u8 nid, u8 dct)
1727 {
1728 struct mem_ctl_info *mci;
1729 struct amd64_pvt *pvt;
1730 u64 cs_base, cs_mask;
1731 int cs_found = -EINVAL;
1732 int csrow;
1733
1734 mci = edac_mc_find(nid);
1735 if (!mci)
1736 return cs_found;
1737
1738 pvt = mci->pvt_info;
1739
1740 edac_dbg(1, "input addr: 0x%llx, DCT: %d\n", in_addr, dct);
1741
1742 for_each_chip_select(csrow, dct, pvt) {
1743 if (!csrow_enabled(csrow, dct, pvt))
1744 continue;
1745
1746 get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
1747
1748 edac_dbg(1, " CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
1749 csrow, cs_base, cs_mask);
1750
1751 cs_mask = ~cs_mask;
1752
1753 edac_dbg(1, " (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx\n",
1754 (in_addr & cs_mask), (cs_base & cs_mask));
1755
1756 if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
1757 if (pvt->fam == 0x15 && pvt->model >= 0x30) {
1758 cs_found = csrow;
1759 break;
1760 }
1761 cs_found = f10_process_possible_spare(pvt, dct, csrow);
1762
1763 edac_dbg(1, " MATCH csrow=%d\n", cs_found);
1764 break;
1765 }
1766 }
1767 return cs_found;
1768 }
1769
1770 /*
1771 * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is
1772 * swapped with a region located at the bottom of memory so that the GPU can use
1773 * the interleaved region and thus two channels.
1774 */
1775 static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)
1776 {
1777 u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;
1778
1779 if (pvt->fam == 0x10) {
1780 /* only revC3 and revE have that feature */
1781 if (pvt->model < 4 || (pvt->model < 0xa && pvt->stepping < 3))
1782 return sys_addr;
1783 }
1784
1785 amd64_read_pci_cfg(pvt->F2, SWAP_INTLV_REG, &swap_reg);
1786
1787 if (!(swap_reg & 0x1))
1788 return sys_addr;
1789
1790 swap_base = (swap_reg >> 3) & 0x7f;
1791 swap_limit = (swap_reg >> 11) & 0x7f;
1792 rgn_size = (swap_reg >> 20) & 0x7f;
1793 tmp_addr = sys_addr >> 27;
1794
1795 if (!(sys_addr >> 34) &&
1796 (((tmp_addr >= swap_base) &&
1797 (tmp_addr <= swap_limit)) ||
1798 (tmp_addr < rgn_size)))
1799 return sys_addr ^ (u64)swap_base << 27;
1800
1801 return sys_addr;
1802 }
1803
1804 /* For a given @dram_range, check if @sys_addr falls within it. */
1805 static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
1806 u64 sys_addr, int *chan_sel)
1807 {
1808 int cs_found = -EINVAL;
1809 u64 chan_addr;
1810 u32 dct_sel_base;
1811 u8 channel;
1812 bool high_range = false;
1813
1814 u8 node_id = dram_dst_node(pvt, range);
1815 u8 intlv_en = dram_intlv_en(pvt, range);
1816 u32 intlv_sel = dram_intlv_sel(pvt, range);
1817
1818 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1819 range, sys_addr, get_dram_limit(pvt, range));
1820
1821 if (dhar_valid(pvt) &&
1822 dhar_base(pvt) <= sys_addr &&
1823 sys_addr < BIT_64(32)) {
1824 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1825 sys_addr);
1826 return -EINVAL;
1827 }
1828
1829 if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en)))
1830 return -EINVAL;
1831
1832 sys_addr = f1x_swap_interleaved_region(pvt, sys_addr);
1833
1834 dct_sel_base = dct_sel_baseaddr(pvt);
1835
1836 /*
1837 * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
1838 * select between DCT0 and DCT1.
1839 */
1840 if (dct_high_range_enabled(pvt) &&
1841 !dct_ganging_enabled(pvt) &&
1842 ((sys_addr >> 27) >= (dct_sel_base >> 11)))
1843 high_range = true;
1844
1845 channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en);
1846
1847 chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr,
1848 high_range, dct_sel_base);
1849
1850 /* Remove node interleaving, see F1x120 */
1851 if (intlv_en)
1852 chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) |
1853 (chan_addr & 0xfff);
1854
1855 /* remove channel interleave */
1856 if (dct_interleave_enabled(pvt) &&
1857 !dct_high_range_enabled(pvt) &&
1858 !dct_ganging_enabled(pvt)) {
1859
1860 if (dct_sel_interleave_addr(pvt) != 1) {
1861 if (dct_sel_interleave_addr(pvt) == 0x3)
1862 /* hash 9 */
1863 chan_addr = ((chan_addr >> 10) << 9) |
1864 (chan_addr & 0x1ff);
1865 else
1866 /* A[6] or hash 6 */
1867 chan_addr = ((chan_addr >> 7) << 6) |
1868 (chan_addr & 0x3f);
1869 } else
1870 /* A[12] */
1871 chan_addr = ((chan_addr >> 13) << 12) |
1872 (chan_addr & 0xfff);
1873 }
1874
1875 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
1876
1877 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel);
1878
1879 if (cs_found >= 0)
1880 *chan_sel = channel;
1881
1882 return cs_found;
1883 }
1884
1885 static int f15_m30h_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
1886 u64 sys_addr, int *chan_sel)
1887 {
1888 int cs_found = -EINVAL;
1889 int num_dcts_intlv = 0;
1890 u64 chan_addr, chan_offset;
1891 u64 dct_base, dct_limit;
1892 u32 dct_cont_base_reg, dct_cont_limit_reg, tmp;
1893 u8 channel, alias_channel, leg_mmio_hole, dct_sel, dct_offset_en;
1894
1895 u64 dhar_offset = f10_dhar_offset(pvt);
1896 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1897 u8 node_id = dram_dst_node(pvt, range);
1898 u8 intlv_en = dram_intlv_en(pvt, range);
1899
1900 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_BASE, &dct_cont_base_reg);
1901 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_LIMIT, &dct_cont_limit_reg);
1902
1903 dct_offset_en = (u8) ((dct_cont_base_reg >> 3) & BIT(0));
1904 dct_sel = (u8) ((dct_cont_base_reg >> 4) & 0x7);
1905
1906 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1907 range, sys_addr, get_dram_limit(pvt, range));
1908
1909 if (!(get_dram_base(pvt, range) <= sys_addr) &&
1910 !(get_dram_limit(pvt, range) >= sys_addr))
1911 return -EINVAL;
1912
1913 if (dhar_valid(pvt) &&
1914 dhar_base(pvt) <= sys_addr &&
1915 sys_addr < BIT_64(32)) {
1916 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1917 sys_addr);
1918 return -EINVAL;
1919 }
1920
1921 /* Verify sys_addr is within DCT Range. */
1922 dct_base = (u64) dct_sel_baseaddr(pvt);
1923 dct_limit = (dct_cont_limit_reg >> 11) & 0x1FFF;
1924
1925 if (!(dct_cont_base_reg & BIT(0)) &&
1926 !(dct_base <= (sys_addr >> 27) &&
1927 dct_limit >= (sys_addr >> 27)))
1928 return -EINVAL;
1929
1930 /* Verify number of dct's that participate in channel interleaving. */
1931 num_dcts_intlv = (int) hweight8(intlv_en);
1932
1933 if (!(num_dcts_intlv % 2 == 0) || (num_dcts_intlv > 4))
1934 return -EINVAL;
1935
1936 if (pvt->model >= 0x60)
1937 channel = f1x_determine_channel(pvt, sys_addr, false, intlv_en);
1938 else
1939 channel = f15_m30h_determine_channel(pvt, sys_addr, intlv_en,
1940 num_dcts_intlv, dct_sel);
1941
1942 /* Verify we stay within the MAX number of channels allowed */
1943 if (channel > 3)
1944 return -EINVAL;
1945
1946 leg_mmio_hole = (u8) (dct_cont_base_reg >> 1 & BIT(0));
1947
1948 /* Get normalized DCT addr */
1949 if (leg_mmio_hole && (sys_addr >= BIT_64(32)))
1950 chan_offset = dhar_offset;
1951 else
1952 chan_offset = dct_base << 27;
1953
1954 chan_addr = sys_addr - chan_offset;
1955
1956 /* remove channel interleave */
1957 if (num_dcts_intlv == 2) {
1958 if (intlv_addr == 0x4)
1959 chan_addr = ((chan_addr >> 9) << 8) |
1960 (chan_addr & 0xff);
1961 else if (intlv_addr == 0x5)
1962 chan_addr = ((chan_addr >> 10) << 9) |
1963 (chan_addr & 0x1ff);
1964 else
1965 return -EINVAL;
1966
1967 } else if (num_dcts_intlv == 4) {
1968 if (intlv_addr == 0x4)
1969 chan_addr = ((chan_addr >> 10) << 8) |
1970 (chan_addr & 0xff);
1971 else if (intlv_addr == 0x5)
1972 chan_addr = ((chan_addr >> 11) << 9) |
1973 (chan_addr & 0x1ff);
1974 else
1975 return -EINVAL;
1976 }
1977
1978 if (dct_offset_en) {
1979 amd64_read_pci_cfg(pvt->F1,
1980 DRAM_CONT_HIGH_OFF + (int) channel * 4,
1981 &tmp);
1982 chan_addr += (u64) ((tmp >> 11) & 0xfff) << 27;
1983 }
1984
1985 f15h_select_dct(pvt, channel);
1986
1987 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
1988
1989 /*
1990 * Find Chip select:
1991 * if channel = 3, then alias it to 1. This is because, in F15 M30h,
1992 * there is support for 4 DCT's, but only 2 are currently functional.
1993 * They are DCT0 and DCT3. But we have read all registers of DCT3 into
1994 * pvt->csels[1]. So we need to use '1' here to get correct info.
1995 * Refer F15 M30h BKDG Section 2.10 and 2.10.3 for clarifications.
1996 */
1997 alias_channel = (channel == 3) ? 1 : channel;
1998
1999 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, alias_channel);
2000
2001 if (cs_found >= 0)
2002 *chan_sel = alias_channel;
2003
2004 return cs_found;
2005 }
2006
2007 static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt,
2008 u64 sys_addr,
2009 int *chan_sel)
2010 {
2011 int cs_found = -EINVAL;
2012 unsigned range;
2013
2014 for (range = 0; range < DRAM_RANGES; range++) {
2015 if (!dram_rw(pvt, range))
2016 continue;
2017
2018 if (pvt->fam == 0x15 && pvt->model >= 0x30)
2019 cs_found = f15_m30h_match_to_this_node(pvt, range,
2020 sys_addr,
2021 chan_sel);
2022
2023 else if ((get_dram_base(pvt, range) <= sys_addr) &&
2024 (get_dram_limit(pvt, range) >= sys_addr)) {
2025 cs_found = f1x_match_to_this_node(pvt, range,
2026 sys_addr, chan_sel);
2027 if (cs_found >= 0)
2028 break;
2029 }
2030 }
2031 return cs_found;
2032 }
2033
2034 /*
2035 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
2036 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
2037 *
2038 * The @sys_addr is usually an error address received from the hardware
2039 * (MCX_ADDR).
2040 */
2041 static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
2042 struct err_info *err)
2043 {
2044 struct amd64_pvt *pvt = mci->pvt_info;
2045
2046 error_address_to_page_and_offset(sys_addr, err);
2047
2048 err->csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &err->channel);
2049 if (err->csrow < 0) {
2050 err->err_code = ERR_CSROW;
2051 return;
2052 }
2053
2054 /*
2055 * We need the syndromes for channel detection only when we're
2056 * ganged. Otherwise @chan should already contain the channel at
2057 * this point.
2058 */
2059 if (dct_ganging_enabled(pvt))
2060 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
2061 }
2062
2063 /*
2064 * debug routine to display the memory sizes of all logical DIMMs and its
2065 * CSROWs
2066 */
2067 static void debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
2068 {
2069 int dimm, size0, size1;
2070 u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
2071 u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0;
2072
2073 if (pvt->fam == 0xf) {
2074 /* K8 families < revF not supported yet */
2075 if (pvt->ext_model < K8_REV_F)
2076 return;
2077 else
2078 WARN_ON(ctrl != 0);
2079 }
2080
2081 if (pvt->fam == 0x10) {
2082 dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1
2083 : pvt->dbam0;
2084 dcsb = (ctrl && !dct_ganging_enabled(pvt)) ?
2085 pvt->csels[1].csbases :
2086 pvt->csels[0].csbases;
2087 } else if (ctrl) {
2088 dbam = pvt->dbam0;
2089 dcsb = pvt->csels[1].csbases;
2090 }
2091 edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n",
2092 ctrl, dbam);
2093
2094 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
2095
2096 /* Dump memory sizes for DIMM and its CSROWs */
2097 for (dimm = 0; dimm < 4; dimm++) {
2098
2099 size0 = 0;
2100 if (dcsb[dimm*2] & DCSB_CS_ENABLE)
2101 /*
2102 * For F15m60h, we need multiplier for LRDIMM cs_size
2103 * calculation. We pass dimm value to the dbam_to_cs
2104 * mapper so we can find the multiplier from the
2105 * corresponding DCSM.
2106 */
2107 size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
2108 DBAM_DIMM(dimm, dbam),
2109 dimm);
2110
2111 size1 = 0;
2112 if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
2113 size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
2114 DBAM_DIMM(dimm, dbam),
2115 dimm);
2116
2117 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
2118 dimm * 2, size0,
2119 dimm * 2 + 1, size1);
2120 }
2121 }
2122
2123 static struct amd64_family_type family_types[] = {
2124 [K8_CPUS] = {
2125 .ctl_name = "K8",
2126 .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
2127 .f2_id = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
2128 .ops = {
2129 .early_channel_count = k8_early_channel_count,
2130 .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow,
2131 .dbam_to_cs = k8_dbam_to_chip_select,
2132 }
2133 },
2134 [F10_CPUS] = {
2135 .ctl_name = "F10h",
2136 .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
2137 .f2_id = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
2138 .ops = {
2139 .early_channel_count = f1x_early_channel_count,
2140 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2141 .dbam_to_cs = f10_dbam_to_chip_select,
2142 }
2143 },
2144 [F15_CPUS] = {
2145 .ctl_name = "F15h",
2146 .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1,
2147 .f2_id = PCI_DEVICE_ID_AMD_15H_NB_F2,
2148 .ops = {
2149 .early_channel_count = f1x_early_channel_count,
2150 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2151 .dbam_to_cs = f15_dbam_to_chip_select,
2152 }
2153 },
2154 [F15_M30H_CPUS] = {
2155 .ctl_name = "F15h_M30h",
2156 .f1_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1,
2157 .f2_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F2,
2158 .ops = {
2159 .early_channel_count = f1x_early_channel_count,
2160 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2161 .dbam_to_cs = f16_dbam_to_chip_select,
2162 }
2163 },
2164 [F15_M60H_CPUS] = {
2165 .ctl_name = "F15h_M60h",
2166 .f1_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1,
2167 .f2_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F2,
2168 .ops = {
2169 .early_channel_count = f1x_early_channel_count,
2170 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2171 .dbam_to_cs = f15_m60h_dbam_to_chip_select,
2172 }
2173 },
2174 [F16_CPUS] = {
2175 .ctl_name = "F16h",
2176 .f1_id = PCI_DEVICE_ID_AMD_16H_NB_F1,
2177 .f2_id = PCI_DEVICE_ID_AMD_16H_NB_F2,
2178 .ops = {
2179 .early_channel_count = f1x_early_channel_count,
2180 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2181 .dbam_to_cs = f16_dbam_to_chip_select,
2182 }
2183 },
2184 [F16_M30H_CPUS] = {
2185 .ctl_name = "F16h_M30h",
2186 .f1_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F1,
2187 .f2_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F2,
2188 .ops = {
2189 .early_channel_count = f1x_early_channel_count,
2190 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2191 .dbam_to_cs = f16_dbam_to_chip_select,
2192 }
2193 },
2194 [F17_CPUS] = {
2195 .ctl_name = "F17h",
2196 .f0_id = PCI_DEVICE_ID_AMD_17H_DF_F0,
2197 .f6_id = PCI_DEVICE_ID_AMD_17H_DF_F6,
2198 .ops = {
2199 .early_channel_count = f17_early_channel_count,
2200 .dbam_to_cs = f17_base_addr_to_cs_size,
2201 }
2202 },
2203 };
2204
2205 /*
2206 * These are tables of eigenvectors (one per line) which can be used for the
2207 * construction of the syndrome tables. The modified syndrome search algorithm
2208 * uses those to find the symbol in error and thus the DIMM.
2209 *
2210 * Algorithm courtesy of Ross LaFetra from AMD.
2211 */
2212 static const u16 x4_vectors[] = {
2213 0x2f57, 0x1afe, 0x66cc, 0xdd88,
2214 0x11eb, 0x3396, 0x7f4c, 0xeac8,
2215 0x0001, 0x0002, 0x0004, 0x0008,
2216 0x1013, 0x3032, 0x4044, 0x8088,
2217 0x106b, 0x30d6, 0x70fc, 0xe0a8,
2218 0x4857, 0xc4fe, 0x13cc, 0x3288,
2219 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
2220 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
2221 0x15c1, 0x2a42, 0x89ac, 0x4758,
2222 0x2b03, 0x1602, 0x4f0c, 0xca08,
2223 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
2224 0x8ba7, 0x465e, 0x244c, 0x1cc8,
2225 0x2b87, 0x164e, 0x642c, 0xdc18,
2226 0x40b9, 0x80de, 0x1094, 0x20e8,
2227 0x27db, 0x1eb6, 0x9dac, 0x7b58,
2228 0x11c1, 0x2242, 0x84ac, 0x4c58,
2229 0x1be5, 0x2d7a, 0x5e34, 0xa718,
2230 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
2231 0x4c97, 0xc87e, 0x11fc, 0x33a8,
2232 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
2233 0x16b3, 0x3d62, 0x4f34, 0x8518,
2234 0x1e2f, 0x391a, 0x5cac, 0xf858,
2235 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
2236 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
2237 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
2238 0x4397, 0xc27e, 0x17fc, 0x3ea8,
2239 0x1617, 0x3d3e, 0x6464, 0xb8b8,
2240 0x23ff, 0x12aa, 0xab6c, 0x56d8,
2241 0x2dfb, 0x1ba6, 0x913c, 0x7328,
2242 0x185d, 0x2ca6, 0x7914, 0x9e28,
2243 0x171b, 0x3e36, 0x7d7c, 0xebe8,
2244 0x4199, 0x82ee, 0x19f4, 0x2e58,
2245 0x4807, 0xc40e, 0x130c, 0x3208,
2246 0x1905, 0x2e0a, 0x5804, 0xac08,
2247 0x213f, 0x132a, 0xadfc, 0x5ba8,
2248 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
2249 };
2250
2251 static const u16 x8_vectors[] = {
2252 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
2253 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
2254 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
2255 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
2256 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
2257 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
2258 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
2259 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
2260 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
2261 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
2262 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
2263 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
2264 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
2265 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
2266 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
2267 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
2268 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
2269 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
2270 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
2271 };
2272
2273 static int decode_syndrome(u16 syndrome, const u16 *vectors, unsigned num_vecs,
2274 unsigned v_dim)
2275 {
2276 unsigned int i, err_sym;
2277
2278 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
2279 u16 s = syndrome;
2280 unsigned v_idx = err_sym * v_dim;
2281 unsigned v_end = (err_sym + 1) * v_dim;
2282
2283 /* walk over all 16 bits of the syndrome */
2284 for (i = 1; i < (1U << 16); i <<= 1) {
2285
2286 /* if bit is set in that eigenvector... */
2287 if (v_idx < v_end && vectors[v_idx] & i) {
2288 u16 ev_comp = vectors[v_idx++];
2289
2290 /* ... and bit set in the modified syndrome, */
2291 if (s & i) {
2292 /* remove it. */
2293 s ^= ev_comp;
2294
2295 if (!s)
2296 return err_sym;
2297 }
2298
2299 } else if (s & i)
2300 /* can't get to zero, move to next symbol */
2301 break;
2302 }
2303 }
2304
2305 edac_dbg(0, "syndrome(%x) not found\n", syndrome);
2306 return -1;
2307 }
2308
2309 static int map_err_sym_to_channel(int err_sym, int sym_size)
2310 {
2311 if (sym_size == 4)
2312 switch (err_sym) {
2313 case 0x20:
2314 case 0x21:
2315 return 0;
2316 break;
2317 case 0x22:
2318 case 0x23:
2319 return 1;
2320 break;
2321 default:
2322 return err_sym >> 4;
2323 break;
2324 }
2325 /* x8 symbols */
2326 else
2327 switch (err_sym) {
2328 /* imaginary bits not in a DIMM */
2329 case 0x10:
2330 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
2331 err_sym);
2332 return -1;
2333 break;
2334
2335 case 0x11:
2336 return 0;
2337 break;
2338 case 0x12:
2339 return 1;
2340 break;
2341 default:
2342 return err_sym >> 3;
2343 break;
2344 }
2345 return -1;
2346 }
2347
2348 static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
2349 {
2350 struct amd64_pvt *pvt = mci->pvt_info;
2351 int err_sym = -1;
2352
2353 if (pvt->ecc_sym_sz == 8)
2354 err_sym = decode_syndrome(syndrome, x8_vectors,
2355 ARRAY_SIZE(x8_vectors),
2356 pvt->ecc_sym_sz);
2357 else if (pvt->ecc_sym_sz == 4)
2358 err_sym = decode_syndrome(syndrome, x4_vectors,
2359 ARRAY_SIZE(x4_vectors),
2360 pvt->ecc_sym_sz);
2361 else {
2362 amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz);
2363 return err_sym;
2364 }
2365
2366 return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
2367 }
2368
2369 static void __log_ecc_error(struct mem_ctl_info *mci, struct err_info *err,
2370 u8 ecc_type)
2371 {
2372 enum hw_event_mc_err_type err_type;
2373 const char *string;
2374
2375 if (ecc_type == 2)
2376 err_type = HW_EVENT_ERR_CORRECTED;
2377 else if (ecc_type == 1)
2378 err_type = HW_EVENT_ERR_UNCORRECTED;
2379 else if (ecc_type == 3)
2380 err_type = HW_EVENT_ERR_DEFERRED;
2381 else {
2382 WARN(1, "Something is rotten in the state of Denmark.\n");
2383 return;
2384 }
2385
2386 switch (err->err_code) {
2387 case DECODE_OK:
2388 string = "";
2389 break;
2390 case ERR_NODE:
2391 string = "Failed to map error addr to a node";
2392 break;
2393 case ERR_CSROW:
2394 string = "Failed to map error addr to a csrow";
2395 break;
2396 case ERR_CHANNEL:
2397 string = "Unknown syndrome - possible error reporting race";
2398 break;
2399 case ERR_SYND:
2400 string = "MCA_SYND not valid - unknown syndrome and csrow";
2401 break;
2402 case ERR_NORM_ADDR:
2403 string = "Cannot decode normalized address";
2404 break;
2405 default:
2406 string = "WTF error";
2407 break;
2408 }
2409
2410 edac_mc_handle_error(err_type, mci, 1,
2411 err->page, err->offset, err->syndrome,
2412 err->csrow, err->channel, -1,
2413 string, "");
2414 }
2415
2416 static inline void decode_bus_error(int node_id, struct mce *m)
2417 {
2418 struct mem_ctl_info *mci;
2419 struct amd64_pvt *pvt;
2420 u8 ecc_type = (m->status >> 45) & 0x3;
2421 u8 xec = XEC(m->status, 0x1f);
2422 u16 ec = EC(m->status);
2423 u64 sys_addr;
2424 struct err_info err;
2425
2426 mci = edac_mc_find(node_id);
2427 if (!mci)
2428 return;
2429
2430 pvt = mci->pvt_info;
2431
2432 /* Bail out early if this was an 'observed' error */
2433 if (PP(ec) == NBSL_PP_OBS)
2434 return;
2435
2436 /* Do only ECC errors */
2437 if (xec && xec != F10_NBSL_EXT_ERR_ECC)
2438 return;
2439
2440 memset(&err, 0, sizeof(err));
2441
2442 sys_addr = get_error_address(pvt, m);
2443
2444 if (ecc_type == 2)
2445 err.syndrome = extract_syndrome(m->status);
2446
2447 pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, &err);
2448
2449 __log_ecc_error(mci, &err, ecc_type);
2450 }
2451
2452 /*
2453 * To find the UMC channel represented by this bank we need to match on its
2454 * instance_id. The instance_id of a bank is held in the lower 32 bits of its
2455 * IPID.
2456 */
2457 static int find_umc_channel(struct amd64_pvt *pvt, struct mce *m)
2458 {
2459 u32 umc_instance_id[] = {0x50f00, 0x150f00};
2460 u32 instance_id = m->ipid & GENMASK(31, 0);
2461 int i, channel = -1;
2462
2463 for (i = 0; i < ARRAY_SIZE(umc_instance_id); i++)
2464 if (umc_instance_id[i] == instance_id)
2465 channel = i;
2466
2467 return channel;
2468 }
2469
2470 static void decode_umc_error(int node_id, struct mce *m)
2471 {
2472 u8 ecc_type = (m->status >> 45) & 0x3;
2473 struct mem_ctl_info *mci;
2474 struct amd64_pvt *pvt;
2475 struct err_info err;
2476 u64 sys_addr;
2477
2478 mci = edac_mc_find(node_id);
2479 if (!mci)
2480 return;
2481
2482 pvt = mci->pvt_info;
2483
2484 memset(&err, 0, sizeof(err));
2485
2486 if (m->status & MCI_STATUS_DEFERRED)
2487 ecc_type = 3;
2488
2489 err.channel = find_umc_channel(pvt, m);
2490 if (err.channel < 0) {
2491 err.err_code = ERR_CHANNEL;
2492 goto log_error;
2493 }
2494
2495 if (umc_normaddr_to_sysaddr(m->addr, pvt->mc_node_id, err.channel, &sys_addr)) {
2496 err.err_code = ERR_NORM_ADDR;
2497 goto log_error;
2498 }
2499
2500 error_address_to_page_and_offset(sys_addr, &err);
2501
2502 if (!(m->status & MCI_STATUS_SYNDV)) {
2503 err.err_code = ERR_SYND;
2504 goto log_error;
2505 }
2506
2507 if (ecc_type == 2) {
2508 u8 length = (m->synd >> 18) & 0x3f;
2509
2510 if (length)
2511 err.syndrome = (m->synd >> 32) & GENMASK(length - 1, 0);
2512 else
2513 err.err_code = ERR_CHANNEL;
2514 }
2515
2516 err.csrow = m->synd & 0x7;
2517
2518 log_error:
2519 __log_ecc_error(mci, &err, ecc_type);
2520 }
2521
2522 /*
2523 * Use pvt->F3 which contains the F3 CPU PCI device to get the related
2524 * F1 (AddrMap) and F2 (Dct) devices. Return negative value on error.
2525 * Reserve F0 and F6 on systems with a UMC.
2526 */
2527 static int
2528 reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 pci_id1, u16 pci_id2)
2529 {
2530 if (pvt->umc) {
2531 pvt->F0 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3);
2532 if (!pvt->F0) {
2533 amd64_err("F0 not found, device 0x%x (broken BIOS?)\n", pci_id1);
2534 return -ENODEV;
2535 }
2536
2537 pvt->F6 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3);
2538 if (!pvt->F6) {
2539 pci_dev_put(pvt->F0);
2540 pvt->F0 = NULL;
2541
2542 amd64_err("F6 not found: device 0x%x (broken BIOS?)\n", pci_id2);
2543 return -ENODEV;
2544 }
2545
2546 edac_dbg(1, "F0: %s\n", pci_name(pvt->F0));
2547 edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
2548 edac_dbg(1, "F6: %s\n", pci_name(pvt->F6));
2549
2550 return 0;
2551 }
2552
2553 /* Reserve the ADDRESS MAP Device */
2554 pvt->F1 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3);
2555 if (!pvt->F1) {
2556 amd64_err("F1 not found: device 0x%x (broken BIOS?)\n", pci_id1);
2557 return -ENODEV;
2558 }
2559
2560 /* Reserve the DCT Device */
2561 pvt->F2 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3);
2562 if (!pvt->F2) {
2563 pci_dev_put(pvt->F1);
2564 pvt->F1 = NULL;
2565
2566 amd64_err("F2 not found: device 0x%x (broken BIOS?)\n", pci_id2);
2567 return -ENODEV;
2568 }
2569
2570 edac_dbg(1, "F1: %s\n", pci_name(pvt->F1));
2571 edac_dbg(1, "F2: %s\n", pci_name(pvt->F2));
2572 edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
2573
2574 return 0;
2575 }
2576
2577 static void free_mc_sibling_devs(struct amd64_pvt *pvt)
2578 {
2579 if (pvt->umc) {
2580 pci_dev_put(pvt->F0);
2581 pci_dev_put(pvt->F6);
2582 } else {
2583 pci_dev_put(pvt->F1);
2584 pci_dev_put(pvt->F2);
2585 }
2586 }
2587
2588 static void determine_ecc_sym_sz(struct amd64_pvt *pvt)
2589 {
2590 pvt->ecc_sym_sz = 4;
2591
2592 if (pvt->umc) {
2593 u8 i;
2594
2595 for (i = 0; i < NUM_UMCS; i++) {
2596 /* Check enabled channels only: */
2597 if ((pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) &&
2598 (pvt->umc[i].ecc_ctrl & BIT(7))) {
2599 pvt->ecc_sym_sz = 8;
2600 break;
2601 }
2602 }
2603
2604 return;
2605 }
2606
2607 if (pvt->fam >= 0x10) {
2608 u32 tmp;
2609
2610 amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
2611 /* F16h has only DCT0, so no need to read dbam1. */
2612 if (pvt->fam != 0x16)
2613 amd64_read_dct_pci_cfg(pvt, 1, DBAM0, &pvt->dbam1);
2614
2615 /* F10h, revD and later can do x8 ECC too. */
2616 if ((pvt->fam > 0x10 || pvt->model > 7) && tmp & BIT(25))
2617 pvt->ecc_sym_sz = 8;
2618 }
2619 }
2620
2621 /*
2622 * Retrieve the hardware registers of the memory controller.
2623 */
2624 static void __read_mc_regs_df(struct amd64_pvt *pvt)
2625 {
2626 u8 nid = pvt->mc_node_id;
2627 struct amd64_umc *umc;
2628 u32 i, umc_base;
2629
2630 /* Read registers from each UMC */
2631 for (i = 0; i < NUM_UMCS; i++) {
2632
2633 umc_base = get_umc_base(i);
2634 umc = &pvt->umc[i];
2635
2636 amd_smn_read(nid, umc_base + UMCCH_DIMM_CFG, &umc->dimm_cfg);
2637 amd_smn_read(nid, umc_base + UMCCH_UMC_CFG, &umc->umc_cfg);
2638 amd_smn_read(nid, umc_base + UMCCH_SDP_CTRL, &umc->sdp_ctrl);
2639 amd_smn_read(nid, umc_base + UMCCH_ECC_CTRL, &umc->ecc_ctrl);
2640 amd_smn_read(nid, umc_base + UMCCH_UMC_CAP_HI, &umc->umc_cap_hi);
2641 }
2642 }
2643
2644 /*
2645 * Retrieve the hardware registers of the memory controller (this includes the
2646 * 'Address Map' and 'Misc' device regs)
2647 */
2648 static void read_mc_regs(struct amd64_pvt *pvt)
2649 {
2650 unsigned int range;
2651 u64 msr_val;
2652
2653 /*
2654 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
2655 * those are Read-As-Zero.
2656 */
2657 rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
2658 edac_dbg(0, " TOP_MEM: 0x%016llx\n", pvt->top_mem);
2659
2660 /* Check first whether TOP_MEM2 is enabled: */
2661 rdmsrl(MSR_K8_SYSCFG, msr_val);
2662 if (msr_val & BIT(21)) {
2663 rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
2664 edac_dbg(0, " TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
2665 } else {
2666 edac_dbg(0, " TOP_MEM2 disabled\n");
2667 }
2668
2669 if (pvt->umc) {
2670 __read_mc_regs_df(pvt);
2671 amd64_read_pci_cfg(pvt->F0, DF_DHAR, &pvt->dhar);
2672
2673 goto skip;
2674 }
2675
2676 amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
2677
2678 read_dram_ctl_register(pvt);
2679
2680 for (range = 0; range < DRAM_RANGES; range++) {
2681 u8 rw;
2682
2683 /* read settings for this DRAM range */
2684 read_dram_base_limit_regs(pvt, range);
2685
2686 rw = dram_rw(pvt, range);
2687 if (!rw)
2688 continue;
2689
2690 edac_dbg(1, " DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
2691 range,
2692 get_dram_base(pvt, range),
2693 get_dram_limit(pvt, range));
2694
2695 edac_dbg(1, " IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
2696 dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
2697 (rw & 0x1) ? "R" : "-",
2698 (rw & 0x2) ? "W" : "-",
2699 dram_intlv_sel(pvt, range),
2700 dram_dst_node(pvt, range));
2701 }
2702
2703 amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
2704 amd64_read_dct_pci_cfg(pvt, 0, DBAM0, &pvt->dbam0);
2705
2706 amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
2707
2708 amd64_read_dct_pci_cfg(pvt, 0, DCLR0, &pvt->dclr0);
2709 amd64_read_dct_pci_cfg(pvt, 0, DCHR0, &pvt->dchr0);
2710
2711 if (!dct_ganging_enabled(pvt)) {
2712 amd64_read_dct_pci_cfg(pvt, 1, DCLR0, &pvt->dclr1);
2713 amd64_read_dct_pci_cfg(pvt, 1, DCHR0, &pvt->dchr1);
2714 }
2715
2716 skip:
2717 read_dct_base_mask(pvt);
2718
2719 determine_memory_type(pvt);
2720 edac_dbg(1, " DIMM type: %s\n", edac_mem_types[pvt->dram_type]);
2721
2722 determine_ecc_sym_sz(pvt);
2723
2724 dump_misc_regs(pvt);
2725 }
2726
2727 /*
2728 * NOTE: CPU Revision Dependent code
2729 *
2730 * Input:
2731 * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
2732 * k8 private pointer to -->
2733 * DRAM Bank Address mapping register
2734 * node_id
2735 * DCL register where dual_channel_active is
2736 *
2737 * The DBAM register consists of 4 sets of 4 bits each definitions:
2738 *
2739 * Bits: CSROWs
2740 * 0-3 CSROWs 0 and 1
2741 * 4-7 CSROWs 2 and 3
2742 * 8-11 CSROWs 4 and 5
2743 * 12-15 CSROWs 6 and 7
2744 *
2745 * Values range from: 0 to 15
2746 * The meaning of the values depends on CPU revision and dual-channel state,
2747 * see relevant BKDG more info.
2748 *
2749 * The memory controller provides for total of only 8 CSROWs in its current
2750 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
2751 * single channel or two (2) DIMMs in dual channel mode.
2752 *
2753 * The following code logic collapses the various tables for CSROW based on CPU
2754 * revision.
2755 *
2756 * Returns:
2757 * The number of PAGE_SIZE pages on the specified CSROW number it
2758 * encompasses
2759 *
2760 */
2761 static u32 get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr_orig)
2762 {
2763 u32 dbam = dct ? pvt->dbam1 : pvt->dbam0;
2764 int csrow_nr = csrow_nr_orig;
2765 u32 cs_mode, nr_pages;
2766
2767 if (!pvt->umc)
2768 csrow_nr >>= 1;
2769
2770 cs_mode = DBAM_DIMM(csrow_nr, dbam);
2771
2772 nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode, csrow_nr);
2773 nr_pages <<= 20 - PAGE_SHIFT;
2774
2775 edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n",
2776 csrow_nr_orig, dct, cs_mode);
2777 edac_dbg(0, "nr_pages/channel: %u\n", nr_pages);
2778
2779 return nr_pages;
2780 }
2781
2782 /*
2783 * Initialize the array of csrow attribute instances, based on the values
2784 * from pci config hardware registers.
2785 */
2786 static int init_csrows(struct mem_ctl_info *mci)
2787 {
2788 struct amd64_pvt *pvt = mci->pvt_info;
2789 enum edac_type edac_mode = EDAC_NONE;
2790 struct csrow_info *csrow;
2791 struct dimm_info *dimm;
2792 int i, j, empty = 1;
2793 int nr_pages = 0;
2794 u32 val;
2795
2796 if (!pvt->umc) {
2797 amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
2798
2799 pvt->nbcfg = val;
2800
2801 edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2802 pvt->mc_node_id, val,
2803 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
2804 }
2805
2806 /*
2807 * We iterate over DCT0 here but we look at DCT1 in parallel, if needed.
2808 */
2809 for_each_chip_select(i, 0, pvt) {
2810 bool row_dct0 = !!csrow_enabled(i, 0, pvt);
2811 bool row_dct1 = false;
2812
2813 if (pvt->fam != 0xf)
2814 row_dct1 = !!csrow_enabled(i, 1, pvt);
2815
2816 if (!row_dct0 && !row_dct1)
2817 continue;
2818
2819 csrow = mci->csrows[i];
2820 empty = 0;
2821
2822 edac_dbg(1, "MC node: %d, csrow: %d\n",
2823 pvt->mc_node_id, i);
2824
2825 if (row_dct0) {
2826 nr_pages = get_csrow_nr_pages(pvt, 0, i);
2827 csrow->channels[0]->dimm->nr_pages = nr_pages;
2828 }
2829
2830 /* K8 has only one DCT */
2831 if (pvt->fam != 0xf && row_dct1) {
2832 int row_dct1_pages = get_csrow_nr_pages(pvt, 1, i);
2833
2834 csrow->channels[1]->dimm->nr_pages = row_dct1_pages;
2835 nr_pages += row_dct1_pages;
2836 }
2837
2838 edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages);
2839
2840 /* Determine DIMM ECC mode: */
2841 if (pvt->umc) {
2842 if (mci->edac_ctl_cap & EDAC_FLAG_S4ECD4ED)
2843 edac_mode = EDAC_S4ECD4ED;
2844 else if (mci->edac_ctl_cap & EDAC_FLAG_SECDED)
2845 edac_mode = EDAC_SECDED;
2846
2847 } else if (pvt->nbcfg & NBCFG_ECC_ENABLE) {
2848 edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL)
2849 ? EDAC_S4ECD4ED
2850 : EDAC_SECDED;
2851 }
2852
2853 for (j = 0; j < pvt->channel_count; j++) {
2854 dimm = csrow->channels[j]->dimm;
2855 dimm->mtype = pvt->dram_type;
2856 dimm->edac_mode = edac_mode;
2857 }
2858 }
2859
2860 return empty;
2861 }
2862
2863 /* get all cores on this DCT */
2864 static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid)
2865 {
2866 int cpu;
2867
2868 for_each_online_cpu(cpu)
2869 if (amd_get_nb_id(cpu) == nid)
2870 cpumask_set_cpu(cpu, mask);
2871 }
2872
2873 /* check MCG_CTL on all the cpus on this node */
2874 static bool nb_mce_bank_enabled_on_node(u16 nid)
2875 {
2876 cpumask_var_t mask;
2877 int cpu, nbe;
2878 bool ret = false;
2879
2880 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
2881 amd64_warn("%s: Error allocating mask\n", __func__);
2882 return false;
2883 }
2884
2885 get_cpus_on_this_dct_cpumask(mask, nid);
2886
2887 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
2888
2889 for_each_cpu(cpu, mask) {
2890 struct msr *reg = per_cpu_ptr(msrs, cpu);
2891 nbe = reg->l & MSR_MCGCTL_NBE;
2892
2893 edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
2894 cpu, reg->q,
2895 (nbe ? "enabled" : "disabled"));
2896
2897 if (!nbe)
2898 goto out;
2899 }
2900 ret = true;
2901
2902 out:
2903 free_cpumask_var(mask);
2904 return ret;
2905 }
2906
2907 static int toggle_ecc_err_reporting(struct ecc_settings *s, u16 nid, bool on)
2908 {
2909 cpumask_var_t cmask;
2910 int cpu;
2911
2912 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
2913 amd64_warn("%s: error allocating mask\n", __func__);
2914 return -ENOMEM;
2915 }
2916
2917 get_cpus_on_this_dct_cpumask(cmask, nid);
2918
2919 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2920
2921 for_each_cpu(cpu, cmask) {
2922
2923 struct msr *reg = per_cpu_ptr(msrs, cpu);
2924
2925 if (on) {
2926 if (reg->l & MSR_MCGCTL_NBE)
2927 s->flags.nb_mce_enable = 1;
2928
2929 reg->l |= MSR_MCGCTL_NBE;
2930 } else {
2931 /*
2932 * Turn off NB MCE reporting only when it was off before
2933 */
2934 if (!s->flags.nb_mce_enable)
2935 reg->l &= ~MSR_MCGCTL_NBE;
2936 }
2937 }
2938 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2939
2940 free_cpumask_var(cmask);
2941
2942 return 0;
2943 }
2944
2945 static bool enable_ecc_error_reporting(struct ecc_settings *s, u16 nid,
2946 struct pci_dev *F3)
2947 {
2948 bool ret = true;
2949 u32 value, mask = 0x3; /* UECC/CECC enable */
2950
2951 if (toggle_ecc_err_reporting(s, nid, ON)) {
2952 amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
2953 return false;
2954 }
2955
2956 amd64_read_pci_cfg(F3, NBCTL, &value);
2957
2958 s->old_nbctl = value & mask;
2959 s->nbctl_valid = true;
2960
2961 value |= mask;
2962 amd64_write_pci_cfg(F3, NBCTL, value);
2963
2964 amd64_read_pci_cfg(F3, NBCFG, &value);
2965
2966 edac_dbg(0, "1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2967 nid, value, !!(value & NBCFG_ECC_ENABLE));
2968
2969 if (!(value & NBCFG_ECC_ENABLE)) {
2970 amd64_warn("DRAM ECC disabled on this node, enabling...\n");
2971
2972 s->flags.nb_ecc_prev = 0;
2973
2974 /* Attempt to turn on DRAM ECC Enable */
2975 value |= NBCFG_ECC_ENABLE;
2976 amd64_write_pci_cfg(F3, NBCFG, value);
2977
2978 amd64_read_pci_cfg(F3, NBCFG, &value);
2979
2980 if (!(value & NBCFG_ECC_ENABLE)) {
2981 amd64_warn("Hardware rejected DRAM ECC enable,"
2982 "check memory DIMM configuration.\n");
2983 ret = false;
2984 } else {
2985 amd64_info("Hardware accepted DRAM ECC Enable\n");
2986 }
2987 } else {
2988 s->flags.nb_ecc_prev = 1;
2989 }
2990
2991 edac_dbg(0, "2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2992 nid, value, !!(value & NBCFG_ECC_ENABLE));
2993
2994 return ret;
2995 }
2996
2997 static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid,
2998 struct pci_dev *F3)
2999 {
3000 u32 value, mask = 0x3; /* UECC/CECC enable */
3001
3002 if (!s->nbctl_valid)
3003 return;
3004
3005 amd64_read_pci_cfg(F3, NBCTL, &value);
3006 value &= ~mask;
3007 value |= s->old_nbctl;
3008
3009 amd64_write_pci_cfg(F3, NBCTL, value);
3010
3011 /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
3012 if (!s->flags.nb_ecc_prev) {
3013 amd64_read_pci_cfg(F3, NBCFG, &value);
3014 value &= ~NBCFG_ECC_ENABLE;
3015 amd64_write_pci_cfg(F3, NBCFG, value);
3016 }
3017
3018 /* restore the NB Enable MCGCTL bit */
3019 if (toggle_ecc_err_reporting(s, nid, OFF))
3020 amd64_warn("Error restoring NB MCGCTL settings!\n");
3021 }
3022
3023 /*
3024 * EDAC requires that the BIOS have ECC enabled before
3025 * taking over the processing of ECC errors. A command line
3026 * option allows to force-enable hardware ECC later in
3027 * enable_ecc_error_reporting().
3028 */
3029 static const char *ecc_msg =
3030 "ECC disabled in the BIOS or no ECC capability, module will not load.\n"
3031 " Either enable ECC checking or force module loading by setting "
3032 "'ecc_enable_override'.\n"
3033 " (Note that use of the override may cause unknown side effects.)\n";
3034
3035 static bool ecc_enabled(struct pci_dev *F3, u16 nid)
3036 {
3037 bool nb_mce_en = false;
3038 u8 ecc_en = 0, i;
3039 u32 value;
3040
3041 if (boot_cpu_data.x86 >= 0x17) {
3042 u8 umc_en_mask = 0, ecc_en_mask = 0;
3043
3044 for (i = 0; i < NUM_UMCS; i++) {
3045 u32 base = get_umc_base(i);
3046
3047 /* Only check enabled UMCs. */
3048 if (amd_smn_read(nid, base + UMCCH_SDP_CTRL, &value))
3049 continue;
3050
3051 if (!(value & UMC_SDP_INIT))
3052 continue;
3053
3054 umc_en_mask |= BIT(i);
3055
3056 if (amd_smn_read(nid, base + UMCCH_UMC_CAP_HI, &value))
3057 continue;
3058
3059 if (value & UMC_ECC_ENABLED)
3060 ecc_en_mask |= BIT(i);
3061 }
3062
3063 /* Check whether at least one UMC is enabled: */
3064 if (umc_en_mask)
3065 ecc_en = umc_en_mask == ecc_en_mask;
3066 else
3067 edac_dbg(0, "Node %d: No enabled UMCs.\n", nid);
3068
3069 /* Assume UMC MCA banks are enabled. */
3070 nb_mce_en = true;
3071 } else {
3072 amd64_read_pci_cfg(F3, NBCFG, &value);
3073
3074 ecc_en = !!(value & NBCFG_ECC_ENABLE);
3075
3076 nb_mce_en = nb_mce_bank_enabled_on_node(nid);
3077 if (!nb_mce_en)
3078 edac_dbg(0, "NB MCE bank disabled, set MSR 0x%08x[4] on node %d to enable.\n",
3079 MSR_IA32_MCG_CTL, nid);
3080 }
3081
3082 amd64_info("Node %d: DRAM ECC %s.\n",
3083 nid, (ecc_en ? "enabled" : "disabled"));
3084
3085 if (!ecc_en || !nb_mce_en) {
3086 amd64_info("%s", ecc_msg);
3087 return false;
3088 }
3089 return true;
3090 }
3091
3092 static inline void
3093 f17h_determine_edac_ctl_cap(struct mem_ctl_info *mci, struct amd64_pvt *pvt)
3094 {
3095 u8 i, ecc_en = 1, cpk_en = 1;
3096
3097 for (i = 0; i < NUM_UMCS; i++) {
3098 if (pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) {
3099 ecc_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_ENABLED);
3100 cpk_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_CHIPKILL_CAP);
3101 }
3102 }
3103
3104 /* Set chipkill only if ECC is enabled: */
3105 if (ecc_en) {
3106 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
3107
3108 if (cpk_en)
3109 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
3110 }
3111 }
3112
3113 static void setup_mci_misc_attrs(struct mem_ctl_info *mci,
3114 struct amd64_family_type *fam)
3115 {
3116 struct amd64_pvt *pvt = mci->pvt_info;
3117
3118 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
3119 mci->edac_ctl_cap = EDAC_FLAG_NONE;
3120
3121 if (pvt->umc) {
3122 f17h_determine_edac_ctl_cap(mci, pvt);
3123 } else {
3124 if (pvt->nbcap & NBCAP_SECDED)
3125 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
3126
3127 if (pvt->nbcap & NBCAP_CHIPKILL)
3128 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
3129 }
3130
3131 mci->edac_cap = determine_edac_cap(pvt);
3132 mci->mod_name = EDAC_MOD_STR;
3133 mci->ctl_name = fam->ctl_name;
3134 mci->dev_name = pci_name(pvt->F3);
3135 mci->ctl_page_to_phys = NULL;
3136
3137 /* memory scrubber interface */
3138 mci->set_sdram_scrub_rate = set_scrub_rate;
3139 mci->get_sdram_scrub_rate = get_scrub_rate;
3140 }
3141
3142 /*
3143 * returns a pointer to the family descriptor on success, NULL otherwise.
3144 */
3145 static struct amd64_family_type *per_family_init(struct amd64_pvt *pvt)
3146 {
3147 struct amd64_family_type *fam_type = NULL;
3148
3149 pvt->ext_model = boot_cpu_data.x86_model >> 4;
3150 pvt->stepping = boot_cpu_data.x86_stepping;
3151 pvt->model = boot_cpu_data.x86_model;
3152 pvt->fam = boot_cpu_data.x86;
3153
3154 switch (pvt->fam) {
3155 case 0xf:
3156 fam_type = &family_types[K8_CPUS];
3157 pvt->ops = &family_types[K8_CPUS].ops;
3158 break;
3159
3160 case 0x10:
3161 fam_type = &family_types[F10_CPUS];
3162 pvt->ops = &family_types[F10_CPUS].ops;
3163 break;
3164
3165 case 0x15:
3166 if (pvt->model == 0x30) {
3167 fam_type = &family_types[F15_M30H_CPUS];
3168 pvt->ops = &family_types[F15_M30H_CPUS].ops;
3169 break;
3170 } else if (pvt->model == 0x60) {
3171 fam_type = &family_types[F15_M60H_CPUS];
3172 pvt->ops = &family_types[F15_M60H_CPUS].ops;
3173 break;
3174 }
3175
3176 fam_type = &family_types[F15_CPUS];
3177 pvt->ops = &family_types[F15_CPUS].ops;
3178 break;
3179
3180 case 0x16:
3181 if (pvt->model == 0x30) {
3182 fam_type = &family_types[F16_M30H_CPUS];
3183 pvt->ops = &family_types[F16_M30H_CPUS].ops;
3184 break;
3185 }
3186 fam_type = &family_types[F16_CPUS];
3187 pvt->ops = &family_types[F16_CPUS].ops;
3188 break;
3189
3190 case 0x17:
3191 fam_type = &family_types[F17_CPUS];
3192 pvt->ops = &family_types[F17_CPUS].ops;
3193 break;
3194
3195 default:
3196 amd64_err("Unsupported family!\n");
3197 return NULL;
3198 }
3199
3200 amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name,
3201 (pvt->fam == 0xf ?
3202 (pvt->ext_model >= K8_REV_F ? "revF or later "
3203 : "revE or earlier ")
3204 : ""), pvt->mc_node_id);
3205 return fam_type;
3206 }
3207
3208 static const struct attribute_group *amd64_edac_attr_groups[] = {
3209 #ifdef CONFIG_EDAC_DEBUG
3210 &amd64_edac_dbg_group,
3211 #endif
3212 #ifdef CONFIG_EDAC_AMD64_ERROR_INJECTION
3213 &amd64_edac_inj_group,
3214 #endif
3215 NULL
3216 };
3217
3218 static int init_one_instance(unsigned int nid)
3219 {
3220 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
3221 struct amd64_family_type *fam_type = NULL;
3222 struct mem_ctl_info *mci = NULL;
3223 struct edac_mc_layer layers[2];
3224 struct amd64_pvt *pvt = NULL;
3225 u16 pci_id1, pci_id2;
3226 int err = 0, ret;
3227
3228 ret = -ENOMEM;
3229 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
3230 if (!pvt)
3231 goto err_ret;
3232
3233 pvt->mc_node_id = nid;
3234 pvt->F3 = F3;
3235
3236 ret = -EINVAL;
3237 fam_type = per_family_init(pvt);
3238 if (!fam_type)
3239 goto err_free;
3240
3241 if (pvt->fam >= 0x17) {
3242 pvt->umc = kcalloc(NUM_UMCS, sizeof(struct amd64_umc), GFP_KERNEL);
3243 if (!pvt->umc) {
3244 ret = -ENOMEM;
3245 goto err_free;
3246 }
3247
3248 pci_id1 = fam_type->f0_id;
3249 pci_id2 = fam_type->f6_id;
3250 } else {
3251 pci_id1 = fam_type->f1_id;
3252 pci_id2 = fam_type->f2_id;
3253 }
3254
3255 err = reserve_mc_sibling_devs(pvt, pci_id1, pci_id2);
3256 if (err)
3257 goto err_post_init;
3258
3259 read_mc_regs(pvt);
3260
3261 /*
3262 * We need to determine how many memory channels there are. Then use
3263 * that information for calculating the size of the dynamic instance
3264 * tables in the 'mci' structure.
3265 */
3266 ret = -EINVAL;
3267 pvt->channel_count = pvt->ops->early_channel_count(pvt);
3268 if (pvt->channel_count < 0)
3269 goto err_siblings;
3270
3271 ret = -ENOMEM;
3272 layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
3273 layers[0].size = pvt->csels[0].b_cnt;
3274 layers[0].is_virt_csrow = true;
3275 layers[1].type = EDAC_MC_LAYER_CHANNEL;
3276
3277 /*
3278 * Always allocate two channels since we can have setups with DIMMs on
3279 * only one channel. Also, this simplifies handling later for the price
3280 * of a couple of KBs tops.
3281 */
3282 layers[1].size = 2;
3283 layers[1].is_virt_csrow = false;
3284
3285 mci = edac_mc_alloc(nid, ARRAY_SIZE(layers), layers, 0);
3286 if (!mci)
3287 goto err_siblings;
3288
3289 mci->pvt_info = pvt;
3290 mci->pdev = &pvt->F3->dev;
3291
3292 setup_mci_misc_attrs(mci, fam_type);
3293
3294 if (init_csrows(mci))
3295 mci->edac_cap = EDAC_FLAG_NONE;
3296
3297 ret = -ENODEV;
3298 if (edac_mc_add_mc_with_groups(mci, amd64_edac_attr_groups)) {
3299 edac_dbg(1, "failed edac_mc_add_mc()\n");
3300 goto err_add_mc;
3301 }
3302
3303 return 0;
3304
3305 err_add_mc:
3306 edac_mc_free(mci);
3307
3308 err_siblings:
3309 free_mc_sibling_devs(pvt);
3310
3311 err_post_init:
3312 if (pvt->fam >= 0x17)
3313 kfree(pvt->umc);
3314
3315 err_free:
3316 kfree(pvt);
3317
3318 err_ret:
3319 return ret;
3320 }
3321
3322 static int probe_one_instance(unsigned int nid)
3323 {
3324 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
3325 struct ecc_settings *s;
3326 int ret;
3327
3328 ret = -ENOMEM;
3329 s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
3330 if (!s)
3331 goto err_out;
3332
3333 ecc_stngs[nid] = s;
3334
3335 if (!ecc_enabled(F3, nid)) {
3336 ret = 0;
3337
3338 if (!ecc_enable_override)
3339 goto err_enable;
3340
3341 if (boot_cpu_data.x86 >= 0x17) {
3342 amd64_warn("Forcing ECC on is not recommended on newer systems. Please enable ECC in BIOS.");
3343 goto err_enable;
3344 } else
3345 amd64_warn("Forcing ECC on!\n");
3346
3347 if (!enable_ecc_error_reporting(s, nid, F3))
3348 goto err_enable;
3349 }
3350
3351 ret = init_one_instance(nid);
3352 if (ret < 0) {
3353 amd64_err("Error probing instance: %d\n", nid);
3354
3355 if (boot_cpu_data.x86 < 0x17)
3356 restore_ecc_error_reporting(s, nid, F3);
3357
3358 goto err_enable;
3359 }
3360
3361 return ret;
3362
3363 err_enable:
3364 kfree(s);
3365 ecc_stngs[nid] = NULL;
3366
3367 err_out:
3368 return ret;
3369 }
3370
3371 static void remove_one_instance(unsigned int nid)
3372 {
3373 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
3374 struct ecc_settings *s = ecc_stngs[nid];
3375 struct mem_ctl_info *mci;
3376 struct amd64_pvt *pvt;
3377
3378 mci = find_mci_by_dev(&F3->dev);
3379 WARN_ON(!mci);
3380
3381 /* Remove from EDAC CORE tracking list */
3382 mci = edac_mc_del_mc(&F3->dev);
3383 if (!mci)
3384 return;
3385
3386 pvt = mci->pvt_info;
3387
3388 restore_ecc_error_reporting(s, nid, F3);
3389
3390 free_mc_sibling_devs(pvt);
3391
3392 kfree(ecc_stngs[nid]);
3393 ecc_stngs[nid] = NULL;
3394
3395 /* Free the EDAC CORE resources */
3396 mci->pvt_info = NULL;
3397
3398 kfree(pvt);
3399 edac_mc_free(mci);
3400 }
3401
3402 static void setup_pci_device(void)
3403 {
3404 struct mem_ctl_info *mci;
3405 struct amd64_pvt *pvt;
3406
3407 if (pci_ctl)
3408 return;
3409
3410 mci = edac_mc_find(0);
3411 if (!mci)
3412 return;
3413
3414 pvt = mci->pvt_info;
3415 if (pvt->umc)
3416 pci_ctl = edac_pci_create_generic_ctl(&pvt->F0->dev, EDAC_MOD_STR);
3417 else
3418 pci_ctl = edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR);
3419 if (!pci_ctl) {
3420 pr_warn("%s(): Unable to create PCI control\n", __func__);
3421 pr_warn("%s(): PCI error report via EDAC not set\n", __func__);
3422 }
3423 }
3424
3425 static const struct x86_cpu_id amd64_cpuids[] = {
3426 { X86_VENDOR_AMD, 0xF, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
3427 { X86_VENDOR_AMD, 0x10, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
3428 { X86_VENDOR_AMD, 0x15, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
3429 { X86_VENDOR_AMD, 0x16, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
3430 { X86_VENDOR_AMD, 0x17, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
3431 { }
3432 };
3433 MODULE_DEVICE_TABLE(x86cpu, amd64_cpuids);
3434
3435 static int __init amd64_edac_init(void)
3436 {
3437 const char *owner;
3438 int err = -ENODEV;
3439 int i;
3440
3441 owner = edac_get_owner();
3442 if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR)))
3443 return -EBUSY;
3444
3445 if (!x86_match_cpu(amd64_cpuids))
3446 return -ENODEV;
3447
3448 if (amd_cache_northbridges() < 0)
3449 return -ENODEV;
3450
3451 opstate_init();
3452
3453 err = -ENOMEM;
3454 ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL);
3455 if (!ecc_stngs)
3456 goto err_free;
3457
3458 msrs = msrs_alloc();
3459 if (!msrs)
3460 goto err_free;
3461
3462 for (i = 0; i < amd_nb_num(); i++) {
3463 err = probe_one_instance(i);
3464 if (err) {
3465 /* unwind properly */
3466 while (--i >= 0)
3467 remove_one_instance(i);
3468
3469 goto err_pci;
3470 }
3471 }
3472
3473 if (!edac_has_mcs()) {
3474 err = -ENODEV;
3475 goto err_pci;
3476 }
3477
3478 /* register stuff with EDAC MCE */
3479 if (report_gart_errors)
3480 amd_report_gart_errors(true);
3481
3482 if (boot_cpu_data.x86 >= 0x17)
3483 amd_register_ecc_decoder(decode_umc_error);
3484 else
3485 amd_register_ecc_decoder(decode_bus_error);
3486
3487 setup_pci_device();
3488
3489 #ifdef CONFIG_X86_32
3490 amd64_err("%s on 32-bit is unsupported. USE AT YOUR OWN RISK!\n", EDAC_MOD_STR);
3491 #endif
3492
3493 printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION);
3494
3495 return 0;
3496
3497 err_pci:
3498 msrs_free(msrs);
3499 msrs = NULL;
3500
3501 err_free:
3502 kfree(ecc_stngs);
3503 ecc_stngs = NULL;
3504
3505 return err;
3506 }
3507
3508 static void __exit amd64_edac_exit(void)
3509 {
3510 int i;
3511
3512 if (pci_ctl)
3513 edac_pci_release_generic_ctl(pci_ctl);
3514
3515 /* unregister from EDAC MCE */
3516 amd_report_gart_errors(false);
3517
3518 if (boot_cpu_data.x86 >= 0x17)
3519 amd_unregister_ecc_decoder(decode_umc_error);
3520 else
3521 amd_unregister_ecc_decoder(decode_bus_error);
3522
3523 for (i = 0; i < amd_nb_num(); i++)
3524 remove_one_instance(i);
3525
3526 kfree(ecc_stngs);
3527 ecc_stngs = NULL;
3528
3529 msrs_free(msrs);
3530 msrs = NULL;
3531 }
3532
3533 module_init(amd64_edac_init);
3534 module_exit(amd64_edac_exit);
3535
3536 MODULE_LICENSE("GPL");
3537 MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
3538 "Dave Peterson, Thayne Harbaugh");
3539 MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
3540 EDAC_AMD64_VERSION);
3541
3542 module_param(edac_op_state, int, 0444);
3543 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
Linux with added FPC-III support