?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
btrfs: replace remaining do_div calls with div_u64 variants
[linux/fpc-iii.git] / drivers / edac / amd64_edac.c
blob17638d7cf5c279a3b2fd63aa2250939a8c059388
1 #include "amd64_edac.h"
2 #include <asm/amd_nb.h>
3
4 static struct edac_pci_ctl_info *pci_ctl;
5
6 static int report_gart_errors;
7 module_param(report_gart_errors, int, 0644);
8
9 /*
10 * Set by command line parameter. If BIOS has enabled the ECC, this override is
11 * cleared to prevent re-enabling the hardware by this driver.
12 */
13 static int ecc_enable_override;
14 module_param(ecc_enable_override, int, 0644);
15
16 static struct msr __percpu *msrs;
17
18 /*
19 * count successfully initialized driver instances for setup_pci_device()
20 */
21 static atomic_t drv_instances = ATOMIC_INIT(0);
22
23 /* Per-node driver instances */
24 static struct mem_ctl_info **mcis;
25 static struct ecc_settings **ecc_stngs;
26
27 /*
28 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
29 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
30 * or higher value'.
31 *
32 *FIXME: Produce a better mapping/linearisation.
33 */
34 static const struct scrubrate {
35 u32 scrubval; /* bit pattern for scrub rate */
36 u32 bandwidth; /* bandwidth consumed (bytes/sec) */
37 } scrubrates[] = {
38 { 0x01, 1600000000UL},
39 { 0x02, 800000000UL},
40 { 0x03, 400000000UL},
41 { 0x04, 200000000UL},
42 { 0x05, 100000000UL},
43 { 0x06, 50000000UL},
44 { 0x07, 25000000UL},
45 { 0x08, 12284069UL},
46 { 0x09, 6274509UL},
47 { 0x0A, 3121951UL},
48 { 0x0B, 1560975UL},
49 { 0x0C, 781440UL},
50 { 0x0D, 390720UL},
51 { 0x0E, 195300UL},
52 { 0x0F, 97650UL},
53 { 0x10, 48854UL},
54 { 0x11, 24427UL},
55 { 0x12, 12213UL},
56 { 0x13, 6101UL},
57 { 0x14, 3051UL},
58 { 0x15, 1523UL},
59 { 0x16, 761UL},
60 { 0x00, 0UL}, /* scrubbing off */
61 };
62
63 int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
64 u32 *val, const char *func)
65 {
66 int err = 0;
67
68 err = pci_read_config_dword(pdev, offset, val);
69 if (err)
70 amd64_warn("%s: error reading F%dx%03x.\n",
71 func, PCI_FUNC(pdev->devfn), offset);
72
73 return err;
74 }
75
76 int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
77 u32 val, const char *func)
78 {
79 int err = 0;
80
81 err = pci_write_config_dword(pdev, offset, val);
82 if (err)
83 amd64_warn("%s: error writing to F%dx%03x.\n",
84 func, PCI_FUNC(pdev->devfn), offset);
85
86 return err;
87 }
88
89 /*
90 * Select DCT to which PCI cfg accesses are routed
91 */
92 static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct)
93 {
94 u32 reg = 0;
95
96 amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, &reg);
97 reg &= (pvt->model == 0x30) ? ~3 : ~1;
98 reg |= dct;
99 amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
100 }
101
102 /*
103 *
104 * Depending on the family, F2 DCT reads need special handling:
105 *
106 * K8: has a single DCT only and no address offsets >= 0x100
107 *
108 * F10h: each DCT has its own set of regs
109 * DCT0 -> F2x040..
110 * DCT1 -> F2x140..
111 *
112 * F16h: has only 1 DCT
113 *
114 * F15h: we select which DCT we access using F1x10C[DctCfgSel]
115 */
116 static inline int amd64_read_dct_pci_cfg(struct amd64_pvt *pvt, u8 dct,
117 int offset, u32 *val)
118 {
119 switch (pvt->fam) {
120 case 0xf:
121 if (dct || offset >= 0x100)
122 return -EINVAL;
123 break;
124
125 case 0x10:
126 if (dct) {
127 /*
128 * Note: If ganging is enabled, barring the regs
129 * F2x[1,0]98 and F2x[1,0]9C; reads reads to F2x1xx
130 * return 0. (cf. Section 2.8.1 F10h BKDG)
131 */
132 if (dct_ganging_enabled(pvt))
133 return 0;
134
135 offset += 0x100;
136 }
137 break;
138
139 case 0x15:
140 /*
141 * F15h: F2x1xx addresses do not map explicitly to DCT1.
142 * We should select which DCT we access using F1x10C[DctCfgSel]
143 */
144 dct = (dct && pvt->model == 0x30) ? 3 : dct;
145 f15h_select_dct(pvt, dct);
146 break;
147
148 case 0x16:
149 if (dct)
150 return -EINVAL;
151 break;
152
153 default:
154 break;
155 }
156 return amd64_read_pci_cfg(pvt->F2, offset, val);
157 }
158
159 /*
160 * Memory scrubber control interface. For K8, memory scrubbing is handled by
161 * hardware and can involve L2 cache, dcache as well as the main memory. With
162 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
163 * functionality.
164 *
165 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
166 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
167 * bytes/sec for the setting.
168 *
169 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
170 * other archs, we might not have access to the caches directly.
171 */
172
173 /*
174 * scan the scrub rate mapping table for a close or matching bandwidth value to
175 * issue. If requested is too big, then use last maximum value found.
176 */
177 static int __set_scrub_rate(struct pci_dev *ctl, u32 new_bw, u32 min_rate)
178 {
179 u32 scrubval;
180 int i;
181
182 /*
183 * map the configured rate (new_bw) to a value specific to the AMD64
184 * memory controller and apply to register. Search for the first
185 * bandwidth entry that is greater or equal than the setting requested
186 * and program that. If at last entry, turn off DRAM scrubbing.
187 *
188 * If no suitable bandwidth is found, turn off DRAM scrubbing entirely
189 * by falling back to the last element in scrubrates[].
190 */
191 for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) {
192 /*
193 * skip scrub rates which aren't recommended
194 * (see F10 BKDG, F3x58)
195 */
196 if (scrubrates[i].scrubval < min_rate)
197 continue;
198
199 if (scrubrates[i].bandwidth <= new_bw)
200 break;
201 }
202
203 scrubval = scrubrates[i].scrubval;
204
205 pci_write_bits32(ctl, SCRCTRL, scrubval, 0x001F);
206
207 if (scrubval)
208 return scrubrates[i].bandwidth;
209
210 return 0;
211 }
212
213 static int set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
214 {
215 struct amd64_pvt *pvt = mci->pvt_info;
216 u32 min_scrubrate = 0x5;
217
218 if (pvt->fam == 0xf)
219 min_scrubrate = 0x0;
220
221 /* Erratum #505 */
222 if (pvt->fam == 0x15 && pvt->model < 0x10)
223 f15h_select_dct(pvt, 0);
224
225 return __set_scrub_rate(pvt->F3, bw, min_scrubrate);
226 }
227
228 static int get_scrub_rate(struct mem_ctl_info *mci)
229 {
230 struct amd64_pvt *pvt = mci->pvt_info;
231 u32 scrubval = 0;
232 int i, retval = -EINVAL;
233
234 /* Erratum #505 */
235 if (pvt->fam == 0x15 && pvt->model < 0x10)
236 f15h_select_dct(pvt, 0);
237
238 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
239
240 scrubval = scrubval & 0x001F;
241
242 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
243 if (scrubrates[i].scrubval == scrubval) {
244 retval = scrubrates[i].bandwidth;
245 break;
246 }
247 }
248 return retval;
249 }
250
251 /*
252 * returns true if the SysAddr given by sys_addr matches the
253 * DRAM base/limit associated with node_id
254 */
255 static bool base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, u8 nid)
256 {
257 u64 addr;
258
259 /* The K8 treats this as a 40-bit value. However, bits 63-40 will be
260 * all ones if the most significant implemented address bit is 1.
261 * Here we discard bits 63-40. See section 3.4.2 of AMD publication
262 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
263 * Application Programming.
264 */
265 addr = sys_addr & 0x000000ffffffffffull;
266
267 return ((addr >= get_dram_base(pvt, nid)) &&
268 (addr <= get_dram_limit(pvt, nid)));
269 }
270
271 /*
272 * Attempt to map a SysAddr to a node. On success, return a pointer to the
273 * mem_ctl_info structure for the node that the SysAddr maps to.
274 *
275 * On failure, return NULL.
276 */
277 static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
278 u64 sys_addr)
279 {
280 struct amd64_pvt *pvt;
281 u8 node_id;
282 u32 intlv_en, bits;
283
284 /*
285 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
286 * 3.4.4.2) registers to map the SysAddr to a node ID.
287 */
288 pvt = mci->pvt_info;
289
290 /*
291 * The value of this field should be the same for all DRAM Base
292 * registers. Therefore we arbitrarily choose to read it from the
293 * register for node 0.
294 */
295 intlv_en = dram_intlv_en(pvt, 0);
296
297 if (intlv_en == 0) {
298 for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
299 if (base_limit_match(pvt, sys_addr, node_id))
300 goto found;
301 }
302 goto err_no_match;
303 }
304
305 if (unlikely((intlv_en != 0x01) &&
306 (intlv_en != 0x03) &&
307 (intlv_en != 0x07))) {
308 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
309 return NULL;
310 }
311
312 bits = (((u32) sys_addr) >> 12) & intlv_en;
313
314 for (node_id = 0; ; ) {
315 if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
316 break; /* intlv_sel field matches */
317
318 if (++node_id >= DRAM_RANGES)
319 goto err_no_match;
320 }
321
322 /* sanity test for sys_addr */
323 if (unlikely(!base_limit_match(pvt, sys_addr, node_id))) {
324 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
325 "range for node %d with node interleaving enabled.\n",
326 __func__, sys_addr, node_id);
327 return NULL;
328 }
329
330 found:
331 return edac_mc_find((int)node_id);
332
333 err_no_match:
334 edac_dbg(2, "sys_addr 0x%lx doesn't match any node\n",
335 (unsigned long)sys_addr);
336
337 return NULL;
338 }
339
340 /*
341 * compute the CS base address of the @csrow on the DRAM controller @dct.
342 * For details see F2x[5C:40] in the processor's BKDG
343 */
344 static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
345 u64 *base, u64 *mask)
346 {
347 u64 csbase, csmask, base_bits, mask_bits;
348 u8 addr_shift;
349
350 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
351 csbase = pvt->csels[dct].csbases[csrow];
352 csmask = pvt->csels[dct].csmasks[csrow];
353 base_bits = GENMASK_ULL(31, 21) | GENMASK_ULL(15, 9);
354 mask_bits = GENMASK_ULL(29, 21) | GENMASK_ULL(15, 9);
355 addr_shift = 4;
356
357 /*
358 * F16h and F15h, models 30h and later need two addr_shift values:
359 * 8 for high and 6 for low (cf. F16h BKDG).
360 */
361 } else if (pvt->fam == 0x16 ||
362 (pvt->fam == 0x15 && pvt->model >= 0x30)) {
363 csbase = pvt->csels[dct].csbases[csrow];
364 csmask = pvt->csels[dct].csmasks[csrow >> 1];
365
366 *base = (csbase & GENMASK_ULL(15, 5)) << 6;
367 *base |= (csbase & GENMASK_ULL(30, 19)) << 8;
368
369 *mask = ~0ULL;
370 /* poke holes for the csmask */
371 *mask &= ~((GENMASK_ULL(15, 5) << 6) |
372 (GENMASK_ULL(30, 19) << 8));
373
374 *mask |= (csmask & GENMASK_ULL(15, 5)) << 6;
375 *mask |= (csmask & GENMASK_ULL(30, 19)) << 8;
376
377 return;
378 } else {
379 csbase = pvt->csels[dct].csbases[csrow];
380 csmask = pvt->csels[dct].csmasks[csrow >> 1];
381 addr_shift = 8;
382
383 if (pvt->fam == 0x15)
384 base_bits = mask_bits =
385 GENMASK_ULL(30,19) | GENMASK_ULL(13,5);
386 else
387 base_bits = mask_bits =
388 GENMASK_ULL(28,19) | GENMASK_ULL(13,5);
389 }
390
391 *base = (csbase & base_bits) << addr_shift;
392
393 *mask = ~0ULL;
394 /* poke holes for the csmask */
395 *mask &= ~(mask_bits << addr_shift);
396 /* OR them in */
397 *mask |= (csmask & mask_bits) << addr_shift;
398 }
399
400 #define for_each_chip_select(i, dct, pvt) \
401 for (i = 0; i < pvt->csels[dct].b_cnt; i++)
402
403 #define chip_select_base(i, dct, pvt) \
404 pvt->csels[dct].csbases[i]
405
406 #define for_each_chip_select_mask(i, dct, pvt) \
407 for (i = 0; i < pvt->csels[dct].m_cnt; i++)
408
409 /*
410 * @input_addr is an InputAddr associated with the node given by mci. Return the
411 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
412 */
413 static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
414 {
415 struct amd64_pvt *pvt;
416 int csrow;
417 u64 base, mask;
418
419 pvt = mci->pvt_info;
420
421 for_each_chip_select(csrow, 0, pvt) {
422 if (!csrow_enabled(csrow, 0, pvt))
423 continue;
424
425 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
426
427 mask = ~mask;
428
429 if ((input_addr & mask) == (base & mask)) {
430 edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)\n",
431 (unsigned long)input_addr, csrow,
432 pvt->mc_node_id);
433
434 return csrow;
435 }
436 }
437 edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)\n",
438 (unsigned long)input_addr, pvt->mc_node_id);
439
440 return -1;
441 }
442
443 /*
444 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
445 * for the node represented by mci. Info is passed back in *hole_base,
446 * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if
447 * info is invalid. Info may be invalid for either of the following reasons:
448 *
449 * - The revision of the node is not E or greater. In this case, the DRAM Hole
450 * Address Register does not exist.
451 *
452 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
453 * indicating that its contents are not valid.
454 *
455 * The values passed back in *hole_base, *hole_offset, and *hole_size are
456 * complete 32-bit values despite the fact that the bitfields in the DHAR
457 * only represent bits 31-24 of the base and offset values.
458 */
459 int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
460 u64 *hole_offset, u64 *hole_size)
461 {
462 struct amd64_pvt *pvt = mci->pvt_info;
463
464 /* only revE and later have the DRAM Hole Address Register */
465 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_E) {
466 edac_dbg(1, " revision %d for node %d does not support DHAR\n",
467 pvt->ext_model, pvt->mc_node_id);
468 return 1;
469 }
470
471 /* valid for Fam10h and above */
472 if (pvt->fam >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
473 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this system\n");
474 return 1;
475 }
476
477 if (!dhar_valid(pvt)) {
478 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this node %d\n",
479 pvt->mc_node_id);
480 return 1;
481 }
482
483 /* This node has Memory Hoisting */
484
485 /* +------------------+--------------------+--------------------+-----
486 * | memory | DRAM hole | relocated |
487 * | [0, (x - 1)] | [x, 0xffffffff] | addresses from |
488 * | | | DRAM hole |
489 * | | | [0x100000000, |
490 * | | | (0x100000000+ |
491 * | | | (0xffffffff-x))] |
492 * +------------------+--------------------+--------------------+-----
493 *
494 * Above is a diagram of physical memory showing the DRAM hole and the
495 * relocated addresses from the DRAM hole. As shown, the DRAM hole
496 * starts at address x (the base address) and extends through address
497 * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the
498 * addresses in the hole so that they start at 0x100000000.
499 */
500
501 *hole_base = dhar_base(pvt);
502 *hole_size = (1ULL << 32) - *hole_base;
503
504 *hole_offset = (pvt->fam > 0xf) ? f10_dhar_offset(pvt)
505 : k8_dhar_offset(pvt);
506
507 edac_dbg(1, " DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
508 pvt->mc_node_id, (unsigned long)*hole_base,
509 (unsigned long)*hole_offset, (unsigned long)*hole_size);
510
511 return 0;
512 }
513 EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info);
514
515 /*
516 * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is
517 * assumed that sys_addr maps to the node given by mci.
518 *
519 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
520 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
521 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
522 * then it is also involved in translating a SysAddr to a DramAddr. Sections
523 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
524 * These parts of the documentation are unclear. I interpret them as follows:
525 *
526 * When node n receives a SysAddr, it processes the SysAddr as follows:
527 *
528 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
529 * Limit registers for node n. If the SysAddr is not within the range
530 * specified by the base and limit values, then node n ignores the Sysaddr
531 * (since it does not map to node n). Otherwise continue to step 2 below.
532 *
533 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
534 * disabled so skip to step 3 below. Otherwise see if the SysAddr is within
535 * the range of relocated addresses (starting at 0x100000000) from the DRAM
536 * hole. If not, skip to step 3 below. Else get the value of the
537 * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
538 * offset defined by this value from the SysAddr.
539 *
540 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
541 * Base register for node n. To obtain the DramAddr, subtract the base
542 * address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
543 */
544 static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
545 {
546 struct amd64_pvt *pvt = mci->pvt_info;
547 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
548 int ret;
549
550 dram_base = get_dram_base(pvt, pvt->mc_node_id);
551
552 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
553 &hole_size);
554 if (!ret) {
555 if ((sys_addr >= (1ULL << 32)) &&
556 (sys_addr < ((1ULL << 32) + hole_size))) {
557 /* use DHAR to translate SysAddr to DramAddr */
558 dram_addr = sys_addr - hole_offset;
559
560 edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
561 (unsigned long)sys_addr,
562 (unsigned long)dram_addr);
563
564 return dram_addr;
565 }
566 }
567
568 /*
569 * Translate the SysAddr to a DramAddr as shown near the start of
570 * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8
571 * only deals with 40-bit values. Therefore we discard bits 63-40 of
572 * sys_addr below. If bit 39 of sys_addr is 1 then the bits we
573 * discard are all 1s. Otherwise the bits we discard are all 0s. See
574 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
575 * Programmer's Manual Volume 1 Application Programming.
576 */
577 dram_addr = (sys_addr & GENMASK_ULL(39, 0)) - dram_base;
578
579 edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
580 (unsigned long)sys_addr, (unsigned long)dram_addr);
581 return dram_addr;
582 }
583
584 /*
585 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
586 * (section 3.4.4.1). Return the number of bits from a SysAddr that are used
587 * for node interleaving.
588 */
589 static int num_node_interleave_bits(unsigned intlv_en)
590 {
591 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
592 int n;
593
594 BUG_ON(intlv_en > 7);
595 n = intlv_shift_table[intlv_en];
596 return n;
597 }
598
599 /* Translate the DramAddr given by @dram_addr to an InputAddr. */
600 static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
601 {
602 struct amd64_pvt *pvt;
603 int intlv_shift;
604 u64 input_addr;
605
606 pvt = mci->pvt_info;
607
608 /*
609 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
610 * concerning translating a DramAddr to an InputAddr.
611 */
612 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
613 input_addr = ((dram_addr >> intlv_shift) & GENMASK_ULL(35, 12)) +
614 (dram_addr & 0xfff);
615
616 edac_dbg(2, " Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
617 intlv_shift, (unsigned long)dram_addr,
618 (unsigned long)input_addr);
619
620 return input_addr;
621 }
622
623 /*
624 * Translate the SysAddr represented by @sys_addr to an InputAddr. It is
625 * assumed that @sys_addr maps to the node given by mci.
626 */
627 static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
628 {
629 u64 input_addr;
630
631 input_addr =
632 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
633
634 edac_dbg(2, "SysAdddr 0x%lx translates to InputAddr 0x%lx\n",
635 (unsigned long)sys_addr, (unsigned long)input_addr);
636
637 return input_addr;
638 }
639
640 /* Map the Error address to a PAGE and PAGE OFFSET. */
641 static inline void error_address_to_page_and_offset(u64 error_address,
642 struct err_info *err)
643 {
644 err->page = (u32) (error_address >> PAGE_SHIFT);
645 err->offset = ((u32) error_address) & ~PAGE_MASK;
646 }
647
648 /*
649 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
650 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
651 * of a node that detected an ECC memory error. mci represents the node that
652 * the error address maps to (possibly different from the node that detected
653 * the error). Return the number of the csrow that sys_addr maps to, or -1 on
654 * error.
655 */
656 static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
657 {
658 int csrow;
659
660 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
661
662 if (csrow == -1)
663 amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
664 "address 0x%lx\n", (unsigned long)sys_addr);
665 return csrow;
666 }
667
668 static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
669
670 /*
671 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
672 * are ECC capable.
673 */
674 static unsigned long determine_edac_cap(struct amd64_pvt *pvt)
675 {
676 u8 bit;
677 unsigned long edac_cap = EDAC_FLAG_NONE;
678
679 bit = (pvt->fam > 0xf || pvt->ext_model >= K8_REV_F)
680 ? 19
681 : 17;
682
683 if (pvt->dclr0 & BIT(bit))
684 edac_cap = EDAC_FLAG_SECDED;
685
686 return edac_cap;
687 }
688
689 static void debug_display_dimm_sizes(struct amd64_pvt *, u8);
690
691 static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan)
692 {
693 edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
694
695 if (pvt->dram_type == MEM_LRDDR3) {
696 u32 dcsm = pvt->csels[chan].csmasks[0];
697 /*
698 * It's assumed all LRDIMMs in a DCT are going to be of
699 * same 'type' until proven otherwise. So, use a cs
700 * value of '0' here to get dcsm value.
701 */
702 edac_dbg(1, " LRDIMM %dx rank multiply\n", (dcsm & 0x3));
703 }
704
705 edac_dbg(1, "All DIMMs support ECC:%s\n",
706 (dclr & BIT(19)) ? "yes" : "no");
707
708
709 edac_dbg(1, " PAR/ERR parity: %s\n",
710 (dclr & BIT(8)) ? "enabled" : "disabled");
711
712 if (pvt->fam == 0x10)
713 edac_dbg(1, " DCT 128bit mode width: %s\n",
714 (dclr & BIT(11)) ? "128b" : "64b");
715
716 edac_dbg(1, " x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
717 (dclr & BIT(12)) ? "yes" : "no",
718 (dclr & BIT(13)) ? "yes" : "no",
719 (dclr & BIT(14)) ? "yes" : "no",
720 (dclr & BIT(15)) ? "yes" : "no");
721 }
722
723 /* Display and decode various NB registers for debug purposes. */
724 static void dump_misc_regs(struct amd64_pvt *pvt)
725 {
726 edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
727
728 edac_dbg(1, " NB two channel DRAM capable: %s\n",
729 (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
730
731 edac_dbg(1, " ECC capable: %s, ChipKill ECC capable: %s\n",
732 (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
733 (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
734
735 debug_dump_dramcfg_low(pvt, pvt->dclr0, 0);
736
737 edac_dbg(1, "F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
738
739 edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x\n",
740 pvt->dhar, dhar_base(pvt),
741 (pvt->fam == 0xf) ? k8_dhar_offset(pvt)
742 : f10_dhar_offset(pvt));
743
744 edac_dbg(1, " DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
745
746 debug_display_dimm_sizes(pvt, 0);
747
748 /* everything below this point is Fam10h and above */
749 if (pvt->fam == 0xf)
750 return;
751
752 debug_display_dimm_sizes(pvt, 1);
753
754 amd64_info("using %s syndromes.\n", ((pvt->ecc_sym_sz == 8) ? "x8" : "x4"));
755
756 /* Only if NOT ganged does dclr1 have valid info */
757 if (!dct_ganging_enabled(pvt))
758 debug_dump_dramcfg_low(pvt, pvt->dclr1, 1);
759 }
760
761 /*
762 * See BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
763 */
764 static void prep_chip_selects(struct amd64_pvt *pvt)
765 {
766 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
767 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
768 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
769 } else if (pvt->fam == 0x15 && pvt->model == 0x30) {
770 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 4;
771 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 2;
772 } else {
773 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
774 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
775 }
776 }
777
778 /*
779 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
780 */
781 static void read_dct_base_mask(struct amd64_pvt *pvt)
782 {
783 int cs;
784
785 prep_chip_selects(pvt);
786
787 for_each_chip_select(cs, 0, pvt) {
788 int reg0 = DCSB0 + (cs * 4);
789 int reg1 = DCSB1 + (cs * 4);
790 u32 *base0 = &pvt->csels[0].csbases[cs];
791 u32 *base1 = &pvt->csels[1].csbases[cs];
792
793 if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, base0))
794 edac_dbg(0, " DCSB0[%d]=0x%08x reg: F2x%x\n",
795 cs, *base0, reg0);
796
797 if (pvt->fam == 0xf)
798 continue;
799
800 if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, base1))
801 edac_dbg(0, " DCSB1[%d]=0x%08x reg: F2x%x\n",
802 cs, *base1, (pvt->fam == 0x10) ? reg1
803 : reg0);
804 }
805
806 for_each_chip_select_mask(cs, 0, pvt) {
807 int reg0 = DCSM0 + (cs * 4);
808 int reg1 = DCSM1 + (cs * 4);
809 u32 *mask0 = &pvt->csels[0].csmasks[cs];
810 u32 *mask1 = &pvt->csels[1].csmasks[cs];
811
812 if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, mask0))
813 edac_dbg(0, " DCSM0[%d]=0x%08x reg: F2x%x\n",
814 cs, *mask0, reg0);
815
816 if (pvt->fam == 0xf)
817 continue;
818
819 if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, mask1))
820 edac_dbg(0, " DCSM1[%d]=0x%08x reg: F2x%x\n",
821 cs, *mask1, (pvt->fam == 0x10) ? reg1
822 : reg0);
823 }
824 }
825
826 static void determine_memory_type(struct amd64_pvt *pvt)
827 {
828 u32 dram_ctrl, dcsm;
829
830 switch (pvt->fam) {
831 case 0xf:
832 if (pvt->ext_model >= K8_REV_F)
833 goto ddr3;
834
835 pvt->dram_type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
836 return;
837
838 case 0x10:
839 if (pvt->dchr0 & DDR3_MODE)
840 goto ddr3;
841
842 pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
843 return;
844
845 case 0x15:
846 if (pvt->model < 0x60)
847 goto ddr3;
848
849 /*
850 * Model 0x60h needs special handling:
851 *
852 * We use a Chip Select value of '0' to obtain dcsm.
853 * Theoretically, it is possible to populate LRDIMMs of different
854 * 'Rank' value on a DCT. But this is not the common case. So,
855 * it's reasonable to assume all DIMMs are going to be of same
856 * 'type' until proven otherwise.
857 */
858 amd64_read_dct_pci_cfg(pvt, 0, DRAM_CONTROL, &dram_ctrl);
859 dcsm = pvt->csels[0].csmasks[0];
860
861 if (((dram_ctrl >> 8) & 0x7) == 0x2)
862 pvt->dram_type = MEM_DDR4;
863 else if (pvt->dclr0 & BIT(16))
864 pvt->dram_type = MEM_DDR3;
865 else if (dcsm & 0x3)
866 pvt->dram_type = MEM_LRDDR3;
867 else
868 pvt->dram_type = MEM_RDDR3;
869
870 return;
871
872 case 0x16:
873 goto ddr3;
874
875 default:
876 WARN(1, KERN_ERR "%s: Family??? 0x%x\n", __func__, pvt->fam);
877 pvt->dram_type = MEM_EMPTY;
878 }
879 return;
880
881 ddr3:
882 pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
883 }
884
885 /* Get the number of DCT channels the memory controller is using. */
886 static int k8_early_channel_count(struct amd64_pvt *pvt)
887 {
888 int flag;
889
890 if (pvt->ext_model >= K8_REV_F)
891 /* RevF (NPT) and later */
892 flag = pvt->dclr0 & WIDTH_128;
893 else
894 /* RevE and earlier */
895 flag = pvt->dclr0 & REVE_WIDTH_128;
896
897 /* not used */
898 pvt->dclr1 = 0;
899
900 return (flag) ? 2 : 1;
901 }
902
903 /* On F10h and later ErrAddr is MC4_ADDR[47:1] */
904 static u64 get_error_address(struct amd64_pvt *pvt, struct mce *m)
905 {
906 u64 addr;
907 u8 start_bit = 1;
908 u8 end_bit = 47;
909
910 if (pvt->fam == 0xf) {
911 start_bit = 3;
912 end_bit = 39;
913 }
914
915 addr = m->addr & GENMASK_ULL(end_bit, start_bit);
916
917 /*
918 * Erratum 637 workaround
919 */
920 if (pvt->fam == 0x15) {
921 struct amd64_pvt *pvt;
922 u64 cc6_base, tmp_addr;
923 u32 tmp;
924 u16 mce_nid;
925 u8 intlv_en;
926
927 if ((addr & GENMASK_ULL(47, 24)) >> 24 != 0x00fdf7)
928 return addr;
929
930 mce_nid = amd_get_nb_id(m->extcpu);
931 pvt = mcis[mce_nid]->pvt_info;
932
933 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp);
934 intlv_en = tmp >> 21 & 0x7;
935
936 /* add [47:27] + 3 trailing bits */
937 cc6_base = (tmp & GENMASK_ULL(20, 0)) << 3;
938
939 /* reverse and add DramIntlvEn */
940 cc6_base |= intlv_en ^ 0x7;
941
942 /* pin at [47:24] */
943 cc6_base <<= 24;
944
945 if (!intlv_en)
946 return cc6_base | (addr & GENMASK_ULL(23, 0));
947
948 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp);
949
950 /* faster log2 */
951 tmp_addr = (addr & GENMASK_ULL(23, 12)) << __fls(intlv_en + 1);
952
953 /* OR DramIntlvSel into bits [14:12] */
954 tmp_addr |= (tmp & GENMASK_ULL(23, 21)) >> 9;
955
956 /* add remaining [11:0] bits from original MC4_ADDR */
957 tmp_addr |= addr & GENMASK_ULL(11, 0);
958
959 return cc6_base | tmp_addr;
960 }
961
962 return addr;
963 }
964
965 static struct pci_dev *pci_get_related_function(unsigned int vendor,
966 unsigned int device,
967 struct pci_dev *related)
968 {
969 struct pci_dev *dev = NULL;
970
971 while ((dev = pci_get_device(vendor, device, dev))) {
972 if (pci_domain_nr(dev->bus) == pci_domain_nr(related->bus) &&
973 (dev->bus->number == related->bus->number) &&
974 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
975 break;
976 }
977
978 return dev;
979 }
980
981 static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
982 {
983 struct amd_northbridge *nb;
984 struct pci_dev *f1 = NULL;
985 unsigned int pci_func;
986 int off = range << 3;
987 u32 llim;
988
989 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo);
990 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
991
992 if (pvt->fam == 0xf)
993 return;
994
995 if (!dram_rw(pvt, range))
996 return;
997
998 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi);
999 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
1000
1001 /* F15h: factor in CC6 save area by reading dst node's limit reg */
1002 if (pvt->fam != 0x15)
1003 return;
1004
1005 nb = node_to_amd_nb(dram_dst_node(pvt, range));
1006 if (WARN_ON(!nb))
1007 return;
1008
1009 if (pvt->model == 0x60)
1010 pci_func = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1;
1011 else if (pvt->model == 0x30)
1012 pci_func = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1;
1013 else
1014 pci_func = PCI_DEVICE_ID_AMD_15H_NB_F1;
1015
1016 f1 = pci_get_related_function(nb->misc->vendor, pci_func, nb->misc);
1017 if (WARN_ON(!f1))
1018 return;
1019
1020 amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim);
1021
1022 pvt->ranges[range].lim.lo &= GENMASK_ULL(15, 0);
1023
1024 /* {[39:27],111b} */
1025 pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16;
1026
1027 pvt->ranges[range].lim.hi &= GENMASK_ULL(7, 0);
1028
1029 /* [47:40] */
1030 pvt->ranges[range].lim.hi |= llim >> 13;
1031
1032 pci_dev_put(f1);
1033 }
1034
1035 static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
1036 struct err_info *err)
1037 {
1038 struct amd64_pvt *pvt = mci->pvt_info;
1039
1040 error_address_to_page_and_offset(sys_addr, err);
1041
1042 /*
1043 * Find out which node the error address belongs to. This may be
1044 * different from the node that detected the error.
1045 */
1046 err->src_mci = find_mc_by_sys_addr(mci, sys_addr);
1047 if (!err->src_mci) {
1048 amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
1049 (unsigned long)sys_addr);
1050 err->err_code = ERR_NODE;
1051 return;
1052 }
1053
1054 /* Now map the sys_addr to a CSROW */
1055 err->csrow = sys_addr_to_csrow(err->src_mci, sys_addr);
1056 if (err->csrow < 0) {
1057 err->err_code = ERR_CSROW;
1058 return;
1059 }
1060
1061 /* CHIPKILL enabled */
1062 if (pvt->nbcfg & NBCFG_CHIPKILL) {
1063 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
1064 if (err->channel < 0) {
1065 /*
1066 * Syndrome didn't map, so we don't know which of the
1067 * 2 DIMMs is in error. So we need to ID 'both' of them
1068 * as suspect.
1069 */
1070 amd64_mc_warn(err->src_mci, "unknown syndrome 0x%04x - "
1071 "possible error reporting race\n",
1072 err->syndrome);
1073 err->err_code = ERR_CHANNEL;
1074 return;
1075 }
1076 } else {
1077 /*
1078 * non-chipkill ecc mode
1079 *
1080 * The k8 documentation is unclear about how to determine the
1081 * channel number when using non-chipkill memory. This method
1082 * was obtained from email communication with someone at AMD.
1083 * (Wish the email was placed in this comment - norsk)
1084 */
1085 err->channel = ((sys_addr & BIT(3)) != 0);
1086 }
1087 }
1088
1089 static int ddr2_cs_size(unsigned i, bool dct_width)
1090 {
1091 unsigned shift = 0;
1092
1093 if (i <= 2)
1094 shift = i;
1095 else if (!(i & 0x1))
1096 shift = i >> 1;
1097 else
1098 shift = (i + 1) >> 1;
1099
1100 return 128 << (shift + !!dct_width);
1101 }
1102
1103 static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1104 unsigned cs_mode, int cs_mask_nr)
1105 {
1106 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1107
1108 if (pvt->ext_model >= K8_REV_F) {
1109 WARN_ON(cs_mode > 11);
1110 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1111 }
1112 else if (pvt->ext_model >= K8_REV_D) {
1113 unsigned diff;
1114 WARN_ON(cs_mode > 10);
1115
1116 /*
1117 * the below calculation, besides trying to win an obfuscated C
1118 * contest, maps cs_mode values to DIMM chip select sizes. The
1119 * mappings are:
1120 *
1121 * cs_mode CS size (mb)
1122 * ======= ============
1123 * 0 32
1124 * 1 64
1125 * 2 128
1126 * 3 128
1127 * 4 256
1128 * 5 512
1129 * 6 256
1130 * 7 512
1131 * 8 1024
1132 * 9 1024
1133 * 10 2048
1134 *
1135 * Basically, it calculates a value with which to shift the
1136 * smallest CS size of 32MB.
1137 *
1138 * ddr[23]_cs_size have a similar purpose.
1139 */
1140 diff = cs_mode/3 + (unsigned)(cs_mode > 5);
1141
1142 return 32 << (cs_mode - diff);
1143 }
1144 else {
1145 WARN_ON(cs_mode > 6);
1146 return 32 << cs_mode;
1147 }
1148 }
1149
1150 /*
1151 * Get the number of DCT channels in use.
1152 *
1153 * Return:
1154 * number of Memory Channels in operation
1155 * Pass back:
1156 * contents of the DCL0_LOW register
1157 */
1158 static int f1x_early_channel_count(struct amd64_pvt *pvt)
1159 {
1160 int i, j, channels = 0;
1161
1162 /* On F10h, if we are in 128 bit mode, then we are using 2 channels */
1163 if (pvt->fam == 0x10 && (pvt->dclr0 & WIDTH_128))
1164 return 2;
1165
1166 /*
1167 * Need to check if in unganged mode: In such, there are 2 channels,
1168 * but they are not in 128 bit mode and thus the above 'dclr0' status
1169 * bit will be OFF.
1170 *
1171 * Need to check DCT0[0] and DCT1[0] to see if only one of them has
1172 * their CSEnable bit on. If so, then SINGLE DIMM case.
1173 */
1174 edac_dbg(0, "Data width is not 128 bits - need more decoding\n");
1175
1176 /*
1177 * Check DRAM Bank Address Mapping values for each DIMM to see if there
1178 * is more than just one DIMM present in unganged mode. Need to check
1179 * both controllers since DIMMs can be placed in either one.
1180 */
1181 for (i = 0; i < 2; i++) {
1182 u32 dbam = (i ? pvt->dbam1 : pvt->dbam0);
1183
1184 for (j = 0; j < 4; j++) {
1185 if (DBAM_DIMM(j, dbam) > 0) {
1186 channels++;
1187 break;
1188 }
1189 }
1190 }
1191
1192 if (channels > 2)
1193 channels = 2;
1194
1195 amd64_info("MCT channel count: %d\n", channels);
1196
1197 return channels;
1198 }
1199
1200 static int ddr3_cs_size(unsigned i, bool dct_width)
1201 {
1202 unsigned shift = 0;
1203 int cs_size = 0;
1204
1205 if (i == 0 || i == 3 || i == 4)
1206 cs_size = -1;
1207 else if (i <= 2)
1208 shift = i;
1209 else if (i == 12)
1210 shift = 7;
1211 else if (!(i & 0x1))
1212 shift = i >> 1;
1213 else
1214 shift = (i + 1) >> 1;
1215
1216 if (cs_size != -1)
1217 cs_size = (128 * (1 << !!dct_width)) << shift;
1218
1219 return cs_size;
1220 }
1221
1222 static int ddr3_lrdimm_cs_size(unsigned i, unsigned rank_multiply)
1223 {
1224 unsigned shift = 0;
1225 int cs_size = 0;
1226
1227 if (i < 4 || i == 6)
1228 cs_size = -1;
1229 else if (i == 12)
1230 shift = 7;
1231 else if (!(i & 0x1))
1232 shift = i >> 1;
1233 else
1234 shift = (i + 1) >> 1;
1235
1236 if (cs_size != -1)
1237 cs_size = rank_multiply * (128 << shift);
1238
1239 return cs_size;
1240 }
1241
1242 static int ddr4_cs_size(unsigned i)
1243 {
1244 int cs_size = 0;
1245
1246 if (i == 0)
1247 cs_size = -1;
1248 else if (i == 1)
1249 cs_size = 1024;
1250 else
1251 /* Min cs_size = 1G */
1252 cs_size = 1024 * (1 << (i >> 1));
1253
1254 return cs_size;
1255 }
1256
1257 static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1258 unsigned cs_mode, int cs_mask_nr)
1259 {
1260 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1261
1262 WARN_ON(cs_mode > 11);
1263
1264 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
1265 return ddr3_cs_size(cs_mode, dclr & WIDTH_128);
1266 else
1267 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1268 }
1269
1270 /*
1271 * F15h supports only 64bit DCT interfaces
1272 */
1273 static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1274 unsigned cs_mode, int cs_mask_nr)
1275 {
1276 WARN_ON(cs_mode > 12);
1277
1278 return ddr3_cs_size(cs_mode, false);
1279 }
1280
1281 /* F15h M60h supports DDR4 mapping as well.. */
1282 static int f15_m60h_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1283 unsigned cs_mode, int cs_mask_nr)
1284 {
1285 int cs_size;
1286 u32 dcsm = pvt->csels[dct].csmasks[cs_mask_nr];
1287
1288 WARN_ON(cs_mode > 12);
1289
1290 if (pvt->dram_type == MEM_DDR4) {
1291 if (cs_mode > 9)
1292 return -1;
1293
1294 cs_size = ddr4_cs_size(cs_mode);
1295 } else if (pvt->dram_type == MEM_LRDDR3) {
1296 unsigned rank_multiply = dcsm & 0xf;
1297
1298 if (rank_multiply == 3)
1299 rank_multiply = 4;
1300 cs_size = ddr3_lrdimm_cs_size(cs_mode, rank_multiply);
1301 } else {
1302 /* Minimum cs size is 512mb for F15hM60h*/
1303 if (cs_mode == 0x1)
1304 return -1;
1305
1306 cs_size = ddr3_cs_size(cs_mode, false);
1307 }
1308
1309 return cs_size;
1310 }
1311
1312 /*
1313 * F16h and F15h model 30h have only limited cs_modes.
1314 */
1315 static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1316 unsigned cs_mode, int cs_mask_nr)
1317 {
1318 WARN_ON(cs_mode > 12);
1319
1320 if (cs_mode == 6 || cs_mode == 8 ||
1321 cs_mode == 9 || cs_mode == 12)
1322 return -1;
1323 else
1324 return ddr3_cs_size(cs_mode, false);
1325 }
1326
1327 static void read_dram_ctl_register(struct amd64_pvt *pvt)
1328 {
1329
1330 if (pvt->fam == 0xf)
1331 return;
1332
1333 if (!amd64_read_pci_cfg(pvt->F2, DCT_SEL_LO, &pvt->dct_sel_lo)) {
1334 edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
1335 pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
1336
1337 edac_dbg(0, " DCTs operate in %s mode\n",
1338 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"));
1339
1340 if (!dct_ganging_enabled(pvt))
1341 edac_dbg(0, " Address range split per DCT: %s\n",
1342 (dct_high_range_enabled(pvt) ? "yes" : "no"));
1343
1344 edac_dbg(0, " data interleave for ECC: %s, DRAM cleared since last warm reset: %s\n",
1345 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
1346 (dct_memory_cleared(pvt) ? "yes" : "no"));
1347
1348 edac_dbg(0, " channel interleave: %s, "
1349 "interleave bits selector: 0x%x\n",
1350 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
1351 dct_sel_interleave_addr(pvt));
1352 }
1353
1354 amd64_read_pci_cfg(pvt->F2, DCT_SEL_HI, &pvt->dct_sel_hi);
1355 }
1356
1357 /*
1358 * Determine channel (DCT) based on the interleaving mode (see F15h M30h BKDG,
1359 * 2.10.12 Memory Interleaving Modes).
1360 */
1361 static u8 f15_m30h_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1362 u8 intlv_en, int num_dcts_intlv,
1363 u32 dct_sel)
1364 {
1365 u8 channel = 0;
1366 u8 select;
1367
1368 if (!(intlv_en))
1369 return (u8)(dct_sel);
1370
1371 if (num_dcts_intlv == 2) {
1372 select = (sys_addr >> 8) & 0x3;
1373 channel = select ? 0x3 : 0;
1374 } else if (num_dcts_intlv == 4) {
1375 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1376 switch (intlv_addr) {
1377 case 0x4:
1378 channel = (sys_addr >> 8) & 0x3;
1379 break;
1380 case 0x5:
1381 channel = (sys_addr >> 9) & 0x3;
1382 break;
1383 }
1384 }
1385 return channel;
1386 }
1387
1388 /*
1389 * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
1390 * Interleaving Modes.
1391 */
1392 static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1393 bool hi_range_sel, u8 intlv_en)
1394 {
1395 u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
1396
1397 if (dct_ganging_enabled(pvt))
1398 return 0;
1399
1400 if (hi_range_sel)
1401 return dct_sel_high;
1402
1403 /*
1404 * see F2x110[DctSelIntLvAddr] - channel interleave mode
1405 */
1406 if (dct_interleave_enabled(pvt)) {
1407 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1408
1409 /* return DCT select function: 0=DCT0, 1=DCT1 */
1410 if (!intlv_addr)
1411 return sys_addr >> 6 & 1;
1412
1413 if (intlv_addr & 0x2) {
1414 u8 shift = intlv_addr & 0x1 ? 9 : 6;
1415 u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2;
1416
1417 return ((sys_addr >> shift) & 1) ^ temp;
1418 }
1419
1420 return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
1421 }
1422
1423 if (dct_high_range_enabled(pvt))
1424 return ~dct_sel_high & 1;
1425
1426 return 0;
1427 }
1428
1429 /* Convert the sys_addr to the normalized DCT address */
1430 static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range,
1431 u64 sys_addr, bool hi_rng,
1432 u32 dct_sel_base_addr)
1433 {
1434 u64 chan_off;
1435 u64 dram_base = get_dram_base(pvt, range);
1436 u64 hole_off = f10_dhar_offset(pvt);
1437 u64 dct_sel_base_off = (pvt->dct_sel_hi & 0xFFFFFC00) << 16;
1438
1439 if (hi_rng) {
1440 /*
1441 * if
1442 * base address of high range is below 4Gb
1443 * (bits [47:27] at [31:11])
1444 * DRAM address space on this DCT is hoisted above 4Gb &&
1445 * sys_addr > 4Gb
1446 *
1447 * remove hole offset from sys_addr
1448 * else
1449 * remove high range offset from sys_addr
1450 */
1451 if ((!(dct_sel_base_addr >> 16) ||
1452 dct_sel_base_addr < dhar_base(pvt)) &&
1453 dhar_valid(pvt) &&
1454 (sys_addr >= BIT_64(32)))
1455 chan_off = hole_off;
1456 else
1457 chan_off = dct_sel_base_off;
1458 } else {
1459 /*
1460 * if
1461 * we have a valid hole &&
1462 * sys_addr > 4Gb
1463 *
1464 * remove hole
1465 * else
1466 * remove dram base to normalize to DCT address
1467 */
1468 if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))
1469 chan_off = hole_off;
1470 else
1471 chan_off = dram_base;
1472 }
1473
1474 return (sys_addr & GENMASK_ULL(47,6)) - (chan_off & GENMASK_ULL(47,23));
1475 }
1476
1477 /*
1478 * checks if the csrow passed in is marked as SPARED, if so returns the new
1479 * spare row
1480 */
1481 static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
1482 {
1483 int tmp_cs;
1484
1485 if (online_spare_swap_done(pvt, dct) &&
1486 csrow == online_spare_bad_dramcs(pvt, dct)) {
1487
1488 for_each_chip_select(tmp_cs, dct, pvt) {
1489 if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
1490 csrow = tmp_cs;
1491 break;
1492 }
1493 }
1494 }
1495 return csrow;
1496 }
1497
1498 /*
1499 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
1500 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
1501 *
1502 * Return:
1503 * -EINVAL: NOT FOUND
1504 * 0..csrow = Chip-Select Row
1505 */
1506 static int f1x_lookup_addr_in_dct(u64 in_addr, u8 nid, u8 dct)
1507 {
1508 struct mem_ctl_info *mci;
1509 struct amd64_pvt *pvt;
1510 u64 cs_base, cs_mask;
1511 int cs_found = -EINVAL;
1512 int csrow;
1513
1514 mci = mcis[nid];
1515 if (!mci)
1516 return cs_found;
1517
1518 pvt = mci->pvt_info;
1519
1520 edac_dbg(1, "input addr: 0x%llx, DCT: %d\n", in_addr, dct);
1521
1522 for_each_chip_select(csrow, dct, pvt) {
1523 if (!csrow_enabled(csrow, dct, pvt))
1524 continue;
1525
1526 get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
1527
1528 edac_dbg(1, " CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
1529 csrow, cs_base, cs_mask);
1530
1531 cs_mask = ~cs_mask;
1532
1533 edac_dbg(1, " (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx\n",
1534 (in_addr & cs_mask), (cs_base & cs_mask));
1535
1536 if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
1537 if (pvt->fam == 0x15 && pvt->model >= 0x30) {
1538 cs_found = csrow;
1539 break;
1540 }
1541 cs_found = f10_process_possible_spare(pvt, dct, csrow);
1542
1543 edac_dbg(1, " MATCH csrow=%d\n", cs_found);
1544 break;
1545 }
1546 }
1547 return cs_found;
1548 }
1549
1550 /*
1551 * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is
1552 * swapped with a region located at the bottom of memory so that the GPU can use
1553 * the interleaved region and thus two channels.
1554 */
1555 static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)
1556 {
1557 u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;
1558
1559 if (pvt->fam == 0x10) {
1560 /* only revC3 and revE have that feature */
1561 if (pvt->model < 4 || (pvt->model < 0xa && pvt->stepping < 3))
1562 return sys_addr;
1563 }
1564
1565 amd64_read_pci_cfg(pvt->F2, SWAP_INTLV_REG, &swap_reg);
1566
1567 if (!(swap_reg & 0x1))
1568 return sys_addr;
1569
1570 swap_base = (swap_reg >> 3) & 0x7f;
1571 swap_limit = (swap_reg >> 11) & 0x7f;
1572 rgn_size = (swap_reg >> 20) & 0x7f;
1573 tmp_addr = sys_addr >> 27;
1574
1575 if (!(sys_addr >> 34) &&
1576 (((tmp_addr >= swap_base) &&
1577 (tmp_addr <= swap_limit)) ||
1578 (tmp_addr < rgn_size)))
1579 return sys_addr ^ (u64)swap_base << 27;
1580
1581 return sys_addr;
1582 }
1583
1584 /* For a given @dram_range, check if @sys_addr falls within it. */
1585 static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
1586 u64 sys_addr, int *chan_sel)
1587 {
1588 int cs_found = -EINVAL;
1589 u64 chan_addr;
1590 u32 dct_sel_base;
1591 u8 channel;
1592 bool high_range = false;
1593
1594 u8 node_id = dram_dst_node(pvt, range);
1595 u8 intlv_en = dram_intlv_en(pvt, range);
1596 u32 intlv_sel = dram_intlv_sel(pvt, range);
1597
1598 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1599 range, sys_addr, get_dram_limit(pvt, range));
1600
1601 if (dhar_valid(pvt) &&
1602 dhar_base(pvt) <= sys_addr &&
1603 sys_addr < BIT_64(32)) {
1604 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1605 sys_addr);
1606 return -EINVAL;
1607 }
1608
1609 if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en)))
1610 return -EINVAL;
1611
1612 sys_addr = f1x_swap_interleaved_region(pvt, sys_addr);
1613
1614 dct_sel_base = dct_sel_baseaddr(pvt);
1615
1616 /*
1617 * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
1618 * select between DCT0 and DCT1.
1619 */
1620 if (dct_high_range_enabled(pvt) &&
1621 !dct_ganging_enabled(pvt) &&
1622 ((sys_addr >> 27) >= (dct_sel_base >> 11)))
1623 high_range = true;
1624
1625 channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en);
1626
1627 chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr,
1628 high_range, dct_sel_base);
1629
1630 /* Remove node interleaving, see F1x120 */
1631 if (intlv_en)
1632 chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) |
1633 (chan_addr & 0xfff);
1634
1635 /* remove channel interleave */
1636 if (dct_interleave_enabled(pvt) &&
1637 !dct_high_range_enabled(pvt) &&
1638 !dct_ganging_enabled(pvt)) {
1639
1640 if (dct_sel_interleave_addr(pvt) != 1) {
1641 if (dct_sel_interleave_addr(pvt) == 0x3)
1642 /* hash 9 */
1643 chan_addr = ((chan_addr >> 10) << 9) |
1644 (chan_addr & 0x1ff);
1645 else
1646 /* A[6] or hash 6 */
1647 chan_addr = ((chan_addr >> 7) << 6) |
1648 (chan_addr & 0x3f);
1649 } else
1650 /* A[12] */
1651 chan_addr = ((chan_addr >> 13) << 12) |
1652 (chan_addr & 0xfff);
1653 }
1654
1655 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
1656
1657 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel);
1658
1659 if (cs_found >= 0)
1660 *chan_sel = channel;
1661
1662 return cs_found;
1663 }
1664
1665 static int f15_m30h_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
1666 u64 sys_addr, int *chan_sel)
1667 {
1668 int cs_found = -EINVAL;
1669 int num_dcts_intlv = 0;
1670 u64 chan_addr, chan_offset;
1671 u64 dct_base, dct_limit;
1672 u32 dct_cont_base_reg, dct_cont_limit_reg, tmp;
1673 u8 channel, alias_channel, leg_mmio_hole, dct_sel, dct_offset_en;
1674
1675 u64 dhar_offset = f10_dhar_offset(pvt);
1676 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1677 u8 node_id = dram_dst_node(pvt, range);
1678 u8 intlv_en = dram_intlv_en(pvt, range);
1679
1680 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_BASE, &dct_cont_base_reg);
1681 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_LIMIT, &dct_cont_limit_reg);
1682
1683 dct_offset_en = (u8) ((dct_cont_base_reg >> 3) & BIT(0));
1684 dct_sel = (u8) ((dct_cont_base_reg >> 4) & 0x7);
1685
1686 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1687 range, sys_addr, get_dram_limit(pvt, range));
1688
1689 if (!(get_dram_base(pvt, range) <= sys_addr) &&
1690 !(get_dram_limit(pvt, range) >= sys_addr))
1691 return -EINVAL;
1692
1693 if (dhar_valid(pvt) &&
1694 dhar_base(pvt) <= sys_addr &&
1695 sys_addr < BIT_64(32)) {
1696 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1697 sys_addr);
1698 return -EINVAL;
1699 }
1700
1701 /* Verify sys_addr is within DCT Range. */
1702 dct_base = (u64) dct_sel_baseaddr(pvt);
1703 dct_limit = (dct_cont_limit_reg >> 11) & 0x1FFF;
1704
1705 if (!(dct_cont_base_reg & BIT(0)) &&
1706 !(dct_base <= (sys_addr >> 27) &&
1707 dct_limit >= (sys_addr >> 27)))
1708 return -EINVAL;
1709
1710 /* Verify number of dct's that participate in channel interleaving. */
1711 num_dcts_intlv = (int) hweight8(intlv_en);
1712
1713 if (!(num_dcts_intlv % 2 == 0) || (num_dcts_intlv > 4))
1714 return -EINVAL;
1715
1716 channel = f15_m30h_determine_channel(pvt, sys_addr, intlv_en,
1717 num_dcts_intlv, dct_sel);
1718
1719 /* Verify we stay within the MAX number of channels allowed */
1720 if (channel > 3)
1721 return -EINVAL;
1722
1723 leg_mmio_hole = (u8) (dct_cont_base_reg >> 1 & BIT(0));
1724
1725 /* Get normalized DCT addr */
1726 if (leg_mmio_hole && (sys_addr >= BIT_64(32)))
1727 chan_offset = dhar_offset;
1728 else
1729 chan_offset = dct_base << 27;
1730
1731 chan_addr = sys_addr - chan_offset;
1732
1733 /* remove channel interleave */
1734 if (num_dcts_intlv == 2) {
1735 if (intlv_addr == 0x4)
1736 chan_addr = ((chan_addr >> 9) << 8) |
1737 (chan_addr & 0xff);
1738 else if (intlv_addr == 0x5)
1739 chan_addr = ((chan_addr >> 10) << 9) |
1740 (chan_addr & 0x1ff);
1741 else
1742 return -EINVAL;
1743
1744 } else if (num_dcts_intlv == 4) {
1745 if (intlv_addr == 0x4)
1746 chan_addr = ((chan_addr >> 10) << 8) |
1747 (chan_addr & 0xff);
1748 else if (intlv_addr == 0x5)
1749 chan_addr = ((chan_addr >> 11) << 9) |
1750 (chan_addr & 0x1ff);
1751 else
1752 return -EINVAL;
1753 }
1754
1755 if (dct_offset_en) {
1756 amd64_read_pci_cfg(pvt->F1,
1757 DRAM_CONT_HIGH_OFF + (int) channel * 4,
1758 &tmp);
1759 chan_addr += (u64) ((tmp >> 11) & 0xfff) << 27;
1760 }
1761
1762 f15h_select_dct(pvt, channel);
1763
1764 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
1765
1766 /*
1767 * Find Chip select:
1768 * if channel = 3, then alias it to 1. This is because, in F15 M30h,
1769 * there is support for 4 DCT's, but only 2 are currently functional.
1770 * They are DCT0 and DCT3. But we have read all registers of DCT3 into
1771 * pvt->csels[1]. So we need to use '1' here to get correct info.
1772 * Refer F15 M30h BKDG Section 2.10 and 2.10.3 for clarifications.
1773 */
1774 alias_channel = (channel == 3) ? 1 : channel;
1775
1776 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, alias_channel);
1777
1778 if (cs_found >= 0)
1779 *chan_sel = alias_channel;
1780
1781 return cs_found;
1782 }
1783
1784 static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt,
1785 u64 sys_addr,
1786 int *chan_sel)
1787 {
1788 int cs_found = -EINVAL;
1789 unsigned range;
1790
1791 for (range = 0; range < DRAM_RANGES; range++) {
1792 if (!dram_rw(pvt, range))
1793 continue;
1794
1795 if (pvt->fam == 0x15 && pvt->model >= 0x30)
1796 cs_found = f15_m30h_match_to_this_node(pvt, range,
1797 sys_addr,
1798 chan_sel);
1799
1800 else if ((get_dram_base(pvt, range) <= sys_addr) &&
1801 (get_dram_limit(pvt, range) >= sys_addr)) {
1802 cs_found = f1x_match_to_this_node(pvt, range,
1803 sys_addr, chan_sel);
1804 if (cs_found >= 0)
1805 break;
1806 }
1807 }
1808 return cs_found;
1809 }
1810
1811 /*
1812 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
1813 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
1814 *
1815 * The @sys_addr is usually an error address received from the hardware
1816 * (MCX_ADDR).
1817 */
1818 static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
1819 struct err_info *err)
1820 {
1821 struct amd64_pvt *pvt = mci->pvt_info;
1822
1823 error_address_to_page_and_offset(sys_addr, err);
1824
1825 err->csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &err->channel);
1826 if (err->csrow < 0) {
1827 err->err_code = ERR_CSROW;
1828 return;
1829 }
1830
1831 /*
1832 * We need the syndromes for channel detection only when we're
1833 * ganged. Otherwise @chan should already contain the channel at
1834 * this point.
1835 */
1836 if (dct_ganging_enabled(pvt))
1837 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
1838 }
1839
1840 /*
1841 * debug routine to display the memory sizes of all logical DIMMs and its
1842 * CSROWs
1843 */
1844 static void debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
1845 {
1846 int dimm, size0, size1;
1847 u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
1848 u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0;
1849
1850 if (pvt->fam == 0xf) {
1851 /* K8 families < revF not supported yet */
1852 if (pvt->ext_model < K8_REV_F)
1853 return;
1854 else
1855 WARN_ON(ctrl != 0);
1856 }
1857
1858 if (pvt->fam == 0x10) {
1859 dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1
1860 : pvt->dbam0;
1861 dcsb = (ctrl && !dct_ganging_enabled(pvt)) ?
1862 pvt->csels[1].csbases :
1863 pvt->csels[0].csbases;
1864 } else if (ctrl) {
1865 dbam = pvt->dbam0;
1866 dcsb = pvt->csels[1].csbases;
1867 }
1868 edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n",
1869 ctrl, dbam);
1870
1871 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
1872
1873 /* Dump memory sizes for DIMM and its CSROWs */
1874 for (dimm = 0; dimm < 4; dimm++) {
1875
1876 size0 = 0;
1877 if (dcsb[dimm*2] & DCSB_CS_ENABLE)
1878 /* For f15m60h, need multiplier for LRDIMM cs_size
1879 * calculation. We pass 'dimm' value to the dbam_to_cs
1880 * mapper so we can find the multiplier from the
1881 * corresponding DCSM.
1882 */
1883 size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
1884 DBAM_DIMM(dimm, dbam),
1885 dimm);
1886
1887 size1 = 0;
1888 if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
1889 size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
1890 DBAM_DIMM(dimm, dbam),
1891 dimm);
1892
1893 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
1894 dimm * 2, size0,
1895 dimm * 2 + 1, size1);
1896 }
1897 }
1898
1899 static struct amd64_family_type family_types[] = {
1900 [K8_CPUS] = {
1901 .ctl_name = "K8",
1902 .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
1903 .f3_id = PCI_DEVICE_ID_AMD_K8_NB_MISC,
1904 .ops = {
1905 .early_channel_count = k8_early_channel_count,
1906 .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow,
1907 .dbam_to_cs = k8_dbam_to_chip_select,
1908 }
1909 },
1910 [F10_CPUS] = {
1911 .ctl_name = "F10h",
1912 .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
1913 .f3_id = PCI_DEVICE_ID_AMD_10H_NB_MISC,
1914 .ops = {
1915 .early_channel_count = f1x_early_channel_count,
1916 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1917 .dbam_to_cs = f10_dbam_to_chip_select,
1918 }
1919 },
1920 [F15_CPUS] = {
1921 .ctl_name = "F15h",
1922 .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1,
1923 .f3_id = PCI_DEVICE_ID_AMD_15H_NB_F3,
1924 .ops = {
1925 .early_channel_count = f1x_early_channel_count,
1926 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1927 .dbam_to_cs = f15_dbam_to_chip_select,
1928 }
1929 },
1930 [F15_M30H_CPUS] = {
1931 .ctl_name = "F15h_M30h",
1932 .f1_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1,
1933 .f3_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F3,
1934 .ops = {
1935 .early_channel_count = f1x_early_channel_count,
1936 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1937 .dbam_to_cs = f16_dbam_to_chip_select,
1938 }
1939 },
1940 [F15_M60H_CPUS] = {
1941 .ctl_name = "F15h_M60h",
1942 .f1_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1,
1943 .f3_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F3,
1944 .ops = {
1945 .early_channel_count = f1x_early_channel_count,
1946 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1947 .dbam_to_cs = f15_m60h_dbam_to_chip_select,
1948 }
1949 },
1950 [F16_CPUS] = {
1951 .ctl_name = "F16h",
1952 .f1_id = PCI_DEVICE_ID_AMD_16H_NB_F1,
1953 .f3_id = PCI_DEVICE_ID_AMD_16H_NB_F3,
1954 .ops = {
1955 .early_channel_count = f1x_early_channel_count,
1956 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1957 .dbam_to_cs = f16_dbam_to_chip_select,
1958 }
1959 },
1960 [F16_M30H_CPUS] = {
1961 .ctl_name = "F16h_M30h",
1962 .f1_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F1,
1963 .f3_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F3,
1964 .ops = {
1965 .early_channel_count = f1x_early_channel_count,
1966 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1967 .dbam_to_cs = f16_dbam_to_chip_select,
1968 }
1969 },
1970 };
1971
1972 /*
1973 * These are tables of eigenvectors (one per line) which can be used for the
1974 * construction of the syndrome tables. The modified syndrome search algorithm
1975 * uses those to find the symbol in error and thus the DIMM.
1976 *
1977 * Algorithm courtesy of Ross LaFetra from AMD.
1978 */
1979 static const u16 x4_vectors[] = {
1980 0x2f57, 0x1afe, 0x66cc, 0xdd88,
1981 0x11eb, 0x3396, 0x7f4c, 0xeac8,
1982 0x0001, 0x0002, 0x0004, 0x0008,
1983 0x1013, 0x3032, 0x4044, 0x8088,
1984 0x106b, 0x30d6, 0x70fc, 0xe0a8,
1985 0x4857, 0xc4fe, 0x13cc, 0x3288,
1986 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
1987 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
1988 0x15c1, 0x2a42, 0x89ac, 0x4758,
1989 0x2b03, 0x1602, 0x4f0c, 0xca08,
1990 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
1991 0x8ba7, 0x465e, 0x244c, 0x1cc8,
1992 0x2b87, 0x164e, 0x642c, 0xdc18,
1993 0x40b9, 0x80de, 0x1094, 0x20e8,
1994 0x27db, 0x1eb6, 0x9dac, 0x7b58,
1995 0x11c1, 0x2242, 0x84ac, 0x4c58,
1996 0x1be5, 0x2d7a, 0x5e34, 0xa718,
1997 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
1998 0x4c97, 0xc87e, 0x11fc, 0x33a8,
1999 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
2000 0x16b3, 0x3d62, 0x4f34, 0x8518,
2001 0x1e2f, 0x391a, 0x5cac, 0xf858,
2002 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
2003 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
2004 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
2005 0x4397, 0xc27e, 0x17fc, 0x3ea8,
2006 0x1617, 0x3d3e, 0x6464, 0xb8b8,
2007 0x23ff, 0x12aa, 0xab6c, 0x56d8,
2008 0x2dfb, 0x1ba6, 0x913c, 0x7328,
2009 0x185d, 0x2ca6, 0x7914, 0x9e28,
2010 0x171b, 0x3e36, 0x7d7c, 0xebe8,
2011 0x4199, 0x82ee, 0x19f4, 0x2e58,
2012 0x4807, 0xc40e, 0x130c, 0x3208,
2013 0x1905, 0x2e0a, 0x5804, 0xac08,
2014 0x213f, 0x132a, 0xadfc, 0x5ba8,
2015 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
2016 };
2017
2018 static const u16 x8_vectors[] = {
2019 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
2020 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
2021 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
2022 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
2023 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
2024 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
2025 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
2026 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
2027 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
2028 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
2029 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
2030 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
2031 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
2032 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
2033 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
2034 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
2035 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
2036 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
2037 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
2038 };
2039
2040 static int decode_syndrome(u16 syndrome, const u16 *vectors, unsigned num_vecs,
2041 unsigned v_dim)
2042 {
2043 unsigned int i, err_sym;
2044
2045 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
2046 u16 s = syndrome;
2047 unsigned v_idx = err_sym * v_dim;
2048 unsigned v_end = (err_sym + 1) * v_dim;
2049
2050 /* walk over all 16 bits of the syndrome */
2051 for (i = 1; i < (1U << 16); i <<= 1) {
2052
2053 /* if bit is set in that eigenvector... */
2054 if (v_idx < v_end && vectors[v_idx] & i) {
2055 u16 ev_comp = vectors[v_idx++];
2056
2057 /* ... and bit set in the modified syndrome, */
2058 if (s & i) {
2059 /* remove it. */
2060 s ^= ev_comp;
2061
2062 if (!s)
2063 return err_sym;
2064 }
2065
2066 } else if (s & i)
2067 /* can't get to zero, move to next symbol */
2068 break;
2069 }
2070 }
2071
2072 edac_dbg(0, "syndrome(%x) not found\n", syndrome);
2073 return -1;
2074 }
2075
2076 static int map_err_sym_to_channel(int err_sym, int sym_size)
2077 {
2078 if (sym_size == 4)
2079 switch (err_sym) {
2080 case 0x20:
2081 case 0x21:
2082 return 0;
2083 break;
2084 case 0x22:
2085 case 0x23:
2086 return 1;
2087 break;
2088 default:
2089 return err_sym >> 4;
2090 break;
2091 }
2092 /* x8 symbols */
2093 else
2094 switch (err_sym) {
2095 /* imaginary bits not in a DIMM */
2096 case 0x10:
2097 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
2098 err_sym);
2099 return -1;
2100 break;
2101
2102 case 0x11:
2103 return 0;
2104 break;
2105 case 0x12:
2106 return 1;
2107 break;
2108 default:
2109 return err_sym >> 3;
2110 break;
2111 }
2112 return -1;
2113 }
2114
2115 static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
2116 {
2117 struct amd64_pvt *pvt = mci->pvt_info;
2118 int err_sym = -1;
2119
2120 if (pvt->ecc_sym_sz == 8)
2121 err_sym = decode_syndrome(syndrome, x8_vectors,
2122 ARRAY_SIZE(x8_vectors),
2123 pvt->ecc_sym_sz);
2124 else if (pvt->ecc_sym_sz == 4)
2125 err_sym = decode_syndrome(syndrome, x4_vectors,
2126 ARRAY_SIZE(x4_vectors),
2127 pvt->ecc_sym_sz);
2128 else {
2129 amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz);
2130 return err_sym;
2131 }
2132
2133 return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
2134 }
2135
2136 static void __log_bus_error(struct mem_ctl_info *mci, struct err_info *err,
2137 u8 ecc_type)
2138 {
2139 enum hw_event_mc_err_type err_type;
2140 const char *string;
2141
2142 if (ecc_type == 2)
2143 err_type = HW_EVENT_ERR_CORRECTED;
2144 else if (ecc_type == 1)
2145 err_type = HW_EVENT_ERR_UNCORRECTED;
2146 else {
2147 WARN(1, "Something is rotten in the state of Denmark.\n");
2148 return;
2149 }
2150
2151 switch (err->err_code) {
2152 case DECODE_OK:
2153 string = "";
2154 break;
2155 case ERR_NODE:
2156 string = "Failed to map error addr to a node";
2157 break;
2158 case ERR_CSROW:
2159 string = "Failed to map error addr to a csrow";
2160 break;
2161 case ERR_CHANNEL:
2162 string = "unknown syndrome - possible error reporting race";
2163 break;
2164 default:
2165 string = "WTF error";
2166 break;
2167 }
2168
2169 edac_mc_handle_error(err_type, mci, 1,
2170 err->page, err->offset, err->syndrome,
2171 err->csrow, err->channel, -1,
2172 string, "");
2173 }
2174
2175 static inline void decode_bus_error(int node_id, struct mce *m)
2176 {
2177 struct mem_ctl_info *mci = mcis[node_id];
2178 struct amd64_pvt *pvt = mci->pvt_info;
2179 u8 ecc_type = (m->status >> 45) & 0x3;
2180 u8 xec = XEC(m->status, 0x1f);
2181 u16 ec = EC(m->status);
2182 u64 sys_addr;
2183 struct err_info err;
2184
2185 /* Bail out early if this was an 'observed' error */
2186 if (PP(ec) == NBSL_PP_OBS)
2187 return;
2188
2189 /* Do only ECC errors */
2190 if (xec && xec != F10_NBSL_EXT_ERR_ECC)
2191 return;
2192
2193 memset(&err, 0, sizeof(err));
2194
2195 sys_addr = get_error_address(pvt, m);
2196
2197 if (ecc_type == 2)
2198 err.syndrome = extract_syndrome(m->status);
2199
2200 pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, &err);
2201
2202 __log_bus_error(mci, &err, ecc_type);
2203 }
2204
2205 /*
2206 * Use pvt->F2 which contains the F2 CPU PCI device to get the related
2207 * F1 (AddrMap) and F3 (Misc) devices. Return negative value on error.
2208 */
2209 static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f3_id)
2210 {
2211 /* Reserve the ADDRESS MAP Device */
2212 pvt->F1 = pci_get_related_function(pvt->F2->vendor, f1_id, pvt->F2);
2213 if (!pvt->F1) {
2214 amd64_err("error address map device not found: "
2215 "vendor %x device 0x%x (broken BIOS?)\n",
2216 PCI_VENDOR_ID_AMD, f1_id);
2217 return -ENODEV;
2218 }
2219
2220 /* Reserve the MISC Device */
2221 pvt->F3 = pci_get_related_function(pvt->F2->vendor, f3_id, pvt->F2);
2222 if (!pvt->F3) {
2223 pci_dev_put(pvt->F1);
2224 pvt->F1 = NULL;
2225
2226 amd64_err("error F3 device not found: "
2227 "vendor %x device 0x%x (broken BIOS?)\n",
2228 PCI_VENDOR_ID_AMD, f3_id);
2229
2230 return -ENODEV;
2231 }
2232 edac_dbg(1, "F1: %s\n", pci_name(pvt->F1));
2233 edac_dbg(1, "F2: %s\n", pci_name(pvt->F2));
2234 edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
2235
2236 return 0;
2237 }
2238
2239 static void free_mc_sibling_devs(struct amd64_pvt *pvt)
2240 {
2241 pci_dev_put(pvt->F1);
2242 pci_dev_put(pvt->F3);
2243 }
2244
2245 /*
2246 * Retrieve the hardware registers of the memory controller (this includes the
2247 * 'Address Map' and 'Misc' device regs)
2248 */
2249 static void read_mc_regs(struct amd64_pvt *pvt)
2250 {
2251 unsigned range;
2252 u64 msr_val;
2253 u32 tmp;
2254
2255 /*
2256 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
2257 * those are Read-As-Zero
2258 */
2259 rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
2260 edac_dbg(0, " TOP_MEM: 0x%016llx\n", pvt->top_mem);
2261
2262 /* check first whether TOP_MEM2 is enabled */
2263 rdmsrl(MSR_K8_SYSCFG, msr_val);
2264 if (msr_val & (1U << 21)) {
2265 rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
2266 edac_dbg(0, " TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
2267 } else
2268 edac_dbg(0, " TOP_MEM2 disabled\n");
2269
2270 amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
2271
2272 read_dram_ctl_register(pvt);
2273
2274 for (range = 0; range < DRAM_RANGES; range++) {
2275 u8 rw;
2276
2277 /* read settings for this DRAM range */
2278 read_dram_base_limit_regs(pvt, range);
2279
2280 rw = dram_rw(pvt, range);
2281 if (!rw)
2282 continue;
2283
2284 edac_dbg(1, " DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
2285 range,
2286 get_dram_base(pvt, range),
2287 get_dram_limit(pvt, range));
2288
2289 edac_dbg(1, " IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
2290 dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
2291 (rw & 0x1) ? "R" : "-",
2292 (rw & 0x2) ? "W" : "-",
2293 dram_intlv_sel(pvt, range),
2294 dram_dst_node(pvt, range));
2295 }
2296
2297 read_dct_base_mask(pvt);
2298
2299 amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
2300 amd64_read_dct_pci_cfg(pvt, 0, DBAM0, &pvt->dbam0);
2301
2302 amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
2303
2304 amd64_read_dct_pci_cfg(pvt, 0, DCLR0, &pvt->dclr0);
2305 amd64_read_dct_pci_cfg(pvt, 0, DCHR0, &pvt->dchr0);
2306
2307 if (!dct_ganging_enabled(pvt)) {
2308 amd64_read_dct_pci_cfg(pvt, 1, DCLR0, &pvt->dclr1);
2309 amd64_read_dct_pci_cfg(pvt, 1, DCHR0, &pvt->dchr1);
2310 }
2311
2312 pvt->ecc_sym_sz = 4;
2313 determine_memory_type(pvt);
2314 edac_dbg(1, " DIMM type: %s\n", edac_mem_types[pvt->dram_type]);
2315
2316 if (pvt->fam >= 0x10) {
2317 amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
2318 /* F16h has only DCT0, so no need to read dbam1 */
2319 if (pvt->fam != 0x16)
2320 amd64_read_dct_pci_cfg(pvt, 1, DBAM0, &pvt->dbam1);
2321
2322 /* F10h, revD and later can do x8 ECC too */
2323 if ((pvt->fam > 0x10 || pvt->model > 7) && tmp & BIT(25))
2324 pvt->ecc_sym_sz = 8;
2325 }
2326 dump_misc_regs(pvt);
2327 }
2328
2329 /*
2330 * NOTE: CPU Revision Dependent code
2331 *
2332 * Input:
2333 * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
2334 * k8 private pointer to -->
2335 * DRAM Bank Address mapping register
2336 * node_id
2337 * DCL register where dual_channel_active is
2338 *
2339 * The DBAM register consists of 4 sets of 4 bits each definitions:
2340 *
2341 * Bits: CSROWs
2342 * 0-3 CSROWs 0 and 1
2343 * 4-7 CSROWs 2 and 3
2344 * 8-11 CSROWs 4 and 5
2345 * 12-15 CSROWs 6 and 7
2346 *
2347 * Values range from: 0 to 15
2348 * The meaning of the values depends on CPU revision and dual-channel state,
2349 * see relevant BKDG more info.
2350 *
2351 * The memory controller provides for total of only 8 CSROWs in its current
2352 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
2353 * single channel or two (2) DIMMs in dual channel mode.
2354 *
2355 * The following code logic collapses the various tables for CSROW based on CPU
2356 * revision.
2357 *
2358 * Returns:
2359 * The number of PAGE_SIZE pages on the specified CSROW number it
2360 * encompasses
2361 *
2362 */
2363 static u32 get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr)
2364 {
2365 u32 cs_mode, nr_pages;
2366 u32 dbam = dct ? pvt->dbam1 : pvt->dbam0;
2367
2368
2369 /*
2370 * The math on this doesn't look right on the surface because x/2*4 can
2371 * be simplified to x*2 but this expression makes use of the fact that
2372 * it is integral math where 1/2=0. This intermediate value becomes the
2373 * number of bits to shift the DBAM register to extract the proper CSROW
2374 * field.
2375 */
2376 cs_mode = DBAM_DIMM(csrow_nr / 2, dbam);
2377
2378 nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode, (csrow_nr / 2))
2379 << (20 - PAGE_SHIFT);
2380
2381 edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n",
2382 csrow_nr, dct, cs_mode);
2383 edac_dbg(0, "nr_pages/channel: %u\n", nr_pages);
2384
2385 return nr_pages;
2386 }
2387
2388 /*
2389 * Initialize the array of csrow attribute instances, based on the values
2390 * from pci config hardware registers.
2391 */
2392 static int init_csrows(struct mem_ctl_info *mci)
2393 {
2394 struct amd64_pvt *pvt = mci->pvt_info;
2395 struct csrow_info *csrow;
2396 struct dimm_info *dimm;
2397 enum edac_type edac_mode;
2398 int i, j, empty = 1;
2399 int nr_pages = 0;
2400 u32 val;
2401
2402 amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
2403
2404 pvt->nbcfg = val;
2405
2406 edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2407 pvt->mc_node_id, val,
2408 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
2409
2410 /*
2411 * We iterate over DCT0 here but we look at DCT1 in parallel, if needed.
2412 */
2413 for_each_chip_select(i, 0, pvt) {
2414 bool row_dct0 = !!csrow_enabled(i, 0, pvt);
2415 bool row_dct1 = false;
2416
2417 if (pvt->fam != 0xf)
2418 row_dct1 = !!csrow_enabled(i, 1, pvt);
2419
2420 if (!row_dct0 && !row_dct1)
2421 continue;
2422
2423 csrow = mci->csrows[i];
2424 empty = 0;
2425
2426 edac_dbg(1, "MC node: %d, csrow: %d\n",
2427 pvt->mc_node_id, i);
2428
2429 if (row_dct0) {
2430 nr_pages = get_csrow_nr_pages(pvt, 0, i);
2431 csrow->channels[0]->dimm->nr_pages = nr_pages;
2432 }
2433
2434 /* K8 has only one DCT */
2435 if (pvt->fam != 0xf && row_dct1) {
2436 int row_dct1_pages = get_csrow_nr_pages(pvt, 1, i);
2437
2438 csrow->channels[1]->dimm->nr_pages = row_dct1_pages;
2439 nr_pages += row_dct1_pages;
2440 }
2441
2442 edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages);
2443
2444 /*
2445 * determine whether CHIPKILL or JUST ECC or NO ECC is operating
2446 */
2447 if (pvt->nbcfg & NBCFG_ECC_ENABLE)
2448 edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL) ?
2449 EDAC_S4ECD4ED : EDAC_SECDED;
2450 else
2451 edac_mode = EDAC_NONE;
2452
2453 for (j = 0; j < pvt->channel_count; j++) {
2454 dimm = csrow->channels[j]->dimm;
2455 dimm->mtype = pvt->dram_type;
2456 dimm->edac_mode = edac_mode;
2457 }
2458 }
2459
2460 return empty;
2461 }
2462
2463 /* get all cores on this DCT */
2464 static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid)
2465 {
2466 int cpu;
2467
2468 for_each_online_cpu(cpu)
2469 if (amd_get_nb_id(cpu) == nid)
2470 cpumask_set_cpu(cpu, mask);
2471 }
2472
2473 /* check MCG_CTL on all the cpus on this node */
2474 static bool nb_mce_bank_enabled_on_node(u16 nid)
2475 {
2476 cpumask_var_t mask;
2477 int cpu, nbe;
2478 bool ret = false;
2479
2480 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
2481 amd64_warn("%s: Error allocating mask\n", __func__);
2482 return false;
2483 }
2484
2485 get_cpus_on_this_dct_cpumask(mask, nid);
2486
2487 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
2488
2489 for_each_cpu(cpu, mask) {
2490 struct msr *reg = per_cpu_ptr(msrs, cpu);
2491 nbe = reg->l & MSR_MCGCTL_NBE;
2492
2493 edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
2494 cpu, reg->q,
2495 (nbe ? "enabled" : "disabled"));
2496
2497 if (!nbe)
2498 goto out;
2499 }
2500 ret = true;
2501
2502 out:
2503 free_cpumask_var(mask);
2504 return ret;
2505 }
2506
2507 static int toggle_ecc_err_reporting(struct ecc_settings *s, u16 nid, bool on)
2508 {
2509 cpumask_var_t cmask;
2510 int cpu;
2511
2512 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
2513 amd64_warn("%s: error allocating mask\n", __func__);
2514 return false;
2515 }
2516
2517 get_cpus_on_this_dct_cpumask(cmask, nid);
2518
2519 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2520
2521 for_each_cpu(cpu, cmask) {
2522
2523 struct msr *reg = per_cpu_ptr(msrs, cpu);
2524
2525 if (on) {
2526 if (reg->l & MSR_MCGCTL_NBE)
2527 s->flags.nb_mce_enable = 1;
2528
2529 reg->l |= MSR_MCGCTL_NBE;
2530 } else {
2531 /*
2532 * Turn off NB MCE reporting only when it was off before
2533 */
2534 if (!s->flags.nb_mce_enable)
2535 reg->l &= ~MSR_MCGCTL_NBE;
2536 }
2537 }
2538 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2539
2540 free_cpumask_var(cmask);
2541
2542 return 0;
2543 }
2544
2545 static bool enable_ecc_error_reporting(struct ecc_settings *s, u16 nid,
2546 struct pci_dev *F3)
2547 {
2548 bool ret = true;
2549 u32 value, mask = 0x3; /* UECC/CECC enable */
2550
2551 if (toggle_ecc_err_reporting(s, nid, ON)) {
2552 amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
2553 return false;
2554 }
2555
2556 amd64_read_pci_cfg(F3, NBCTL, &value);
2557
2558 s->old_nbctl = value & mask;
2559 s->nbctl_valid = true;
2560
2561 value |= mask;
2562 amd64_write_pci_cfg(F3, NBCTL, value);
2563
2564 amd64_read_pci_cfg(F3, NBCFG, &value);
2565
2566 edac_dbg(0, "1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2567 nid, value, !!(value & NBCFG_ECC_ENABLE));
2568
2569 if (!(value & NBCFG_ECC_ENABLE)) {
2570 amd64_warn("DRAM ECC disabled on this node, enabling...\n");
2571
2572 s->flags.nb_ecc_prev = 0;
2573
2574 /* Attempt to turn on DRAM ECC Enable */
2575 value |= NBCFG_ECC_ENABLE;
2576 amd64_write_pci_cfg(F3, NBCFG, value);
2577
2578 amd64_read_pci_cfg(F3, NBCFG, &value);
2579
2580 if (!(value & NBCFG_ECC_ENABLE)) {
2581 amd64_warn("Hardware rejected DRAM ECC enable,"
2582 "check memory DIMM configuration.\n");
2583 ret = false;
2584 } else {
2585 amd64_info("Hardware accepted DRAM ECC Enable\n");
2586 }
2587 } else {
2588 s->flags.nb_ecc_prev = 1;
2589 }
2590
2591 edac_dbg(0, "2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2592 nid, value, !!(value & NBCFG_ECC_ENABLE));
2593
2594 return ret;
2595 }
2596
2597 static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid,
2598 struct pci_dev *F3)
2599 {
2600 u32 value, mask = 0x3; /* UECC/CECC enable */
2601
2602
2603 if (!s->nbctl_valid)
2604 return;
2605
2606 amd64_read_pci_cfg(F3, NBCTL, &value);
2607 value &= ~mask;
2608 value |= s->old_nbctl;
2609
2610 amd64_write_pci_cfg(F3, NBCTL, value);
2611
2612 /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
2613 if (!s->flags.nb_ecc_prev) {
2614 amd64_read_pci_cfg(F3, NBCFG, &value);
2615 value &= ~NBCFG_ECC_ENABLE;
2616 amd64_write_pci_cfg(F3, NBCFG, value);
2617 }
2618
2619 /* restore the NB Enable MCGCTL bit */
2620 if (toggle_ecc_err_reporting(s, nid, OFF))
2621 amd64_warn("Error restoring NB MCGCTL settings!\n");
2622 }
2623
2624 /*
2625 * EDAC requires that the BIOS have ECC enabled before
2626 * taking over the processing of ECC errors. A command line
2627 * option allows to force-enable hardware ECC later in
2628 * enable_ecc_error_reporting().
2629 */
2630 static const char *ecc_msg =
2631 "ECC disabled in the BIOS or no ECC capability, module will not load.\n"
2632 " Either enable ECC checking or force module loading by setting "
2633 "'ecc_enable_override'.\n"
2634 " (Note that use of the override may cause unknown side effects.)\n";
2635
2636 static bool ecc_enabled(struct pci_dev *F3, u16 nid)
2637 {
2638 u32 value;
2639 u8 ecc_en = 0;
2640 bool nb_mce_en = false;
2641
2642 amd64_read_pci_cfg(F3, NBCFG, &value);
2643
2644 ecc_en = !!(value & NBCFG_ECC_ENABLE);
2645 amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled"));
2646
2647 nb_mce_en = nb_mce_bank_enabled_on_node(nid);
2648 if (!nb_mce_en)
2649 amd64_notice("NB MCE bank disabled, set MSR "
2650 "0x%08x[4] on node %d to enable.\n",
2651 MSR_IA32_MCG_CTL, nid);
2652
2653 if (!ecc_en || !nb_mce_en) {
2654 amd64_notice("%s", ecc_msg);
2655 return false;
2656 }
2657 return true;
2658 }
2659
2660 static int set_mc_sysfs_attrs(struct mem_ctl_info *mci)
2661 {
2662 struct amd64_pvt *pvt = mci->pvt_info;
2663 int rc;
2664
2665 rc = amd64_create_sysfs_dbg_files(mci);
2666 if (rc < 0)
2667 return rc;
2668
2669 if (pvt->fam >= 0x10) {
2670 rc = amd64_create_sysfs_inject_files(mci);
2671 if (rc < 0)
2672 return rc;
2673 }
2674
2675 return 0;
2676 }
2677
2678 static void del_mc_sysfs_attrs(struct mem_ctl_info *mci)
2679 {
2680 struct amd64_pvt *pvt = mci->pvt_info;
2681
2682 amd64_remove_sysfs_dbg_files(mci);
2683
2684 if (pvt->fam >= 0x10)
2685 amd64_remove_sysfs_inject_files(mci);
2686 }
2687
2688 static void setup_mci_misc_attrs(struct mem_ctl_info *mci,
2689 struct amd64_family_type *fam)
2690 {
2691 struct amd64_pvt *pvt = mci->pvt_info;
2692
2693 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
2694 mci->edac_ctl_cap = EDAC_FLAG_NONE;
2695
2696 if (pvt->nbcap & NBCAP_SECDED)
2697 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
2698
2699 if (pvt->nbcap & NBCAP_CHIPKILL)
2700 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
2701
2702 mci->edac_cap = determine_edac_cap(pvt);
2703 mci->mod_name = EDAC_MOD_STR;
2704 mci->mod_ver = EDAC_AMD64_VERSION;
2705 mci->ctl_name = fam->ctl_name;
2706 mci->dev_name = pci_name(pvt->F2);
2707 mci->ctl_page_to_phys = NULL;
2708
2709 /* memory scrubber interface */
2710 mci->set_sdram_scrub_rate = set_scrub_rate;
2711 mci->get_sdram_scrub_rate = get_scrub_rate;
2712 }
2713
2714 /*
2715 * returns a pointer to the family descriptor on success, NULL otherwise.
2716 */
2717 static struct amd64_family_type *per_family_init(struct amd64_pvt *pvt)
2718 {
2719 struct amd64_family_type *fam_type = NULL;
2720
2721 pvt->ext_model = boot_cpu_data.x86_model >> 4;
2722 pvt->stepping = boot_cpu_data.x86_mask;
2723 pvt->model = boot_cpu_data.x86_model;
2724 pvt->fam = boot_cpu_data.x86;
2725
2726 switch (pvt->fam) {
2727 case 0xf:
2728 fam_type = &family_types[K8_CPUS];
2729 pvt->ops = &family_types[K8_CPUS].ops;
2730 break;
2731
2732 case 0x10:
2733 fam_type = &family_types[F10_CPUS];
2734 pvt->ops = &family_types[F10_CPUS].ops;
2735 break;
2736
2737 case 0x15:
2738 if (pvt->model == 0x30) {
2739 fam_type = &family_types[F15_M30H_CPUS];
2740 pvt->ops = &family_types[F15_M30H_CPUS].ops;
2741 break;
2742 } else if (pvt->model == 0x60) {
2743 fam_type = &family_types[F15_M60H_CPUS];
2744 pvt->ops = &family_types[F15_M60H_CPUS].ops;
2745 break;
2746 }
2747
2748 fam_type = &family_types[F15_CPUS];
2749 pvt->ops = &family_types[F15_CPUS].ops;
2750 break;
2751
2752 case 0x16:
2753 if (pvt->model == 0x30) {
2754 fam_type = &family_types[F16_M30H_CPUS];
2755 pvt->ops = &family_types[F16_M30H_CPUS].ops;
2756 break;
2757 }
2758 fam_type = &family_types[F16_CPUS];
2759 pvt->ops = &family_types[F16_CPUS].ops;
2760 break;
2761
2762 default:
2763 amd64_err("Unsupported family!\n");
2764 return NULL;
2765 }
2766
2767 amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name,
2768 (pvt->fam == 0xf ?
2769 (pvt->ext_model >= K8_REV_F ? "revF or later "
2770 : "revE or earlier ")
2771 : ""), pvt->mc_node_id);
2772 return fam_type;
2773 }
2774
2775 static int init_one_instance(struct pci_dev *F2)
2776 {
2777 struct amd64_pvt *pvt = NULL;
2778 struct amd64_family_type *fam_type = NULL;
2779 struct mem_ctl_info *mci = NULL;
2780 struct edac_mc_layer layers[2];
2781 int err = 0, ret;
2782 u16 nid = amd_get_node_id(F2);
2783
2784 ret = -ENOMEM;
2785 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
2786 if (!pvt)
2787 goto err_ret;
2788
2789 pvt->mc_node_id = nid;
2790 pvt->F2 = F2;
2791
2792 ret = -EINVAL;
2793 fam_type = per_family_init(pvt);
2794 if (!fam_type)
2795 goto err_free;
2796
2797 ret = -ENODEV;
2798 err = reserve_mc_sibling_devs(pvt, fam_type->f1_id, fam_type->f3_id);
2799 if (err)
2800 goto err_free;
2801
2802 read_mc_regs(pvt);
2803
2804 /*
2805 * We need to determine how many memory channels there are. Then use
2806 * that information for calculating the size of the dynamic instance
2807 * tables in the 'mci' structure.
2808 */
2809 ret = -EINVAL;
2810 pvt->channel_count = pvt->ops->early_channel_count(pvt);
2811 if (pvt->channel_count < 0)
2812 goto err_siblings;
2813
2814 ret = -ENOMEM;
2815 layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
2816 layers[0].size = pvt->csels[0].b_cnt;
2817 layers[0].is_virt_csrow = true;
2818 layers[1].type = EDAC_MC_LAYER_CHANNEL;
2819
2820 /*
2821 * Always allocate two channels since we can have setups with DIMMs on
2822 * only one channel. Also, this simplifies handling later for the price
2823 * of a couple of KBs tops.
2824 */
2825 layers[1].size = 2;
2826 layers[1].is_virt_csrow = false;
2827
2828 mci = edac_mc_alloc(nid, ARRAY_SIZE(layers), layers, 0);
2829 if (!mci)
2830 goto err_siblings;
2831
2832 mci->pvt_info = pvt;
2833 mci->pdev = &pvt->F2->dev;
2834
2835 setup_mci_misc_attrs(mci, fam_type);
2836
2837 if (init_csrows(mci))
2838 mci->edac_cap = EDAC_FLAG_NONE;
2839
2840 ret = -ENODEV;
2841 if (edac_mc_add_mc(mci)) {
2842 edac_dbg(1, "failed edac_mc_add_mc()\n");
2843 goto err_add_mc;
2844 }
2845 if (set_mc_sysfs_attrs(mci)) {
2846 edac_dbg(1, "failed edac_mc_add_mc()\n");
2847 goto err_add_sysfs;
2848 }
2849
2850 /* register stuff with EDAC MCE */
2851 if (report_gart_errors)
2852 amd_report_gart_errors(true);
2853
2854 amd_register_ecc_decoder(decode_bus_error);
2855
2856 mcis[nid] = mci;
2857
2858 atomic_inc(&drv_instances);
2859
2860 return 0;
2861
2862 err_add_sysfs:
2863 edac_mc_del_mc(mci->pdev);
2864 err_add_mc:
2865 edac_mc_free(mci);
2866
2867 err_siblings:
2868 free_mc_sibling_devs(pvt);
2869
2870 err_free:
2871 kfree(pvt);
2872
2873 err_ret:
2874 return ret;
2875 }
2876
2877 static int probe_one_instance(struct pci_dev *pdev,
2878 const struct pci_device_id *mc_type)
2879 {
2880 u16 nid = amd_get_node_id(pdev);
2881 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2882 struct ecc_settings *s;
2883 int ret = 0;
2884
2885 ret = pci_enable_device(pdev);
2886 if (ret < 0) {
2887 edac_dbg(0, "ret=%d\n", ret);
2888 return -EIO;
2889 }
2890
2891 ret = -ENOMEM;
2892 s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
2893 if (!s)
2894 goto err_out;
2895
2896 ecc_stngs[nid] = s;
2897
2898 if (!ecc_enabled(F3, nid)) {
2899 ret = -ENODEV;
2900
2901 if (!ecc_enable_override)
2902 goto err_enable;
2903
2904 amd64_warn("Forcing ECC on!\n");
2905
2906 if (!enable_ecc_error_reporting(s, nid, F3))
2907 goto err_enable;
2908 }
2909
2910 ret = init_one_instance(pdev);
2911 if (ret < 0) {
2912 amd64_err("Error probing instance: %d\n", nid);
2913 restore_ecc_error_reporting(s, nid, F3);
2914 }
2915
2916 return ret;
2917
2918 err_enable:
2919 kfree(s);
2920 ecc_stngs[nid] = NULL;
2921
2922 err_out:
2923 return ret;
2924 }
2925
2926 static void remove_one_instance(struct pci_dev *pdev)
2927 {
2928 struct mem_ctl_info *mci;
2929 struct amd64_pvt *pvt;
2930 u16 nid = amd_get_node_id(pdev);
2931 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2932 struct ecc_settings *s = ecc_stngs[nid];
2933
2934 mci = find_mci_by_dev(&pdev->dev);
2935 WARN_ON(!mci);
2936
2937 del_mc_sysfs_attrs(mci);
2938 /* Remove from EDAC CORE tracking list */
2939 mci = edac_mc_del_mc(&pdev->dev);
2940 if (!mci)
2941 return;
2942
2943 pvt = mci->pvt_info;
2944
2945 restore_ecc_error_reporting(s, nid, F3);
2946
2947 free_mc_sibling_devs(pvt);
2948
2949 /* unregister from EDAC MCE */
2950 amd_report_gart_errors(false);
2951 amd_unregister_ecc_decoder(decode_bus_error);
2952
2953 kfree(ecc_stngs[nid]);
2954 ecc_stngs[nid] = NULL;
2955
2956 /* Free the EDAC CORE resources */
2957 mci->pvt_info = NULL;
2958 mcis[nid] = NULL;
2959
2960 kfree(pvt);
2961 edac_mc_free(mci);
2962 }
2963
2964 /*
2965 * This table is part of the interface for loading drivers for PCI devices. The
2966 * PCI core identifies what devices are on a system during boot, and then
2967 * inquiry this table to see if this driver is for a given device found.
2968 */
2969 static const struct pci_device_id amd64_pci_table[] = {
2970 { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_K8_NB_MEMCTL) },
2971 { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_10H_NB_DRAM) },
2972 { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_15H_NB_F2) },
2973 { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_15H_M30H_NB_F2) },
2974 { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_15H_M60H_NB_F2) },
2975 { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_16H_NB_F2) },
2976 { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_16H_M30H_NB_F2) },
2977 {0, }
2978 };
2979 MODULE_DEVICE_TABLE(pci, amd64_pci_table);
2980
2981 static struct pci_driver amd64_pci_driver = {
2982 .name = EDAC_MOD_STR,
2983 .probe = probe_one_instance,
2984 .remove = remove_one_instance,
2985 .id_table = amd64_pci_table,
2986 };
2987
2988 static void setup_pci_device(void)
2989 {
2990 struct mem_ctl_info *mci;
2991 struct amd64_pvt *pvt;
2992
2993 if (pci_ctl)
2994 return;
2995
2996 mci = mcis[0];
2997 if (!mci)
2998 return;
2999
3000 pvt = mci->pvt_info;
3001 pci_ctl = edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR);
3002 if (!pci_ctl) {
3003 pr_warn("%s(): Unable to create PCI control\n", __func__);
3004 pr_warn("%s(): PCI error report via EDAC not set\n", __func__);
3005 }
3006 }
3007
3008 static int __init amd64_edac_init(void)
3009 {
3010 int err = -ENODEV;
3011
3012 printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION);
3013
3014 opstate_init();
3015
3016 if (amd_cache_northbridges() < 0)
3017 goto err_ret;
3018
3019 err = -ENOMEM;
3020 mcis = kzalloc(amd_nb_num() * sizeof(mcis[0]), GFP_KERNEL);
3021 ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL);
3022 if (!(mcis && ecc_stngs))
3023 goto err_free;
3024
3025 msrs = msrs_alloc();
3026 if (!msrs)
3027 goto err_free;
3028
3029 err = pci_register_driver(&amd64_pci_driver);
3030 if (err)
3031 goto err_pci;
3032
3033 err = -ENODEV;
3034 if (!atomic_read(&drv_instances))
3035 goto err_no_instances;
3036
3037 setup_pci_device();
3038
3039 #ifdef CONFIG_X86_32
3040 amd64_err("%s on 32-bit is unsupported. USE AT YOUR OWN RISK!\n", EDAC_MOD_STR);
3041 #endif
3042
3043 return 0;
3044
3045 err_no_instances:
3046 pci_unregister_driver(&amd64_pci_driver);
3047
3048 err_pci:
3049 msrs_free(msrs);
3050 msrs = NULL;
3051
3052 err_free:
3053 kfree(mcis);
3054 mcis = NULL;
3055
3056 kfree(ecc_stngs);
3057 ecc_stngs = NULL;
3058
3059 err_ret:
3060 return err;
3061 }
3062
3063 static void __exit amd64_edac_exit(void)
3064 {
3065 if (pci_ctl)
3066 edac_pci_release_generic_ctl(pci_ctl);
3067
3068 pci_unregister_driver(&amd64_pci_driver);
3069
3070 kfree(ecc_stngs);
3071 ecc_stngs = NULL;
3072
3073 kfree(mcis);
3074 mcis = NULL;
3075
3076 msrs_free(msrs);
3077 msrs = NULL;
3078 }
3079
3080 module_init(amd64_edac_init);
3081 module_exit(amd64_edac_exit);
3082
3083 MODULE_LICENSE("GPL");
3084 MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
3085 "Dave Peterson, Thayne Harbaugh");
3086 MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
3087 EDAC_AMD64_VERSION);
3088
3089 module_param(edac_op_state, int, 0444);
3090 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
Linux with added FPC-III support