?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
tty: Remove to support serial for S5P6442
[zen-stable.git] / drivers / edac / amd64_edac.c
blob31e71c4fc83171e7da846740533f63131189327b
1 #include "amd64_edac.h"
2 #include <asm/amd_nb.h>
3
4 static struct edac_pci_ctl_info *amd64_ctl_pci;
5
6 static int report_gart_errors;
7 module_param(report_gart_errors, int, 0644);
8
9 /*
10 * Set by command line parameter. If BIOS has enabled the ECC, this override is
11 * cleared to prevent re-enabling the hardware by this driver.
12 */
13 static int ecc_enable_override;
14 module_param(ecc_enable_override, int, 0644);
15
16 static struct msr __percpu *msrs;
17
18 /*
19 * count successfully initialized driver instances for setup_pci_device()
20 */
21 static atomic_t drv_instances = ATOMIC_INIT(0);
22
23 /* Per-node driver instances */
24 static struct mem_ctl_info **mcis;
25 static struct ecc_settings **ecc_stngs;
26
27 /*
28 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
29 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
30 * or higher value'.
31 *
32 *FIXME: Produce a better mapping/linearisation.
33 */
34 struct scrubrate {
35 u32 scrubval; /* bit pattern for scrub rate */
36 u32 bandwidth; /* bandwidth consumed (bytes/sec) */
37 } scrubrates[] = {
38 { 0x01, 1600000000UL},
39 { 0x02, 800000000UL},
40 { 0x03, 400000000UL},
41 { 0x04, 200000000UL},
42 { 0x05, 100000000UL},
43 { 0x06, 50000000UL},
44 { 0x07, 25000000UL},
45 { 0x08, 12284069UL},
46 { 0x09, 6274509UL},
47 { 0x0A, 3121951UL},
48 { 0x0B, 1560975UL},
49 { 0x0C, 781440UL},
50 { 0x0D, 390720UL},
51 { 0x0E, 195300UL},
52 { 0x0F, 97650UL},
53 { 0x10, 48854UL},
54 { 0x11, 24427UL},
55 { 0x12, 12213UL},
56 { 0x13, 6101UL},
57 { 0x14, 3051UL},
58 { 0x15, 1523UL},
59 { 0x16, 761UL},
60 { 0x00, 0UL}, /* scrubbing off */
61 };
62
63 static int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
64 u32 *val, const char *func)
65 {
66 int err = 0;
67
68 err = pci_read_config_dword(pdev, offset, val);
69 if (err)
70 amd64_warn("%s: error reading F%dx%03x.\n",
71 func, PCI_FUNC(pdev->devfn), offset);
72
73 return err;
74 }
75
76 int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
77 u32 val, const char *func)
78 {
79 int err = 0;
80
81 err = pci_write_config_dword(pdev, offset, val);
82 if (err)
83 amd64_warn("%s: error writing to F%dx%03x.\n",
84 func, PCI_FUNC(pdev->devfn), offset);
85
86 return err;
87 }
88
89 /*
90 *
91 * Depending on the family, F2 DCT reads need special handling:
92 *
93 * K8: has a single DCT only
94 *
95 * F10h: each DCT has its own set of regs
96 * DCT0 -> F2x040..
97 * DCT1 -> F2x140..
98 *
99 * F15h: we select which DCT we access using F1x10C[DctCfgSel]
100 *
101 */
102 static int k8_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
103 const char *func)
104 {
105 if (addr >= 0x100)
106 return -EINVAL;
107
108 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
109 }
110
111 static int f10_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
112 const char *func)
113 {
114 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
115 }
116
117 static int f15_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
118 const char *func)
119 {
120 u32 reg = 0;
121 u8 dct = 0;
122
123 if (addr >= 0x140 && addr <= 0x1a0) {
124 dct = 1;
125 addr -= 0x100;
126 }
127
128 amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, &reg);
129 reg &= 0xfffffffe;
130 reg |= dct;
131 amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
132
133 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
134 }
135
136 /*
137 * Memory scrubber control interface. For K8, memory scrubbing is handled by
138 * hardware and can involve L2 cache, dcache as well as the main memory. With
139 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
140 * functionality.
141 *
142 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
143 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
144 * bytes/sec for the setting.
145 *
146 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
147 * other archs, we might not have access to the caches directly.
148 */
149
150 /*
151 * scan the scrub rate mapping table for a close or matching bandwidth value to
152 * issue. If requested is too big, then use last maximum value found.
153 */
154 static int __amd64_set_scrub_rate(struct pci_dev *ctl, u32 new_bw, u32 min_rate)
155 {
156 u32 scrubval;
157 int i;
158
159 /*
160 * map the configured rate (new_bw) to a value specific to the AMD64
161 * memory controller and apply to register. Search for the first
162 * bandwidth entry that is greater or equal than the setting requested
163 * and program that. If at last entry, turn off DRAM scrubbing.
164 */
165 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
166 /*
167 * skip scrub rates which aren't recommended
168 * (see F10 BKDG, F3x58)
169 */
170 if (scrubrates[i].scrubval < min_rate)
171 continue;
172
173 if (scrubrates[i].bandwidth <= new_bw)
174 break;
175
176 /*
177 * if no suitable bandwidth found, turn off DRAM scrubbing
178 * entirely by falling back to the last element in the
179 * scrubrates array.
180 */
181 }
182
183 scrubval = scrubrates[i].scrubval;
184
185 pci_write_bits32(ctl, SCRCTRL, scrubval, 0x001F);
186
187 if (scrubval)
188 return scrubrates[i].bandwidth;
189
190 return 0;
191 }
192
193 static int amd64_set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
194 {
195 struct amd64_pvt *pvt = mci->pvt_info;
196 u32 min_scrubrate = 0x5;
197
198 if (boot_cpu_data.x86 == 0xf)
199 min_scrubrate = 0x0;
200
201 return __amd64_set_scrub_rate(pvt->F3, bw, min_scrubrate);
202 }
203
204 static int amd64_get_scrub_rate(struct mem_ctl_info *mci)
205 {
206 struct amd64_pvt *pvt = mci->pvt_info;
207 u32 scrubval = 0;
208 int i, retval = -EINVAL;
209
210 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
211
212 scrubval = scrubval & 0x001F;
213
214 amd64_debug("pci-read, sdram scrub control value: %d\n", scrubval);
215
216 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
217 if (scrubrates[i].scrubval == scrubval) {
218 retval = scrubrates[i].bandwidth;
219 break;
220 }
221 }
222 return retval;
223 }
224
225 /*
226 * returns true if the SysAddr given by sys_addr matches the
227 * DRAM base/limit associated with node_id
228 */
229 static bool amd64_base_limit_match(struct amd64_pvt *pvt, u64 sys_addr,
230 unsigned nid)
231 {
232 u64 addr;
233
234 /* The K8 treats this as a 40-bit value. However, bits 63-40 will be
235 * all ones if the most significant implemented address bit is 1.
236 * Here we discard bits 63-40. See section 3.4.2 of AMD publication
237 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
238 * Application Programming.
239 */
240 addr = sys_addr & 0x000000ffffffffffull;
241
242 return ((addr >= get_dram_base(pvt, nid)) &&
243 (addr <= get_dram_limit(pvt, nid)));
244 }
245
246 /*
247 * Attempt to map a SysAddr to a node. On success, return a pointer to the
248 * mem_ctl_info structure for the node that the SysAddr maps to.
249 *
250 * On failure, return NULL.
251 */
252 static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
253 u64 sys_addr)
254 {
255 struct amd64_pvt *pvt;
256 unsigned node_id;
257 u32 intlv_en, bits;
258
259 /*
260 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
261 * 3.4.4.2) registers to map the SysAddr to a node ID.
262 */
263 pvt = mci->pvt_info;
264
265 /*
266 * The value of this field should be the same for all DRAM Base
267 * registers. Therefore we arbitrarily choose to read it from the
268 * register for node 0.
269 */
270 intlv_en = dram_intlv_en(pvt, 0);
271
272 if (intlv_en == 0) {
273 for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
274 if (amd64_base_limit_match(pvt, sys_addr, node_id))
275 goto found;
276 }
277 goto err_no_match;
278 }
279
280 if (unlikely((intlv_en != 0x01) &&
281 (intlv_en != 0x03) &&
282 (intlv_en != 0x07))) {
283 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
284 return NULL;
285 }
286
287 bits = (((u32) sys_addr) >> 12) & intlv_en;
288
289 for (node_id = 0; ; ) {
290 if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
291 break; /* intlv_sel field matches */
292
293 if (++node_id >= DRAM_RANGES)
294 goto err_no_match;
295 }
296
297 /* sanity test for sys_addr */
298 if (unlikely(!amd64_base_limit_match(pvt, sys_addr, node_id))) {
299 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
300 "range for node %d with node interleaving enabled.\n",
301 __func__, sys_addr, node_id);
302 return NULL;
303 }
304
305 found:
306 return edac_mc_find((int)node_id);
307
308 err_no_match:
309 debugf2("sys_addr 0x%lx doesn't match any node\n",
310 (unsigned long)sys_addr);
311
312 return NULL;
313 }
314
315 /*
316 * compute the CS base address of the @csrow on the DRAM controller @dct.
317 * For details see F2x[5C:40] in the processor's BKDG
318 */
319 static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
320 u64 *base, u64 *mask)
321 {
322 u64 csbase, csmask, base_bits, mask_bits;
323 u8 addr_shift;
324
325 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) {
326 csbase = pvt->csels[dct].csbases[csrow];
327 csmask = pvt->csels[dct].csmasks[csrow];
328 base_bits = GENMASK(21, 31) | GENMASK(9, 15);
329 mask_bits = GENMASK(21, 29) | GENMASK(9, 15);
330 addr_shift = 4;
331 } else {
332 csbase = pvt->csels[dct].csbases[csrow];
333 csmask = pvt->csels[dct].csmasks[csrow >> 1];
334 addr_shift = 8;
335
336 if (boot_cpu_data.x86 == 0x15)
337 base_bits = mask_bits = GENMASK(19,30) | GENMASK(5,13);
338 else
339 base_bits = mask_bits = GENMASK(19,28) | GENMASK(5,13);
340 }
341
342 *base = (csbase & base_bits) << addr_shift;
343
344 *mask = ~0ULL;
345 /* poke holes for the csmask */
346 *mask &= ~(mask_bits << addr_shift);
347 /* OR them in */
348 *mask |= (csmask & mask_bits) << addr_shift;
349 }
350
351 #define for_each_chip_select(i, dct, pvt) \
352 for (i = 0; i < pvt->csels[dct].b_cnt; i++)
353
354 #define chip_select_base(i, dct, pvt) \
355 pvt->csels[dct].csbases[i]
356
357 #define for_each_chip_select_mask(i, dct, pvt) \
358 for (i = 0; i < pvt->csels[dct].m_cnt; i++)
359
360 /*
361 * @input_addr is an InputAddr associated with the node given by mci. Return the
362 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
363 */
364 static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
365 {
366 struct amd64_pvt *pvt;
367 int csrow;
368 u64 base, mask;
369
370 pvt = mci->pvt_info;
371
372 for_each_chip_select(csrow, 0, pvt) {
373 if (!csrow_enabled(csrow, 0, pvt))
374 continue;
375
376 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
377
378 mask = ~mask;
379
380 if ((input_addr & mask) == (base & mask)) {
381 debugf2("InputAddr 0x%lx matches csrow %d (node %d)\n",
382 (unsigned long)input_addr, csrow,
383 pvt->mc_node_id);
384
385 return csrow;
386 }
387 }
388 debugf2("no matching csrow for InputAddr 0x%lx (MC node %d)\n",
389 (unsigned long)input_addr, pvt->mc_node_id);
390
391 return -1;
392 }
393
394 /*
395 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
396 * for the node represented by mci. Info is passed back in *hole_base,
397 * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if
398 * info is invalid. Info may be invalid for either of the following reasons:
399 *
400 * - The revision of the node is not E or greater. In this case, the DRAM Hole
401 * Address Register does not exist.
402 *
403 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
404 * indicating that its contents are not valid.
405 *
406 * The values passed back in *hole_base, *hole_offset, and *hole_size are
407 * complete 32-bit values despite the fact that the bitfields in the DHAR
408 * only represent bits 31-24 of the base and offset values.
409 */
410 int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
411 u64 *hole_offset, u64 *hole_size)
412 {
413 struct amd64_pvt *pvt = mci->pvt_info;
414 u64 base;
415
416 /* only revE and later have the DRAM Hole Address Register */
417 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_E) {
418 debugf1(" revision %d for node %d does not support DHAR\n",
419 pvt->ext_model, pvt->mc_node_id);
420 return 1;
421 }
422
423 /* valid for Fam10h and above */
424 if (boot_cpu_data.x86 >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
425 debugf1(" Dram Memory Hoisting is DISABLED on this system\n");
426 return 1;
427 }
428
429 if (!dhar_valid(pvt)) {
430 debugf1(" Dram Memory Hoisting is DISABLED on this node %d\n",
431 pvt->mc_node_id);
432 return 1;
433 }
434
435 /* This node has Memory Hoisting */
436
437 /* +------------------+--------------------+--------------------+-----
438 * | memory | DRAM hole | relocated |
439 * | [0, (x - 1)] | [x, 0xffffffff] | addresses from |
440 * | | | DRAM hole |
441 * | | | [0x100000000, |
442 * | | | (0x100000000+ |
443 * | | | (0xffffffff-x))] |
444 * +------------------+--------------------+--------------------+-----
445 *
446 * Above is a diagram of physical memory showing the DRAM hole and the
447 * relocated addresses from the DRAM hole. As shown, the DRAM hole
448 * starts at address x (the base address) and extends through address
449 * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the
450 * addresses in the hole so that they start at 0x100000000.
451 */
452
453 base = dhar_base(pvt);
454
455 *hole_base = base;
456 *hole_size = (0x1ull << 32) - base;
457
458 if (boot_cpu_data.x86 > 0xf)
459 *hole_offset = f10_dhar_offset(pvt);
460 else
461 *hole_offset = k8_dhar_offset(pvt);
462
463 debugf1(" DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
464 pvt->mc_node_id, (unsigned long)*hole_base,
465 (unsigned long)*hole_offset, (unsigned long)*hole_size);
466
467 return 0;
468 }
469 EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info);
470
471 /*
472 * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is
473 * assumed that sys_addr maps to the node given by mci.
474 *
475 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
476 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
477 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
478 * then it is also involved in translating a SysAddr to a DramAddr. Sections
479 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
480 * These parts of the documentation are unclear. I interpret them as follows:
481 *
482 * When node n receives a SysAddr, it processes the SysAddr as follows:
483 *
484 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
485 * Limit registers for node n. If the SysAddr is not within the range
486 * specified by the base and limit values, then node n ignores the Sysaddr
487 * (since it does not map to node n). Otherwise continue to step 2 below.
488 *
489 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
490 * disabled so skip to step 3 below. Otherwise see if the SysAddr is within
491 * the range of relocated addresses (starting at 0x100000000) from the DRAM
492 * hole. If not, skip to step 3 below. Else get the value of the
493 * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
494 * offset defined by this value from the SysAddr.
495 *
496 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
497 * Base register for node n. To obtain the DramAddr, subtract the base
498 * address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
499 */
500 static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
501 {
502 struct amd64_pvt *pvt = mci->pvt_info;
503 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
504 int ret = 0;
505
506 dram_base = get_dram_base(pvt, pvt->mc_node_id);
507
508 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
509 &hole_size);
510 if (!ret) {
511 if ((sys_addr >= (1ull << 32)) &&
512 (sys_addr < ((1ull << 32) + hole_size))) {
513 /* use DHAR to translate SysAddr to DramAddr */
514 dram_addr = sys_addr - hole_offset;
515
516 debugf2("using DHAR to translate SysAddr 0x%lx to "
517 "DramAddr 0x%lx\n",
518 (unsigned long)sys_addr,
519 (unsigned long)dram_addr);
520
521 return dram_addr;
522 }
523 }
524
525 /*
526 * Translate the SysAddr to a DramAddr as shown near the start of
527 * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8
528 * only deals with 40-bit values. Therefore we discard bits 63-40 of
529 * sys_addr below. If bit 39 of sys_addr is 1 then the bits we
530 * discard are all 1s. Otherwise the bits we discard are all 0s. See
531 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
532 * Programmer's Manual Volume 1 Application Programming.
533 */
534 dram_addr = (sys_addr & GENMASK(0, 39)) - dram_base;
535
536 debugf2("using DRAM Base register to translate SysAddr 0x%lx to "
537 "DramAddr 0x%lx\n", (unsigned long)sys_addr,
538 (unsigned long)dram_addr);
539 return dram_addr;
540 }
541
542 /*
543 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
544 * (section 3.4.4.1). Return the number of bits from a SysAddr that are used
545 * for node interleaving.
546 */
547 static int num_node_interleave_bits(unsigned intlv_en)
548 {
549 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
550 int n;
551
552 BUG_ON(intlv_en > 7);
553 n = intlv_shift_table[intlv_en];
554 return n;
555 }
556
557 /* Translate the DramAddr given by @dram_addr to an InputAddr. */
558 static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
559 {
560 struct amd64_pvt *pvt;
561 int intlv_shift;
562 u64 input_addr;
563
564 pvt = mci->pvt_info;
565
566 /*
567 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
568 * concerning translating a DramAddr to an InputAddr.
569 */
570 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
571 input_addr = ((dram_addr >> intlv_shift) & GENMASK(12, 35)) +
572 (dram_addr & 0xfff);
573
574 debugf2(" Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
575 intlv_shift, (unsigned long)dram_addr,
576 (unsigned long)input_addr);
577
578 return input_addr;
579 }
580
581 /*
582 * Translate the SysAddr represented by @sys_addr to an InputAddr. It is
583 * assumed that @sys_addr maps to the node given by mci.
584 */
585 static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
586 {
587 u64 input_addr;
588
589 input_addr =
590 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
591
592 debugf2("SysAdddr 0x%lx translates to InputAddr 0x%lx\n",
593 (unsigned long)sys_addr, (unsigned long)input_addr);
594
595 return input_addr;
596 }
597
598
599 /*
600 * @input_addr is an InputAddr associated with the node represented by mci.
601 * Translate @input_addr to a DramAddr and return the result.
602 */
603 static u64 input_addr_to_dram_addr(struct mem_ctl_info *mci, u64 input_addr)
604 {
605 struct amd64_pvt *pvt;
606 unsigned node_id, intlv_shift;
607 u64 bits, dram_addr;
608 u32 intlv_sel;
609
610 /*
611 * Near the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
612 * shows how to translate a DramAddr to an InputAddr. Here we reverse
613 * this procedure. When translating from a DramAddr to an InputAddr, the
614 * bits used for node interleaving are discarded. Here we recover these
615 * bits from the IntlvSel field of the DRAM Limit register (section
616 * 3.4.4.2) for the node that input_addr is associated with.
617 */
618 pvt = mci->pvt_info;
619 node_id = pvt->mc_node_id;
620
621 BUG_ON(node_id > 7);
622
623 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
624 if (intlv_shift == 0) {
625 debugf1(" InputAddr 0x%lx translates to DramAddr of "
626 "same value\n", (unsigned long)input_addr);
627
628 return input_addr;
629 }
630
631 bits = ((input_addr & GENMASK(12, 35)) << intlv_shift) +
632 (input_addr & 0xfff);
633
634 intlv_sel = dram_intlv_sel(pvt, node_id) & ((1 << intlv_shift) - 1);
635 dram_addr = bits + (intlv_sel << 12);
636
637 debugf1("InputAddr 0x%lx translates to DramAddr 0x%lx "
638 "(%d node interleave bits)\n", (unsigned long)input_addr,
639 (unsigned long)dram_addr, intlv_shift);
640
641 return dram_addr;
642 }
643
644 /*
645 * @dram_addr is a DramAddr that maps to the node represented by mci. Convert
646 * @dram_addr to a SysAddr.
647 */
648 static u64 dram_addr_to_sys_addr(struct mem_ctl_info *mci, u64 dram_addr)
649 {
650 struct amd64_pvt *pvt = mci->pvt_info;
651 u64 hole_base, hole_offset, hole_size, base, sys_addr;
652 int ret = 0;
653
654 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
655 &hole_size);
656 if (!ret) {
657 if ((dram_addr >= hole_base) &&
658 (dram_addr < (hole_base + hole_size))) {
659 sys_addr = dram_addr + hole_offset;
660
661 debugf1("using DHAR to translate DramAddr 0x%lx to "
662 "SysAddr 0x%lx\n", (unsigned long)dram_addr,
663 (unsigned long)sys_addr);
664
665 return sys_addr;
666 }
667 }
668
669 base = get_dram_base(pvt, pvt->mc_node_id);
670 sys_addr = dram_addr + base;
671
672 /*
673 * The sys_addr we have computed up to this point is a 40-bit value
674 * because the k8 deals with 40-bit values. However, the value we are
675 * supposed to return is a full 64-bit physical address. The AMD
676 * x86-64 architecture specifies that the most significant implemented
677 * address bit through bit 63 of a physical address must be either all
678 * 0s or all 1s. Therefore we sign-extend the 40-bit sys_addr to a
679 * 64-bit value below. See section 3.4.2 of AMD publication 24592:
680 * AMD x86-64 Architecture Programmer's Manual Volume 1 Application
681 * Programming.
682 */
683 sys_addr |= ~((sys_addr & (1ull << 39)) - 1);
684
685 debugf1(" Node %d, DramAddr 0x%lx to SysAddr 0x%lx\n",
686 pvt->mc_node_id, (unsigned long)dram_addr,
687 (unsigned long)sys_addr);
688
689 return sys_addr;
690 }
691
692 /*
693 * @input_addr is an InputAddr associated with the node given by mci. Translate
694 * @input_addr to a SysAddr.
695 */
696 static inline u64 input_addr_to_sys_addr(struct mem_ctl_info *mci,
697 u64 input_addr)
698 {
699 return dram_addr_to_sys_addr(mci,
700 input_addr_to_dram_addr(mci, input_addr));
701 }
702
703 /*
704 * Find the minimum and maximum InputAddr values that map to the given @csrow.
705 * Pass back these values in *input_addr_min and *input_addr_max.
706 */
707 static void find_csrow_limits(struct mem_ctl_info *mci, int csrow,
708 u64 *input_addr_min, u64 *input_addr_max)
709 {
710 struct amd64_pvt *pvt;
711 u64 base, mask;
712
713 pvt = mci->pvt_info;
714 BUG_ON((csrow < 0) || (csrow >= pvt->csels[0].b_cnt));
715
716 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
717
718 *input_addr_min = base & ~mask;
719 *input_addr_max = base | mask;
720 }
721
722 /* Map the Error address to a PAGE and PAGE OFFSET. */
723 static inline void error_address_to_page_and_offset(u64 error_address,
724 u32 *page, u32 *offset)
725 {
726 *page = (u32) (error_address >> PAGE_SHIFT);
727 *offset = ((u32) error_address) & ~PAGE_MASK;
728 }
729
730 /*
731 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
732 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
733 * of a node that detected an ECC memory error. mci represents the node that
734 * the error address maps to (possibly different from the node that detected
735 * the error). Return the number of the csrow that sys_addr maps to, or -1 on
736 * error.
737 */
738 static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
739 {
740 int csrow;
741
742 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
743
744 if (csrow == -1)
745 amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
746 "address 0x%lx\n", (unsigned long)sys_addr);
747 return csrow;
748 }
749
750 static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
751
752 /*
753 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
754 * are ECC capable.
755 */
756 static enum edac_type amd64_determine_edac_cap(struct amd64_pvt *pvt)
757 {
758 u8 bit;
759 enum dev_type edac_cap = EDAC_FLAG_NONE;
760
761 bit = (boot_cpu_data.x86 > 0xf || pvt->ext_model >= K8_REV_F)
762 ? 19
763 : 17;
764
765 if (pvt->dclr0 & BIT(bit))
766 edac_cap = EDAC_FLAG_SECDED;
767
768 return edac_cap;
769 }
770
771 static void amd64_debug_display_dimm_sizes(struct amd64_pvt *, u8);
772
773 static void amd64_dump_dramcfg_low(u32 dclr, int chan)
774 {
775 debugf1("F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
776
777 debugf1(" DIMM type: %sbuffered; all DIMMs support ECC: %s\n",
778 (dclr & BIT(16)) ? "un" : "",
779 (dclr & BIT(19)) ? "yes" : "no");
780
781 debugf1(" PAR/ERR parity: %s\n",
782 (dclr & BIT(8)) ? "enabled" : "disabled");
783
784 if (boot_cpu_data.x86 == 0x10)
785 debugf1(" DCT 128bit mode width: %s\n",
786 (dclr & BIT(11)) ? "128b" : "64b");
787
788 debugf1(" x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
789 (dclr & BIT(12)) ? "yes" : "no",
790 (dclr & BIT(13)) ? "yes" : "no",
791 (dclr & BIT(14)) ? "yes" : "no",
792 (dclr & BIT(15)) ? "yes" : "no");
793 }
794
795 /* Display and decode various NB registers for debug purposes. */
796 static void dump_misc_regs(struct amd64_pvt *pvt)
797 {
798 debugf1("F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
799
800 debugf1(" NB two channel DRAM capable: %s\n",
801 (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
802
803 debugf1(" ECC capable: %s, ChipKill ECC capable: %s\n",
804 (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
805 (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
806
807 amd64_dump_dramcfg_low(pvt->dclr0, 0);
808
809 debugf1("F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
810
811 debugf1("F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, "
812 "offset: 0x%08x\n",
813 pvt->dhar, dhar_base(pvt),
814 (boot_cpu_data.x86 == 0xf) ? k8_dhar_offset(pvt)
815 : f10_dhar_offset(pvt));
816
817 debugf1(" DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
818
819 amd64_debug_display_dimm_sizes(pvt, 0);
820
821 /* everything below this point is Fam10h and above */
822 if (boot_cpu_data.x86 == 0xf)
823 return;
824
825 amd64_debug_display_dimm_sizes(pvt, 1);
826
827 amd64_info("using %s syndromes.\n", ((pvt->ecc_sym_sz == 8) ? "x8" : "x4"));
828
829 /* Only if NOT ganged does dclr1 have valid info */
830 if (!dct_ganging_enabled(pvt))
831 amd64_dump_dramcfg_low(pvt->dclr1, 1);
832 }
833
834 /*
835 * see BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
836 */
837 static void prep_chip_selects(struct amd64_pvt *pvt)
838 {
839 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) {
840 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
841 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
842 } else {
843 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
844 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
845 }
846 }
847
848 /*
849 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
850 */
851 static void read_dct_base_mask(struct amd64_pvt *pvt)
852 {
853 int cs;
854
855 prep_chip_selects(pvt);
856
857 for_each_chip_select(cs, 0, pvt) {
858 int reg0 = DCSB0 + (cs * 4);
859 int reg1 = DCSB1 + (cs * 4);
860 u32 *base0 = &pvt->csels[0].csbases[cs];
861 u32 *base1 = &pvt->csels[1].csbases[cs];
862
863 if (!amd64_read_dct_pci_cfg(pvt, reg0, base0))
864 debugf0(" DCSB0[%d]=0x%08x reg: F2x%x\n",
865 cs, *base0, reg0);
866
867 if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt))
868 continue;
869
870 if (!amd64_read_dct_pci_cfg(pvt, reg1, base1))
871 debugf0(" DCSB1[%d]=0x%08x reg: F2x%x\n",
872 cs, *base1, reg1);
873 }
874
875 for_each_chip_select_mask(cs, 0, pvt) {
876 int reg0 = DCSM0 + (cs * 4);
877 int reg1 = DCSM1 + (cs * 4);
878 u32 *mask0 = &pvt->csels[0].csmasks[cs];
879 u32 *mask1 = &pvt->csels[1].csmasks[cs];
880
881 if (!amd64_read_dct_pci_cfg(pvt, reg0, mask0))
882 debugf0(" DCSM0[%d]=0x%08x reg: F2x%x\n",
883 cs, *mask0, reg0);
884
885 if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt))
886 continue;
887
888 if (!amd64_read_dct_pci_cfg(pvt, reg1, mask1))
889 debugf0(" DCSM1[%d]=0x%08x reg: F2x%x\n",
890 cs, *mask1, reg1);
891 }
892 }
893
894 static enum mem_type amd64_determine_memory_type(struct amd64_pvt *pvt, int cs)
895 {
896 enum mem_type type;
897
898 /* F15h supports only DDR3 */
899 if (boot_cpu_data.x86 >= 0x15)
900 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
901 else if (boot_cpu_data.x86 == 0x10 || pvt->ext_model >= K8_REV_F) {
902 if (pvt->dchr0 & DDR3_MODE)
903 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
904 else
905 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
906 } else {
907 type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
908 }
909
910 amd64_info("CS%d: %s\n", cs, edac_mem_types[type]);
911
912 return type;
913 }
914
915 /* Get the number of DCT channels the memory controller is using. */
916 static int k8_early_channel_count(struct amd64_pvt *pvt)
917 {
918 int flag;
919
920 if (pvt->ext_model >= K8_REV_F)
921 /* RevF (NPT) and later */
922 flag = pvt->dclr0 & WIDTH_128;
923 else
924 /* RevE and earlier */
925 flag = pvt->dclr0 & REVE_WIDTH_128;
926
927 /* not used */
928 pvt->dclr1 = 0;
929
930 return (flag) ? 2 : 1;
931 }
932
933 /* On F10h and later ErrAddr is MC4_ADDR[47:1] */
934 static u64 get_error_address(struct mce *m)
935 {
936 u8 start_bit = 1;
937 u8 end_bit = 47;
938
939 if (boot_cpu_data.x86 == 0xf) {
940 start_bit = 3;
941 end_bit = 39;
942 }
943
944 return m->addr & GENMASK(start_bit, end_bit);
945 }
946
947 static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
948 {
949 int off = range << 3;
950
951 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo);
952 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
953
954 if (boot_cpu_data.x86 == 0xf)
955 return;
956
957 if (!dram_rw(pvt, range))
958 return;
959
960 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi);
961 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
962 }
963
964 static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
965 u16 syndrome)
966 {
967 struct mem_ctl_info *src_mci;
968 struct amd64_pvt *pvt = mci->pvt_info;
969 int channel, csrow;
970 u32 page, offset;
971
972 /* CHIPKILL enabled */
973 if (pvt->nbcfg & NBCFG_CHIPKILL) {
974 channel = get_channel_from_ecc_syndrome(mci, syndrome);
975 if (channel < 0) {
976 /*
977 * Syndrome didn't map, so we don't know which of the
978 * 2 DIMMs is in error. So we need to ID 'both' of them
979 * as suspect.
980 */
981 amd64_mc_warn(mci, "unknown syndrome 0x%04x - possible "
982 "error reporting race\n", syndrome);
983 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
984 return;
985 }
986 } else {
987 /*
988 * non-chipkill ecc mode
989 *
990 * The k8 documentation is unclear about how to determine the
991 * channel number when using non-chipkill memory. This method
992 * was obtained from email communication with someone at AMD.
993 * (Wish the email was placed in this comment - norsk)
994 */
995 channel = ((sys_addr & BIT(3)) != 0);
996 }
997
998 /*
999 * Find out which node the error address belongs to. This may be
1000 * different from the node that detected the error.
1001 */
1002 src_mci = find_mc_by_sys_addr(mci, sys_addr);
1003 if (!src_mci) {
1004 amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
1005 (unsigned long)sys_addr);
1006 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1007 return;
1008 }
1009
1010 /* Now map the sys_addr to a CSROW */
1011 csrow = sys_addr_to_csrow(src_mci, sys_addr);
1012 if (csrow < 0) {
1013 edac_mc_handle_ce_no_info(src_mci, EDAC_MOD_STR);
1014 } else {
1015 error_address_to_page_and_offset(sys_addr, &page, &offset);
1016
1017 edac_mc_handle_ce(src_mci, page, offset, syndrome, csrow,
1018 channel, EDAC_MOD_STR);
1019 }
1020 }
1021
1022 static int ddr2_cs_size(unsigned i, bool dct_width)
1023 {
1024 unsigned shift = 0;
1025
1026 if (i <= 2)
1027 shift = i;
1028 else if (!(i & 0x1))
1029 shift = i >> 1;
1030 else
1031 shift = (i + 1) >> 1;
1032
1033 return 128 << (shift + !!dct_width);
1034 }
1035
1036 static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1037 unsigned cs_mode)
1038 {
1039 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1040
1041 if (pvt->ext_model >= K8_REV_F) {
1042 WARN_ON(cs_mode > 11);
1043 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1044 }
1045 else if (pvt->ext_model >= K8_REV_D) {
1046 WARN_ON(cs_mode > 10);
1047
1048 if (cs_mode == 3 || cs_mode == 8)
1049 return 32 << (cs_mode - 1);
1050 else
1051 return 32 << cs_mode;
1052 }
1053 else {
1054 WARN_ON(cs_mode > 6);
1055 return 32 << cs_mode;
1056 }
1057 }
1058
1059 /*
1060 * Get the number of DCT channels in use.
1061 *
1062 * Return:
1063 * number of Memory Channels in operation
1064 * Pass back:
1065 * contents of the DCL0_LOW register
1066 */
1067 static int f1x_early_channel_count(struct amd64_pvt *pvt)
1068 {
1069 int i, j, channels = 0;
1070
1071 /* On F10h, if we are in 128 bit mode, then we are using 2 channels */
1072 if (boot_cpu_data.x86 == 0x10 && (pvt->dclr0 & WIDTH_128))
1073 return 2;
1074
1075 /*
1076 * Need to check if in unganged mode: In such, there are 2 channels,
1077 * but they are not in 128 bit mode and thus the above 'dclr0' status
1078 * bit will be OFF.
1079 *
1080 * Need to check DCT0[0] and DCT1[0] to see if only one of them has
1081 * their CSEnable bit on. If so, then SINGLE DIMM case.
1082 */
1083 debugf0("Data width is not 128 bits - need more decoding\n");
1084
1085 /*
1086 * Check DRAM Bank Address Mapping values for each DIMM to see if there
1087 * is more than just one DIMM present in unganged mode. Need to check
1088 * both controllers since DIMMs can be placed in either one.
1089 */
1090 for (i = 0; i < 2; i++) {
1091 u32 dbam = (i ? pvt->dbam1 : pvt->dbam0);
1092
1093 for (j = 0; j < 4; j++) {
1094 if (DBAM_DIMM(j, dbam) > 0) {
1095 channels++;
1096 break;
1097 }
1098 }
1099 }
1100
1101 if (channels > 2)
1102 channels = 2;
1103
1104 amd64_info("MCT channel count: %d\n", channels);
1105
1106 return channels;
1107 }
1108
1109 static int ddr3_cs_size(unsigned i, bool dct_width)
1110 {
1111 unsigned shift = 0;
1112 int cs_size = 0;
1113
1114 if (i == 0 || i == 3 || i == 4)
1115 cs_size = -1;
1116 else if (i <= 2)
1117 shift = i;
1118 else if (i == 12)
1119 shift = 7;
1120 else if (!(i & 0x1))
1121 shift = i >> 1;
1122 else
1123 shift = (i + 1) >> 1;
1124
1125 if (cs_size != -1)
1126 cs_size = (128 * (1 << !!dct_width)) << shift;
1127
1128 return cs_size;
1129 }
1130
1131 static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1132 unsigned cs_mode)
1133 {
1134 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1135
1136 WARN_ON(cs_mode > 11);
1137
1138 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
1139 return ddr3_cs_size(cs_mode, dclr & WIDTH_128);
1140 else
1141 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1142 }
1143
1144 /*
1145 * F15h supports only 64bit DCT interfaces
1146 */
1147 static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1148 unsigned cs_mode)
1149 {
1150 WARN_ON(cs_mode > 12);
1151
1152 return ddr3_cs_size(cs_mode, false);
1153 }
1154
1155 static void read_dram_ctl_register(struct amd64_pvt *pvt)
1156 {
1157
1158 if (boot_cpu_data.x86 == 0xf)
1159 return;
1160
1161 if (!amd64_read_dct_pci_cfg(pvt, DCT_SEL_LO, &pvt->dct_sel_lo)) {
1162 debugf0("F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
1163 pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
1164
1165 debugf0(" DCTs operate in %s mode.\n",
1166 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"));
1167
1168 if (!dct_ganging_enabled(pvt))
1169 debugf0(" Address range split per DCT: %s\n",
1170 (dct_high_range_enabled(pvt) ? "yes" : "no"));
1171
1172 debugf0(" data interleave for ECC: %s, "
1173 "DRAM cleared since last warm reset: %s\n",
1174 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
1175 (dct_memory_cleared(pvt) ? "yes" : "no"));
1176
1177 debugf0(" channel interleave: %s, "
1178 "interleave bits selector: 0x%x\n",
1179 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
1180 dct_sel_interleave_addr(pvt));
1181 }
1182
1183 amd64_read_dct_pci_cfg(pvt, DCT_SEL_HI, &pvt->dct_sel_hi);
1184 }
1185
1186 /*
1187 * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
1188 * Interleaving Modes.
1189 */
1190 static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1191 bool hi_range_sel, u8 intlv_en)
1192 {
1193 u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
1194
1195 if (dct_ganging_enabled(pvt))
1196 return 0;
1197
1198 if (hi_range_sel)
1199 return dct_sel_high;
1200
1201 /*
1202 * see F2x110[DctSelIntLvAddr] - channel interleave mode
1203 */
1204 if (dct_interleave_enabled(pvt)) {
1205 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1206
1207 /* return DCT select function: 0=DCT0, 1=DCT1 */
1208 if (!intlv_addr)
1209 return sys_addr >> 6 & 1;
1210
1211 if (intlv_addr & 0x2) {
1212 u8 shift = intlv_addr & 0x1 ? 9 : 6;
1213 u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2;
1214
1215 return ((sys_addr >> shift) & 1) ^ temp;
1216 }
1217
1218 return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
1219 }
1220
1221 if (dct_high_range_enabled(pvt))
1222 return ~dct_sel_high & 1;
1223
1224 return 0;
1225 }
1226
1227 /* Convert the sys_addr to the normalized DCT address */
1228 static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, unsigned range,
1229 u64 sys_addr, bool hi_rng,
1230 u32 dct_sel_base_addr)
1231 {
1232 u64 chan_off;
1233 u64 dram_base = get_dram_base(pvt, range);
1234 u64 hole_off = f10_dhar_offset(pvt);
1235 u64 dct_sel_base_off = (pvt->dct_sel_hi & 0xFFFFFC00) << 16;
1236
1237 if (hi_rng) {
1238 /*
1239 * if
1240 * base address of high range is below 4Gb
1241 * (bits [47:27] at [31:11])
1242 * DRAM address space on this DCT is hoisted above 4Gb &&
1243 * sys_addr > 4Gb
1244 *
1245 * remove hole offset from sys_addr
1246 * else
1247 * remove high range offset from sys_addr
1248 */
1249 if ((!(dct_sel_base_addr >> 16) ||
1250 dct_sel_base_addr < dhar_base(pvt)) &&
1251 dhar_valid(pvt) &&
1252 (sys_addr >= BIT_64(32)))
1253 chan_off = hole_off;
1254 else
1255 chan_off = dct_sel_base_off;
1256 } else {
1257 /*
1258 * if
1259 * we have a valid hole &&
1260 * sys_addr > 4Gb
1261 *
1262 * remove hole
1263 * else
1264 * remove dram base to normalize to DCT address
1265 */
1266 if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))
1267 chan_off = hole_off;
1268 else
1269 chan_off = dram_base;
1270 }
1271
1272 return (sys_addr & GENMASK(6,47)) - (chan_off & GENMASK(23,47));
1273 }
1274
1275 /*
1276 * checks if the csrow passed in is marked as SPARED, if so returns the new
1277 * spare row
1278 */
1279 static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
1280 {
1281 int tmp_cs;
1282
1283 if (online_spare_swap_done(pvt, dct) &&
1284 csrow == online_spare_bad_dramcs(pvt, dct)) {
1285
1286 for_each_chip_select(tmp_cs, dct, pvt) {
1287 if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
1288 csrow = tmp_cs;
1289 break;
1290 }
1291 }
1292 }
1293 return csrow;
1294 }
1295
1296 /*
1297 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
1298 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
1299 *
1300 * Return:
1301 * -EINVAL: NOT FOUND
1302 * 0..csrow = Chip-Select Row
1303 */
1304 static int f1x_lookup_addr_in_dct(u64 in_addr, u32 nid, u8 dct)
1305 {
1306 struct mem_ctl_info *mci;
1307 struct amd64_pvt *pvt;
1308 u64 cs_base, cs_mask;
1309 int cs_found = -EINVAL;
1310 int csrow;
1311
1312 mci = mcis[nid];
1313 if (!mci)
1314 return cs_found;
1315
1316 pvt = mci->pvt_info;
1317
1318 debugf1("input addr: 0x%llx, DCT: %d\n", in_addr, dct);
1319
1320 for_each_chip_select(csrow, dct, pvt) {
1321 if (!csrow_enabled(csrow, dct, pvt))
1322 continue;
1323
1324 get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
1325
1326 debugf1(" CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
1327 csrow, cs_base, cs_mask);
1328
1329 cs_mask = ~cs_mask;
1330
1331 debugf1(" (InputAddr & ~CSMask)=0x%llx "
1332 "(CSBase & ~CSMask)=0x%llx\n",
1333 (in_addr & cs_mask), (cs_base & cs_mask));
1334
1335 if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
1336 cs_found = f10_process_possible_spare(pvt, dct, csrow);
1337
1338 debugf1(" MATCH csrow=%d\n", cs_found);
1339 break;
1340 }
1341 }
1342 return cs_found;
1343 }
1344
1345 /*
1346 * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is
1347 * swapped with a region located at the bottom of memory so that the GPU can use
1348 * the interleaved region and thus two channels.
1349 */
1350 static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)
1351 {
1352 u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;
1353
1354 if (boot_cpu_data.x86 == 0x10) {
1355 /* only revC3 and revE have that feature */
1356 if (boot_cpu_data.x86_model < 4 ||
1357 (boot_cpu_data.x86_model < 0xa &&
1358 boot_cpu_data.x86_mask < 3))
1359 return sys_addr;
1360 }
1361
1362 amd64_read_dct_pci_cfg(pvt, SWAP_INTLV_REG, &swap_reg);
1363
1364 if (!(swap_reg & 0x1))
1365 return sys_addr;
1366
1367 swap_base = (swap_reg >> 3) & 0x7f;
1368 swap_limit = (swap_reg >> 11) & 0x7f;
1369 rgn_size = (swap_reg >> 20) & 0x7f;
1370 tmp_addr = sys_addr >> 27;
1371
1372 if (!(sys_addr >> 34) &&
1373 (((tmp_addr >= swap_base) &&
1374 (tmp_addr <= swap_limit)) ||
1375 (tmp_addr < rgn_size)))
1376 return sys_addr ^ (u64)swap_base << 27;
1377
1378 return sys_addr;
1379 }
1380
1381 /* For a given @dram_range, check if @sys_addr falls within it. */
1382 static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
1383 u64 sys_addr, int *nid, int *chan_sel)
1384 {
1385 int cs_found = -EINVAL;
1386 u64 chan_addr;
1387 u32 dct_sel_base;
1388 u8 channel;
1389 bool high_range = false;
1390
1391 u8 node_id = dram_dst_node(pvt, range);
1392 u8 intlv_en = dram_intlv_en(pvt, range);
1393 u32 intlv_sel = dram_intlv_sel(pvt, range);
1394
1395 debugf1("(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1396 range, sys_addr, get_dram_limit(pvt, range));
1397
1398 if (dhar_valid(pvt) &&
1399 dhar_base(pvt) <= sys_addr &&
1400 sys_addr < BIT_64(32)) {
1401 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1402 sys_addr);
1403 return -EINVAL;
1404 }
1405
1406 if (intlv_en &&
1407 (intlv_sel != ((sys_addr >> 12) & intlv_en))) {
1408 amd64_warn("Botched intlv bits, en: 0x%x, sel: 0x%x\n",
1409 intlv_en, intlv_sel);
1410 return -EINVAL;
1411 }
1412
1413 sys_addr = f1x_swap_interleaved_region(pvt, sys_addr);
1414
1415 dct_sel_base = dct_sel_baseaddr(pvt);
1416
1417 /*
1418 * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
1419 * select between DCT0 and DCT1.
1420 */
1421 if (dct_high_range_enabled(pvt) &&
1422 !dct_ganging_enabled(pvt) &&
1423 ((sys_addr >> 27) >= (dct_sel_base >> 11)))
1424 high_range = true;
1425
1426 channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en);
1427
1428 chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr,
1429 high_range, dct_sel_base);
1430
1431 /* Remove node interleaving, see F1x120 */
1432 if (intlv_en)
1433 chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) |
1434 (chan_addr & 0xfff);
1435
1436 /* remove channel interleave */
1437 if (dct_interleave_enabled(pvt) &&
1438 !dct_high_range_enabled(pvt) &&
1439 !dct_ganging_enabled(pvt)) {
1440
1441 if (dct_sel_interleave_addr(pvt) != 1) {
1442 if (dct_sel_interleave_addr(pvt) == 0x3)
1443 /* hash 9 */
1444 chan_addr = ((chan_addr >> 10) << 9) |
1445 (chan_addr & 0x1ff);
1446 else
1447 /* A[6] or hash 6 */
1448 chan_addr = ((chan_addr >> 7) << 6) |
1449 (chan_addr & 0x3f);
1450 } else
1451 /* A[12] */
1452 chan_addr = ((chan_addr >> 13) << 12) |
1453 (chan_addr & 0xfff);
1454 }
1455
1456 debugf1(" Normalized DCT addr: 0x%llx\n", chan_addr);
1457
1458 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel);
1459
1460 if (cs_found >= 0) {
1461 *nid = node_id;
1462 *chan_sel = channel;
1463 }
1464 return cs_found;
1465 }
1466
1467 static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt, u64 sys_addr,
1468 int *node, int *chan_sel)
1469 {
1470 int cs_found = -EINVAL;
1471 unsigned range;
1472
1473 for (range = 0; range < DRAM_RANGES; range++) {
1474
1475 if (!dram_rw(pvt, range))
1476 continue;
1477
1478 if ((get_dram_base(pvt, range) <= sys_addr) &&
1479 (get_dram_limit(pvt, range) >= sys_addr)) {
1480
1481 cs_found = f1x_match_to_this_node(pvt, range,
1482 sys_addr, node,
1483 chan_sel);
1484 if (cs_found >= 0)
1485 break;
1486 }
1487 }
1488 return cs_found;
1489 }
1490
1491 /*
1492 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
1493 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
1494 *
1495 * The @sys_addr is usually an error address received from the hardware
1496 * (MCX_ADDR).
1497 */
1498 static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
1499 u16 syndrome)
1500 {
1501 struct amd64_pvt *pvt = mci->pvt_info;
1502 u32 page, offset;
1503 int nid, csrow, chan = 0;
1504
1505 csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &nid, &chan);
1506
1507 if (csrow < 0) {
1508 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1509 return;
1510 }
1511
1512 error_address_to_page_and_offset(sys_addr, &page, &offset);
1513
1514 /*
1515 * We need the syndromes for channel detection only when we're
1516 * ganged. Otherwise @chan should already contain the channel at
1517 * this point.
1518 */
1519 if (dct_ganging_enabled(pvt))
1520 chan = get_channel_from_ecc_syndrome(mci, syndrome);
1521
1522 if (chan >= 0)
1523 edac_mc_handle_ce(mci, page, offset, syndrome, csrow, chan,
1524 EDAC_MOD_STR);
1525 else
1526 /*
1527 * Channel unknown, report all channels on this CSROW as failed.
1528 */
1529 for (chan = 0; chan < mci->csrows[csrow].nr_channels; chan++)
1530 edac_mc_handle_ce(mci, page, offset, syndrome,
1531 csrow, chan, EDAC_MOD_STR);
1532 }
1533
1534 /*
1535 * debug routine to display the memory sizes of all logical DIMMs and its
1536 * CSROWs
1537 */
1538 static void amd64_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
1539 {
1540 int dimm, size0, size1, factor = 0;
1541 u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
1542 u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0;
1543
1544 if (boot_cpu_data.x86 == 0xf) {
1545 if (pvt->dclr0 & WIDTH_128)
1546 factor = 1;
1547
1548 /* K8 families < revF not supported yet */
1549 if (pvt->ext_model < K8_REV_F)
1550 return;
1551 else
1552 WARN_ON(ctrl != 0);
1553 }
1554
1555 dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1 : pvt->dbam0;
1556 dcsb = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->csels[1].csbases
1557 : pvt->csels[0].csbases;
1558
1559 debugf1("F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n", ctrl, dbam);
1560
1561 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
1562
1563 /* Dump memory sizes for DIMM and its CSROWs */
1564 for (dimm = 0; dimm < 4; dimm++) {
1565
1566 size0 = 0;
1567 if (dcsb[dimm*2] & DCSB_CS_ENABLE)
1568 size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
1569 DBAM_DIMM(dimm, dbam));
1570
1571 size1 = 0;
1572 if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
1573 size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
1574 DBAM_DIMM(dimm, dbam));
1575
1576 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
1577 dimm * 2, size0 << factor,
1578 dimm * 2 + 1, size1 << factor);
1579 }
1580 }
1581
1582 static struct amd64_family_type amd64_family_types[] = {
1583 [K8_CPUS] = {
1584 .ctl_name = "K8",
1585 .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
1586 .f3_id = PCI_DEVICE_ID_AMD_K8_NB_MISC,
1587 .ops = {
1588 .early_channel_count = k8_early_channel_count,
1589 .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow,
1590 .dbam_to_cs = k8_dbam_to_chip_select,
1591 .read_dct_pci_cfg = k8_read_dct_pci_cfg,
1592 }
1593 },
1594 [F10_CPUS] = {
1595 .ctl_name = "F10h",
1596 .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
1597 .f3_id = PCI_DEVICE_ID_AMD_10H_NB_MISC,
1598 .ops = {
1599 .early_channel_count = f1x_early_channel_count,
1600 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1601 .dbam_to_cs = f10_dbam_to_chip_select,
1602 .read_dct_pci_cfg = f10_read_dct_pci_cfg,
1603 }
1604 },
1605 [F15_CPUS] = {
1606 .ctl_name = "F15h",
1607 .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1,
1608 .f3_id = PCI_DEVICE_ID_AMD_15H_NB_F3,
1609 .ops = {
1610 .early_channel_count = f1x_early_channel_count,
1611 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1612 .dbam_to_cs = f15_dbam_to_chip_select,
1613 .read_dct_pci_cfg = f15_read_dct_pci_cfg,
1614 }
1615 },
1616 };
1617
1618 static struct pci_dev *pci_get_related_function(unsigned int vendor,
1619 unsigned int device,
1620 struct pci_dev *related)
1621 {
1622 struct pci_dev *dev = NULL;
1623
1624 dev = pci_get_device(vendor, device, dev);
1625 while (dev) {
1626 if ((dev->bus->number == related->bus->number) &&
1627 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
1628 break;
1629 dev = pci_get_device(vendor, device, dev);
1630 }
1631
1632 return dev;
1633 }
1634
1635 /*
1636 * These are tables of eigenvectors (one per line) which can be used for the
1637 * construction of the syndrome tables. The modified syndrome search algorithm
1638 * uses those to find the symbol in error and thus the DIMM.
1639 *
1640 * Algorithm courtesy of Ross LaFetra from AMD.
1641 */
1642 static u16 x4_vectors[] = {
1643 0x2f57, 0x1afe, 0x66cc, 0xdd88,
1644 0x11eb, 0x3396, 0x7f4c, 0xeac8,
1645 0x0001, 0x0002, 0x0004, 0x0008,
1646 0x1013, 0x3032, 0x4044, 0x8088,
1647 0x106b, 0x30d6, 0x70fc, 0xe0a8,
1648 0x4857, 0xc4fe, 0x13cc, 0x3288,
1649 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
1650 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
1651 0x15c1, 0x2a42, 0x89ac, 0x4758,
1652 0x2b03, 0x1602, 0x4f0c, 0xca08,
1653 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
1654 0x8ba7, 0x465e, 0x244c, 0x1cc8,
1655 0x2b87, 0x164e, 0x642c, 0xdc18,
1656 0x40b9, 0x80de, 0x1094, 0x20e8,
1657 0x27db, 0x1eb6, 0x9dac, 0x7b58,
1658 0x11c1, 0x2242, 0x84ac, 0x4c58,
1659 0x1be5, 0x2d7a, 0x5e34, 0xa718,
1660 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
1661 0x4c97, 0xc87e, 0x11fc, 0x33a8,
1662 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
1663 0x16b3, 0x3d62, 0x4f34, 0x8518,
1664 0x1e2f, 0x391a, 0x5cac, 0xf858,
1665 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
1666 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
1667 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
1668 0x4397, 0xc27e, 0x17fc, 0x3ea8,
1669 0x1617, 0x3d3e, 0x6464, 0xb8b8,
1670 0x23ff, 0x12aa, 0xab6c, 0x56d8,
1671 0x2dfb, 0x1ba6, 0x913c, 0x7328,
1672 0x185d, 0x2ca6, 0x7914, 0x9e28,
1673 0x171b, 0x3e36, 0x7d7c, 0xebe8,
1674 0x4199, 0x82ee, 0x19f4, 0x2e58,
1675 0x4807, 0xc40e, 0x130c, 0x3208,
1676 0x1905, 0x2e0a, 0x5804, 0xac08,
1677 0x213f, 0x132a, 0xadfc, 0x5ba8,
1678 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
1679 };
1680
1681 static u16 x8_vectors[] = {
1682 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
1683 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
1684 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
1685 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
1686 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
1687 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
1688 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
1689 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
1690 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
1691 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
1692 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
1693 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
1694 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
1695 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
1696 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
1697 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
1698 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
1699 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
1700 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
1701 };
1702
1703 static int decode_syndrome(u16 syndrome, u16 *vectors, unsigned num_vecs,
1704 unsigned v_dim)
1705 {
1706 unsigned int i, err_sym;
1707
1708 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
1709 u16 s = syndrome;
1710 unsigned v_idx = err_sym * v_dim;
1711 unsigned v_end = (err_sym + 1) * v_dim;
1712
1713 /* walk over all 16 bits of the syndrome */
1714 for (i = 1; i < (1U << 16); i <<= 1) {
1715
1716 /* if bit is set in that eigenvector... */
1717 if (v_idx < v_end && vectors[v_idx] & i) {
1718 u16 ev_comp = vectors[v_idx++];
1719
1720 /* ... and bit set in the modified syndrome, */
1721 if (s & i) {
1722 /* remove it. */
1723 s ^= ev_comp;
1724
1725 if (!s)
1726 return err_sym;
1727 }
1728
1729 } else if (s & i)
1730 /* can't get to zero, move to next symbol */
1731 break;
1732 }
1733 }
1734
1735 debugf0("syndrome(%x) not found\n", syndrome);
1736 return -1;
1737 }
1738
1739 static int map_err_sym_to_channel(int err_sym, int sym_size)
1740 {
1741 if (sym_size == 4)
1742 switch (err_sym) {
1743 case 0x20:
1744 case 0x21:
1745 return 0;
1746 break;
1747 case 0x22:
1748 case 0x23:
1749 return 1;
1750 break;
1751 default:
1752 return err_sym >> 4;
1753 break;
1754 }
1755 /* x8 symbols */
1756 else
1757 switch (err_sym) {
1758 /* imaginary bits not in a DIMM */
1759 case 0x10:
1760 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
1761 err_sym);
1762 return -1;
1763 break;
1764
1765 case 0x11:
1766 return 0;
1767 break;
1768 case 0x12:
1769 return 1;
1770 break;
1771 default:
1772 return err_sym >> 3;
1773 break;
1774 }
1775 return -1;
1776 }
1777
1778 static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
1779 {
1780 struct amd64_pvt *pvt = mci->pvt_info;
1781 int err_sym = -1;
1782
1783 if (pvt->ecc_sym_sz == 8)
1784 err_sym = decode_syndrome(syndrome, x8_vectors,
1785 ARRAY_SIZE(x8_vectors),
1786 pvt->ecc_sym_sz);
1787 else if (pvt->ecc_sym_sz == 4)
1788 err_sym = decode_syndrome(syndrome, x4_vectors,
1789 ARRAY_SIZE(x4_vectors),
1790 pvt->ecc_sym_sz);
1791 else {
1792 amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz);
1793 return err_sym;
1794 }
1795
1796 return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
1797 }
1798
1799 /*
1800 * Handle any Correctable Errors (CEs) that have occurred. Check for valid ERROR
1801 * ADDRESS and process.
1802 */
1803 static void amd64_handle_ce(struct mem_ctl_info *mci, struct mce *m)
1804 {
1805 struct amd64_pvt *pvt = mci->pvt_info;
1806 u64 sys_addr;
1807 u16 syndrome;
1808
1809 /* Ensure that the Error Address is VALID */
1810 if (!(m->status & MCI_STATUS_ADDRV)) {
1811 amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n");
1812 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1813 return;
1814 }
1815
1816 sys_addr = get_error_address(m);
1817 syndrome = extract_syndrome(m->status);
1818
1819 amd64_mc_err(mci, "CE ERROR_ADDRESS= 0x%llx\n", sys_addr);
1820
1821 pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, syndrome);
1822 }
1823
1824 /* Handle any Un-correctable Errors (UEs) */
1825 static void amd64_handle_ue(struct mem_ctl_info *mci, struct mce *m)
1826 {
1827 struct mem_ctl_info *log_mci, *src_mci = NULL;
1828 int csrow;
1829 u64 sys_addr;
1830 u32 page, offset;
1831
1832 log_mci = mci;
1833
1834 if (!(m->status & MCI_STATUS_ADDRV)) {
1835 amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n");
1836 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1837 return;
1838 }
1839
1840 sys_addr = get_error_address(m);
1841
1842 /*
1843 * Find out which node the error address belongs to. This may be
1844 * different from the node that detected the error.
1845 */
1846 src_mci = find_mc_by_sys_addr(mci, sys_addr);
1847 if (!src_mci) {
1848 amd64_mc_err(mci, "ERROR ADDRESS (0x%lx) NOT mapped to a MC\n",
1849 (unsigned long)sys_addr);
1850 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1851 return;
1852 }
1853
1854 log_mci = src_mci;
1855
1856 csrow = sys_addr_to_csrow(log_mci, sys_addr);
1857 if (csrow < 0) {
1858 amd64_mc_err(mci, "ERROR_ADDRESS (0x%lx) NOT mapped to CS\n",
1859 (unsigned long)sys_addr);
1860 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1861 } else {
1862 error_address_to_page_and_offset(sys_addr, &page, &offset);
1863 edac_mc_handle_ue(log_mci, page, offset, csrow, EDAC_MOD_STR);
1864 }
1865 }
1866
1867 static inline void __amd64_decode_bus_error(struct mem_ctl_info *mci,
1868 struct mce *m)
1869 {
1870 u16 ec = EC(m->status);
1871 u8 xec = XEC(m->status, 0x1f);
1872 u8 ecc_type = (m->status >> 45) & 0x3;
1873
1874 /* Bail early out if this was an 'observed' error */
1875 if (PP(ec) == NBSL_PP_OBS)
1876 return;
1877
1878 /* Do only ECC errors */
1879 if (xec && xec != F10_NBSL_EXT_ERR_ECC)
1880 return;
1881
1882 if (ecc_type == 2)
1883 amd64_handle_ce(mci, m);
1884 else if (ecc_type == 1)
1885 amd64_handle_ue(mci, m);
1886 }
1887
1888 void amd64_decode_bus_error(int node_id, struct mce *m, u32 nbcfg)
1889 {
1890 struct mem_ctl_info *mci = mcis[node_id];
1891
1892 __amd64_decode_bus_error(mci, m);
1893 }
1894
1895 /*
1896 * Use pvt->F2 which contains the F2 CPU PCI device to get the related
1897 * F1 (AddrMap) and F3 (Misc) devices. Return negative value on error.
1898 */
1899 static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f3_id)
1900 {
1901 /* Reserve the ADDRESS MAP Device */
1902 pvt->F1 = pci_get_related_function(pvt->F2->vendor, f1_id, pvt->F2);
1903 if (!pvt->F1) {
1904 amd64_err("error address map device not found: "
1905 "vendor %x device 0x%x (broken BIOS?)\n",
1906 PCI_VENDOR_ID_AMD, f1_id);
1907 return -ENODEV;
1908 }
1909
1910 /* Reserve the MISC Device */
1911 pvt->F3 = pci_get_related_function(pvt->F2->vendor, f3_id, pvt->F2);
1912 if (!pvt->F3) {
1913 pci_dev_put(pvt->F1);
1914 pvt->F1 = NULL;
1915
1916 amd64_err("error F3 device not found: "
1917 "vendor %x device 0x%x (broken BIOS?)\n",
1918 PCI_VENDOR_ID_AMD, f3_id);
1919
1920 return -ENODEV;
1921 }
1922 debugf1("F1: %s\n", pci_name(pvt->F1));
1923 debugf1("F2: %s\n", pci_name(pvt->F2));
1924 debugf1("F3: %s\n", pci_name(pvt->F3));
1925
1926 return 0;
1927 }
1928
1929 static void free_mc_sibling_devs(struct amd64_pvt *pvt)
1930 {
1931 pci_dev_put(pvt->F1);
1932 pci_dev_put(pvt->F3);
1933 }
1934
1935 /*
1936 * Retrieve the hardware registers of the memory controller (this includes the
1937 * 'Address Map' and 'Misc' device regs)
1938 */
1939 static void read_mc_regs(struct amd64_pvt *pvt)
1940 {
1941 struct cpuinfo_x86 *c = &boot_cpu_data;
1942 u64 msr_val;
1943 u32 tmp;
1944 unsigned range;
1945
1946 /*
1947 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
1948 * those are Read-As-Zero
1949 */
1950 rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
1951 debugf0(" TOP_MEM: 0x%016llx\n", pvt->top_mem);
1952
1953 /* check first whether TOP_MEM2 is enabled */
1954 rdmsrl(MSR_K8_SYSCFG, msr_val);
1955 if (msr_val & (1U << 21)) {
1956 rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
1957 debugf0(" TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
1958 } else
1959 debugf0(" TOP_MEM2 disabled.\n");
1960
1961 amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
1962
1963 read_dram_ctl_register(pvt);
1964
1965 for (range = 0; range < DRAM_RANGES; range++) {
1966 u8 rw;
1967
1968 /* read settings for this DRAM range */
1969 read_dram_base_limit_regs(pvt, range);
1970
1971 rw = dram_rw(pvt, range);
1972 if (!rw)
1973 continue;
1974
1975 debugf1(" DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
1976 range,
1977 get_dram_base(pvt, range),
1978 get_dram_limit(pvt, range));
1979
1980 debugf1(" IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
1981 dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
1982 (rw & 0x1) ? "R" : "-",
1983 (rw & 0x2) ? "W" : "-",
1984 dram_intlv_sel(pvt, range),
1985 dram_dst_node(pvt, range));
1986 }
1987
1988 read_dct_base_mask(pvt);
1989
1990 amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
1991 amd64_read_dct_pci_cfg(pvt, DBAM0, &pvt->dbam0);
1992
1993 amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
1994
1995 amd64_read_dct_pci_cfg(pvt, DCLR0, &pvt->dclr0);
1996 amd64_read_dct_pci_cfg(pvt, DCHR0, &pvt->dchr0);
1997
1998 if (!dct_ganging_enabled(pvt)) {
1999 amd64_read_dct_pci_cfg(pvt, DCLR1, &pvt->dclr1);
2000 amd64_read_dct_pci_cfg(pvt, DCHR1, &pvt->dchr1);
2001 }
2002
2003 pvt->ecc_sym_sz = 4;
2004
2005 if (c->x86 >= 0x10) {
2006 amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
2007 amd64_read_dct_pci_cfg(pvt, DBAM1, &pvt->dbam1);
2008
2009 /* F10h, revD and later can do x8 ECC too */
2010 if ((c->x86 > 0x10 || c->x86_model > 7) && tmp & BIT(25))
2011 pvt->ecc_sym_sz = 8;
2012 }
2013 dump_misc_regs(pvt);
2014 }
2015
2016 /*
2017 * NOTE: CPU Revision Dependent code
2018 *
2019 * Input:
2020 * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
2021 * k8 private pointer to -->
2022 * DRAM Bank Address mapping register
2023 * node_id
2024 * DCL register where dual_channel_active is
2025 *
2026 * The DBAM register consists of 4 sets of 4 bits each definitions:
2027 *
2028 * Bits: CSROWs
2029 * 0-3 CSROWs 0 and 1
2030 * 4-7 CSROWs 2 and 3
2031 * 8-11 CSROWs 4 and 5
2032 * 12-15 CSROWs 6 and 7
2033 *
2034 * Values range from: 0 to 15
2035 * The meaning of the values depends on CPU revision and dual-channel state,
2036 * see relevant BKDG more info.
2037 *
2038 * The memory controller provides for total of only 8 CSROWs in its current
2039 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
2040 * single channel or two (2) DIMMs in dual channel mode.
2041 *
2042 * The following code logic collapses the various tables for CSROW based on CPU
2043 * revision.
2044 *
2045 * Returns:
2046 * The number of PAGE_SIZE pages on the specified CSROW number it
2047 * encompasses
2048 *
2049 */
2050 static u32 amd64_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr)
2051 {
2052 u32 cs_mode, nr_pages;
2053
2054 /*
2055 * The math on this doesn't look right on the surface because x/2*4 can
2056 * be simplified to x*2 but this expression makes use of the fact that
2057 * it is integral math where 1/2=0. This intermediate value becomes the
2058 * number of bits to shift the DBAM register to extract the proper CSROW
2059 * field.
2060 */
2061 cs_mode = (pvt->dbam0 >> ((csrow_nr / 2) * 4)) & 0xF;
2062
2063 nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode) << (20 - PAGE_SHIFT);
2064
2065 /*
2066 * If dual channel then double the memory size of single channel.
2067 * Channel count is 1 or 2
2068 */
2069 nr_pages <<= (pvt->channel_count - 1);
2070
2071 debugf0(" (csrow=%d) DBAM map index= %d\n", csrow_nr, cs_mode);
2072 debugf0(" nr_pages= %u channel-count = %d\n",
2073 nr_pages, pvt->channel_count);
2074
2075 return nr_pages;
2076 }
2077
2078 /*
2079 * Initialize the array of csrow attribute instances, based on the values
2080 * from pci config hardware registers.
2081 */
2082 static int init_csrows(struct mem_ctl_info *mci)
2083 {
2084 struct csrow_info *csrow;
2085 struct amd64_pvt *pvt = mci->pvt_info;
2086 u64 input_addr_min, input_addr_max, sys_addr, base, mask;
2087 u32 val;
2088 int i, empty = 1;
2089
2090 amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
2091
2092 pvt->nbcfg = val;
2093
2094 debugf0("node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2095 pvt->mc_node_id, val,
2096 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
2097
2098 for_each_chip_select(i, 0, pvt) {
2099 csrow = &mci->csrows[i];
2100
2101 if (!csrow_enabled(i, 0, pvt)) {
2102 debugf1("----CSROW %d EMPTY for node %d\n", i,
2103 pvt->mc_node_id);
2104 continue;
2105 }
2106
2107 debugf1("----CSROW %d VALID for MC node %d\n",
2108 i, pvt->mc_node_id);
2109
2110 empty = 0;
2111 csrow->nr_pages = amd64_csrow_nr_pages(pvt, 0, i);
2112 find_csrow_limits(mci, i, &input_addr_min, &input_addr_max);
2113 sys_addr = input_addr_to_sys_addr(mci, input_addr_min);
2114 csrow->first_page = (u32) (sys_addr >> PAGE_SHIFT);
2115 sys_addr = input_addr_to_sys_addr(mci, input_addr_max);
2116 csrow->last_page = (u32) (sys_addr >> PAGE_SHIFT);
2117
2118 get_cs_base_and_mask(pvt, i, 0, &base, &mask);
2119 csrow->page_mask = ~mask;
2120 /* 8 bytes of resolution */
2121
2122 csrow->mtype = amd64_determine_memory_type(pvt, i);
2123
2124 debugf1(" for MC node %d csrow %d:\n", pvt->mc_node_id, i);
2125 debugf1(" input_addr_min: 0x%lx input_addr_max: 0x%lx\n",
2126 (unsigned long)input_addr_min,
2127 (unsigned long)input_addr_max);
2128 debugf1(" sys_addr: 0x%lx page_mask: 0x%lx\n",
2129 (unsigned long)sys_addr, csrow->page_mask);
2130 debugf1(" nr_pages: %u first_page: 0x%lx "
2131 "last_page: 0x%lx\n",
2132 (unsigned)csrow->nr_pages,
2133 csrow->first_page, csrow->last_page);
2134
2135 /*
2136 * determine whether CHIPKILL or JUST ECC or NO ECC is operating
2137 */
2138 if (pvt->nbcfg & NBCFG_ECC_ENABLE)
2139 csrow->edac_mode =
2140 (pvt->nbcfg & NBCFG_CHIPKILL) ?
2141 EDAC_S4ECD4ED : EDAC_SECDED;
2142 else
2143 csrow->edac_mode = EDAC_NONE;
2144 }
2145
2146 return empty;
2147 }
2148
2149 /* get all cores on this DCT */
2150 static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, unsigned nid)
2151 {
2152 int cpu;
2153
2154 for_each_online_cpu(cpu)
2155 if (amd_get_nb_id(cpu) == nid)
2156 cpumask_set_cpu(cpu, mask);
2157 }
2158
2159 /* check MCG_CTL on all the cpus on this node */
2160 static bool amd64_nb_mce_bank_enabled_on_node(unsigned nid)
2161 {
2162 cpumask_var_t mask;
2163 int cpu, nbe;
2164 bool ret = false;
2165
2166 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
2167 amd64_warn("%s: Error allocating mask\n", __func__);
2168 return false;
2169 }
2170
2171 get_cpus_on_this_dct_cpumask(mask, nid);
2172
2173 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
2174
2175 for_each_cpu(cpu, mask) {
2176 struct msr *reg = per_cpu_ptr(msrs, cpu);
2177 nbe = reg->l & MSR_MCGCTL_NBE;
2178
2179 debugf0("core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
2180 cpu, reg->q,
2181 (nbe ? "enabled" : "disabled"));
2182
2183 if (!nbe)
2184 goto out;
2185 }
2186 ret = true;
2187
2188 out:
2189 free_cpumask_var(mask);
2190 return ret;
2191 }
2192
2193 static int toggle_ecc_err_reporting(struct ecc_settings *s, u8 nid, bool on)
2194 {
2195 cpumask_var_t cmask;
2196 int cpu;
2197
2198 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
2199 amd64_warn("%s: error allocating mask\n", __func__);
2200 return false;
2201 }
2202
2203 get_cpus_on_this_dct_cpumask(cmask, nid);
2204
2205 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2206
2207 for_each_cpu(cpu, cmask) {
2208
2209 struct msr *reg = per_cpu_ptr(msrs, cpu);
2210
2211 if (on) {
2212 if (reg->l & MSR_MCGCTL_NBE)
2213 s->flags.nb_mce_enable = 1;
2214
2215 reg->l |= MSR_MCGCTL_NBE;
2216 } else {
2217 /*
2218 * Turn off NB MCE reporting only when it was off before
2219 */
2220 if (!s->flags.nb_mce_enable)
2221 reg->l &= ~MSR_MCGCTL_NBE;
2222 }
2223 }
2224 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2225
2226 free_cpumask_var(cmask);
2227
2228 return 0;
2229 }
2230
2231 static bool enable_ecc_error_reporting(struct ecc_settings *s, u8 nid,
2232 struct pci_dev *F3)
2233 {
2234 bool ret = true;
2235 u32 value, mask = 0x3; /* UECC/CECC enable */
2236
2237 if (toggle_ecc_err_reporting(s, nid, ON)) {
2238 amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
2239 return false;
2240 }
2241
2242 amd64_read_pci_cfg(F3, NBCTL, &value);
2243
2244 s->old_nbctl = value & mask;
2245 s->nbctl_valid = true;
2246
2247 value |= mask;
2248 amd64_write_pci_cfg(F3, NBCTL, value);
2249
2250 amd64_read_pci_cfg(F3, NBCFG, &value);
2251
2252 debugf0("1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2253 nid, value, !!(value & NBCFG_ECC_ENABLE));
2254
2255 if (!(value & NBCFG_ECC_ENABLE)) {
2256 amd64_warn("DRAM ECC disabled on this node, enabling...\n");
2257
2258 s->flags.nb_ecc_prev = 0;
2259
2260 /* Attempt to turn on DRAM ECC Enable */
2261 value |= NBCFG_ECC_ENABLE;
2262 amd64_write_pci_cfg(F3, NBCFG, value);
2263
2264 amd64_read_pci_cfg(F3, NBCFG, &value);
2265
2266 if (!(value & NBCFG_ECC_ENABLE)) {
2267 amd64_warn("Hardware rejected DRAM ECC enable,"
2268 "check memory DIMM configuration.\n");
2269 ret = false;
2270 } else {
2271 amd64_info("Hardware accepted DRAM ECC Enable\n");
2272 }
2273 } else {
2274 s->flags.nb_ecc_prev = 1;
2275 }
2276
2277 debugf0("2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2278 nid, value, !!(value & NBCFG_ECC_ENABLE));
2279
2280 return ret;
2281 }
2282
2283 static void restore_ecc_error_reporting(struct ecc_settings *s, u8 nid,
2284 struct pci_dev *F3)
2285 {
2286 u32 value, mask = 0x3; /* UECC/CECC enable */
2287
2288
2289 if (!s->nbctl_valid)
2290 return;
2291
2292 amd64_read_pci_cfg(F3, NBCTL, &value);
2293 value &= ~mask;
2294 value |= s->old_nbctl;
2295
2296 amd64_write_pci_cfg(F3, NBCTL, value);
2297
2298 /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
2299 if (!s->flags.nb_ecc_prev) {
2300 amd64_read_pci_cfg(F3, NBCFG, &value);
2301 value &= ~NBCFG_ECC_ENABLE;
2302 amd64_write_pci_cfg(F3, NBCFG, value);
2303 }
2304
2305 /* restore the NB Enable MCGCTL bit */
2306 if (toggle_ecc_err_reporting(s, nid, OFF))
2307 amd64_warn("Error restoring NB MCGCTL settings!\n");
2308 }
2309
2310 /*
2311 * EDAC requires that the BIOS have ECC enabled before
2312 * taking over the processing of ECC errors. A command line
2313 * option allows to force-enable hardware ECC later in
2314 * enable_ecc_error_reporting().
2315 */
2316 static const char *ecc_msg =
2317 "ECC disabled in the BIOS or no ECC capability, module will not load.\n"
2318 " Either enable ECC checking or force module loading by setting "
2319 "'ecc_enable_override'.\n"
2320 " (Note that use of the override may cause unknown side effects.)\n";
2321
2322 static bool ecc_enabled(struct pci_dev *F3, u8 nid)
2323 {
2324 u32 value;
2325 u8 ecc_en = 0;
2326 bool nb_mce_en = false;
2327
2328 amd64_read_pci_cfg(F3, NBCFG, &value);
2329
2330 ecc_en = !!(value & NBCFG_ECC_ENABLE);
2331 amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled"));
2332
2333 nb_mce_en = amd64_nb_mce_bank_enabled_on_node(nid);
2334 if (!nb_mce_en)
2335 amd64_notice("NB MCE bank disabled, set MSR "
2336 "0x%08x[4] on node %d to enable.\n",
2337 MSR_IA32_MCG_CTL, nid);
2338
2339 if (!ecc_en || !nb_mce_en) {
2340 amd64_notice("%s", ecc_msg);
2341 return false;
2342 }
2343 return true;
2344 }
2345
2346 struct mcidev_sysfs_attribute sysfs_attrs[ARRAY_SIZE(amd64_dbg_attrs) +
2347 ARRAY_SIZE(amd64_inj_attrs) +
2348 1];
2349
2350 struct mcidev_sysfs_attribute terminator = { .attr = { .name = NULL } };
2351
2352 static void set_mc_sysfs_attrs(struct mem_ctl_info *mci)
2353 {
2354 unsigned int i = 0, j = 0;
2355
2356 for (; i < ARRAY_SIZE(amd64_dbg_attrs); i++)
2357 sysfs_attrs[i] = amd64_dbg_attrs[i];
2358
2359 if (boot_cpu_data.x86 >= 0x10)
2360 for (j = 0; j < ARRAY_SIZE(amd64_inj_attrs); j++, i++)
2361 sysfs_attrs[i] = amd64_inj_attrs[j];
2362
2363 sysfs_attrs[i] = terminator;
2364
2365 mci->mc_driver_sysfs_attributes = sysfs_attrs;
2366 }
2367
2368 static void setup_mci_misc_attrs(struct mem_ctl_info *mci,
2369 struct amd64_family_type *fam)
2370 {
2371 struct amd64_pvt *pvt = mci->pvt_info;
2372
2373 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
2374 mci->edac_ctl_cap = EDAC_FLAG_NONE;
2375
2376 if (pvt->nbcap & NBCAP_SECDED)
2377 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
2378
2379 if (pvt->nbcap & NBCAP_CHIPKILL)
2380 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
2381
2382 mci->edac_cap = amd64_determine_edac_cap(pvt);
2383 mci->mod_name = EDAC_MOD_STR;
2384 mci->mod_ver = EDAC_AMD64_VERSION;
2385 mci->ctl_name = fam->ctl_name;
2386 mci->dev_name = pci_name(pvt->F2);
2387 mci->ctl_page_to_phys = NULL;
2388
2389 /* memory scrubber interface */
2390 mci->set_sdram_scrub_rate = amd64_set_scrub_rate;
2391 mci->get_sdram_scrub_rate = amd64_get_scrub_rate;
2392 }
2393
2394 /*
2395 * returns a pointer to the family descriptor on success, NULL otherwise.
2396 */
2397 static struct amd64_family_type *amd64_per_family_init(struct amd64_pvt *pvt)
2398 {
2399 u8 fam = boot_cpu_data.x86;
2400 struct amd64_family_type *fam_type = NULL;
2401
2402 switch (fam) {
2403 case 0xf:
2404 fam_type = &amd64_family_types[K8_CPUS];
2405 pvt->ops = &amd64_family_types[K8_CPUS].ops;
2406 break;
2407
2408 case 0x10:
2409 fam_type = &amd64_family_types[F10_CPUS];
2410 pvt->ops = &amd64_family_types[F10_CPUS].ops;
2411 break;
2412
2413 case 0x15:
2414 fam_type = &amd64_family_types[F15_CPUS];
2415 pvt->ops = &amd64_family_types[F15_CPUS].ops;
2416 break;
2417
2418 default:
2419 amd64_err("Unsupported family!\n");
2420 return NULL;
2421 }
2422
2423 pvt->ext_model = boot_cpu_data.x86_model >> 4;
2424
2425 amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name,
2426 (fam == 0xf ?
2427 (pvt->ext_model >= K8_REV_F ? "revF or later "
2428 : "revE or earlier ")
2429 : ""), pvt->mc_node_id);
2430 return fam_type;
2431 }
2432
2433 static int amd64_init_one_instance(struct pci_dev *F2)
2434 {
2435 struct amd64_pvt *pvt = NULL;
2436 struct amd64_family_type *fam_type = NULL;
2437 struct mem_ctl_info *mci = NULL;
2438 int err = 0, ret;
2439 u8 nid = get_node_id(F2);
2440
2441 ret = -ENOMEM;
2442 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
2443 if (!pvt)
2444 goto err_ret;
2445
2446 pvt->mc_node_id = nid;
2447 pvt->F2 = F2;
2448
2449 ret = -EINVAL;
2450 fam_type = amd64_per_family_init(pvt);
2451 if (!fam_type)
2452 goto err_free;
2453
2454 ret = -ENODEV;
2455 err = reserve_mc_sibling_devs(pvt, fam_type->f1_id, fam_type->f3_id);
2456 if (err)
2457 goto err_free;
2458
2459 read_mc_regs(pvt);
2460
2461 /*
2462 * We need to determine how many memory channels there are. Then use
2463 * that information for calculating the size of the dynamic instance
2464 * tables in the 'mci' structure.
2465 */
2466 ret = -EINVAL;
2467 pvt->channel_count = pvt->ops->early_channel_count(pvt);
2468 if (pvt->channel_count < 0)
2469 goto err_siblings;
2470
2471 ret = -ENOMEM;
2472 mci = edac_mc_alloc(0, pvt->csels[0].b_cnt, pvt->channel_count, nid);
2473 if (!mci)
2474 goto err_siblings;
2475
2476 mci->pvt_info = pvt;
2477 mci->dev = &pvt->F2->dev;
2478
2479 setup_mci_misc_attrs(mci, fam_type);
2480
2481 if (init_csrows(mci))
2482 mci->edac_cap = EDAC_FLAG_NONE;
2483
2484 set_mc_sysfs_attrs(mci);
2485
2486 ret = -ENODEV;
2487 if (edac_mc_add_mc(mci)) {
2488 debugf1("failed edac_mc_add_mc()\n");
2489 goto err_add_mc;
2490 }
2491
2492 /* register stuff with EDAC MCE */
2493 if (report_gart_errors)
2494 amd_report_gart_errors(true);
2495
2496 amd_register_ecc_decoder(amd64_decode_bus_error);
2497
2498 mcis[nid] = mci;
2499
2500 atomic_inc(&drv_instances);
2501
2502 return 0;
2503
2504 err_add_mc:
2505 edac_mc_free(mci);
2506
2507 err_siblings:
2508 free_mc_sibling_devs(pvt);
2509
2510 err_free:
2511 kfree(pvt);
2512
2513 err_ret:
2514 return ret;
2515 }
2516
2517 static int __devinit amd64_probe_one_instance(struct pci_dev *pdev,
2518 const struct pci_device_id *mc_type)
2519 {
2520 u8 nid = get_node_id(pdev);
2521 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2522 struct ecc_settings *s;
2523 int ret = 0;
2524
2525 ret = pci_enable_device(pdev);
2526 if (ret < 0) {
2527 debugf0("ret=%d\n", ret);
2528 return -EIO;
2529 }
2530
2531 ret = -ENOMEM;
2532 s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
2533 if (!s)
2534 goto err_out;
2535
2536 ecc_stngs[nid] = s;
2537
2538 if (!ecc_enabled(F3, nid)) {
2539 ret = -ENODEV;
2540
2541 if (!ecc_enable_override)
2542 goto err_enable;
2543
2544 amd64_warn("Forcing ECC on!\n");
2545
2546 if (!enable_ecc_error_reporting(s, nid, F3))
2547 goto err_enable;
2548 }
2549
2550 ret = amd64_init_one_instance(pdev);
2551 if (ret < 0) {
2552 amd64_err("Error probing instance: %d\n", nid);
2553 restore_ecc_error_reporting(s, nid, F3);
2554 }
2555
2556 return ret;
2557
2558 err_enable:
2559 kfree(s);
2560 ecc_stngs[nid] = NULL;
2561
2562 err_out:
2563 return ret;
2564 }
2565
2566 static void __devexit amd64_remove_one_instance(struct pci_dev *pdev)
2567 {
2568 struct mem_ctl_info *mci;
2569 struct amd64_pvt *pvt;
2570 u8 nid = get_node_id(pdev);
2571 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2572 struct ecc_settings *s = ecc_stngs[nid];
2573
2574 /* Remove from EDAC CORE tracking list */
2575 mci = edac_mc_del_mc(&pdev->dev);
2576 if (!mci)
2577 return;
2578
2579 pvt = mci->pvt_info;
2580
2581 restore_ecc_error_reporting(s, nid, F3);
2582
2583 free_mc_sibling_devs(pvt);
2584
2585 /* unregister from EDAC MCE */
2586 amd_report_gart_errors(false);
2587 amd_unregister_ecc_decoder(amd64_decode_bus_error);
2588
2589 kfree(ecc_stngs[nid]);
2590 ecc_stngs[nid] = NULL;
2591
2592 /* Free the EDAC CORE resources */
2593 mci->pvt_info = NULL;
2594 mcis[nid] = NULL;
2595
2596 kfree(pvt);
2597 edac_mc_free(mci);
2598 }
2599
2600 /*
2601 * This table is part of the interface for loading drivers for PCI devices. The
2602 * PCI core identifies what devices are on a system during boot, and then
2603 * inquiry this table to see if this driver is for a given device found.
2604 */
2605 static const struct pci_device_id amd64_pci_table[] __devinitdata = {
2606 {
2607 .vendor = PCI_VENDOR_ID_AMD,
2608 .device = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
2609 .subvendor = PCI_ANY_ID,
2610 .subdevice = PCI_ANY_ID,
2611 .class = 0,
2612 .class_mask = 0,
2613 },
2614 {
2615 .vendor = PCI_VENDOR_ID_AMD,
2616 .device = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
2617 .subvendor = PCI_ANY_ID,
2618 .subdevice = PCI_ANY_ID,
2619 .class = 0,
2620 .class_mask = 0,
2621 },
2622 {
2623 .vendor = PCI_VENDOR_ID_AMD,
2624 .device = PCI_DEVICE_ID_AMD_15H_NB_F2,
2625 .subvendor = PCI_ANY_ID,
2626 .subdevice = PCI_ANY_ID,
2627 .class = 0,
2628 .class_mask = 0,
2629 },
2630
2631 {0, }
2632 };
2633 MODULE_DEVICE_TABLE(pci, amd64_pci_table);
2634
2635 static struct pci_driver amd64_pci_driver = {
2636 .name = EDAC_MOD_STR,
2637 .probe = amd64_probe_one_instance,
2638 .remove = __devexit_p(amd64_remove_one_instance),
2639 .id_table = amd64_pci_table,
2640 };
2641
2642 static void setup_pci_device(void)
2643 {
2644 struct mem_ctl_info *mci;
2645 struct amd64_pvt *pvt;
2646
2647 if (amd64_ctl_pci)
2648 return;
2649
2650 mci = mcis[0];
2651 if (mci) {
2652
2653 pvt = mci->pvt_info;
2654 amd64_ctl_pci =
2655 edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR);
2656
2657 if (!amd64_ctl_pci) {
2658 pr_warning("%s(): Unable to create PCI control\n",
2659 __func__);
2660
2661 pr_warning("%s(): PCI error report via EDAC not set\n",
2662 __func__);
2663 }
2664 }
2665 }
2666
2667 static int __init amd64_edac_init(void)
2668 {
2669 int err = -ENODEV;
2670
2671 printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION);
2672
2673 opstate_init();
2674
2675 if (amd_cache_northbridges() < 0)
2676 goto err_ret;
2677
2678 err = -ENOMEM;
2679 mcis = kzalloc(amd_nb_num() * sizeof(mcis[0]), GFP_KERNEL);
2680 ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL);
2681 if (!(mcis && ecc_stngs))
2682 goto err_free;
2683
2684 msrs = msrs_alloc();
2685 if (!msrs)
2686 goto err_free;
2687
2688 err = pci_register_driver(&amd64_pci_driver);
2689 if (err)
2690 goto err_pci;
2691
2692 err = -ENODEV;
2693 if (!atomic_read(&drv_instances))
2694 goto err_no_instances;
2695
2696 setup_pci_device();
2697 return 0;
2698
2699 err_no_instances:
2700 pci_unregister_driver(&amd64_pci_driver);
2701
2702 err_pci:
2703 msrs_free(msrs);
2704 msrs = NULL;
2705
2706 err_free:
2707 kfree(mcis);
2708 mcis = NULL;
2709
2710 kfree(ecc_stngs);
2711 ecc_stngs = NULL;
2712
2713 err_ret:
2714 return err;
2715 }
2716
2717 static void __exit amd64_edac_exit(void)
2718 {
2719 if (amd64_ctl_pci)
2720 edac_pci_release_generic_ctl(amd64_ctl_pci);
2721
2722 pci_unregister_driver(&amd64_pci_driver);
2723
2724 kfree(ecc_stngs);
2725 ecc_stngs = NULL;
2726
2727 kfree(mcis);
2728 mcis = NULL;
2729
2730 msrs_free(msrs);
2731 msrs = NULL;
2732 }
2733
2734 module_init(amd64_edac_init);
2735 module_exit(amd64_edac_exit);
2736
2737 MODULE_LICENSE("GPL");
2738 MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
2739 "Dave Peterson, Thayne Harbaugh");
2740 MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
2741 EDAC_AMD64_VERSION);
2742
2743 module_param(edac_op_state, int, 0444);
2744 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");