?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
include: replace linux/module.h with "struct module" wherever possible
[linux-2.6/next.git] / drivers / edac / amd64_edac.c
blob9a8bebcf6b177fa79e9635ecde71f05b2578e798
1 #include "amd64_edac.h"
2 #include <asm/amd_nb.h>
3
4 static struct edac_pci_ctl_info *amd64_ctl_pci;
5
6 static int report_gart_errors;
7 module_param(report_gart_errors, int, 0644);
8
9 /*
10 * Set by command line parameter. If BIOS has enabled the ECC, this override is
11 * cleared to prevent re-enabling the hardware by this driver.
12 */
13 static int ecc_enable_override;
14 module_param(ecc_enable_override, int, 0644);
15
16 static struct msr __percpu *msrs;
17
18 /*
19 * count successfully initialized driver instances for setup_pci_device()
20 */
21 static atomic_t drv_instances = ATOMIC_INIT(0);
22
23 /* Per-node driver instances */
24 static struct mem_ctl_info **mcis;
25 static struct ecc_settings **ecc_stngs;
26
27 /*
28 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
29 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
30 * or higher value'.
31 *
32 *FIXME: Produce a better mapping/linearisation.
33 */
34 struct scrubrate {
35 u32 scrubval; /* bit pattern for scrub rate */
36 u32 bandwidth; /* bandwidth consumed (bytes/sec) */
37 } scrubrates[] = {
38 { 0x01, 1600000000UL},
39 { 0x02, 800000000UL},
40 { 0x03, 400000000UL},
41 { 0x04, 200000000UL},
42 { 0x05, 100000000UL},
43 { 0x06, 50000000UL},
44 { 0x07, 25000000UL},
45 { 0x08, 12284069UL},
46 { 0x09, 6274509UL},
47 { 0x0A, 3121951UL},
48 { 0x0B, 1560975UL},
49 { 0x0C, 781440UL},
50 { 0x0D, 390720UL},
51 { 0x0E, 195300UL},
52 { 0x0F, 97650UL},
53 { 0x10, 48854UL},
54 { 0x11, 24427UL},
55 { 0x12, 12213UL},
56 { 0x13, 6101UL},
57 { 0x14, 3051UL},
58 { 0x15, 1523UL},
59 { 0x16, 761UL},
60 { 0x00, 0UL}, /* scrubbing off */
61 };
62
63 static int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
64 u32 *val, const char *func)
65 {
66 int err = 0;
67
68 err = pci_read_config_dword(pdev, offset, val);
69 if (err)
70 amd64_warn("%s: error reading F%dx%03x.\n",
71 func, PCI_FUNC(pdev->devfn), offset);
72
73 return err;
74 }
75
76 int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
77 u32 val, const char *func)
78 {
79 int err = 0;
80
81 err = pci_write_config_dword(pdev, offset, val);
82 if (err)
83 amd64_warn("%s: error writing to F%dx%03x.\n",
84 func, PCI_FUNC(pdev->devfn), offset);
85
86 return err;
87 }
88
89 /*
90 *
91 * Depending on the family, F2 DCT reads need special handling:
92 *
93 * K8: has a single DCT only
94 *
95 * F10h: each DCT has its own set of regs
96 * DCT0 -> F2x040..
97 * DCT1 -> F2x140..
98 *
99 * F15h: we select which DCT we access using F1x10C[DctCfgSel]
100 *
101 */
102 static int k8_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
103 const char *func)
104 {
105 if (addr >= 0x100)
106 return -EINVAL;
107
108 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
109 }
110
111 static int f10_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
112 const char *func)
113 {
114 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
115 }
116
117 static int f15_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
118 const char *func)
119 {
120 u32 reg = 0;
121 u8 dct = 0;
122
123 if (addr >= 0x140 && addr <= 0x1a0) {
124 dct = 1;
125 addr -= 0x100;
126 }
127
128 amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, &reg);
129 reg &= 0xfffffffe;
130 reg |= dct;
131 amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
132
133 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
134 }
135
136 /*
137 * Memory scrubber control interface. For K8, memory scrubbing is handled by
138 * hardware and can involve L2 cache, dcache as well as the main memory. With
139 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
140 * functionality.
141 *
142 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
143 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
144 * bytes/sec for the setting.
145 *
146 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
147 * other archs, we might not have access to the caches directly.
148 */
149
150 /*
151 * scan the scrub rate mapping table for a close or matching bandwidth value to
152 * issue. If requested is too big, then use last maximum value found.
153 */
154 static int __amd64_set_scrub_rate(struct pci_dev *ctl, u32 new_bw, u32 min_rate)
155 {
156 u32 scrubval;
157 int i;
158
159 /*
160 * map the configured rate (new_bw) to a value specific to the AMD64
161 * memory controller and apply to register. Search for the first
162 * bandwidth entry that is greater or equal than the setting requested
163 * and program that. If at last entry, turn off DRAM scrubbing.
164 */
165 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
166 /*
167 * skip scrub rates which aren't recommended
168 * (see F10 BKDG, F3x58)
169 */
170 if (scrubrates[i].scrubval < min_rate)
171 continue;
172
173 if (scrubrates[i].bandwidth <= new_bw)
174 break;
175
176 /*
177 * if no suitable bandwidth found, turn off DRAM scrubbing
178 * entirely by falling back to the last element in the
179 * scrubrates array.
180 */
181 }
182
183 scrubval = scrubrates[i].scrubval;
184
185 pci_write_bits32(ctl, SCRCTRL, scrubval, 0x001F);
186
187 if (scrubval)
188 return scrubrates[i].bandwidth;
189
190 return 0;
191 }
192
193 static int amd64_set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
194 {
195 struct amd64_pvt *pvt = mci->pvt_info;
196 u32 min_scrubrate = 0x5;
197
198 if (boot_cpu_data.x86 == 0xf)
199 min_scrubrate = 0x0;
200
201 return __amd64_set_scrub_rate(pvt->F3, bw, min_scrubrate);
202 }
203
204 static int amd64_get_scrub_rate(struct mem_ctl_info *mci)
205 {
206 struct amd64_pvt *pvt = mci->pvt_info;
207 u32 scrubval = 0;
208 int i, retval = -EINVAL;
209
210 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
211
212 scrubval = scrubval & 0x001F;
213
214 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
215 if (scrubrates[i].scrubval == scrubval) {
216 retval = scrubrates[i].bandwidth;
217 break;
218 }
219 }
220 return retval;
221 }
222
223 /*
224 * returns true if the SysAddr given by sys_addr matches the
225 * DRAM base/limit associated with node_id
226 */
227 static bool amd64_base_limit_match(struct amd64_pvt *pvt, u64 sys_addr,
228 unsigned nid)
229 {
230 u64 addr;
231
232 /* The K8 treats this as a 40-bit value. However, bits 63-40 will be
233 * all ones if the most significant implemented address bit is 1.
234 * Here we discard bits 63-40. See section 3.4.2 of AMD publication
235 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
236 * Application Programming.
237 */
238 addr = sys_addr & 0x000000ffffffffffull;
239
240 return ((addr >= get_dram_base(pvt, nid)) &&
241 (addr <= get_dram_limit(pvt, nid)));
242 }
243
244 /*
245 * Attempt to map a SysAddr to a node. On success, return a pointer to the
246 * mem_ctl_info structure for the node that the SysAddr maps to.
247 *
248 * On failure, return NULL.
249 */
250 static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
251 u64 sys_addr)
252 {
253 struct amd64_pvt *pvt;
254 unsigned node_id;
255 u32 intlv_en, bits;
256
257 /*
258 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
259 * 3.4.4.2) registers to map the SysAddr to a node ID.
260 */
261 pvt = mci->pvt_info;
262
263 /*
264 * The value of this field should be the same for all DRAM Base
265 * registers. Therefore we arbitrarily choose to read it from the
266 * register for node 0.
267 */
268 intlv_en = dram_intlv_en(pvt, 0);
269
270 if (intlv_en == 0) {
271 for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
272 if (amd64_base_limit_match(pvt, sys_addr, node_id))
273 goto found;
274 }
275 goto err_no_match;
276 }
277
278 if (unlikely((intlv_en != 0x01) &&
279 (intlv_en != 0x03) &&
280 (intlv_en != 0x07))) {
281 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
282 return NULL;
283 }
284
285 bits = (((u32) sys_addr) >> 12) & intlv_en;
286
287 for (node_id = 0; ; ) {
288 if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
289 break; /* intlv_sel field matches */
290
291 if (++node_id >= DRAM_RANGES)
292 goto err_no_match;
293 }
294
295 /* sanity test for sys_addr */
296 if (unlikely(!amd64_base_limit_match(pvt, sys_addr, node_id))) {
297 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
298 "range for node %d with node interleaving enabled.\n",
299 __func__, sys_addr, node_id);
300 return NULL;
301 }
302
303 found:
304 return edac_mc_find((int)node_id);
305
306 err_no_match:
307 debugf2("sys_addr 0x%lx doesn't match any node\n",
308 (unsigned long)sys_addr);
309
310 return NULL;
311 }
312
313 /*
314 * compute the CS base address of the @csrow on the DRAM controller @dct.
315 * For details see F2x[5C:40] in the processor's BKDG
316 */
317 static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
318 u64 *base, u64 *mask)
319 {
320 u64 csbase, csmask, base_bits, mask_bits;
321 u8 addr_shift;
322
323 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) {
324 csbase = pvt->csels[dct].csbases[csrow];
325 csmask = pvt->csels[dct].csmasks[csrow];
326 base_bits = GENMASK(21, 31) | GENMASK(9, 15);
327 mask_bits = GENMASK(21, 29) | GENMASK(9, 15);
328 addr_shift = 4;
329 } else {
330 csbase = pvt->csels[dct].csbases[csrow];
331 csmask = pvt->csels[dct].csmasks[csrow >> 1];
332 addr_shift = 8;
333
334 if (boot_cpu_data.x86 == 0x15)
335 base_bits = mask_bits = GENMASK(19,30) | GENMASK(5,13);
336 else
337 base_bits = mask_bits = GENMASK(19,28) | GENMASK(5,13);
338 }
339
340 *base = (csbase & base_bits) << addr_shift;
341
342 *mask = ~0ULL;
343 /* poke holes for the csmask */
344 *mask &= ~(mask_bits << addr_shift);
345 /* OR them in */
346 *mask |= (csmask & mask_bits) << addr_shift;
347 }
348
349 #define for_each_chip_select(i, dct, pvt) \
350 for (i = 0; i < pvt->csels[dct].b_cnt; i++)
351
352 #define chip_select_base(i, dct, pvt) \
353 pvt->csels[dct].csbases[i]
354
355 #define for_each_chip_select_mask(i, dct, pvt) \
356 for (i = 0; i < pvt->csels[dct].m_cnt; i++)
357
358 /*
359 * @input_addr is an InputAddr associated with the node given by mci. Return the
360 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
361 */
362 static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
363 {
364 struct amd64_pvt *pvt;
365 int csrow;
366 u64 base, mask;
367
368 pvt = mci->pvt_info;
369
370 for_each_chip_select(csrow, 0, pvt) {
371 if (!csrow_enabled(csrow, 0, pvt))
372 continue;
373
374 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
375
376 mask = ~mask;
377
378 if ((input_addr & mask) == (base & mask)) {
379 debugf2("InputAddr 0x%lx matches csrow %d (node %d)\n",
380 (unsigned long)input_addr, csrow,
381 pvt->mc_node_id);
382
383 return csrow;
384 }
385 }
386 debugf2("no matching csrow for InputAddr 0x%lx (MC node %d)\n",
387 (unsigned long)input_addr, pvt->mc_node_id);
388
389 return -1;
390 }
391
392 /*
393 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
394 * for the node represented by mci. Info is passed back in *hole_base,
395 * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if
396 * info is invalid. Info may be invalid for either of the following reasons:
397 *
398 * - The revision of the node is not E or greater. In this case, the DRAM Hole
399 * Address Register does not exist.
400 *
401 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
402 * indicating that its contents are not valid.
403 *
404 * The values passed back in *hole_base, *hole_offset, and *hole_size are
405 * complete 32-bit values despite the fact that the bitfields in the DHAR
406 * only represent bits 31-24 of the base and offset values.
407 */
408 int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
409 u64 *hole_offset, u64 *hole_size)
410 {
411 struct amd64_pvt *pvt = mci->pvt_info;
412 u64 base;
413
414 /* only revE and later have the DRAM Hole Address Register */
415 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_E) {
416 debugf1(" revision %d for node %d does not support DHAR\n",
417 pvt->ext_model, pvt->mc_node_id);
418 return 1;
419 }
420
421 /* valid for Fam10h and above */
422 if (boot_cpu_data.x86 >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
423 debugf1(" Dram Memory Hoisting is DISABLED on this system\n");
424 return 1;
425 }
426
427 if (!dhar_valid(pvt)) {
428 debugf1(" Dram Memory Hoisting is DISABLED on this node %d\n",
429 pvt->mc_node_id);
430 return 1;
431 }
432
433 /* This node has Memory Hoisting */
434
435 /* +------------------+--------------------+--------------------+-----
436 * | memory | DRAM hole | relocated |
437 * | [0, (x - 1)] | [x, 0xffffffff] | addresses from |
438 * | | | DRAM hole |
439 * | | | [0x100000000, |
440 * | | | (0x100000000+ |
441 * | | | (0xffffffff-x))] |
442 * +------------------+--------------------+--------------------+-----
443 *
444 * Above is a diagram of physical memory showing the DRAM hole and the
445 * relocated addresses from the DRAM hole. As shown, the DRAM hole
446 * starts at address x (the base address) and extends through address
447 * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the
448 * addresses in the hole so that they start at 0x100000000.
449 */
450
451 base = dhar_base(pvt);
452
453 *hole_base = base;
454 *hole_size = (0x1ull << 32) - base;
455
456 if (boot_cpu_data.x86 > 0xf)
457 *hole_offset = f10_dhar_offset(pvt);
458 else
459 *hole_offset = k8_dhar_offset(pvt);
460
461 debugf1(" DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
462 pvt->mc_node_id, (unsigned long)*hole_base,
463 (unsigned long)*hole_offset, (unsigned long)*hole_size);
464
465 return 0;
466 }
467 EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info);
468
469 /*
470 * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is
471 * assumed that sys_addr maps to the node given by mci.
472 *
473 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
474 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
475 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
476 * then it is also involved in translating a SysAddr to a DramAddr. Sections
477 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
478 * These parts of the documentation are unclear. I interpret them as follows:
479 *
480 * When node n receives a SysAddr, it processes the SysAddr as follows:
481 *
482 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
483 * Limit registers for node n. If the SysAddr is not within the range
484 * specified by the base and limit values, then node n ignores the Sysaddr
485 * (since it does not map to node n). Otherwise continue to step 2 below.
486 *
487 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
488 * disabled so skip to step 3 below. Otherwise see if the SysAddr is within
489 * the range of relocated addresses (starting at 0x100000000) from the DRAM
490 * hole. If not, skip to step 3 below. Else get the value of the
491 * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
492 * offset defined by this value from the SysAddr.
493 *
494 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
495 * Base register for node n. To obtain the DramAddr, subtract the base
496 * address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
497 */
498 static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
499 {
500 struct amd64_pvt *pvt = mci->pvt_info;
501 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
502 int ret = 0;
503
504 dram_base = get_dram_base(pvt, pvt->mc_node_id);
505
506 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
507 &hole_size);
508 if (!ret) {
509 if ((sys_addr >= (1ull << 32)) &&
510 (sys_addr < ((1ull << 32) + hole_size))) {
511 /* use DHAR to translate SysAddr to DramAddr */
512 dram_addr = sys_addr - hole_offset;
513
514 debugf2("using DHAR to translate SysAddr 0x%lx to "
515 "DramAddr 0x%lx\n",
516 (unsigned long)sys_addr,
517 (unsigned long)dram_addr);
518
519 return dram_addr;
520 }
521 }
522
523 /*
524 * Translate the SysAddr to a DramAddr as shown near the start of
525 * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8
526 * only deals with 40-bit values. Therefore we discard bits 63-40 of
527 * sys_addr below. If bit 39 of sys_addr is 1 then the bits we
528 * discard are all 1s. Otherwise the bits we discard are all 0s. See
529 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
530 * Programmer's Manual Volume 1 Application Programming.
531 */
532 dram_addr = (sys_addr & GENMASK(0, 39)) - dram_base;
533
534 debugf2("using DRAM Base register to translate SysAddr 0x%lx to "
535 "DramAddr 0x%lx\n", (unsigned long)sys_addr,
536 (unsigned long)dram_addr);
537 return dram_addr;
538 }
539
540 /*
541 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
542 * (section 3.4.4.1). Return the number of bits from a SysAddr that are used
543 * for node interleaving.
544 */
545 static int num_node_interleave_bits(unsigned intlv_en)
546 {
547 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
548 int n;
549
550 BUG_ON(intlv_en > 7);
551 n = intlv_shift_table[intlv_en];
552 return n;
553 }
554
555 /* Translate the DramAddr given by @dram_addr to an InputAddr. */
556 static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
557 {
558 struct amd64_pvt *pvt;
559 int intlv_shift;
560 u64 input_addr;
561
562 pvt = mci->pvt_info;
563
564 /*
565 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
566 * concerning translating a DramAddr to an InputAddr.
567 */
568 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
569 input_addr = ((dram_addr >> intlv_shift) & GENMASK(12, 35)) +
570 (dram_addr & 0xfff);
571
572 debugf2(" Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
573 intlv_shift, (unsigned long)dram_addr,
574 (unsigned long)input_addr);
575
576 return input_addr;
577 }
578
579 /*
580 * Translate the SysAddr represented by @sys_addr to an InputAddr. It is
581 * assumed that @sys_addr maps to the node given by mci.
582 */
583 static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
584 {
585 u64 input_addr;
586
587 input_addr =
588 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
589
590 debugf2("SysAdddr 0x%lx translates to InputAddr 0x%lx\n",
591 (unsigned long)sys_addr, (unsigned long)input_addr);
592
593 return input_addr;
594 }
595
596
597 /*
598 * @input_addr is an InputAddr associated with the node represented by mci.
599 * Translate @input_addr to a DramAddr and return the result.
600 */
601 static u64 input_addr_to_dram_addr(struct mem_ctl_info *mci, u64 input_addr)
602 {
603 struct amd64_pvt *pvt;
604 unsigned node_id, intlv_shift;
605 u64 bits, dram_addr;
606 u32 intlv_sel;
607
608 /*
609 * Near the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
610 * shows how to translate a DramAddr to an InputAddr. Here we reverse
611 * this procedure. When translating from a DramAddr to an InputAddr, the
612 * bits used for node interleaving are discarded. Here we recover these
613 * bits from the IntlvSel field of the DRAM Limit register (section
614 * 3.4.4.2) for the node that input_addr is associated with.
615 */
616 pvt = mci->pvt_info;
617 node_id = pvt->mc_node_id;
618
619 BUG_ON(node_id > 7);
620
621 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
622 if (intlv_shift == 0) {
623 debugf1(" InputAddr 0x%lx translates to DramAddr of "
624 "same value\n", (unsigned long)input_addr);
625
626 return input_addr;
627 }
628
629 bits = ((input_addr & GENMASK(12, 35)) << intlv_shift) +
630 (input_addr & 0xfff);
631
632 intlv_sel = dram_intlv_sel(pvt, node_id) & ((1 << intlv_shift) - 1);
633 dram_addr = bits + (intlv_sel << 12);
634
635 debugf1("InputAddr 0x%lx translates to DramAddr 0x%lx "
636 "(%d node interleave bits)\n", (unsigned long)input_addr,
637 (unsigned long)dram_addr, intlv_shift);
638
639 return dram_addr;
640 }
641
642 /*
643 * @dram_addr is a DramAddr that maps to the node represented by mci. Convert
644 * @dram_addr to a SysAddr.
645 */
646 static u64 dram_addr_to_sys_addr(struct mem_ctl_info *mci, u64 dram_addr)
647 {
648 struct amd64_pvt *pvt = mci->pvt_info;
649 u64 hole_base, hole_offset, hole_size, base, sys_addr;
650 int ret = 0;
651
652 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
653 &hole_size);
654 if (!ret) {
655 if ((dram_addr >= hole_base) &&
656 (dram_addr < (hole_base + hole_size))) {
657 sys_addr = dram_addr + hole_offset;
658
659 debugf1("using DHAR to translate DramAddr 0x%lx to "
660 "SysAddr 0x%lx\n", (unsigned long)dram_addr,
661 (unsigned long)sys_addr);
662
663 return sys_addr;
664 }
665 }
666
667 base = get_dram_base(pvt, pvt->mc_node_id);
668 sys_addr = dram_addr + base;
669
670 /*
671 * The sys_addr we have computed up to this point is a 40-bit value
672 * because the k8 deals with 40-bit values. However, the value we are
673 * supposed to return is a full 64-bit physical address. The AMD
674 * x86-64 architecture specifies that the most significant implemented
675 * address bit through bit 63 of a physical address must be either all
676 * 0s or all 1s. Therefore we sign-extend the 40-bit sys_addr to a
677 * 64-bit value below. See section 3.4.2 of AMD publication 24592:
678 * AMD x86-64 Architecture Programmer's Manual Volume 1 Application
679 * Programming.
680 */
681 sys_addr |= ~((sys_addr & (1ull << 39)) - 1);
682
683 debugf1(" Node %d, DramAddr 0x%lx to SysAddr 0x%lx\n",
684 pvt->mc_node_id, (unsigned long)dram_addr,
685 (unsigned long)sys_addr);
686
687 return sys_addr;
688 }
689
690 /*
691 * @input_addr is an InputAddr associated with the node given by mci. Translate
692 * @input_addr to a SysAddr.
693 */
694 static inline u64 input_addr_to_sys_addr(struct mem_ctl_info *mci,
695 u64 input_addr)
696 {
697 return dram_addr_to_sys_addr(mci,
698 input_addr_to_dram_addr(mci, input_addr));
699 }
700
701 /*
702 * Find the minimum and maximum InputAddr values that map to the given @csrow.
703 * Pass back these values in *input_addr_min and *input_addr_max.
704 */
705 static void find_csrow_limits(struct mem_ctl_info *mci, int csrow,
706 u64 *input_addr_min, u64 *input_addr_max)
707 {
708 struct amd64_pvt *pvt;
709 u64 base, mask;
710
711 pvt = mci->pvt_info;
712 BUG_ON((csrow < 0) || (csrow >= pvt->csels[0].b_cnt));
713
714 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
715
716 *input_addr_min = base & ~mask;
717 *input_addr_max = base | mask;
718 }
719
720 /* Map the Error address to a PAGE and PAGE OFFSET. */
721 static inline void error_address_to_page_and_offset(u64 error_address,
722 u32 *page, u32 *offset)
723 {
724 *page = (u32) (error_address >> PAGE_SHIFT);
725 *offset = ((u32) error_address) & ~PAGE_MASK;
726 }
727
728 /*
729 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
730 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
731 * of a node that detected an ECC memory error. mci represents the node that
732 * the error address maps to (possibly different from the node that detected
733 * the error). Return the number of the csrow that sys_addr maps to, or -1 on
734 * error.
735 */
736 static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
737 {
738 int csrow;
739
740 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
741
742 if (csrow == -1)
743 amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
744 "address 0x%lx\n", (unsigned long)sys_addr);
745 return csrow;
746 }
747
748 static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
749
750 /*
751 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
752 * are ECC capable.
753 */
754 static enum edac_type amd64_determine_edac_cap(struct amd64_pvt *pvt)
755 {
756 u8 bit;
757 enum dev_type edac_cap = EDAC_FLAG_NONE;
758
759 bit = (boot_cpu_data.x86 > 0xf || pvt->ext_model >= K8_REV_F)
760 ? 19
761 : 17;
762
763 if (pvt->dclr0 & BIT(bit))
764 edac_cap = EDAC_FLAG_SECDED;
765
766 return edac_cap;
767 }
768
769 static void amd64_debug_display_dimm_sizes(struct amd64_pvt *, u8);
770
771 static void amd64_dump_dramcfg_low(u32 dclr, int chan)
772 {
773 debugf1("F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
774
775 debugf1(" DIMM type: %sbuffered; all DIMMs support ECC: %s\n",
776 (dclr & BIT(16)) ? "un" : "",
777 (dclr & BIT(19)) ? "yes" : "no");
778
779 debugf1(" PAR/ERR parity: %s\n",
780 (dclr & BIT(8)) ? "enabled" : "disabled");
781
782 if (boot_cpu_data.x86 == 0x10)
783 debugf1(" DCT 128bit mode width: %s\n",
784 (dclr & BIT(11)) ? "128b" : "64b");
785
786 debugf1(" x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
787 (dclr & BIT(12)) ? "yes" : "no",
788 (dclr & BIT(13)) ? "yes" : "no",
789 (dclr & BIT(14)) ? "yes" : "no",
790 (dclr & BIT(15)) ? "yes" : "no");
791 }
792
793 /* Display and decode various NB registers for debug purposes. */
794 static void dump_misc_regs(struct amd64_pvt *pvt)
795 {
796 debugf1("F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
797
798 debugf1(" NB two channel DRAM capable: %s\n",
799 (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
800
801 debugf1(" ECC capable: %s, ChipKill ECC capable: %s\n",
802 (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
803 (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
804
805 amd64_dump_dramcfg_low(pvt->dclr0, 0);
806
807 debugf1("F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
808
809 debugf1("F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, "
810 "offset: 0x%08x\n",
811 pvt->dhar, dhar_base(pvt),
812 (boot_cpu_data.x86 == 0xf) ? k8_dhar_offset(pvt)
813 : f10_dhar_offset(pvt));
814
815 debugf1(" DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
816
817 amd64_debug_display_dimm_sizes(pvt, 0);
818
819 /* everything below this point is Fam10h and above */
820 if (boot_cpu_data.x86 == 0xf)
821 return;
822
823 amd64_debug_display_dimm_sizes(pvt, 1);
824
825 amd64_info("using %s syndromes.\n", ((pvt->ecc_sym_sz == 8) ? "x8" : "x4"));
826
827 /* Only if NOT ganged does dclr1 have valid info */
828 if (!dct_ganging_enabled(pvt))
829 amd64_dump_dramcfg_low(pvt->dclr1, 1);
830 }
831
832 /*
833 * see BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
834 */
835 static void prep_chip_selects(struct amd64_pvt *pvt)
836 {
837 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) {
838 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
839 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
840 } else {
841 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
842 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
843 }
844 }
845
846 /*
847 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
848 */
849 static void read_dct_base_mask(struct amd64_pvt *pvt)
850 {
851 int cs;
852
853 prep_chip_selects(pvt);
854
855 for_each_chip_select(cs, 0, pvt) {
856 int reg0 = DCSB0 + (cs * 4);
857 int reg1 = DCSB1 + (cs * 4);
858 u32 *base0 = &pvt->csels[0].csbases[cs];
859 u32 *base1 = &pvt->csels[1].csbases[cs];
860
861 if (!amd64_read_dct_pci_cfg(pvt, reg0, base0))
862 debugf0(" DCSB0[%d]=0x%08x reg: F2x%x\n",
863 cs, *base0, reg0);
864
865 if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt))
866 continue;
867
868 if (!amd64_read_dct_pci_cfg(pvt, reg1, base1))
869 debugf0(" DCSB1[%d]=0x%08x reg: F2x%x\n",
870 cs, *base1, reg1);
871 }
872
873 for_each_chip_select_mask(cs, 0, pvt) {
874 int reg0 = DCSM0 + (cs * 4);
875 int reg1 = DCSM1 + (cs * 4);
876 u32 *mask0 = &pvt->csels[0].csmasks[cs];
877 u32 *mask1 = &pvt->csels[1].csmasks[cs];
878
879 if (!amd64_read_dct_pci_cfg(pvt, reg0, mask0))
880 debugf0(" DCSM0[%d]=0x%08x reg: F2x%x\n",
881 cs, *mask0, reg0);
882
883 if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt))
884 continue;
885
886 if (!amd64_read_dct_pci_cfg(pvt, reg1, mask1))
887 debugf0(" DCSM1[%d]=0x%08x reg: F2x%x\n",
888 cs, *mask1, reg1);
889 }
890 }
891
892 static enum mem_type amd64_determine_memory_type(struct amd64_pvt *pvt, int cs)
893 {
894 enum mem_type type;
895
896 /* F15h supports only DDR3 */
897 if (boot_cpu_data.x86 >= 0x15)
898 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
899 else if (boot_cpu_data.x86 == 0x10 || pvt->ext_model >= K8_REV_F) {
900 if (pvt->dchr0 & DDR3_MODE)
901 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
902 else
903 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
904 } else {
905 type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
906 }
907
908 amd64_info("CS%d: %s\n", cs, edac_mem_types[type]);
909
910 return type;
911 }
912
913 /* Get the number of DCT channels the memory controller is using. */
914 static int k8_early_channel_count(struct amd64_pvt *pvt)
915 {
916 int flag;
917
918 if (pvt->ext_model >= K8_REV_F)
919 /* RevF (NPT) and later */
920 flag = pvt->dclr0 & WIDTH_128;
921 else
922 /* RevE and earlier */
923 flag = pvt->dclr0 & REVE_WIDTH_128;
924
925 /* not used */
926 pvt->dclr1 = 0;
927
928 return (flag) ? 2 : 1;
929 }
930
931 /* On F10h and later ErrAddr is MC4_ADDR[47:1] */
932 static u64 get_error_address(struct mce *m)
933 {
934 struct cpuinfo_x86 *c = &boot_cpu_data;
935 u64 addr;
936 u8 start_bit = 1;
937 u8 end_bit = 47;
938
939 if (c->x86 == 0xf) {
940 start_bit = 3;
941 end_bit = 39;
942 }
943
944 addr = m->addr & GENMASK(start_bit, end_bit);
945
946 /*
947 * Erratum 637 workaround
948 */
949 if (c->x86 == 0x15) {
950 struct amd64_pvt *pvt;
951 u64 cc6_base, tmp_addr;
952 u32 tmp;
953 u8 mce_nid, intlv_en;
954
955 if ((addr & GENMASK(24, 47)) >> 24 != 0x00fdf7)
956 return addr;
957
958 mce_nid = amd_get_nb_id(m->extcpu);
959 pvt = mcis[mce_nid]->pvt_info;
960
961 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp);
962 intlv_en = tmp >> 21 & 0x7;
963
964 /* add [47:27] + 3 trailing bits */
965 cc6_base = (tmp & GENMASK(0, 20)) << 3;
966
967 /* reverse and add DramIntlvEn */
968 cc6_base |= intlv_en ^ 0x7;
969
970 /* pin at [47:24] */
971 cc6_base <<= 24;
972
973 if (!intlv_en)
974 return cc6_base | (addr & GENMASK(0, 23));
975
976 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp);
977
978 /* faster log2 */
979 tmp_addr = (addr & GENMASK(12, 23)) << __fls(intlv_en + 1);
980
981 /* OR DramIntlvSel into bits [14:12] */
982 tmp_addr |= (tmp & GENMASK(21, 23)) >> 9;
983
984 /* add remaining [11:0] bits from original MC4_ADDR */
985 tmp_addr |= addr & GENMASK(0, 11);
986
987 return cc6_base | tmp_addr;
988 }
989
990 return addr;
991 }
992
993 static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
994 {
995 struct cpuinfo_x86 *c = &boot_cpu_data;
996 int off = range << 3;
997
998 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo);
999 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
1000
1001 if (c->x86 == 0xf)
1002 return;
1003
1004 if (!dram_rw(pvt, range))
1005 return;
1006
1007 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi);
1008 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
1009
1010 /* Factor in CC6 save area by reading dst node's limit reg */
1011 if (c->x86 == 0x15) {
1012 struct pci_dev *f1 = NULL;
1013 u8 nid = dram_dst_node(pvt, range);
1014 u32 llim;
1015
1016 f1 = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(0x18 + nid, 1));
1017 if (WARN_ON(!f1))
1018 return;
1019
1020 amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim);
1021
1022 pvt->ranges[range].lim.lo &= GENMASK(0, 15);
1023
1024 /* {[39:27],111b} */
1025 pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16;
1026
1027 pvt->ranges[range].lim.hi &= GENMASK(0, 7);
1028
1029 /* [47:40] */
1030 pvt->ranges[range].lim.hi |= llim >> 13;
1031
1032 pci_dev_put(f1);
1033 }
1034 }
1035
1036 static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
1037 u16 syndrome)
1038 {
1039 struct mem_ctl_info *src_mci;
1040 struct amd64_pvt *pvt = mci->pvt_info;
1041 int channel, csrow;
1042 u32 page, offset;
1043
1044 /* CHIPKILL enabled */
1045 if (pvt->nbcfg & NBCFG_CHIPKILL) {
1046 channel = get_channel_from_ecc_syndrome(mci, syndrome);
1047 if (channel < 0) {
1048 /*
1049 * Syndrome didn't map, so we don't know which of the
1050 * 2 DIMMs is in error. So we need to ID 'both' of them
1051 * as suspect.
1052 */
1053 amd64_mc_warn(mci, "unknown syndrome 0x%04x - possible "
1054 "error reporting race\n", syndrome);
1055 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1056 return;
1057 }
1058 } else {
1059 /*
1060 * non-chipkill ecc mode
1061 *
1062 * The k8 documentation is unclear about how to determine the
1063 * channel number when using non-chipkill memory. This method
1064 * was obtained from email communication with someone at AMD.
1065 * (Wish the email was placed in this comment - norsk)
1066 */
1067 channel = ((sys_addr & BIT(3)) != 0);
1068 }
1069
1070 /*
1071 * Find out which node the error address belongs to. This may be
1072 * different from the node that detected the error.
1073 */
1074 src_mci = find_mc_by_sys_addr(mci, sys_addr);
1075 if (!src_mci) {
1076 amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
1077 (unsigned long)sys_addr);
1078 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1079 return;
1080 }
1081
1082 /* Now map the sys_addr to a CSROW */
1083 csrow = sys_addr_to_csrow(src_mci, sys_addr);
1084 if (csrow < 0) {
1085 edac_mc_handle_ce_no_info(src_mci, EDAC_MOD_STR);
1086 } else {
1087 error_address_to_page_and_offset(sys_addr, &page, &offset);
1088
1089 edac_mc_handle_ce(src_mci, page, offset, syndrome, csrow,
1090 channel, EDAC_MOD_STR);
1091 }
1092 }
1093
1094 static int ddr2_cs_size(unsigned i, bool dct_width)
1095 {
1096 unsigned shift = 0;
1097
1098 if (i <= 2)
1099 shift = i;
1100 else if (!(i & 0x1))
1101 shift = i >> 1;
1102 else
1103 shift = (i + 1) >> 1;
1104
1105 return 128 << (shift + !!dct_width);
1106 }
1107
1108 static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1109 unsigned cs_mode)
1110 {
1111 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1112
1113 if (pvt->ext_model >= K8_REV_F) {
1114 WARN_ON(cs_mode > 11);
1115 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1116 }
1117 else if (pvt->ext_model >= K8_REV_D) {
1118 WARN_ON(cs_mode > 10);
1119
1120 if (cs_mode == 3 || cs_mode == 8)
1121 return 32 << (cs_mode - 1);
1122 else
1123 return 32 << cs_mode;
1124 }
1125 else {
1126 WARN_ON(cs_mode > 6);
1127 return 32 << cs_mode;
1128 }
1129 }
1130
1131 /*
1132 * Get the number of DCT channels in use.
1133 *
1134 * Return:
1135 * number of Memory Channels in operation
1136 * Pass back:
1137 * contents of the DCL0_LOW register
1138 */
1139 static int f1x_early_channel_count(struct amd64_pvt *pvt)
1140 {
1141 int i, j, channels = 0;
1142
1143 /* On F10h, if we are in 128 bit mode, then we are using 2 channels */
1144 if (boot_cpu_data.x86 == 0x10 && (pvt->dclr0 & WIDTH_128))
1145 return 2;
1146
1147 /*
1148 * Need to check if in unganged mode: In such, there are 2 channels,
1149 * but they are not in 128 bit mode and thus the above 'dclr0' status
1150 * bit will be OFF.
1151 *
1152 * Need to check DCT0[0] and DCT1[0] to see if only one of them has
1153 * their CSEnable bit on. If so, then SINGLE DIMM case.
1154 */
1155 debugf0("Data width is not 128 bits - need more decoding\n");
1156
1157 /*
1158 * Check DRAM Bank Address Mapping values for each DIMM to see if there
1159 * is more than just one DIMM present in unganged mode. Need to check
1160 * both controllers since DIMMs can be placed in either one.
1161 */
1162 for (i = 0; i < 2; i++) {
1163 u32 dbam = (i ? pvt->dbam1 : pvt->dbam0);
1164
1165 for (j = 0; j < 4; j++) {
1166 if (DBAM_DIMM(j, dbam) > 0) {
1167 channels++;
1168 break;
1169 }
1170 }
1171 }
1172
1173 if (channels > 2)
1174 channels = 2;
1175
1176 amd64_info("MCT channel count: %d\n", channels);
1177
1178 return channels;
1179 }
1180
1181 static int ddr3_cs_size(unsigned i, bool dct_width)
1182 {
1183 unsigned shift = 0;
1184 int cs_size = 0;
1185
1186 if (i == 0 || i == 3 || i == 4)
1187 cs_size = -1;
1188 else if (i <= 2)
1189 shift = i;
1190 else if (i == 12)
1191 shift = 7;
1192 else if (!(i & 0x1))
1193 shift = i >> 1;
1194 else
1195 shift = (i + 1) >> 1;
1196
1197 if (cs_size != -1)
1198 cs_size = (128 * (1 << !!dct_width)) << shift;
1199
1200 return cs_size;
1201 }
1202
1203 static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1204 unsigned cs_mode)
1205 {
1206 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1207
1208 WARN_ON(cs_mode > 11);
1209
1210 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
1211 return ddr3_cs_size(cs_mode, dclr & WIDTH_128);
1212 else
1213 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1214 }
1215
1216 /*
1217 * F15h supports only 64bit DCT interfaces
1218 */
1219 static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1220 unsigned cs_mode)
1221 {
1222 WARN_ON(cs_mode > 12);
1223
1224 return ddr3_cs_size(cs_mode, false);
1225 }
1226
1227 static void read_dram_ctl_register(struct amd64_pvt *pvt)
1228 {
1229
1230 if (boot_cpu_data.x86 == 0xf)
1231 return;
1232
1233 if (!amd64_read_dct_pci_cfg(pvt, DCT_SEL_LO, &pvt->dct_sel_lo)) {
1234 debugf0("F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
1235 pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
1236
1237 debugf0(" DCTs operate in %s mode.\n",
1238 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"));
1239
1240 if (!dct_ganging_enabled(pvt))
1241 debugf0(" Address range split per DCT: %s\n",
1242 (dct_high_range_enabled(pvt) ? "yes" : "no"));
1243
1244 debugf0(" data interleave for ECC: %s, "
1245 "DRAM cleared since last warm reset: %s\n",
1246 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
1247 (dct_memory_cleared(pvt) ? "yes" : "no"));
1248
1249 debugf0(" channel interleave: %s, "
1250 "interleave bits selector: 0x%x\n",
1251 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
1252 dct_sel_interleave_addr(pvt));
1253 }
1254
1255 amd64_read_dct_pci_cfg(pvt, DCT_SEL_HI, &pvt->dct_sel_hi);
1256 }
1257
1258 /*
1259 * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
1260 * Interleaving Modes.
1261 */
1262 static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1263 bool hi_range_sel, u8 intlv_en)
1264 {
1265 u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
1266
1267 if (dct_ganging_enabled(pvt))
1268 return 0;
1269
1270 if (hi_range_sel)
1271 return dct_sel_high;
1272
1273 /*
1274 * see F2x110[DctSelIntLvAddr] - channel interleave mode
1275 */
1276 if (dct_interleave_enabled(pvt)) {
1277 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1278
1279 /* return DCT select function: 0=DCT0, 1=DCT1 */
1280 if (!intlv_addr)
1281 return sys_addr >> 6 & 1;
1282
1283 if (intlv_addr & 0x2) {
1284 u8 shift = intlv_addr & 0x1 ? 9 : 6;
1285 u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2;
1286
1287 return ((sys_addr >> shift) & 1) ^ temp;
1288 }
1289
1290 return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
1291 }
1292
1293 if (dct_high_range_enabled(pvt))
1294 return ~dct_sel_high & 1;
1295
1296 return 0;
1297 }
1298
1299 /* Convert the sys_addr to the normalized DCT address */
1300 static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, unsigned range,
1301 u64 sys_addr, bool hi_rng,
1302 u32 dct_sel_base_addr)
1303 {
1304 u64 chan_off;
1305 u64 dram_base = get_dram_base(pvt, range);
1306 u64 hole_off = f10_dhar_offset(pvt);
1307 u64 dct_sel_base_off = (pvt->dct_sel_hi & 0xFFFFFC00) << 16;
1308
1309 if (hi_rng) {
1310 /*
1311 * if
1312 * base address of high range is below 4Gb
1313 * (bits [47:27] at [31:11])
1314 * DRAM address space on this DCT is hoisted above 4Gb &&
1315 * sys_addr > 4Gb
1316 *
1317 * remove hole offset from sys_addr
1318 * else
1319 * remove high range offset from sys_addr
1320 */
1321 if ((!(dct_sel_base_addr >> 16) ||
1322 dct_sel_base_addr < dhar_base(pvt)) &&
1323 dhar_valid(pvt) &&
1324 (sys_addr >= BIT_64(32)))
1325 chan_off = hole_off;
1326 else
1327 chan_off = dct_sel_base_off;
1328 } else {
1329 /*
1330 * if
1331 * we have a valid hole &&
1332 * sys_addr > 4Gb
1333 *
1334 * remove hole
1335 * else
1336 * remove dram base to normalize to DCT address
1337 */
1338 if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))
1339 chan_off = hole_off;
1340 else
1341 chan_off = dram_base;
1342 }
1343
1344 return (sys_addr & GENMASK(6,47)) - (chan_off & GENMASK(23,47));
1345 }
1346
1347 /*
1348 * checks if the csrow passed in is marked as SPARED, if so returns the new
1349 * spare row
1350 */
1351 static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
1352 {
1353 int tmp_cs;
1354
1355 if (online_spare_swap_done(pvt, dct) &&
1356 csrow == online_spare_bad_dramcs(pvt, dct)) {
1357
1358 for_each_chip_select(tmp_cs, dct, pvt) {
1359 if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
1360 csrow = tmp_cs;
1361 break;
1362 }
1363 }
1364 }
1365 return csrow;
1366 }
1367
1368 /*
1369 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
1370 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
1371 *
1372 * Return:
1373 * -EINVAL: NOT FOUND
1374 * 0..csrow = Chip-Select Row
1375 */
1376 static int f1x_lookup_addr_in_dct(u64 in_addr, u32 nid, u8 dct)
1377 {
1378 struct mem_ctl_info *mci;
1379 struct amd64_pvt *pvt;
1380 u64 cs_base, cs_mask;
1381 int cs_found = -EINVAL;
1382 int csrow;
1383
1384 mci = mcis[nid];
1385 if (!mci)
1386 return cs_found;
1387
1388 pvt = mci->pvt_info;
1389
1390 debugf1("input addr: 0x%llx, DCT: %d\n", in_addr, dct);
1391
1392 for_each_chip_select(csrow, dct, pvt) {
1393 if (!csrow_enabled(csrow, dct, pvt))
1394 continue;
1395
1396 get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
1397
1398 debugf1(" CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
1399 csrow, cs_base, cs_mask);
1400
1401 cs_mask = ~cs_mask;
1402
1403 debugf1(" (InputAddr & ~CSMask)=0x%llx "
1404 "(CSBase & ~CSMask)=0x%llx\n",
1405 (in_addr & cs_mask), (cs_base & cs_mask));
1406
1407 if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
1408 cs_found = f10_process_possible_spare(pvt, dct, csrow);
1409
1410 debugf1(" MATCH csrow=%d\n", cs_found);
1411 break;
1412 }
1413 }
1414 return cs_found;
1415 }
1416
1417 /*
1418 * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is
1419 * swapped with a region located at the bottom of memory so that the GPU can use
1420 * the interleaved region and thus two channels.
1421 */
1422 static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)
1423 {
1424 u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;
1425
1426 if (boot_cpu_data.x86 == 0x10) {
1427 /* only revC3 and revE have that feature */
1428 if (boot_cpu_data.x86_model < 4 ||
1429 (boot_cpu_data.x86_model < 0xa &&
1430 boot_cpu_data.x86_mask < 3))
1431 return sys_addr;
1432 }
1433
1434 amd64_read_dct_pci_cfg(pvt, SWAP_INTLV_REG, &swap_reg);
1435
1436 if (!(swap_reg & 0x1))
1437 return sys_addr;
1438
1439 swap_base = (swap_reg >> 3) & 0x7f;
1440 swap_limit = (swap_reg >> 11) & 0x7f;
1441 rgn_size = (swap_reg >> 20) & 0x7f;
1442 tmp_addr = sys_addr >> 27;
1443
1444 if (!(sys_addr >> 34) &&
1445 (((tmp_addr >= swap_base) &&
1446 (tmp_addr <= swap_limit)) ||
1447 (tmp_addr < rgn_size)))
1448 return sys_addr ^ (u64)swap_base << 27;
1449
1450 return sys_addr;
1451 }
1452
1453 /* For a given @dram_range, check if @sys_addr falls within it. */
1454 static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
1455 u64 sys_addr, int *nid, int *chan_sel)
1456 {
1457 int cs_found = -EINVAL;
1458 u64 chan_addr;
1459 u32 dct_sel_base;
1460 u8 channel;
1461 bool high_range = false;
1462
1463 u8 node_id = dram_dst_node(pvt, range);
1464 u8 intlv_en = dram_intlv_en(pvt, range);
1465 u32 intlv_sel = dram_intlv_sel(pvt, range);
1466
1467 debugf1("(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1468 range, sys_addr, get_dram_limit(pvt, range));
1469
1470 if (dhar_valid(pvt) &&
1471 dhar_base(pvt) <= sys_addr &&
1472 sys_addr < BIT_64(32)) {
1473 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1474 sys_addr);
1475 return -EINVAL;
1476 }
1477
1478 if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en)))
1479 return -EINVAL;
1480
1481 sys_addr = f1x_swap_interleaved_region(pvt, sys_addr);
1482
1483 dct_sel_base = dct_sel_baseaddr(pvt);
1484
1485 /*
1486 * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
1487 * select between DCT0 and DCT1.
1488 */
1489 if (dct_high_range_enabled(pvt) &&
1490 !dct_ganging_enabled(pvt) &&
1491 ((sys_addr >> 27) >= (dct_sel_base >> 11)))
1492 high_range = true;
1493
1494 channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en);
1495
1496 chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr,
1497 high_range, dct_sel_base);
1498
1499 /* Remove node interleaving, see F1x120 */
1500 if (intlv_en)
1501 chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) |
1502 (chan_addr & 0xfff);
1503
1504 /* remove channel interleave */
1505 if (dct_interleave_enabled(pvt) &&
1506 !dct_high_range_enabled(pvt) &&
1507 !dct_ganging_enabled(pvt)) {
1508
1509 if (dct_sel_interleave_addr(pvt) != 1) {
1510 if (dct_sel_interleave_addr(pvt) == 0x3)
1511 /* hash 9 */
1512 chan_addr = ((chan_addr >> 10) << 9) |
1513 (chan_addr & 0x1ff);
1514 else
1515 /* A[6] or hash 6 */
1516 chan_addr = ((chan_addr >> 7) << 6) |
1517 (chan_addr & 0x3f);
1518 } else
1519 /* A[12] */
1520 chan_addr = ((chan_addr >> 13) << 12) |
1521 (chan_addr & 0xfff);
1522 }
1523
1524 debugf1(" Normalized DCT addr: 0x%llx\n", chan_addr);
1525
1526 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel);
1527
1528 if (cs_found >= 0) {
1529 *nid = node_id;
1530 *chan_sel = channel;
1531 }
1532 return cs_found;
1533 }
1534
1535 static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt, u64 sys_addr,
1536 int *node, int *chan_sel)
1537 {
1538 int cs_found = -EINVAL;
1539 unsigned range;
1540
1541 for (range = 0; range < DRAM_RANGES; range++) {
1542
1543 if (!dram_rw(pvt, range))
1544 continue;
1545
1546 if ((get_dram_base(pvt, range) <= sys_addr) &&
1547 (get_dram_limit(pvt, range) >= sys_addr)) {
1548
1549 cs_found = f1x_match_to_this_node(pvt, range,
1550 sys_addr, node,
1551 chan_sel);
1552 if (cs_found >= 0)
1553 break;
1554 }
1555 }
1556 return cs_found;
1557 }
1558
1559 /*
1560 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
1561 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
1562 *
1563 * The @sys_addr is usually an error address received from the hardware
1564 * (MCX_ADDR).
1565 */
1566 static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
1567 u16 syndrome)
1568 {
1569 struct amd64_pvt *pvt = mci->pvt_info;
1570 u32 page, offset;
1571 int nid, csrow, chan = 0;
1572
1573 csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &nid, &chan);
1574
1575 if (csrow < 0) {
1576 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1577 return;
1578 }
1579
1580 error_address_to_page_and_offset(sys_addr, &page, &offset);
1581
1582 /*
1583 * We need the syndromes for channel detection only when we're
1584 * ganged. Otherwise @chan should already contain the channel at
1585 * this point.
1586 */
1587 if (dct_ganging_enabled(pvt))
1588 chan = get_channel_from_ecc_syndrome(mci, syndrome);
1589
1590 if (chan >= 0)
1591 edac_mc_handle_ce(mci, page, offset, syndrome, csrow, chan,
1592 EDAC_MOD_STR);
1593 else
1594 /*
1595 * Channel unknown, report all channels on this CSROW as failed.
1596 */
1597 for (chan = 0; chan < mci->csrows[csrow].nr_channels; chan++)
1598 edac_mc_handle_ce(mci, page, offset, syndrome,
1599 csrow, chan, EDAC_MOD_STR);
1600 }
1601
1602 /*
1603 * debug routine to display the memory sizes of all logical DIMMs and its
1604 * CSROWs
1605 */
1606 static void amd64_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
1607 {
1608 int dimm, size0, size1, factor = 0;
1609 u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
1610 u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0;
1611
1612 if (boot_cpu_data.x86 == 0xf) {
1613 if (pvt->dclr0 & WIDTH_128)
1614 factor = 1;
1615
1616 /* K8 families < revF not supported yet */
1617 if (pvt->ext_model < K8_REV_F)
1618 return;
1619 else
1620 WARN_ON(ctrl != 0);
1621 }
1622
1623 dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1 : pvt->dbam0;
1624 dcsb = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->csels[1].csbases
1625 : pvt->csels[0].csbases;
1626
1627 debugf1("F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n", ctrl, dbam);
1628
1629 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
1630
1631 /* Dump memory sizes for DIMM and its CSROWs */
1632 for (dimm = 0; dimm < 4; dimm++) {
1633
1634 size0 = 0;
1635 if (dcsb[dimm*2] & DCSB_CS_ENABLE)
1636 size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
1637 DBAM_DIMM(dimm, dbam));
1638
1639 size1 = 0;
1640 if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
1641 size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
1642 DBAM_DIMM(dimm, dbam));
1643
1644 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
1645 dimm * 2, size0 << factor,
1646 dimm * 2 + 1, size1 << factor);
1647 }
1648 }
1649
1650 static struct amd64_family_type amd64_family_types[] = {
1651 [K8_CPUS] = {
1652 .ctl_name = "K8",
1653 .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
1654 .f3_id = PCI_DEVICE_ID_AMD_K8_NB_MISC,
1655 .ops = {
1656 .early_channel_count = k8_early_channel_count,
1657 .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow,
1658 .dbam_to_cs = k8_dbam_to_chip_select,
1659 .read_dct_pci_cfg = k8_read_dct_pci_cfg,
1660 }
1661 },
1662 [F10_CPUS] = {
1663 .ctl_name = "F10h",
1664 .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
1665 .f3_id = PCI_DEVICE_ID_AMD_10H_NB_MISC,
1666 .ops = {
1667 .early_channel_count = f1x_early_channel_count,
1668 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1669 .dbam_to_cs = f10_dbam_to_chip_select,
1670 .read_dct_pci_cfg = f10_read_dct_pci_cfg,
1671 }
1672 },
1673 [F15_CPUS] = {
1674 .ctl_name = "F15h",
1675 .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1,
1676 .f3_id = PCI_DEVICE_ID_AMD_15H_NB_F3,
1677 .ops = {
1678 .early_channel_count = f1x_early_channel_count,
1679 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1680 .dbam_to_cs = f15_dbam_to_chip_select,
1681 .read_dct_pci_cfg = f15_read_dct_pci_cfg,
1682 }
1683 },
1684 };
1685
1686 static struct pci_dev *pci_get_related_function(unsigned int vendor,
1687 unsigned int device,
1688 struct pci_dev *related)
1689 {
1690 struct pci_dev *dev = NULL;
1691
1692 dev = pci_get_device(vendor, device, dev);
1693 while (dev) {
1694 if ((dev->bus->number == related->bus->number) &&
1695 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
1696 break;
1697 dev = pci_get_device(vendor, device, dev);
1698 }
1699
1700 return dev;
1701 }
1702
1703 /*
1704 * These are tables of eigenvectors (one per line) which can be used for the
1705 * construction of the syndrome tables. The modified syndrome search algorithm
1706 * uses those to find the symbol in error and thus the DIMM.
1707 *
1708 * Algorithm courtesy of Ross LaFetra from AMD.
1709 */
1710 static u16 x4_vectors[] = {
1711 0x2f57, 0x1afe, 0x66cc, 0xdd88,
1712 0x11eb, 0x3396, 0x7f4c, 0xeac8,
1713 0x0001, 0x0002, 0x0004, 0x0008,
1714 0x1013, 0x3032, 0x4044, 0x8088,
1715 0x106b, 0x30d6, 0x70fc, 0xe0a8,
1716 0x4857, 0xc4fe, 0x13cc, 0x3288,
1717 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
1718 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
1719 0x15c1, 0x2a42, 0x89ac, 0x4758,
1720 0x2b03, 0x1602, 0x4f0c, 0xca08,
1721 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
1722 0x8ba7, 0x465e, 0x244c, 0x1cc8,
1723 0x2b87, 0x164e, 0x642c, 0xdc18,
1724 0x40b9, 0x80de, 0x1094, 0x20e8,
1725 0x27db, 0x1eb6, 0x9dac, 0x7b58,
1726 0x11c1, 0x2242, 0x84ac, 0x4c58,
1727 0x1be5, 0x2d7a, 0x5e34, 0xa718,
1728 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
1729 0x4c97, 0xc87e, 0x11fc, 0x33a8,
1730 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
1731 0x16b3, 0x3d62, 0x4f34, 0x8518,
1732 0x1e2f, 0x391a, 0x5cac, 0xf858,
1733 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
1734 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
1735 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
1736 0x4397, 0xc27e, 0x17fc, 0x3ea8,
1737 0x1617, 0x3d3e, 0x6464, 0xb8b8,
1738 0x23ff, 0x12aa, 0xab6c, 0x56d8,
1739 0x2dfb, 0x1ba6, 0x913c, 0x7328,
1740 0x185d, 0x2ca6, 0x7914, 0x9e28,
1741 0x171b, 0x3e36, 0x7d7c, 0xebe8,
1742 0x4199, 0x82ee, 0x19f4, 0x2e58,
1743 0x4807, 0xc40e, 0x130c, 0x3208,
1744 0x1905, 0x2e0a, 0x5804, 0xac08,
1745 0x213f, 0x132a, 0xadfc, 0x5ba8,
1746 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
1747 };
1748
1749 static u16 x8_vectors[] = {
1750 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
1751 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
1752 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
1753 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
1754 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
1755 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
1756 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
1757 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
1758 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
1759 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
1760 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
1761 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
1762 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
1763 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
1764 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
1765 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
1766 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
1767 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
1768 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
1769 };
1770
1771 static int decode_syndrome(u16 syndrome, u16 *vectors, unsigned num_vecs,
1772 unsigned v_dim)
1773 {
1774 unsigned int i, err_sym;
1775
1776 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
1777 u16 s = syndrome;
1778 unsigned v_idx = err_sym * v_dim;
1779 unsigned v_end = (err_sym + 1) * v_dim;
1780
1781 /* walk over all 16 bits of the syndrome */
1782 for (i = 1; i < (1U << 16); i <<= 1) {
1783
1784 /* if bit is set in that eigenvector... */
1785 if (v_idx < v_end && vectors[v_idx] & i) {
1786 u16 ev_comp = vectors[v_idx++];
1787
1788 /* ... and bit set in the modified syndrome, */
1789 if (s & i) {
1790 /* remove it. */
1791 s ^= ev_comp;
1792
1793 if (!s)
1794 return err_sym;
1795 }
1796
1797 } else if (s & i)
1798 /* can't get to zero, move to next symbol */
1799 break;
1800 }
1801 }
1802
1803 debugf0("syndrome(%x) not found\n", syndrome);
1804 return -1;
1805 }
1806
1807 static int map_err_sym_to_channel(int err_sym, int sym_size)
1808 {
1809 if (sym_size == 4)
1810 switch (err_sym) {
1811 case 0x20:
1812 case 0x21:
1813 return 0;
1814 break;
1815 case 0x22:
1816 case 0x23:
1817 return 1;
1818 break;
1819 default:
1820 return err_sym >> 4;
1821 break;
1822 }
1823 /* x8 symbols */
1824 else
1825 switch (err_sym) {
1826 /* imaginary bits not in a DIMM */
1827 case 0x10:
1828 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
1829 err_sym);
1830 return -1;
1831 break;
1832
1833 case 0x11:
1834 return 0;
1835 break;
1836 case 0x12:
1837 return 1;
1838 break;
1839 default:
1840 return err_sym >> 3;
1841 break;
1842 }
1843 return -1;
1844 }
1845
1846 static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
1847 {
1848 struct amd64_pvt *pvt = mci->pvt_info;
1849 int err_sym = -1;
1850
1851 if (pvt->ecc_sym_sz == 8)
1852 err_sym = decode_syndrome(syndrome, x8_vectors,
1853 ARRAY_SIZE(x8_vectors),
1854 pvt->ecc_sym_sz);
1855 else if (pvt->ecc_sym_sz == 4)
1856 err_sym = decode_syndrome(syndrome, x4_vectors,
1857 ARRAY_SIZE(x4_vectors),
1858 pvt->ecc_sym_sz);
1859 else {
1860 amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz);
1861 return err_sym;
1862 }
1863
1864 return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
1865 }
1866
1867 /*
1868 * Handle any Correctable Errors (CEs) that have occurred. Check for valid ERROR
1869 * ADDRESS and process.
1870 */
1871 static void amd64_handle_ce(struct mem_ctl_info *mci, struct mce *m)
1872 {
1873 struct amd64_pvt *pvt = mci->pvt_info;
1874 u64 sys_addr;
1875 u16 syndrome;
1876
1877 /* Ensure that the Error Address is VALID */
1878 if (!(m->status & MCI_STATUS_ADDRV)) {
1879 amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n");
1880 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1881 return;
1882 }
1883
1884 sys_addr = get_error_address(m);
1885 syndrome = extract_syndrome(m->status);
1886
1887 amd64_mc_err(mci, "CE ERROR_ADDRESS= 0x%llx\n", sys_addr);
1888
1889 pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, syndrome);
1890 }
1891
1892 /* Handle any Un-correctable Errors (UEs) */
1893 static void amd64_handle_ue(struct mem_ctl_info *mci, struct mce *m)
1894 {
1895 struct mem_ctl_info *log_mci, *src_mci = NULL;
1896 int csrow;
1897 u64 sys_addr;
1898 u32 page, offset;
1899
1900 log_mci = mci;
1901
1902 if (!(m->status & MCI_STATUS_ADDRV)) {
1903 amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n");
1904 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1905 return;
1906 }
1907
1908 sys_addr = get_error_address(m);
1909
1910 /*
1911 * Find out which node the error address belongs to. This may be
1912 * different from the node that detected the error.
1913 */
1914 src_mci = find_mc_by_sys_addr(mci, sys_addr);
1915 if (!src_mci) {
1916 amd64_mc_err(mci, "ERROR ADDRESS (0x%lx) NOT mapped to a MC\n",
1917 (unsigned long)sys_addr);
1918 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1919 return;
1920 }
1921
1922 log_mci = src_mci;
1923
1924 csrow = sys_addr_to_csrow(log_mci, sys_addr);
1925 if (csrow < 0) {
1926 amd64_mc_err(mci, "ERROR_ADDRESS (0x%lx) NOT mapped to CS\n",
1927 (unsigned long)sys_addr);
1928 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1929 } else {
1930 error_address_to_page_and_offset(sys_addr, &page, &offset);
1931 edac_mc_handle_ue(log_mci, page, offset, csrow, EDAC_MOD_STR);
1932 }
1933 }
1934
1935 static inline void __amd64_decode_bus_error(struct mem_ctl_info *mci,
1936 struct mce *m)
1937 {
1938 u16 ec = EC(m->status);
1939 u8 xec = XEC(m->status, 0x1f);
1940 u8 ecc_type = (m->status >> 45) & 0x3;
1941
1942 /* Bail early out if this was an 'observed' error */
1943 if (PP(ec) == NBSL_PP_OBS)
1944 return;
1945
1946 /* Do only ECC errors */
1947 if (xec && xec != F10_NBSL_EXT_ERR_ECC)
1948 return;
1949
1950 if (ecc_type == 2)
1951 amd64_handle_ce(mci, m);
1952 else if (ecc_type == 1)
1953 amd64_handle_ue(mci, m);
1954 }
1955
1956 void amd64_decode_bus_error(int node_id, struct mce *m, u32 nbcfg)
1957 {
1958 struct mem_ctl_info *mci = mcis[node_id];
1959
1960 __amd64_decode_bus_error(mci, m);
1961 }
1962
1963 /*
1964 * Use pvt->F2 which contains the F2 CPU PCI device to get the related
1965 * F1 (AddrMap) and F3 (Misc) devices. Return negative value on error.
1966 */
1967 static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f3_id)
1968 {
1969 /* Reserve the ADDRESS MAP Device */
1970 pvt->F1 = pci_get_related_function(pvt->F2->vendor, f1_id, pvt->F2);
1971 if (!pvt->F1) {
1972 amd64_err("error address map device not found: "
1973 "vendor %x device 0x%x (broken BIOS?)\n",
1974 PCI_VENDOR_ID_AMD, f1_id);
1975 return -ENODEV;
1976 }
1977
1978 /* Reserve the MISC Device */
1979 pvt->F3 = pci_get_related_function(pvt->F2->vendor, f3_id, pvt->F2);
1980 if (!pvt->F3) {
1981 pci_dev_put(pvt->F1);
1982 pvt->F1 = NULL;
1983
1984 amd64_err("error F3 device not found: "
1985 "vendor %x device 0x%x (broken BIOS?)\n",
1986 PCI_VENDOR_ID_AMD, f3_id);
1987
1988 return -ENODEV;
1989 }
1990 debugf1("F1: %s\n", pci_name(pvt->F1));
1991 debugf1("F2: %s\n", pci_name(pvt->F2));
1992 debugf1("F3: %s\n", pci_name(pvt->F3));
1993
1994 return 0;
1995 }
1996
1997 static void free_mc_sibling_devs(struct amd64_pvt *pvt)
1998 {
1999 pci_dev_put(pvt->F1);
2000 pci_dev_put(pvt->F3);
2001 }
2002
2003 /*
2004 * Retrieve the hardware registers of the memory controller (this includes the
2005 * 'Address Map' and 'Misc' device regs)
2006 */
2007 static void read_mc_regs(struct amd64_pvt *pvt)
2008 {
2009 struct cpuinfo_x86 *c = &boot_cpu_data;
2010 u64 msr_val;
2011 u32 tmp;
2012 unsigned range;
2013
2014 /*
2015 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
2016 * those are Read-As-Zero
2017 */
2018 rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
2019 debugf0(" TOP_MEM: 0x%016llx\n", pvt->top_mem);
2020
2021 /* check first whether TOP_MEM2 is enabled */
2022 rdmsrl(MSR_K8_SYSCFG, msr_val);
2023 if (msr_val & (1U << 21)) {
2024 rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
2025 debugf0(" TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
2026 } else
2027 debugf0(" TOP_MEM2 disabled.\n");
2028
2029 amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
2030
2031 read_dram_ctl_register(pvt);
2032
2033 for (range = 0; range < DRAM_RANGES; range++) {
2034 u8 rw;
2035
2036 /* read settings for this DRAM range */
2037 read_dram_base_limit_regs(pvt, range);
2038
2039 rw = dram_rw(pvt, range);
2040 if (!rw)
2041 continue;
2042
2043 debugf1(" DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
2044 range,
2045 get_dram_base(pvt, range),
2046 get_dram_limit(pvt, range));
2047
2048 debugf1(" IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
2049 dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
2050 (rw & 0x1) ? "R" : "-",
2051 (rw & 0x2) ? "W" : "-",
2052 dram_intlv_sel(pvt, range),
2053 dram_dst_node(pvt, range));
2054 }
2055
2056 read_dct_base_mask(pvt);
2057
2058 amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
2059 amd64_read_dct_pci_cfg(pvt, DBAM0, &pvt->dbam0);
2060
2061 amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
2062
2063 amd64_read_dct_pci_cfg(pvt, DCLR0, &pvt->dclr0);
2064 amd64_read_dct_pci_cfg(pvt, DCHR0, &pvt->dchr0);
2065
2066 if (!dct_ganging_enabled(pvt)) {
2067 amd64_read_dct_pci_cfg(pvt, DCLR1, &pvt->dclr1);
2068 amd64_read_dct_pci_cfg(pvt, DCHR1, &pvt->dchr1);
2069 }
2070
2071 pvt->ecc_sym_sz = 4;
2072
2073 if (c->x86 >= 0x10) {
2074 amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
2075 amd64_read_dct_pci_cfg(pvt, DBAM1, &pvt->dbam1);
2076
2077 /* F10h, revD and later can do x8 ECC too */
2078 if ((c->x86 > 0x10 || c->x86_model > 7) && tmp & BIT(25))
2079 pvt->ecc_sym_sz = 8;
2080 }
2081 dump_misc_regs(pvt);
2082 }
2083
2084 /*
2085 * NOTE: CPU Revision Dependent code
2086 *
2087 * Input:
2088 * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
2089 * k8 private pointer to -->
2090 * DRAM Bank Address mapping register
2091 * node_id
2092 * DCL register where dual_channel_active is
2093 *
2094 * The DBAM register consists of 4 sets of 4 bits each definitions:
2095 *
2096 * Bits: CSROWs
2097 * 0-3 CSROWs 0 and 1
2098 * 4-7 CSROWs 2 and 3
2099 * 8-11 CSROWs 4 and 5
2100 * 12-15 CSROWs 6 and 7
2101 *
2102 * Values range from: 0 to 15
2103 * The meaning of the values depends on CPU revision and dual-channel state,
2104 * see relevant BKDG more info.
2105 *
2106 * The memory controller provides for total of only 8 CSROWs in its current
2107 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
2108 * single channel or two (2) DIMMs in dual channel mode.
2109 *
2110 * The following code logic collapses the various tables for CSROW based on CPU
2111 * revision.
2112 *
2113 * Returns:
2114 * The number of PAGE_SIZE pages on the specified CSROW number it
2115 * encompasses
2116 *
2117 */
2118 static u32 amd64_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr)
2119 {
2120 u32 cs_mode, nr_pages;
2121
2122 /*
2123 * The math on this doesn't look right on the surface because x/2*4 can
2124 * be simplified to x*2 but this expression makes use of the fact that
2125 * it is integral math where 1/2=0. This intermediate value becomes the
2126 * number of bits to shift the DBAM register to extract the proper CSROW
2127 * field.
2128 */
2129 cs_mode = (pvt->dbam0 >> ((csrow_nr / 2) * 4)) & 0xF;
2130
2131 nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode) << (20 - PAGE_SHIFT);
2132
2133 /*
2134 * If dual channel then double the memory size of single channel.
2135 * Channel count is 1 or 2
2136 */
2137 nr_pages <<= (pvt->channel_count - 1);
2138
2139 debugf0(" (csrow=%d) DBAM map index= %d\n", csrow_nr, cs_mode);
2140 debugf0(" nr_pages= %u channel-count = %d\n",
2141 nr_pages, pvt->channel_count);
2142
2143 return nr_pages;
2144 }
2145
2146 /*
2147 * Initialize the array of csrow attribute instances, based on the values
2148 * from pci config hardware registers.
2149 */
2150 static int init_csrows(struct mem_ctl_info *mci)
2151 {
2152 struct csrow_info *csrow;
2153 struct amd64_pvt *pvt = mci->pvt_info;
2154 u64 input_addr_min, input_addr_max, sys_addr, base, mask;
2155 u32 val;
2156 int i, empty = 1;
2157
2158 amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
2159
2160 pvt->nbcfg = val;
2161
2162 debugf0("node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2163 pvt->mc_node_id, val,
2164 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
2165
2166 for_each_chip_select(i, 0, pvt) {
2167 csrow = &mci->csrows[i];
2168
2169 if (!csrow_enabled(i, 0, pvt)) {
2170 debugf1("----CSROW %d EMPTY for node %d\n", i,
2171 pvt->mc_node_id);
2172 continue;
2173 }
2174
2175 debugf1("----CSROW %d VALID for MC node %d\n",
2176 i, pvt->mc_node_id);
2177
2178 empty = 0;
2179 csrow->nr_pages = amd64_csrow_nr_pages(pvt, 0, i);
2180 find_csrow_limits(mci, i, &input_addr_min, &input_addr_max);
2181 sys_addr = input_addr_to_sys_addr(mci, input_addr_min);
2182 csrow->first_page = (u32) (sys_addr >> PAGE_SHIFT);
2183 sys_addr = input_addr_to_sys_addr(mci, input_addr_max);
2184 csrow->last_page = (u32) (sys_addr >> PAGE_SHIFT);
2185
2186 get_cs_base_and_mask(pvt, i, 0, &base, &mask);
2187 csrow->page_mask = ~mask;
2188 /* 8 bytes of resolution */
2189
2190 csrow->mtype = amd64_determine_memory_type(pvt, i);
2191
2192 debugf1(" for MC node %d csrow %d:\n", pvt->mc_node_id, i);
2193 debugf1(" input_addr_min: 0x%lx input_addr_max: 0x%lx\n",
2194 (unsigned long)input_addr_min,
2195 (unsigned long)input_addr_max);
2196 debugf1(" sys_addr: 0x%lx page_mask: 0x%lx\n",
2197 (unsigned long)sys_addr, csrow->page_mask);
2198 debugf1(" nr_pages: %u first_page: 0x%lx "
2199 "last_page: 0x%lx\n",
2200 (unsigned)csrow->nr_pages,
2201 csrow->first_page, csrow->last_page);
2202
2203 /*
2204 * determine whether CHIPKILL or JUST ECC or NO ECC is operating
2205 */
2206 if (pvt->nbcfg & NBCFG_ECC_ENABLE)
2207 csrow->edac_mode =
2208 (pvt->nbcfg & NBCFG_CHIPKILL) ?
2209 EDAC_S4ECD4ED : EDAC_SECDED;
2210 else
2211 csrow->edac_mode = EDAC_NONE;
2212 }
2213
2214 return empty;
2215 }
2216
2217 /* get all cores on this DCT */
2218 static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, unsigned nid)
2219 {
2220 int cpu;
2221
2222 for_each_online_cpu(cpu)
2223 if (amd_get_nb_id(cpu) == nid)
2224 cpumask_set_cpu(cpu, mask);
2225 }
2226
2227 /* check MCG_CTL on all the cpus on this node */
2228 static bool amd64_nb_mce_bank_enabled_on_node(unsigned nid)
2229 {
2230 cpumask_var_t mask;
2231 int cpu, nbe;
2232 bool ret = false;
2233
2234 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
2235 amd64_warn("%s: Error allocating mask\n", __func__);
2236 return false;
2237 }
2238
2239 get_cpus_on_this_dct_cpumask(mask, nid);
2240
2241 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
2242
2243 for_each_cpu(cpu, mask) {
2244 struct msr *reg = per_cpu_ptr(msrs, cpu);
2245 nbe = reg->l & MSR_MCGCTL_NBE;
2246
2247 debugf0("core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
2248 cpu, reg->q,
2249 (nbe ? "enabled" : "disabled"));
2250
2251 if (!nbe)
2252 goto out;
2253 }
2254 ret = true;
2255
2256 out:
2257 free_cpumask_var(mask);
2258 return ret;
2259 }
2260
2261 static int toggle_ecc_err_reporting(struct ecc_settings *s, u8 nid, bool on)
2262 {
2263 cpumask_var_t cmask;
2264 int cpu;
2265
2266 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
2267 amd64_warn("%s: error allocating mask\n", __func__);
2268 return false;
2269 }
2270
2271 get_cpus_on_this_dct_cpumask(cmask, nid);
2272
2273 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2274
2275 for_each_cpu(cpu, cmask) {
2276
2277 struct msr *reg = per_cpu_ptr(msrs, cpu);
2278
2279 if (on) {
2280 if (reg->l & MSR_MCGCTL_NBE)
2281 s->flags.nb_mce_enable = 1;
2282
2283 reg->l |= MSR_MCGCTL_NBE;
2284 } else {
2285 /*
2286 * Turn off NB MCE reporting only when it was off before
2287 */
2288 if (!s->flags.nb_mce_enable)
2289 reg->l &= ~MSR_MCGCTL_NBE;
2290 }
2291 }
2292 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2293
2294 free_cpumask_var(cmask);
2295
2296 return 0;
2297 }
2298
2299 static bool enable_ecc_error_reporting(struct ecc_settings *s, u8 nid,
2300 struct pci_dev *F3)
2301 {
2302 bool ret = true;
2303 u32 value, mask = 0x3; /* UECC/CECC enable */
2304
2305 if (toggle_ecc_err_reporting(s, nid, ON)) {
2306 amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
2307 return false;
2308 }
2309
2310 amd64_read_pci_cfg(F3, NBCTL, &value);
2311
2312 s->old_nbctl = value & mask;
2313 s->nbctl_valid = true;
2314
2315 value |= mask;
2316 amd64_write_pci_cfg(F3, NBCTL, value);
2317
2318 amd64_read_pci_cfg(F3, NBCFG, &value);
2319
2320 debugf0("1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2321 nid, value, !!(value & NBCFG_ECC_ENABLE));
2322
2323 if (!(value & NBCFG_ECC_ENABLE)) {
2324 amd64_warn("DRAM ECC disabled on this node, enabling...\n");
2325
2326 s->flags.nb_ecc_prev = 0;
2327
2328 /* Attempt to turn on DRAM ECC Enable */
2329 value |= NBCFG_ECC_ENABLE;
2330 amd64_write_pci_cfg(F3, NBCFG, value);
2331
2332 amd64_read_pci_cfg(F3, NBCFG, &value);
2333
2334 if (!(value & NBCFG_ECC_ENABLE)) {
2335 amd64_warn("Hardware rejected DRAM ECC enable,"
2336 "check memory DIMM configuration.\n");
2337 ret = false;
2338 } else {
2339 amd64_info("Hardware accepted DRAM ECC Enable\n");
2340 }
2341 } else {
2342 s->flags.nb_ecc_prev = 1;
2343 }
2344
2345 debugf0("2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2346 nid, value, !!(value & NBCFG_ECC_ENABLE));
2347
2348 return ret;
2349 }
2350
2351 static void restore_ecc_error_reporting(struct ecc_settings *s, u8 nid,
2352 struct pci_dev *F3)
2353 {
2354 u32 value, mask = 0x3; /* UECC/CECC enable */
2355
2356
2357 if (!s->nbctl_valid)
2358 return;
2359
2360 amd64_read_pci_cfg(F3, NBCTL, &value);
2361 value &= ~mask;
2362 value |= s->old_nbctl;
2363
2364 amd64_write_pci_cfg(F3, NBCTL, value);
2365
2366 /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
2367 if (!s->flags.nb_ecc_prev) {
2368 amd64_read_pci_cfg(F3, NBCFG, &value);
2369 value &= ~NBCFG_ECC_ENABLE;
2370 amd64_write_pci_cfg(F3, NBCFG, value);
2371 }
2372
2373 /* restore the NB Enable MCGCTL bit */
2374 if (toggle_ecc_err_reporting(s, nid, OFF))
2375 amd64_warn("Error restoring NB MCGCTL settings!\n");
2376 }
2377
2378 /*
2379 * EDAC requires that the BIOS have ECC enabled before
2380 * taking over the processing of ECC errors. A command line
2381 * option allows to force-enable hardware ECC later in
2382 * enable_ecc_error_reporting().
2383 */
2384 static const char *ecc_msg =
2385 "ECC disabled in the BIOS or no ECC capability, module will not load.\n"
2386 " Either enable ECC checking or force module loading by setting "
2387 "'ecc_enable_override'.\n"
2388 " (Note that use of the override may cause unknown side effects.)\n";
2389
2390 static bool ecc_enabled(struct pci_dev *F3, u8 nid)
2391 {
2392 u32 value;
2393 u8 ecc_en = 0;
2394 bool nb_mce_en = false;
2395
2396 amd64_read_pci_cfg(F3, NBCFG, &value);
2397
2398 ecc_en = !!(value & NBCFG_ECC_ENABLE);
2399 amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled"));
2400
2401 nb_mce_en = amd64_nb_mce_bank_enabled_on_node(nid);
2402 if (!nb_mce_en)
2403 amd64_notice("NB MCE bank disabled, set MSR "
2404 "0x%08x[4] on node %d to enable.\n",
2405 MSR_IA32_MCG_CTL, nid);
2406
2407 if (!ecc_en || !nb_mce_en) {
2408 amd64_notice("%s", ecc_msg);
2409 return false;
2410 }
2411 return true;
2412 }
2413
2414 struct mcidev_sysfs_attribute sysfs_attrs[ARRAY_SIZE(amd64_dbg_attrs) +
2415 ARRAY_SIZE(amd64_inj_attrs) +
2416 1];
2417
2418 struct mcidev_sysfs_attribute terminator = { .attr = { .name = NULL } };
2419
2420 static void set_mc_sysfs_attrs(struct mem_ctl_info *mci)
2421 {
2422 unsigned int i = 0, j = 0;
2423
2424 for (; i < ARRAY_SIZE(amd64_dbg_attrs); i++)
2425 sysfs_attrs[i] = amd64_dbg_attrs[i];
2426
2427 if (boot_cpu_data.x86 >= 0x10)
2428 for (j = 0; j < ARRAY_SIZE(amd64_inj_attrs); j++, i++)
2429 sysfs_attrs[i] = amd64_inj_attrs[j];
2430
2431 sysfs_attrs[i] = terminator;
2432
2433 mci->mc_driver_sysfs_attributes = sysfs_attrs;
2434 }
2435
2436 static void setup_mci_misc_attrs(struct mem_ctl_info *mci,
2437 struct amd64_family_type *fam)
2438 {
2439 struct amd64_pvt *pvt = mci->pvt_info;
2440
2441 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
2442 mci->edac_ctl_cap = EDAC_FLAG_NONE;
2443
2444 if (pvt->nbcap & NBCAP_SECDED)
2445 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
2446
2447 if (pvt->nbcap & NBCAP_CHIPKILL)
2448 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
2449
2450 mci->edac_cap = amd64_determine_edac_cap(pvt);
2451 mci->mod_name = EDAC_MOD_STR;
2452 mci->mod_ver = EDAC_AMD64_VERSION;
2453 mci->ctl_name = fam->ctl_name;
2454 mci->dev_name = pci_name(pvt->F2);
2455 mci->ctl_page_to_phys = NULL;
2456
2457 /* memory scrubber interface */
2458 mci->set_sdram_scrub_rate = amd64_set_scrub_rate;
2459 mci->get_sdram_scrub_rate = amd64_get_scrub_rate;
2460 }
2461
2462 /*
2463 * returns a pointer to the family descriptor on success, NULL otherwise.
2464 */
2465 static struct amd64_family_type *amd64_per_family_init(struct amd64_pvt *pvt)
2466 {
2467 u8 fam = boot_cpu_data.x86;
2468 struct amd64_family_type *fam_type = NULL;
2469
2470 switch (fam) {
2471 case 0xf:
2472 fam_type = &amd64_family_types[K8_CPUS];
2473 pvt->ops = &amd64_family_types[K8_CPUS].ops;
2474 break;
2475
2476 case 0x10:
2477 fam_type = &amd64_family_types[F10_CPUS];
2478 pvt->ops = &amd64_family_types[F10_CPUS].ops;
2479 break;
2480
2481 case 0x15:
2482 fam_type = &amd64_family_types[F15_CPUS];
2483 pvt->ops = &amd64_family_types[F15_CPUS].ops;
2484 break;
2485
2486 default:
2487 amd64_err("Unsupported family!\n");
2488 return NULL;
2489 }
2490
2491 pvt->ext_model = boot_cpu_data.x86_model >> 4;
2492
2493 amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name,
2494 (fam == 0xf ?
2495 (pvt->ext_model >= K8_REV_F ? "revF or later "
2496 : "revE or earlier ")
2497 : ""), pvt->mc_node_id);
2498 return fam_type;
2499 }
2500
2501 static int amd64_init_one_instance(struct pci_dev *F2)
2502 {
2503 struct amd64_pvt *pvt = NULL;
2504 struct amd64_family_type *fam_type = NULL;
2505 struct mem_ctl_info *mci = NULL;
2506 int err = 0, ret;
2507 u8 nid = get_node_id(F2);
2508
2509 ret = -ENOMEM;
2510 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
2511 if (!pvt)
2512 goto err_ret;
2513
2514 pvt->mc_node_id = nid;
2515 pvt->F2 = F2;
2516
2517 ret = -EINVAL;
2518 fam_type = amd64_per_family_init(pvt);
2519 if (!fam_type)
2520 goto err_free;
2521
2522 ret = -ENODEV;
2523 err = reserve_mc_sibling_devs(pvt, fam_type->f1_id, fam_type->f3_id);
2524 if (err)
2525 goto err_free;
2526
2527 read_mc_regs(pvt);
2528
2529 /*
2530 * We need to determine how many memory channels there are. Then use
2531 * that information for calculating the size of the dynamic instance
2532 * tables in the 'mci' structure.
2533 */
2534 ret = -EINVAL;
2535 pvt->channel_count = pvt->ops->early_channel_count(pvt);
2536 if (pvt->channel_count < 0)
2537 goto err_siblings;
2538
2539 ret = -ENOMEM;
2540 mci = edac_mc_alloc(0, pvt->csels[0].b_cnt, pvt->channel_count, nid);
2541 if (!mci)
2542 goto err_siblings;
2543
2544 mci->pvt_info = pvt;
2545 mci->dev = &pvt->F2->dev;
2546
2547 setup_mci_misc_attrs(mci, fam_type);
2548
2549 if (init_csrows(mci))
2550 mci->edac_cap = EDAC_FLAG_NONE;
2551
2552 set_mc_sysfs_attrs(mci);
2553
2554 ret = -ENODEV;
2555 if (edac_mc_add_mc(mci)) {
2556 debugf1("failed edac_mc_add_mc()\n");
2557 goto err_add_mc;
2558 }
2559
2560 /* register stuff with EDAC MCE */
2561 if (report_gart_errors)
2562 amd_report_gart_errors(true);
2563
2564 amd_register_ecc_decoder(amd64_decode_bus_error);
2565
2566 mcis[nid] = mci;
2567
2568 atomic_inc(&drv_instances);
2569
2570 return 0;
2571
2572 err_add_mc:
2573 edac_mc_free(mci);
2574
2575 err_siblings:
2576 free_mc_sibling_devs(pvt);
2577
2578 err_free:
2579 kfree(pvt);
2580
2581 err_ret:
2582 return ret;
2583 }
2584
2585 static int __devinit amd64_probe_one_instance(struct pci_dev *pdev,
2586 const struct pci_device_id *mc_type)
2587 {
2588 u8 nid = get_node_id(pdev);
2589 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2590 struct ecc_settings *s;
2591 int ret = 0;
2592
2593 ret = pci_enable_device(pdev);
2594 if (ret < 0) {
2595 debugf0("ret=%d\n", ret);
2596 return -EIO;
2597 }
2598
2599 ret = -ENOMEM;
2600 s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
2601 if (!s)
2602 goto err_out;
2603
2604 ecc_stngs[nid] = s;
2605
2606 if (!ecc_enabled(F3, nid)) {
2607 ret = -ENODEV;
2608
2609 if (!ecc_enable_override)
2610 goto err_enable;
2611
2612 amd64_warn("Forcing ECC on!\n");
2613
2614 if (!enable_ecc_error_reporting(s, nid, F3))
2615 goto err_enable;
2616 }
2617
2618 ret = amd64_init_one_instance(pdev);
2619 if (ret < 0) {
2620 amd64_err("Error probing instance: %d\n", nid);
2621 restore_ecc_error_reporting(s, nid, F3);
2622 }
2623
2624 return ret;
2625
2626 err_enable:
2627 kfree(s);
2628 ecc_stngs[nid] = NULL;
2629
2630 err_out:
2631 return ret;
2632 }
2633
2634 static void __devexit amd64_remove_one_instance(struct pci_dev *pdev)
2635 {
2636 struct mem_ctl_info *mci;
2637 struct amd64_pvt *pvt;
2638 u8 nid = get_node_id(pdev);
2639 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2640 struct ecc_settings *s = ecc_stngs[nid];
2641
2642 /* Remove from EDAC CORE tracking list */
2643 mci = edac_mc_del_mc(&pdev->dev);
2644 if (!mci)
2645 return;
2646
2647 pvt = mci->pvt_info;
2648
2649 restore_ecc_error_reporting(s, nid, F3);
2650
2651 free_mc_sibling_devs(pvt);
2652
2653 /* unregister from EDAC MCE */
2654 amd_report_gart_errors(false);
2655 amd_unregister_ecc_decoder(amd64_decode_bus_error);
2656
2657 kfree(ecc_stngs[nid]);
2658 ecc_stngs[nid] = NULL;
2659
2660 /* Free the EDAC CORE resources */
2661 mci->pvt_info = NULL;
2662 mcis[nid] = NULL;
2663
2664 kfree(pvt);
2665 edac_mc_free(mci);
2666 }
2667
2668 /*
2669 * This table is part of the interface for loading drivers for PCI devices. The
2670 * PCI core identifies what devices are on a system during boot, and then
2671 * inquiry this table to see if this driver is for a given device found.
2672 */
2673 static const struct pci_device_id amd64_pci_table[] __devinitdata = {
2674 {
2675 .vendor = PCI_VENDOR_ID_AMD,
2676 .device = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
2677 .subvendor = PCI_ANY_ID,
2678 .subdevice = PCI_ANY_ID,
2679 .class = 0,
2680 .class_mask = 0,
2681 },
2682 {
2683 .vendor = PCI_VENDOR_ID_AMD,
2684 .device = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
2685 .subvendor = PCI_ANY_ID,
2686 .subdevice = PCI_ANY_ID,
2687 .class = 0,
2688 .class_mask = 0,
2689 },
2690 {
2691 .vendor = PCI_VENDOR_ID_AMD,
2692 .device = PCI_DEVICE_ID_AMD_15H_NB_F2,
2693 .subvendor = PCI_ANY_ID,
2694 .subdevice = PCI_ANY_ID,
2695 .class = 0,
2696 .class_mask = 0,
2697 },
2698
2699 {0, }
2700 };
2701 MODULE_DEVICE_TABLE(pci, amd64_pci_table);
2702
2703 static struct pci_driver amd64_pci_driver = {
2704 .name = EDAC_MOD_STR,
2705 .probe = amd64_probe_one_instance,
2706 .remove = __devexit_p(amd64_remove_one_instance),
2707 .id_table = amd64_pci_table,
2708 };
2709
2710 static void setup_pci_device(void)
2711 {
2712 struct mem_ctl_info *mci;
2713 struct amd64_pvt *pvt;
2714
2715 if (amd64_ctl_pci)
2716 return;
2717
2718 mci = mcis[0];
2719 if (mci) {
2720
2721 pvt = mci->pvt_info;
2722 amd64_ctl_pci =
2723 edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR);
2724
2725 if (!amd64_ctl_pci) {
2726 pr_warning("%s(): Unable to create PCI control\n",
2727 __func__);
2728
2729 pr_warning("%s(): PCI error report via EDAC not set\n",
2730 __func__);
2731 }
2732 }
2733 }
2734
2735 static int __init amd64_edac_init(void)
2736 {
2737 int err = -ENODEV;
2738
2739 printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION);
2740
2741 opstate_init();
2742
2743 if (amd_cache_northbridges() < 0)
2744 goto err_ret;
2745
2746 err = -ENOMEM;
2747 mcis = kzalloc(amd_nb_num() * sizeof(mcis[0]), GFP_KERNEL);
2748 ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL);
2749 if (!(mcis && ecc_stngs))
2750 goto err_free;
2751
2752 msrs = msrs_alloc();
2753 if (!msrs)
2754 goto err_free;
2755
2756 err = pci_register_driver(&amd64_pci_driver);
2757 if (err)
2758 goto err_pci;
2759
2760 err = -ENODEV;
2761 if (!atomic_read(&drv_instances))
2762 goto err_no_instances;
2763
2764 setup_pci_device();
2765 return 0;
2766
2767 err_no_instances:
2768 pci_unregister_driver(&amd64_pci_driver);
2769
2770 err_pci:
2771 msrs_free(msrs);
2772 msrs = NULL;
2773
2774 err_free:
2775 kfree(mcis);
2776 mcis = NULL;
2777
2778 kfree(ecc_stngs);
2779 ecc_stngs = NULL;
2780
2781 err_ret:
2782 return err;
2783 }
2784
2785 static void __exit amd64_edac_exit(void)
2786 {
2787 if (amd64_ctl_pci)
2788 edac_pci_release_generic_ctl(amd64_ctl_pci);
2789
2790 pci_unregister_driver(&amd64_pci_driver);
2791
2792 kfree(ecc_stngs);
2793 ecc_stngs = NULL;
2794
2795 kfree(mcis);
2796 mcis = NULL;
2797
2798 msrs_free(msrs);
2799 msrs = NULL;
2800 }
2801
2802 module_init(amd64_edac_init);
2803 module_exit(amd64_edac_exit);
2804
2805 MODULE_LICENSE("GPL");
2806 MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
2807 "Dave Peterson, Thayne Harbaugh");
2808 MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
2809 EDAC_AMD64_VERSION);
2810
2811 module_param(edac_op_state, int, 0444);
2812 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
linux-next