?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Avoid beyond bounds copy while caching ACL
[zen-stable.git] / drivers / edac / amd64_edac.c
blobc9eee6d33e9a70cf5f851570c48ff3086b93c39a
1 #include "amd64_edac.h"
2 #include <asm/amd_nb.h>
3
4 static struct edac_pci_ctl_info *amd64_ctl_pci;
5
6 static int report_gart_errors;
7 module_param(report_gart_errors, int, 0644);
8
9 /*
10 * Set by command line parameter. If BIOS has enabled the ECC, this override is
11 * cleared to prevent re-enabling the hardware by this driver.
12 */
13 static int ecc_enable_override;
14 module_param(ecc_enable_override, int, 0644);
15
16 static struct msr __percpu *msrs;
17
18 /*
19 * count successfully initialized driver instances for setup_pci_device()
20 */
21 static atomic_t drv_instances = ATOMIC_INIT(0);
22
23 /* Per-node driver instances */
24 static struct mem_ctl_info **mcis;
25 static struct ecc_settings **ecc_stngs;
26
27 /*
28 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
29 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
30 * or higher value'.
31 *
32 *FIXME: Produce a better mapping/linearisation.
33 */
34 struct scrubrate {
35 u32 scrubval; /* bit pattern for scrub rate */
36 u32 bandwidth; /* bandwidth consumed (bytes/sec) */
37 } scrubrates[] = {
38 { 0x01, 1600000000UL},
39 { 0x02, 800000000UL},
40 { 0x03, 400000000UL},
41 { 0x04, 200000000UL},
42 { 0x05, 100000000UL},
43 { 0x06, 50000000UL},
44 { 0x07, 25000000UL},
45 { 0x08, 12284069UL},
46 { 0x09, 6274509UL},
47 { 0x0A, 3121951UL},
48 { 0x0B, 1560975UL},
49 { 0x0C, 781440UL},
50 { 0x0D, 390720UL},
51 { 0x0E, 195300UL},
52 { 0x0F, 97650UL},
53 { 0x10, 48854UL},
54 { 0x11, 24427UL},
55 { 0x12, 12213UL},
56 { 0x13, 6101UL},
57 { 0x14, 3051UL},
58 { 0x15, 1523UL},
59 { 0x16, 761UL},
60 { 0x00, 0UL}, /* scrubbing off */
61 };
62
63 static int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
64 u32 *val, const char *func)
65 {
66 int err = 0;
67
68 err = pci_read_config_dword(pdev, offset, val);
69 if (err)
70 amd64_warn("%s: error reading F%dx%03x.\n",
71 func, PCI_FUNC(pdev->devfn), offset);
72
73 return err;
74 }
75
76 int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
77 u32 val, const char *func)
78 {
79 int err = 0;
80
81 err = pci_write_config_dword(pdev, offset, val);
82 if (err)
83 amd64_warn("%s: error writing to F%dx%03x.\n",
84 func, PCI_FUNC(pdev->devfn), offset);
85
86 return err;
87 }
88
89 /*
90 *
91 * Depending on the family, F2 DCT reads need special handling:
92 *
93 * K8: has a single DCT only
94 *
95 * F10h: each DCT has its own set of regs
96 * DCT0 -> F2x040..
97 * DCT1 -> F2x140..
98 *
99 * F15h: we select which DCT we access using F1x10C[DctCfgSel]
100 *
101 */
102 static int k8_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
103 const char *func)
104 {
105 if (addr >= 0x100)
106 return -EINVAL;
107
108 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
109 }
110
111 static int f10_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
112 const char *func)
113 {
114 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
115 }
116
117 /*
118 * Select DCT to which PCI cfg accesses are routed
119 */
120 static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct)
121 {
122 u32 reg = 0;
123
124 amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, &reg);
125 reg &= 0xfffffffe;
126 reg |= dct;
127 amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
128 }
129
130 static int f15_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
131 const char *func)
132 {
133 u8 dct = 0;
134
135 if (addr >= 0x140 && addr <= 0x1a0) {
136 dct = 1;
137 addr -= 0x100;
138 }
139
140 f15h_select_dct(pvt, dct);
141
142 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
143 }
144
145 /*
146 * Memory scrubber control interface. For K8, memory scrubbing is handled by
147 * hardware and can involve L2 cache, dcache as well as the main memory. With
148 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
149 * functionality.
150 *
151 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
152 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
153 * bytes/sec for the setting.
154 *
155 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
156 * other archs, we might not have access to the caches directly.
157 */
158
159 /*
160 * scan the scrub rate mapping table for a close or matching bandwidth value to
161 * issue. If requested is too big, then use last maximum value found.
162 */
163 static int __amd64_set_scrub_rate(struct pci_dev *ctl, u32 new_bw, u32 min_rate)
164 {
165 u32 scrubval;
166 int i;
167
168 /*
169 * map the configured rate (new_bw) to a value specific to the AMD64
170 * memory controller and apply to register. Search for the first
171 * bandwidth entry that is greater or equal than the setting requested
172 * and program that. If at last entry, turn off DRAM scrubbing.
173 */
174 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
175 /*
176 * skip scrub rates which aren't recommended
177 * (see F10 BKDG, F3x58)
178 */
179 if (scrubrates[i].scrubval < min_rate)
180 continue;
181
182 if (scrubrates[i].bandwidth <= new_bw)
183 break;
184
185 /*
186 * if no suitable bandwidth found, turn off DRAM scrubbing
187 * entirely by falling back to the last element in the
188 * scrubrates array.
189 */
190 }
191
192 scrubval = scrubrates[i].scrubval;
193
194 pci_write_bits32(ctl, SCRCTRL, scrubval, 0x001F);
195
196 if (scrubval)
197 return scrubrates[i].bandwidth;
198
199 return 0;
200 }
201
202 static int amd64_set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
203 {
204 struct amd64_pvt *pvt = mci->pvt_info;
205 u32 min_scrubrate = 0x5;
206
207 if (boot_cpu_data.x86 == 0xf)
208 min_scrubrate = 0x0;
209
210 /* F15h Erratum #505 */
211 if (boot_cpu_data.x86 == 0x15)
212 f15h_select_dct(pvt, 0);
213
214 return __amd64_set_scrub_rate(pvt->F3, bw, min_scrubrate);
215 }
216
217 static int amd64_get_scrub_rate(struct mem_ctl_info *mci)
218 {
219 struct amd64_pvt *pvt = mci->pvt_info;
220 u32 scrubval = 0;
221 int i, retval = -EINVAL;
222
223 /* F15h Erratum #505 */
224 if (boot_cpu_data.x86 == 0x15)
225 f15h_select_dct(pvt, 0);
226
227 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
228
229 scrubval = scrubval & 0x001F;
230
231 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
232 if (scrubrates[i].scrubval == scrubval) {
233 retval = scrubrates[i].bandwidth;
234 break;
235 }
236 }
237 return retval;
238 }
239
240 /*
241 * returns true if the SysAddr given by sys_addr matches the
242 * DRAM base/limit associated with node_id
243 */
244 static bool amd64_base_limit_match(struct amd64_pvt *pvt, u64 sys_addr,
245 unsigned nid)
246 {
247 u64 addr;
248
249 /* The K8 treats this as a 40-bit value. However, bits 63-40 will be
250 * all ones if the most significant implemented address bit is 1.
251 * Here we discard bits 63-40. See section 3.4.2 of AMD publication
252 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
253 * Application Programming.
254 */
255 addr = sys_addr & 0x000000ffffffffffull;
256
257 return ((addr >= get_dram_base(pvt, nid)) &&
258 (addr <= get_dram_limit(pvt, nid)));
259 }
260
261 /*
262 * Attempt to map a SysAddr to a node. On success, return a pointer to the
263 * mem_ctl_info structure for the node that the SysAddr maps to.
264 *
265 * On failure, return NULL.
266 */
267 static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
268 u64 sys_addr)
269 {
270 struct amd64_pvt *pvt;
271 unsigned node_id;
272 u32 intlv_en, bits;
273
274 /*
275 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
276 * 3.4.4.2) registers to map the SysAddr to a node ID.
277 */
278 pvt = mci->pvt_info;
279
280 /*
281 * The value of this field should be the same for all DRAM Base
282 * registers. Therefore we arbitrarily choose to read it from the
283 * register for node 0.
284 */
285 intlv_en = dram_intlv_en(pvt, 0);
286
287 if (intlv_en == 0) {
288 for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
289 if (amd64_base_limit_match(pvt, sys_addr, node_id))
290 goto found;
291 }
292 goto err_no_match;
293 }
294
295 if (unlikely((intlv_en != 0x01) &&
296 (intlv_en != 0x03) &&
297 (intlv_en != 0x07))) {
298 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
299 return NULL;
300 }
301
302 bits = (((u32) sys_addr) >> 12) & intlv_en;
303
304 for (node_id = 0; ; ) {
305 if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
306 break; /* intlv_sel field matches */
307
308 if (++node_id >= DRAM_RANGES)
309 goto err_no_match;
310 }
311
312 /* sanity test for sys_addr */
313 if (unlikely(!amd64_base_limit_match(pvt, sys_addr, node_id))) {
314 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
315 "range for node %d with node interleaving enabled.\n",
316 __func__, sys_addr, node_id);
317 return NULL;
318 }
319
320 found:
321 return edac_mc_find((int)node_id);
322
323 err_no_match:
324 debugf2("sys_addr 0x%lx doesn't match any node\n",
325 (unsigned long)sys_addr);
326
327 return NULL;
328 }
329
330 /*
331 * compute the CS base address of the @csrow on the DRAM controller @dct.
332 * For details see F2x[5C:40] in the processor's BKDG
333 */
334 static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
335 u64 *base, u64 *mask)
336 {
337 u64 csbase, csmask, base_bits, mask_bits;
338 u8 addr_shift;
339
340 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) {
341 csbase = pvt->csels[dct].csbases[csrow];
342 csmask = pvt->csels[dct].csmasks[csrow];
343 base_bits = GENMASK(21, 31) | GENMASK(9, 15);
344 mask_bits = GENMASK(21, 29) | GENMASK(9, 15);
345 addr_shift = 4;
346 } else {
347 csbase = pvt->csels[dct].csbases[csrow];
348 csmask = pvt->csels[dct].csmasks[csrow >> 1];
349 addr_shift = 8;
350
351 if (boot_cpu_data.x86 == 0x15)
352 base_bits = mask_bits = GENMASK(19,30) | GENMASK(5,13);
353 else
354 base_bits = mask_bits = GENMASK(19,28) | GENMASK(5,13);
355 }
356
357 *base = (csbase & base_bits) << addr_shift;
358
359 *mask = ~0ULL;
360 /* poke holes for the csmask */
361 *mask &= ~(mask_bits << addr_shift);
362 /* OR them in */
363 *mask |= (csmask & mask_bits) << addr_shift;
364 }
365
366 #define for_each_chip_select(i, dct, pvt) \
367 for (i = 0; i < pvt->csels[dct].b_cnt; i++)
368
369 #define chip_select_base(i, dct, pvt) \
370 pvt->csels[dct].csbases[i]
371
372 #define for_each_chip_select_mask(i, dct, pvt) \
373 for (i = 0; i < pvt->csels[dct].m_cnt; i++)
374
375 /*
376 * @input_addr is an InputAddr associated with the node given by mci. Return the
377 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
378 */
379 static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
380 {
381 struct amd64_pvt *pvt;
382 int csrow;
383 u64 base, mask;
384
385 pvt = mci->pvt_info;
386
387 for_each_chip_select(csrow, 0, pvt) {
388 if (!csrow_enabled(csrow, 0, pvt))
389 continue;
390
391 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
392
393 mask = ~mask;
394
395 if ((input_addr & mask) == (base & mask)) {
396 debugf2("InputAddr 0x%lx matches csrow %d (node %d)\n",
397 (unsigned long)input_addr, csrow,
398 pvt->mc_node_id);
399
400 return csrow;
401 }
402 }
403 debugf2("no matching csrow for InputAddr 0x%lx (MC node %d)\n",
404 (unsigned long)input_addr, pvt->mc_node_id);
405
406 return -1;
407 }
408
409 /*
410 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
411 * for the node represented by mci. Info is passed back in *hole_base,
412 * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if
413 * info is invalid. Info may be invalid for either of the following reasons:
414 *
415 * - The revision of the node is not E or greater. In this case, the DRAM Hole
416 * Address Register does not exist.
417 *
418 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
419 * indicating that its contents are not valid.
420 *
421 * The values passed back in *hole_base, *hole_offset, and *hole_size are
422 * complete 32-bit values despite the fact that the bitfields in the DHAR
423 * only represent bits 31-24 of the base and offset values.
424 */
425 int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
426 u64 *hole_offset, u64 *hole_size)
427 {
428 struct amd64_pvt *pvt = mci->pvt_info;
429 u64 base;
430
431 /* only revE and later have the DRAM Hole Address Register */
432 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_E) {
433 debugf1(" revision %d for node %d does not support DHAR\n",
434 pvt->ext_model, pvt->mc_node_id);
435 return 1;
436 }
437
438 /* valid for Fam10h and above */
439 if (boot_cpu_data.x86 >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
440 debugf1(" Dram Memory Hoisting is DISABLED on this system\n");
441 return 1;
442 }
443
444 if (!dhar_valid(pvt)) {
445 debugf1(" Dram Memory Hoisting is DISABLED on this node %d\n",
446 pvt->mc_node_id);
447 return 1;
448 }
449
450 /* This node has Memory Hoisting */
451
452 /* +------------------+--------------------+--------------------+-----
453 * | memory | DRAM hole | relocated |
454 * | [0, (x - 1)] | [x, 0xffffffff] | addresses from |
455 * | | | DRAM hole |
456 * | | | [0x100000000, |
457 * | | | (0x100000000+ |
458 * | | | (0xffffffff-x))] |
459 * +------------------+--------------------+--------------------+-----
460 *
461 * Above is a diagram of physical memory showing the DRAM hole and the
462 * relocated addresses from the DRAM hole. As shown, the DRAM hole
463 * starts at address x (the base address) and extends through address
464 * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the
465 * addresses in the hole so that they start at 0x100000000.
466 */
467
468 base = dhar_base(pvt);
469
470 *hole_base = base;
471 *hole_size = (0x1ull << 32) - base;
472
473 if (boot_cpu_data.x86 > 0xf)
474 *hole_offset = f10_dhar_offset(pvt);
475 else
476 *hole_offset = k8_dhar_offset(pvt);
477
478 debugf1(" DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
479 pvt->mc_node_id, (unsigned long)*hole_base,
480 (unsigned long)*hole_offset, (unsigned long)*hole_size);
481
482 return 0;
483 }
484 EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info);
485
486 /*
487 * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is
488 * assumed that sys_addr maps to the node given by mci.
489 *
490 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
491 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
492 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
493 * then it is also involved in translating a SysAddr to a DramAddr. Sections
494 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
495 * These parts of the documentation are unclear. I interpret them as follows:
496 *
497 * When node n receives a SysAddr, it processes the SysAddr as follows:
498 *
499 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
500 * Limit registers for node n. If the SysAddr is not within the range
501 * specified by the base and limit values, then node n ignores the Sysaddr
502 * (since it does not map to node n). Otherwise continue to step 2 below.
503 *
504 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
505 * disabled so skip to step 3 below. Otherwise see if the SysAddr is within
506 * the range of relocated addresses (starting at 0x100000000) from the DRAM
507 * hole. If not, skip to step 3 below. Else get the value of the
508 * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
509 * offset defined by this value from the SysAddr.
510 *
511 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
512 * Base register for node n. To obtain the DramAddr, subtract the base
513 * address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
514 */
515 static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
516 {
517 struct amd64_pvt *pvt = mci->pvt_info;
518 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
519 int ret = 0;
520
521 dram_base = get_dram_base(pvt, pvt->mc_node_id);
522
523 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
524 &hole_size);
525 if (!ret) {
526 if ((sys_addr >= (1ull << 32)) &&
527 (sys_addr < ((1ull << 32) + hole_size))) {
528 /* use DHAR to translate SysAddr to DramAddr */
529 dram_addr = sys_addr - hole_offset;
530
531 debugf2("using DHAR to translate SysAddr 0x%lx to "
532 "DramAddr 0x%lx\n",
533 (unsigned long)sys_addr,
534 (unsigned long)dram_addr);
535
536 return dram_addr;
537 }
538 }
539
540 /*
541 * Translate the SysAddr to a DramAddr as shown near the start of
542 * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8
543 * only deals with 40-bit values. Therefore we discard bits 63-40 of
544 * sys_addr below. If bit 39 of sys_addr is 1 then the bits we
545 * discard are all 1s. Otherwise the bits we discard are all 0s. See
546 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
547 * Programmer's Manual Volume 1 Application Programming.
548 */
549 dram_addr = (sys_addr & GENMASK(0, 39)) - dram_base;
550
551 debugf2("using DRAM Base register to translate SysAddr 0x%lx to "
552 "DramAddr 0x%lx\n", (unsigned long)sys_addr,
553 (unsigned long)dram_addr);
554 return dram_addr;
555 }
556
557 /*
558 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
559 * (section 3.4.4.1). Return the number of bits from a SysAddr that are used
560 * for node interleaving.
561 */
562 static int num_node_interleave_bits(unsigned intlv_en)
563 {
564 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
565 int n;
566
567 BUG_ON(intlv_en > 7);
568 n = intlv_shift_table[intlv_en];
569 return n;
570 }
571
572 /* Translate the DramAddr given by @dram_addr to an InputAddr. */
573 static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
574 {
575 struct amd64_pvt *pvt;
576 int intlv_shift;
577 u64 input_addr;
578
579 pvt = mci->pvt_info;
580
581 /*
582 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
583 * concerning translating a DramAddr to an InputAddr.
584 */
585 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
586 input_addr = ((dram_addr >> intlv_shift) & GENMASK(12, 35)) +
587 (dram_addr & 0xfff);
588
589 debugf2(" Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
590 intlv_shift, (unsigned long)dram_addr,
591 (unsigned long)input_addr);
592
593 return input_addr;
594 }
595
596 /*
597 * Translate the SysAddr represented by @sys_addr to an InputAddr. It is
598 * assumed that @sys_addr maps to the node given by mci.
599 */
600 static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
601 {
602 u64 input_addr;
603
604 input_addr =
605 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
606
607 debugf2("SysAdddr 0x%lx translates to InputAddr 0x%lx\n",
608 (unsigned long)sys_addr, (unsigned long)input_addr);
609
610 return input_addr;
611 }
612
613
614 /*
615 * @input_addr is an InputAddr associated with the node represented by mci.
616 * Translate @input_addr to a DramAddr and return the result.
617 */
618 static u64 input_addr_to_dram_addr(struct mem_ctl_info *mci, u64 input_addr)
619 {
620 struct amd64_pvt *pvt;
621 unsigned node_id, intlv_shift;
622 u64 bits, dram_addr;
623 u32 intlv_sel;
624
625 /*
626 * Near the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
627 * shows how to translate a DramAddr to an InputAddr. Here we reverse
628 * this procedure. When translating from a DramAddr to an InputAddr, the
629 * bits used for node interleaving are discarded. Here we recover these
630 * bits from the IntlvSel field of the DRAM Limit register (section
631 * 3.4.4.2) for the node that input_addr is associated with.
632 */
633 pvt = mci->pvt_info;
634 node_id = pvt->mc_node_id;
635
636 BUG_ON(node_id > 7);
637
638 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
639 if (intlv_shift == 0) {
640 debugf1(" InputAddr 0x%lx translates to DramAddr of "
641 "same value\n", (unsigned long)input_addr);
642
643 return input_addr;
644 }
645
646 bits = ((input_addr & GENMASK(12, 35)) << intlv_shift) +
647 (input_addr & 0xfff);
648
649 intlv_sel = dram_intlv_sel(pvt, node_id) & ((1 << intlv_shift) - 1);
650 dram_addr = bits + (intlv_sel << 12);
651
652 debugf1("InputAddr 0x%lx translates to DramAddr 0x%lx "
653 "(%d node interleave bits)\n", (unsigned long)input_addr,
654 (unsigned long)dram_addr, intlv_shift);
655
656 return dram_addr;
657 }
658
659 /*
660 * @dram_addr is a DramAddr that maps to the node represented by mci. Convert
661 * @dram_addr to a SysAddr.
662 */
663 static u64 dram_addr_to_sys_addr(struct mem_ctl_info *mci, u64 dram_addr)
664 {
665 struct amd64_pvt *pvt = mci->pvt_info;
666 u64 hole_base, hole_offset, hole_size, base, sys_addr;
667 int ret = 0;
668
669 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
670 &hole_size);
671 if (!ret) {
672 if ((dram_addr >= hole_base) &&
673 (dram_addr < (hole_base + hole_size))) {
674 sys_addr = dram_addr + hole_offset;
675
676 debugf1("using DHAR to translate DramAddr 0x%lx to "
677 "SysAddr 0x%lx\n", (unsigned long)dram_addr,
678 (unsigned long)sys_addr);
679
680 return sys_addr;
681 }
682 }
683
684 base = get_dram_base(pvt, pvt->mc_node_id);
685 sys_addr = dram_addr + base;
686
687 /*
688 * The sys_addr we have computed up to this point is a 40-bit value
689 * because the k8 deals with 40-bit values. However, the value we are
690 * supposed to return is a full 64-bit physical address. The AMD
691 * x86-64 architecture specifies that the most significant implemented
692 * address bit through bit 63 of a physical address must be either all
693 * 0s or all 1s. Therefore we sign-extend the 40-bit sys_addr to a
694 * 64-bit value below. See section 3.4.2 of AMD publication 24592:
695 * AMD x86-64 Architecture Programmer's Manual Volume 1 Application
696 * Programming.
697 */
698 sys_addr |= ~((sys_addr & (1ull << 39)) - 1);
699
700 debugf1(" Node %d, DramAddr 0x%lx to SysAddr 0x%lx\n",
701 pvt->mc_node_id, (unsigned long)dram_addr,
702 (unsigned long)sys_addr);
703
704 return sys_addr;
705 }
706
707 /*
708 * @input_addr is an InputAddr associated with the node given by mci. Translate
709 * @input_addr to a SysAddr.
710 */
711 static inline u64 input_addr_to_sys_addr(struct mem_ctl_info *mci,
712 u64 input_addr)
713 {
714 return dram_addr_to_sys_addr(mci,
715 input_addr_to_dram_addr(mci, input_addr));
716 }
717
718 /*
719 * Find the minimum and maximum InputAddr values that map to the given @csrow.
720 * Pass back these values in *input_addr_min and *input_addr_max.
721 */
722 static void find_csrow_limits(struct mem_ctl_info *mci, int csrow,
723 u64 *input_addr_min, u64 *input_addr_max)
724 {
725 struct amd64_pvt *pvt;
726 u64 base, mask;
727
728 pvt = mci->pvt_info;
729 BUG_ON((csrow < 0) || (csrow >= pvt->csels[0].b_cnt));
730
731 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
732
733 *input_addr_min = base & ~mask;
734 *input_addr_max = base | mask;
735 }
736
737 /* Map the Error address to a PAGE and PAGE OFFSET. */
738 static inline void error_address_to_page_and_offset(u64 error_address,
739 u32 *page, u32 *offset)
740 {
741 *page = (u32) (error_address >> PAGE_SHIFT);
742 *offset = ((u32) error_address) & ~PAGE_MASK;
743 }
744
745 /*
746 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
747 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
748 * of a node that detected an ECC memory error. mci represents the node that
749 * the error address maps to (possibly different from the node that detected
750 * the error). Return the number of the csrow that sys_addr maps to, or -1 on
751 * error.
752 */
753 static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
754 {
755 int csrow;
756
757 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
758
759 if (csrow == -1)
760 amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
761 "address 0x%lx\n", (unsigned long)sys_addr);
762 return csrow;
763 }
764
765 static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
766
767 /*
768 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
769 * are ECC capable.
770 */
771 static unsigned long amd64_determine_edac_cap(struct amd64_pvt *pvt)
772 {
773 u8 bit;
774 unsigned long edac_cap = EDAC_FLAG_NONE;
775
776 bit = (boot_cpu_data.x86 > 0xf || pvt->ext_model >= K8_REV_F)
777 ? 19
778 : 17;
779
780 if (pvt->dclr0 & BIT(bit))
781 edac_cap = EDAC_FLAG_SECDED;
782
783 return edac_cap;
784 }
785
786 static void amd64_debug_display_dimm_sizes(struct amd64_pvt *, u8);
787
788 static void amd64_dump_dramcfg_low(u32 dclr, int chan)
789 {
790 debugf1("F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
791
792 debugf1(" DIMM type: %sbuffered; all DIMMs support ECC: %s\n",
793 (dclr & BIT(16)) ? "un" : "",
794 (dclr & BIT(19)) ? "yes" : "no");
795
796 debugf1(" PAR/ERR parity: %s\n",
797 (dclr & BIT(8)) ? "enabled" : "disabled");
798
799 if (boot_cpu_data.x86 == 0x10)
800 debugf1(" DCT 128bit mode width: %s\n",
801 (dclr & BIT(11)) ? "128b" : "64b");
802
803 debugf1(" x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
804 (dclr & BIT(12)) ? "yes" : "no",
805 (dclr & BIT(13)) ? "yes" : "no",
806 (dclr & BIT(14)) ? "yes" : "no",
807 (dclr & BIT(15)) ? "yes" : "no");
808 }
809
810 /* Display and decode various NB registers for debug purposes. */
811 static void dump_misc_regs(struct amd64_pvt *pvt)
812 {
813 debugf1("F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
814
815 debugf1(" NB two channel DRAM capable: %s\n",
816 (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
817
818 debugf1(" ECC capable: %s, ChipKill ECC capable: %s\n",
819 (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
820 (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
821
822 amd64_dump_dramcfg_low(pvt->dclr0, 0);
823
824 debugf1("F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
825
826 debugf1("F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, "
827 "offset: 0x%08x\n",
828 pvt->dhar, dhar_base(pvt),
829 (boot_cpu_data.x86 == 0xf) ? k8_dhar_offset(pvt)
830 : f10_dhar_offset(pvt));
831
832 debugf1(" DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
833
834 amd64_debug_display_dimm_sizes(pvt, 0);
835
836 /* everything below this point is Fam10h and above */
837 if (boot_cpu_data.x86 == 0xf)
838 return;
839
840 amd64_debug_display_dimm_sizes(pvt, 1);
841
842 amd64_info("using %s syndromes.\n", ((pvt->ecc_sym_sz == 8) ? "x8" : "x4"));
843
844 /* Only if NOT ganged does dclr1 have valid info */
845 if (!dct_ganging_enabled(pvt))
846 amd64_dump_dramcfg_low(pvt->dclr1, 1);
847 }
848
849 /*
850 * see BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
851 */
852 static void prep_chip_selects(struct amd64_pvt *pvt)
853 {
854 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) {
855 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
856 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
857 } else {
858 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
859 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
860 }
861 }
862
863 /*
864 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
865 */
866 static void read_dct_base_mask(struct amd64_pvt *pvt)
867 {
868 int cs;
869
870 prep_chip_selects(pvt);
871
872 for_each_chip_select(cs, 0, pvt) {
873 int reg0 = DCSB0 + (cs * 4);
874 int reg1 = DCSB1 + (cs * 4);
875 u32 *base0 = &pvt->csels[0].csbases[cs];
876 u32 *base1 = &pvt->csels[1].csbases[cs];
877
878 if (!amd64_read_dct_pci_cfg(pvt, reg0, base0))
879 debugf0(" DCSB0[%d]=0x%08x reg: F2x%x\n",
880 cs, *base0, reg0);
881
882 if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt))
883 continue;
884
885 if (!amd64_read_dct_pci_cfg(pvt, reg1, base1))
886 debugf0(" DCSB1[%d]=0x%08x reg: F2x%x\n",
887 cs, *base1, reg1);
888 }
889
890 for_each_chip_select_mask(cs, 0, pvt) {
891 int reg0 = DCSM0 + (cs * 4);
892 int reg1 = DCSM1 + (cs * 4);
893 u32 *mask0 = &pvt->csels[0].csmasks[cs];
894 u32 *mask1 = &pvt->csels[1].csmasks[cs];
895
896 if (!amd64_read_dct_pci_cfg(pvt, reg0, mask0))
897 debugf0(" DCSM0[%d]=0x%08x reg: F2x%x\n",
898 cs, *mask0, reg0);
899
900 if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt))
901 continue;
902
903 if (!amd64_read_dct_pci_cfg(pvt, reg1, mask1))
904 debugf0(" DCSM1[%d]=0x%08x reg: F2x%x\n",
905 cs, *mask1, reg1);
906 }
907 }
908
909 static enum mem_type amd64_determine_memory_type(struct amd64_pvt *pvt, int cs)
910 {
911 enum mem_type type;
912
913 /* F15h supports only DDR3 */
914 if (boot_cpu_data.x86 >= 0x15)
915 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
916 else if (boot_cpu_data.x86 == 0x10 || pvt->ext_model >= K8_REV_F) {
917 if (pvt->dchr0 & DDR3_MODE)
918 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
919 else
920 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
921 } else {
922 type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
923 }
924
925 amd64_info("CS%d: %s\n", cs, edac_mem_types[type]);
926
927 return type;
928 }
929
930 /* Get the number of DCT channels the memory controller is using. */
931 static int k8_early_channel_count(struct amd64_pvt *pvt)
932 {
933 int flag;
934
935 if (pvt->ext_model >= K8_REV_F)
936 /* RevF (NPT) and later */
937 flag = pvt->dclr0 & WIDTH_128;
938 else
939 /* RevE and earlier */
940 flag = pvt->dclr0 & REVE_WIDTH_128;
941
942 /* not used */
943 pvt->dclr1 = 0;
944
945 return (flag) ? 2 : 1;
946 }
947
948 /* On F10h and later ErrAddr is MC4_ADDR[47:1] */
949 static u64 get_error_address(struct mce *m)
950 {
951 struct cpuinfo_x86 *c = &boot_cpu_data;
952 u64 addr;
953 u8 start_bit = 1;
954 u8 end_bit = 47;
955
956 if (c->x86 == 0xf) {
957 start_bit = 3;
958 end_bit = 39;
959 }
960
961 addr = m->addr & GENMASK(start_bit, end_bit);
962
963 /*
964 * Erratum 637 workaround
965 */
966 if (c->x86 == 0x15) {
967 struct amd64_pvt *pvt;
968 u64 cc6_base, tmp_addr;
969 u32 tmp;
970 u8 mce_nid, intlv_en;
971
972 if ((addr & GENMASK(24, 47)) >> 24 != 0x00fdf7)
973 return addr;
974
975 mce_nid = amd_get_nb_id(m->extcpu);
976 pvt = mcis[mce_nid]->pvt_info;
977
978 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp);
979 intlv_en = tmp >> 21 & 0x7;
980
981 /* add [47:27] + 3 trailing bits */
982 cc6_base = (tmp & GENMASK(0, 20)) << 3;
983
984 /* reverse and add DramIntlvEn */
985 cc6_base |= intlv_en ^ 0x7;
986
987 /* pin at [47:24] */
988 cc6_base <<= 24;
989
990 if (!intlv_en)
991 return cc6_base | (addr & GENMASK(0, 23));
992
993 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp);
994
995 /* faster log2 */
996 tmp_addr = (addr & GENMASK(12, 23)) << __fls(intlv_en + 1);
997
998 /* OR DramIntlvSel into bits [14:12] */
999 tmp_addr |= (tmp & GENMASK(21, 23)) >> 9;
1000
1001 /* add remaining [11:0] bits from original MC4_ADDR */
1002 tmp_addr |= addr & GENMASK(0, 11);
1003
1004 return cc6_base | tmp_addr;
1005 }
1006
1007 return addr;
1008 }
1009
1010 static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
1011 {
1012 struct cpuinfo_x86 *c = &boot_cpu_data;
1013 int off = range << 3;
1014
1015 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo);
1016 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
1017
1018 if (c->x86 == 0xf)
1019 return;
1020
1021 if (!dram_rw(pvt, range))
1022 return;
1023
1024 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi);
1025 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
1026
1027 /* Factor in CC6 save area by reading dst node's limit reg */
1028 if (c->x86 == 0x15) {
1029 struct pci_dev *f1 = NULL;
1030 u8 nid = dram_dst_node(pvt, range);
1031 u32 llim;
1032
1033 f1 = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(0x18 + nid, 1));
1034 if (WARN_ON(!f1))
1035 return;
1036
1037 amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim);
1038
1039 pvt->ranges[range].lim.lo &= GENMASK(0, 15);
1040
1041 /* {[39:27],111b} */
1042 pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16;
1043
1044 pvt->ranges[range].lim.hi &= GENMASK(0, 7);
1045
1046 /* [47:40] */
1047 pvt->ranges[range].lim.hi |= llim >> 13;
1048
1049 pci_dev_put(f1);
1050 }
1051 }
1052
1053 static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
1054 u16 syndrome)
1055 {
1056 struct mem_ctl_info *src_mci;
1057 struct amd64_pvt *pvt = mci->pvt_info;
1058 int channel, csrow;
1059 u32 page, offset;
1060
1061 /* CHIPKILL enabled */
1062 if (pvt->nbcfg & NBCFG_CHIPKILL) {
1063 channel = get_channel_from_ecc_syndrome(mci, syndrome);
1064 if (channel < 0) {
1065 /*
1066 * Syndrome didn't map, so we don't know which of the
1067 * 2 DIMMs is in error. So we need to ID 'both' of them
1068 * as suspect.
1069 */
1070 amd64_mc_warn(mci, "unknown syndrome 0x%04x - possible "
1071 "error reporting race\n", syndrome);
1072 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1073 return;
1074 }
1075 } else {
1076 /*
1077 * non-chipkill ecc mode
1078 *
1079 * The k8 documentation is unclear about how to determine the
1080 * channel number when using non-chipkill memory. This method
1081 * was obtained from email communication with someone at AMD.
1082 * (Wish the email was placed in this comment - norsk)
1083 */
1084 channel = ((sys_addr & BIT(3)) != 0);
1085 }
1086
1087 /*
1088 * Find out which node the error address belongs to. This may be
1089 * different from the node that detected the error.
1090 */
1091 src_mci = find_mc_by_sys_addr(mci, sys_addr);
1092 if (!src_mci) {
1093 amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
1094 (unsigned long)sys_addr);
1095 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1096 return;
1097 }
1098
1099 /* Now map the sys_addr to a CSROW */
1100 csrow = sys_addr_to_csrow(src_mci, sys_addr);
1101 if (csrow < 0) {
1102 edac_mc_handle_ce_no_info(src_mci, EDAC_MOD_STR);
1103 } else {
1104 error_address_to_page_and_offset(sys_addr, &page, &offset);
1105
1106 edac_mc_handle_ce(src_mci, page, offset, syndrome, csrow,
1107 channel, EDAC_MOD_STR);
1108 }
1109 }
1110
1111 static int ddr2_cs_size(unsigned i, bool dct_width)
1112 {
1113 unsigned shift = 0;
1114
1115 if (i <= 2)
1116 shift = i;
1117 else if (!(i & 0x1))
1118 shift = i >> 1;
1119 else
1120 shift = (i + 1) >> 1;
1121
1122 return 128 << (shift + !!dct_width);
1123 }
1124
1125 static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1126 unsigned cs_mode)
1127 {
1128 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1129
1130 if (pvt->ext_model >= K8_REV_F) {
1131 WARN_ON(cs_mode > 11);
1132 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1133 }
1134 else if (pvt->ext_model >= K8_REV_D) {
1135 WARN_ON(cs_mode > 10);
1136
1137 if (cs_mode == 3 || cs_mode == 8)
1138 return 32 << (cs_mode - 1);
1139 else
1140 return 32 << cs_mode;
1141 }
1142 else {
1143 WARN_ON(cs_mode > 6);
1144 return 32 << cs_mode;
1145 }
1146 }
1147
1148 /*
1149 * Get the number of DCT channels in use.
1150 *
1151 * Return:
1152 * number of Memory Channels in operation
1153 * Pass back:
1154 * contents of the DCL0_LOW register
1155 */
1156 static int f1x_early_channel_count(struct amd64_pvt *pvt)
1157 {
1158 int i, j, channels = 0;
1159
1160 /* On F10h, if we are in 128 bit mode, then we are using 2 channels */
1161 if (boot_cpu_data.x86 == 0x10 && (pvt->dclr0 & WIDTH_128))
1162 return 2;
1163
1164 /*
1165 * Need to check if in unganged mode: In such, there are 2 channels,
1166 * but they are not in 128 bit mode and thus the above 'dclr0' status
1167 * bit will be OFF.
1168 *
1169 * Need to check DCT0[0] and DCT1[0] to see if only one of them has
1170 * their CSEnable bit on. If so, then SINGLE DIMM case.
1171 */
1172 debugf0("Data width is not 128 bits - need more decoding\n");
1173
1174 /*
1175 * Check DRAM Bank Address Mapping values for each DIMM to see if there
1176 * is more than just one DIMM present in unganged mode. Need to check
1177 * both controllers since DIMMs can be placed in either one.
1178 */
1179 for (i = 0; i < 2; i++) {
1180 u32 dbam = (i ? pvt->dbam1 : pvt->dbam0);
1181
1182 for (j = 0; j < 4; j++) {
1183 if (DBAM_DIMM(j, dbam) > 0) {
1184 channels++;
1185 break;
1186 }
1187 }
1188 }
1189
1190 if (channels > 2)
1191 channels = 2;
1192
1193 amd64_info("MCT channel count: %d\n", channels);
1194
1195 return channels;
1196 }
1197
1198 static int ddr3_cs_size(unsigned i, bool dct_width)
1199 {
1200 unsigned shift = 0;
1201 int cs_size = 0;
1202
1203 if (i == 0 || i == 3 || i == 4)
1204 cs_size = -1;
1205 else if (i <= 2)
1206 shift = i;
1207 else if (i == 12)
1208 shift = 7;
1209 else if (!(i & 0x1))
1210 shift = i >> 1;
1211 else
1212 shift = (i + 1) >> 1;
1213
1214 if (cs_size != -1)
1215 cs_size = (128 * (1 << !!dct_width)) << shift;
1216
1217 return cs_size;
1218 }
1219
1220 static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1221 unsigned cs_mode)
1222 {
1223 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1224
1225 WARN_ON(cs_mode > 11);
1226
1227 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
1228 return ddr3_cs_size(cs_mode, dclr & WIDTH_128);
1229 else
1230 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1231 }
1232
1233 /*
1234 * F15h supports only 64bit DCT interfaces
1235 */
1236 static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1237 unsigned cs_mode)
1238 {
1239 WARN_ON(cs_mode > 12);
1240
1241 return ddr3_cs_size(cs_mode, false);
1242 }
1243
1244 static void read_dram_ctl_register(struct amd64_pvt *pvt)
1245 {
1246
1247 if (boot_cpu_data.x86 == 0xf)
1248 return;
1249
1250 if (!amd64_read_dct_pci_cfg(pvt, DCT_SEL_LO, &pvt->dct_sel_lo)) {
1251 debugf0("F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
1252 pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
1253
1254 debugf0(" DCTs operate in %s mode.\n",
1255 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"));
1256
1257 if (!dct_ganging_enabled(pvt))
1258 debugf0(" Address range split per DCT: %s\n",
1259 (dct_high_range_enabled(pvt) ? "yes" : "no"));
1260
1261 debugf0(" data interleave for ECC: %s, "
1262 "DRAM cleared since last warm reset: %s\n",
1263 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
1264 (dct_memory_cleared(pvt) ? "yes" : "no"));
1265
1266 debugf0(" channel interleave: %s, "
1267 "interleave bits selector: 0x%x\n",
1268 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
1269 dct_sel_interleave_addr(pvt));
1270 }
1271
1272 amd64_read_dct_pci_cfg(pvt, DCT_SEL_HI, &pvt->dct_sel_hi);
1273 }
1274
1275 /*
1276 * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
1277 * Interleaving Modes.
1278 */
1279 static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1280 bool hi_range_sel, u8 intlv_en)
1281 {
1282 u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
1283
1284 if (dct_ganging_enabled(pvt))
1285 return 0;
1286
1287 if (hi_range_sel)
1288 return dct_sel_high;
1289
1290 /*
1291 * see F2x110[DctSelIntLvAddr] - channel interleave mode
1292 */
1293 if (dct_interleave_enabled(pvt)) {
1294 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1295
1296 /* return DCT select function: 0=DCT0, 1=DCT1 */
1297 if (!intlv_addr)
1298 return sys_addr >> 6 & 1;
1299
1300 if (intlv_addr & 0x2) {
1301 u8 shift = intlv_addr & 0x1 ? 9 : 6;
1302 u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2;
1303
1304 return ((sys_addr >> shift) & 1) ^ temp;
1305 }
1306
1307 return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
1308 }
1309
1310 if (dct_high_range_enabled(pvt))
1311 return ~dct_sel_high & 1;
1312
1313 return 0;
1314 }
1315
1316 /* Convert the sys_addr to the normalized DCT address */
1317 static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, unsigned range,
1318 u64 sys_addr, bool hi_rng,
1319 u32 dct_sel_base_addr)
1320 {
1321 u64 chan_off;
1322 u64 dram_base = get_dram_base(pvt, range);
1323 u64 hole_off = f10_dhar_offset(pvt);
1324 u64 dct_sel_base_off = (pvt->dct_sel_hi & 0xFFFFFC00) << 16;
1325
1326 if (hi_rng) {
1327 /*
1328 * if
1329 * base address of high range is below 4Gb
1330 * (bits [47:27] at [31:11])
1331 * DRAM address space on this DCT is hoisted above 4Gb &&
1332 * sys_addr > 4Gb
1333 *
1334 * remove hole offset from sys_addr
1335 * else
1336 * remove high range offset from sys_addr
1337 */
1338 if ((!(dct_sel_base_addr >> 16) ||
1339 dct_sel_base_addr < dhar_base(pvt)) &&
1340 dhar_valid(pvt) &&
1341 (sys_addr >= BIT_64(32)))
1342 chan_off = hole_off;
1343 else
1344 chan_off = dct_sel_base_off;
1345 } else {
1346 /*
1347 * if
1348 * we have a valid hole &&
1349 * sys_addr > 4Gb
1350 *
1351 * remove hole
1352 * else
1353 * remove dram base to normalize to DCT address
1354 */
1355 if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))
1356 chan_off = hole_off;
1357 else
1358 chan_off = dram_base;
1359 }
1360
1361 return (sys_addr & GENMASK(6,47)) - (chan_off & GENMASK(23,47));
1362 }
1363
1364 /*
1365 * checks if the csrow passed in is marked as SPARED, if so returns the new
1366 * spare row
1367 */
1368 static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
1369 {
1370 int tmp_cs;
1371
1372 if (online_spare_swap_done(pvt, dct) &&
1373 csrow == online_spare_bad_dramcs(pvt, dct)) {
1374
1375 for_each_chip_select(tmp_cs, dct, pvt) {
1376 if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
1377 csrow = tmp_cs;
1378 break;
1379 }
1380 }
1381 }
1382 return csrow;
1383 }
1384
1385 /*
1386 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
1387 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
1388 *
1389 * Return:
1390 * -EINVAL: NOT FOUND
1391 * 0..csrow = Chip-Select Row
1392 */
1393 static int f1x_lookup_addr_in_dct(u64 in_addr, u32 nid, u8 dct)
1394 {
1395 struct mem_ctl_info *mci;
1396 struct amd64_pvt *pvt;
1397 u64 cs_base, cs_mask;
1398 int cs_found = -EINVAL;
1399 int csrow;
1400
1401 mci = mcis[nid];
1402 if (!mci)
1403 return cs_found;
1404
1405 pvt = mci->pvt_info;
1406
1407 debugf1("input addr: 0x%llx, DCT: %d\n", in_addr, dct);
1408
1409 for_each_chip_select(csrow, dct, pvt) {
1410 if (!csrow_enabled(csrow, dct, pvt))
1411 continue;
1412
1413 get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
1414
1415 debugf1(" CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
1416 csrow, cs_base, cs_mask);
1417
1418 cs_mask = ~cs_mask;
1419
1420 debugf1(" (InputAddr & ~CSMask)=0x%llx "
1421 "(CSBase & ~CSMask)=0x%llx\n",
1422 (in_addr & cs_mask), (cs_base & cs_mask));
1423
1424 if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
1425 cs_found = f10_process_possible_spare(pvt, dct, csrow);
1426
1427 debugf1(" MATCH csrow=%d\n", cs_found);
1428 break;
1429 }
1430 }
1431 return cs_found;
1432 }
1433
1434 /*
1435 * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is
1436 * swapped with a region located at the bottom of memory so that the GPU can use
1437 * the interleaved region and thus two channels.
1438 */
1439 static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)
1440 {
1441 u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;
1442
1443 if (boot_cpu_data.x86 == 0x10) {
1444 /* only revC3 and revE have that feature */
1445 if (boot_cpu_data.x86_model < 4 ||
1446 (boot_cpu_data.x86_model < 0xa &&
1447 boot_cpu_data.x86_mask < 3))
1448 return sys_addr;
1449 }
1450
1451 amd64_read_dct_pci_cfg(pvt, SWAP_INTLV_REG, &swap_reg);
1452
1453 if (!(swap_reg & 0x1))
1454 return sys_addr;
1455
1456 swap_base = (swap_reg >> 3) & 0x7f;
1457 swap_limit = (swap_reg >> 11) & 0x7f;
1458 rgn_size = (swap_reg >> 20) & 0x7f;
1459 tmp_addr = sys_addr >> 27;
1460
1461 if (!(sys_addr >> 34) &&
1462 (((tmp_addr >= swap_base) &&
1463 (tmp_addr <= swap_limit)) ||
1464 (tmp_addr < rgn_size)))
1465 return sys_addr ^ (u64)swap_base << 27;
1466
1467 return sys_addr;
1468 }
1469
1470 /* For a given @dram_range, check if @sys_addr falls within it. */
1471 static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
1472 u64 sys_addr, int *nid, int *chan_sel)
1473 {
1474 int cs_found = -EINVAL;
1475 u64 chan_addr;
1476 u32 dct_sel_base;
1477 u8 channel;
1478 bool high_range = false;
1479
1480 u8 node_id = dram_dst_node(pvt, range);
1481 u8 intlv_en = dram_intlv_en(pvt, range);
1482 u32 intlv_sel = dram_intlv_sel(pvt, range);
1483
1484 debugf1("(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1485 range, sys_addr, get_dram_limit(pvt, range));
1486
1487 if (dhar_valid(pvt) &&
1488 dhar_base(pvt) <= sys_addr &&
1489 sys_addr < BIT_64(32)) {
1490 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1491 sys_addr);
1492 return -EINVAL;
1493 }
1494
1495 if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en)))
1496 return -EINVAL;
1497
1498 sys_addr = f1x_swap_interleaved_region(pvt, sys_addr);
1499
1500 dct_sel_base = dct_sel_baseaddr(pvt);
1501
1502 /*
1503 * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
1504 * select between DCT0 and DCT1.
1505 */
1506 if (dct_high_range_enabled(pvt) &&
1507 !dct_ganging_enabled(pvt) &&
1508 ((sys_addr >> 27) >= (dct_sel_base >> 11)))
1509 high_range = true;
1510
1511 channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en);
1512
1513 chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr,
1514 high_range, dct_sel_base);
1515
1516 /* Remove node interleaving, see F1x120 */
1517 if (intlv_en)
1518 chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) |
1519 (chan_addr & 0xfff);
1520
1521 /* remove channel interleave */
1522 if (dct_interleave_enabled(pvt) &&
1523 !dct_high_range_enabled(pvt) &&
1524 !dct_ganging_enabled(pvt)) {
1525
1526 if (dct_sel_interleave_addr(pvt) != 1) {
1527 if (dct_sel_interleave_addr(pvt) == 0x3)
1528 /* hash 9 */
1529 chan_addr = ((chan_addr >> 10) << 9) |
1530 (chan_addr & 0x1ff);
1531 else
1532 /* A[6] or hash 6 */
1533 chan_addr = ((chan_addr >> 7) << 6) |
1534 (chan_addr & 0x3f);
1535 } else
1536 /* A[12] */
1537 chan_addr = ((chan_addr >> 13) << 12) |
1538 (chan_addr & 0xfff);
1539 }
1540
1541 debugf1(" Normalized DCT addr: 0x%llx\n", chan_addr);
1542
1543 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel);
1544
1545 if (cs_found >= 0) {
1546 *nid = node_id;
1547 *chan_sel = channel;
1548 }
1549 return cs_found;
1550 }
1551
1552 static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt, u64 sys_addr,
1553 int *node, int *chan_sel)
1554 {
1555 int cs_found = -EINVAL;
1556 unsigned range;
1557
1558 for (range = 0; range < DRAM_RANGES; range++) {
1559
1560 if (!dram_rw(pvt, range))
1561 continue;
1562
1563 if ((get_dram_base(pvt, range) <= sys_addr) &&
1564 (get_dram_limit(pvt, range) >= sys_addr)) {
1565
1566 cs_found = f1x_match_to_this_node(pvt, range,
1567 sys_addr, node,
1568 chan_sel);
1569 if (cs_found >= 0)
1570 break;
1571 }
1572 }
1573 return cs_found;
1574 }
1575
1576 /*
1577 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
1578 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
1579 *
1580 * The @sys_addr is usually an error address received from the hardware
1581 * (MCX_ADDR).
1582 */
1583 static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
1584 u16 syndrome)
1585 {
1586 struct amd64_pvt *pvt = mci->pvt_info;
1587 u32 page, offset;
1588 int nid, csrow, chan = 0;
1589
1590 csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &nid, &chan);
1591
1592 if (csrow < 0) {
1593 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1594 return;
1595 }
1596
1597 error_address_to_page_and_offset(sys_addr, &page, &offset);
1598
1599 /*
1600 * We need the syndromes for channel detection only when we're
1601 * ganged. Otherwise @chan should already contain the channel at
1602 * this point.
1603 */
1604 if (dct_ganging_enabled(pvt))
1605 chan = get_channel_from_ecc_syndrome(mci, syndrome);
1606
1607 if (chan >= 0)
1608 edac_mc_handle_ce(mci, page, offset, syndrome, csrow, chan,
1609 EDAC_MOD_STR);
1610 else
1611 /*
1612 * Channel unknown, report all channels on this CSROW as failed.
1613 */
1614 for (chan = 0; chan < mci->csrows[csrow].nr_channels; chan++)
1615 edac_mc_handle_ce(mci, page, offset, syndrome,
1616 csrow, chan, EDAC_MOD_STR);
1617 }
1618
1619 /*
1620 * debug routine to display the memory sizes of all logical DIMMs and its
1621 * CSROWs
1622 */
1623 static void amd64_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
1624 {
1625 int dimm, size0, size1, factor = 0;
1626 u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
1627 u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0;
1628
1629 if (boot_cpu_data.x86 == 0xf) {
1630 if (pvt->dclr0 & WIDTH_128)
1631 factor = 1;
1632
1633 /* K8 families < revF not supported yet */
1634 if (pvt->ext_model < K8_REV_F)
1635 return;
1636 else
1637 WARN_ON(ctrl != 0);
1638 }
1639
1640 dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1 : pvt->dbam0;
1641 dcsb = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->csels[1].csbases
1642 : pvt->csels[0].csbases;
1643
1644 debugf1("F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n", ctrl, dbam);
1645
1646 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
1647
1648 /* Dump memory sizes for DIMM and its CSROWs */
1649 for (dimm = 0; dimm < 4; dimm++) {
1650
1651 size0 = 0;
1652 if (dcsb[dimm*2] & DCSB_CS_ENABLE)
1653 size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
1654 DBAM_DIMM(dimm, dbam));
1655
1656 size1 = 0;
1657 if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
1658 size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
1659 DBAM_DIMM(dimm, dbam));
1660
1661 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
1662 dimm * 2, size0 << factor,
1663 dimm * 2 + 1, size1 << factor);
1664 }
1665 }
1666
1667 static struct amd64_family_type amd64_family_types[] = {
1668 [K8_CPUS] = {
1669 .ctl_name = "K8",
1670 .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
1671 .f3_id = PCI_DEVICE_ID_AMD_K8_NB_MISC,
1672 .ops = {
1673 .early_channel_count = k8_early_channel_count,
1674 .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow,
1675 .dbam_to_cs = k8_dbam_to_chip_select,
1676 .read_dct_pci_cfg = k8_read_dct_pci_cfg,
1677 }
1678 },
1679 [F10_CPUS] = {
1680 .ctl_name = "F10h",
1681 .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
1682 .f3_id = PCI_DEVICE_ID_AMD_10H_NB_MISC,
1683 .ops = {
1684 .early_channel_count = f1x_early_channel_count,
1685 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1686 .dbam_to_cs = f10_dbam_to_chip_select,
1687 .read_dct_pci_cfg = f10_read_dct_pci_cfg,
1688 }
1689 },
1690 [F15_CPUS] = {
1691 .ctl_name = "F15h",
1692 .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1,
1693 .f3_id = PCI_DEVICE_ID_AMD_15H_NB_F3,
1694 .ops = {
1695 .early_channel_count = f1x_early_channel_count,
1696 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1697 .dbam_to_cs = f15_dbam_to_chip_select,
1698 .read_dct_pci_cfg = f15_read_dct_pci_cfg,
1699 }
1700 },
1701 };
1702
1703 static struct pci_dev *pci_get_related_function(unsigned int vendor,
1704 unsigned int device,
1705 struct pci_dev *related)
1706 {
1707 struct pci_dev *dev = NULL;
1708
1709 dev = pci_get_device(vendor, device, dev);
1710 while (dev) {
1711 if ((dev->bus->number == related->bus->number) &&
1712 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
1713 break;
1714 dev = pci_get_device(vendor, device, dev);
1715 }
1716
1717 return dev;
1718 }
1719
1720 /*
1721 * These are tables of eigenvectors (one per line) which can be used for the
1722 * construction of the syndrome tables. The modified syndrome search algorithm
1723 * uses those to find the symbol in error and thus the DIMM.
1724 *
1725 * Algorithm courtesy of Ross LaFetra from AMD.
1726 */
1727 static u16 x4_vectors[] = {
1728 0x2f57, 0x1afe, 0x66cc, 0xdd88,
1729 0x11eb, 0x3396, 0x7f4c, 0xeac8,
1730 0x0001, 0x0002, 0x0004, 0x0008,
1731 0x1013, 0x3032, 0x4044, 0x8088,
1732 0x106b, 0x30d6, 0x70fc, 0xe0a8,
1733 0x4857, 0xc4fe, 0x13cc, 0x3288,
1734 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
1735 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
1736 0x15c1, 0x2a42, 0x89ac, 0x4758,
1737 0x2b03, 0x1602, 0x4f0c, 0xca08,
1738 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
1739 0x8ba7, 0x465e, 0x244c, 0x1cc8,
1740 0x2b87, 0x164e, 0x642c, 0xdc18,
1741 0x40b9, 0x80de, 0x1094, 0x20e8,
1742 0x27db, 0x1eb6, 0x9dac, 0x7b58,
1743 0x11c1, 0x2242, 0x84ac, 0x4c58,
1744 0x1be5, 0x2d7a, 0x5e34, 0xa718,
1745 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
1746 0x4c97, 0xc87e, 0x11fc, 0x33a8,
1747 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
1748 0x16b3, 0x3d62, 0x4f34, 0x8518,
1749 0x1e2f, 0x391a, 0x5cac, 0xf858,
1750 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
1751 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
1752 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
1753 0x4397, 0xc27e, 0x17fc, 0x3ea8,
1754 0x1617, 0x3d3e, 0x6464, 0xb8b8,
1755 0x23ff, 0x12aa, 0xab6c, 0x56d8,
1756 0x2dfb, 0x1ba6, 0x913c, 0x7328,
1757 0x185d, 0x2ca6, 0x7914, 0x9e28,
1758 0x171b, 0x3e36, 0x7d7c, 0xebe8,
1759 0x4199, 0x82ee, 0x19f4, 0x2e58,
1760 0x4807, 0xc40e, 0x130c, 0x3208,
1761 0x1905, 0x2e0a, 0x5804, 0xac08,
1762 0x213f, 0x132a, 0xadfc, 0x5ba8,
1763 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
1764 };
1765
1766 static u16 x8_vectors[] = {
1767 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
1768 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
1769 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
1770 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
1771 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
1772 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
1773 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
1774 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
1775 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
1776 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
1777 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
1778 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
1779 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
1780 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
1781 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
1782 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
1783 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
1784 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
1785 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
1786 };
1787
1788 static int decode_syndrome(u16 syndrome, u16 *vectors, unsigned num_vecs,
1789 unsigned v_dim)
1790 {
1791 unsigned int i, err_sym;
1792
1793 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
1794 u16 s = syndrome;
1795 unsigned v_idx = err_sym * v_dim;
1796 unsigned v_end = (err_sym + 1) * v_dim;
1797
1798 /* walk over all 16 bits of the syndrome */
1799 for (i = 1; i < (1U << 16); i <<= 1) {
1800
1801 /* if bit is set in that eigenvector... */
1802 if (v_idx < v_end && vectors[v_idx] & i) {
1803 u16 ev_comp = vectors[v_idx++];
1804
1805 /* ... and bit set in the modified syndrome, */
1806 if (s & i) {
1807 /* remove it. */
1808 s ^= ev_comp;
1809
1810 if (!s)
1811 return err_sym;
1812 }
1813
1814 } else if (s & i)
1815 /* can't get to zero, move to next symbol */
1816 break;
1817 }
1818 }
1819
1820 debugf0("syndrome(%x) not found\n", syndrome);
1821 return -1;
1822 }
1823
1824 static int map_err_sym_to_channel(int err_sym, int sym_size)
1825 {
1826 if (sym_size == 4)
1827 switch (err_sym) {
1828 case 0x20:
1829 case 0x21:
1830 return 0;
1831 break;
1832 case 0x22:
1833 case 0x23:
1834 return 1;
1835 break;
1836 default:
1837 return err_sym >> 4;
1838 break;
1839 }
1840 /* x8 symbols */
1841 else
1842 switch (err_sym) {
1843 /* imaginary bits not in a DIMM */
1844 case 0x10:
1845 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
1846 err_sym);
1847 return -1;
1848 break;
1849
1850 case 0x11:
1851 return 0;
1852 break;
1853 case 0x12:
1854 return 1;
1855 break;
1856 default:
1857 return err_sym >> 3;
1858 break;
1859 }
1860 return -1;
1861 }
1862
1863 static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
1864 {
1865 struct amd64_pvt *pvt = mci->pvt_info;
1866 int err_sym = -1;
1867
1868 if (pvt->ecc_sym_sz == 8)
1869 err_sym = decode_syndrome(syndrome, x8_vectors,
1870 ARRAY_SIZE(x8_vectors),
1871 pvt->ecc_sym_sz);
1872 else if (pvt->ecc_sym_sz == 4)
1873 err_sym = decode_syndrome(syndrome, x4_vectors,
1874 ARRAY_SIZE(x4_vectors),
1875 pvt->ecc_sym_sz);
1876 else {
1877 amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz);
1878 return err_sym;
1879 }
1880
1881 return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
1882 }
1883
1884 /*
1885 * Handle any Correctable Errors (CEs) that have occurred. Check for valid ERROR
1886 * ADDRESS and process.
1887 */
1888 static void amd64_handle_ce(struct mem_ctl_info *mci, struct mce *m)
1889 {
1890 struct amd64_pvt *pvt = mci->pvt_info;
1891 u64 sys_addr;
1892 u16 syndrome;
1893
1894 /* Ensure that the Error Address is VALID */
1895 if (!(m->status & MCI_STATUS_ADDRV)) {
1896 amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n");
1897 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1898 return;
1899 }
1900
1901 sys_addr = get_error_address(m);
1902 syndrome = extract_syndrome(m->status);
1903
1904 amd64_mc_err(mci, "CE ERROR_ADDRESS= 0x%llx\n", sys_addr);
1905
1906 pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, syndrome);
1907 }
1908
1909 /* Handle any Un-correctable Errors (UEs) */
1910 static void amd64_handle_ue(struct mem_ctl_info *mci, struct mce *m)
1911 {
1912 struct mem_ctl_info *log_mci, *src_mci = NULL;
1913 int csrow;
1914 u64 sys_addr;
1915 u32 page, offset;
1916
1917 log_mci = mci;
1918
1919 if (!(m->status & MCI_STATUS_ADDRV)) {
1920 amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n");
1921 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1922 return;
1923 }
1924
1925 sys_addr = get_error_address(m);
1926
1927 /*
1928 * Find out which node the error address belongs to. This may be
1929 * different from the node that detected the error.
1930 */
1931 src_mci = find_mc_by_sys_addr(mci, sys_addr);
1932 if (!src_mci) {
1933 amd64_mc_err(mci, "ERROR ADDRESS (0x%lx) NOT mapped to a MC\n",
1934 (unsigned long)sys_addr);
1935 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1936 return;
1937 }
1938
1939 log_mci = src_mci;
1940
1941 csrow = sys_addr_to_csrow(log_mci, sys_addr);
1942 if (csrow < 0) {
1943 amd64_mc_err(mci, "ERROR_ADDRESS (0x%lx) NOT mapped to CS\n",
1944 (unsigned long)sys_addr);
1945 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1946 } else {
1947 error_address_to_page_and_offset(sys_addr, &page, &offset);
1948 edac_mc_handle_ue(log_mci, page, offset, csrow, EDAC_MOD_STR);
1949 }
1950 }
1951
1952 static inline void __amd64_decode_bus_error(struct mem_ctl_info *mci,
1953 struct mce *m)
1954 {
1955 u16 ec = EC(m->status);
1956 u8 xec = XEC(m->status, 0x1f);
1957 u8 ecc_type = (m->status >> 45) & 0x3;
1958
1959 /* Bail early out if this was an 'observed' error */
1960 if (PP(ec) == NBSL_PP_OBS)
1961 return;
1962
1963 /* Do only ECC errors */
1964 if (xec && xec != F10_NBSL_EXT_ERR_ECC)
1965 return;
1966
1967 if (ecc_type == 2)
1968 amd64_handle_ce(mci, m);
1969 else if (ecc_type == 1)
1970 amd64_handle_ue(mci, m);
1971 }
1972
1973 void amd64_decode_bus_error(int node_id, struct mce *m)
1974 {
1975 __amd64_decode_bus_error(mcis[node_id], m);
1976 }
1977
1978 /*
1979 * Use pvt->F2 which contains the F2 CPU PCI device to get the related
1980 * F1 (AddrMap) and F3 (Misc) devices. Return negative value on error.
1981 */
1982 static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f3_id)
1983 {
1984 /* Reserve the ADDRESS MAP Device */
1985 pvt->F1 = pci_get_related_function(pvt->F2->vendor, f1_id, pvt->F2);
1986 if (!pvt->F1) {
1987 amd64_err("error address map device not found: "
1988 "vendor %x device 0x%x (broken BIOS?)\n",
1989 PCI_VENDOR_ID_AMD, f1_id);
1990 return -ENODEV;
1991 }
1992
1993 /* Reserve the MISC Device */
1994 pvt->F3 = pci_get_related_function(pvt->F2->vendor, f3_id, pvt->F2);
1995 if (!pvt->F3) {
1996 pci_dev_put(pvt->F1);
1997 pvt->F1 = NULL;
1998
1999 amd64_err("error F3 device not found: "
2000 "vendor %x device 0x%x (broken BIOS?)\n",
2001 PCI_VENDOR_ID_AMD, f3_id);
2002
2003 return -ENODEV;
2004 }
2005 debugf1("F1: %s\n", pci_name(pvt->F1));
2006 debugf1("F2: %s\n", pci_name(pvt->F2));
2007 debugf1("F3: %s\n", pci_name(pvt->F3));
2008
2009 return 0;
2010 }
2011
2012 static void free_mc_sibling_devs(struct amd64_pvt *pvt)
2013 {
2014 pci_dev_put(pvt->F1);
2015 pci_dev_put(pvt->F3);
2016 }
2017
2018 /*
2019 * Retrieve the hardware registers of the memory controller (this includes the
2020 * 'Address Map' and 'Misc' device regs)
2021 */
2022 static void read_mc_regs(struct amd64_pvt *pvt)
2023 {
2024 struct cpuinfo_x86 *c = &boot_cpu_data;
2025 u64 msr_val;
2026 u32 tmp;
2027 unsigned range;
2028
2029 /*
2030 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
2031 * those are Read-As-Zero
2032 */
2033 rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
2034 debugf0(" TOP_MEM: 0x%016llx\n", pvt->top_mem);
2035
2036 /* check first whether TOP_MEM2 is enabled */
2037 rdmsrl(MSR_K8_SYSCFG, msr_val);
2038 if (msr_val & (1U << 21)) {
2039 rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
2040 debugf0(" TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
2041 } else
2042 debugf0(" TOP_MEM2 disabled.\n");
2043
2044 amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
2045
2046 read_dram_ctl_register(pvt);
2047
2048 for (range = 0; range < DRAM_RANGES; range++) {
2049 u8 rw;
2050
2051 /* read settings for this DRAM range */
2052 read_dram_base_limit_regs(pvt, range);
2053
2054 rw = dram_rw(pvt, range);
2055 if (!rw)
2056 continue;
2057
2058 debugf1(" DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
2059 range,
2060 get_dram_base(pvt, range),
2061 get_dram_limit(pvt, range));
2062
2063 debugf1(" IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
2064 dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
2065 (rw & 0x1) ? "R" : "-",
2066 (rw & 0x2) ? "W" : "-",
2067 dram_intlv_sel(pvt, range),
2068 dram_dst_node(pvt, range));
2069 }
2070
2071 read_dct_base_mask(pvt);
2072
2073 amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
2074 amd64_read_dct_pci_cfg(pvt, DBAM0, &pvt->dbam0);
2075
2076 amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
2077
2078 amd64_read_dct_pci_cfg(pvt, DCLR0, &pvt->dclr0);
2079 amd64_read_dct_pci_cfg(pvt, DCHR0, &pvt->dchr0);
2080
2081 if (!dct_ganging_enabled(pvt)) {
2082 amd64_read_dct_pci_cfg(pvt, DCLR1, &pvt->dclr1);
2083 amd64_read_dct_pci_cfg(pvt, DCHR1, &pvt->dchr1);
2084 }
2085
2086 pvt->ecc_sym_sz = 4;
2087
2088 if (c->x86 >= 0x10) {
2089 amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
2090 amd64_read_dct_pci_cfg(pvt, DBAM1, &pvt->dbam1);
2091
2092 /* F10h, revD and later can do x8 ECC too */
2093 if ((c->x86 > 0x10 || c->x86_model > 7) && tmp & BIT(25))
2094 pvt->ecc_sym_sz = 8;
2095 }
2096 dump_misc_regs(pvt);
2097 }
2098
2099 /*
2100 * NOTE: CPU Revision Dependent code
2101 *
2102 * Input:
2103 * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
2104 * k8 private pointer to -->
2105 * DRAM Bank Address mapping register
2106 * node_id
2107 * DCL register where dual_channel_active is
2108 *
2109 * The DBAM register consists of 4 sets of 4 bits each definitions:
2110 *
2111 * Bits: CSROWs
2112 * 0-3 CSROWs 0 and 1
2113 * 4-7 CSROWs 2 and 3
2114 * 8-11 CSROWs 4 and 5
2115 * 12-15 CSROWs 6 and 7
2116 *
2117 * Values range from: 0 to 15
2118 * The meaning of the values depends on CPU revision and dual-channel state,
2119 * see relevant BKDG more info.
2120 *
2121 * The memory controller provides for total of only 8 CSROWs in its current
2122 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
2123 * single channel or two (2) DIMMs in dual channel mode.
2124 *
2125 * The following code logic collapses the various tables for CSROW based on CPU
2126 * revision.
2127 *
2128 * Returns:
2129 * The number of PAGE_SIZE pages on the specified CSROW number it
2130 * encompasses
2131 *
2132 */
2133 static u32 amd64_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr)
2134 {
2135 u32 cs_mode, nr_pages;
2136
2137 /*
2138 * The math on this doesn't look right on the surface because x/2*4 can
2139 * be simplified to x*2 but this expression makes use of the fact that
2140 * it is integral math where 1/2=0. This intermediate value becomes the
2141 * number of bits to shift the DBAM register to extract the proper CSROW
2142 * field.
2143 */
2144 cs_mode = (pvt->dbam0 >> ((csrow_nr / 2) * 4)) & 0xF;
2145
2146 nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode) << (20 - PAGE_SHIFT);
2147
2148 /*
2149 * If dual channel then double the memory size of single channel.
2150 * Channel count is 1 or 2
2151 */
2152 nr_pages <<= (pvt->channel_count - 1);
2153
2154 debugf0(" (csrow=%d) DBAM map index= %d\n", csrow_nr, cs_mode);
2155 debugf0(" nr_pages= %u channel-count = %d\n",
2156 nr_pages, pvt->channel_count);
2157
2158 return nr_pages;
2159 }
2160
2161 /*
2162 * Initialize the array of csrow attribute instances, based on the values
2163 * from pci config hardware registers.
2164 */
2165 static int init_csrows(struct mem_ctl_info *mci)
2166 {
2167 struct csrow_info *csrow;
2168 struct amd64_pvt *pvt = mci->pvt_info;
2169 u64 input_addr_min, input_addr_max, sys_addr, base, mask;
2170 u32 val;
2171 int i, empty = 1;
2172
2173 amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
2174
2175 pvt->nbcfg = val;
2176
2177 debugf0("node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2178 pvt->mc_node_id, val,
2179 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
2180
2181 for_each_chip_select(i, 0, pvt) {
2182 csrow = &mci->csrows[i];
2183
2184 if (!csrow_enabled(i, 0, pvt)) {
2185 debugf1("----CSROW %d EMPTY for node %d\n", i,
2186 pvt->mc_node_id);
2187 continue;
2188 }
2189
2190 debugf1("----CSROW %d VALID for MC node %d\n",
2191 i, pvt->mc_node_id);
2192
2193 empty = 0;
2194 csrow->nr_pages = amd64_csrow_nr_pages(pvt, 0, i);
2195 find_csrow_limits(mci, i, &input_addr_min, &input_addr_max);
2196 sys_addr = input_addr_to_sys_addr(mci, input_addr_min);
2197 csrow->first_page = (u32) (sys_addr >> PAGE_SHIFT);
2198 sys_addr = input_addr_to_sys_addr(mci, input_addr_max);
2199 csrow->last_page = (u32) (sys_addr >> PAGE_SHIFT);
2200
2201 get_cs_base_and_mask(pvt, i, 0, &base, &mask);
2202 csrow->page_mask = ~mask;
2203 /* 8 bytes of resolution */
2204
2205 csrow->mtype = amd64_determine_memory_type(pvt, i);
2206
2207 debugf1(" for MC node %d csrow %d:\n", pvt->mc_node_id, i);
2208 debugf1(" input_addr_min: 0x%lx input_addr_max: 0x%lx\n",
2209 (unsigned long)input_addr_min,
2210 (unsigned long)input_addr_max);
2211 debugf1(" sys_addr: 0x%lx page_mask: 0x%lx\n",
2212 (unsigned long)sys_addr, csrow->page_mask);
2213 debugf1(" nr_pages: %u first_page: 0x%lx "
2214 "last_page: 0x%lx\n",
2215 (unsigned)csrow->nr_pages,
2216 csrow->first_page, csrow->last_page);
2217
2218 /*
2219 * determine whether CHIPKILL or JUST ECC or NO ECC is operating
2220 */
2221 if (pvt->nbcfg & NBCFG_ECC_ENABLE)
2222 csrow->edac_mode =
2223 (pvt->nbcfg & NBCFG_CHIPKILL) ?
2224 EDAC_S4ECD4ED : EDAC_SECDED;
2225 else
2226 csrow->edac_mode = EDAC_NONE;
2227 }
2228
2229 return empty;
2230 }
2231
2232 /* get all cores on this DCT */
2233 static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, unsigned nid)
2234 {
2235 int cpu;
2236
2237 for_each_online_cpu(cpu)
2238 if (amd_get_nb_id(cpu) == nid)
2239 cpumask_set_cpu(cpu, mask);
2240 }
2241
2242 /* check MCG_CTL on all the cpus on this node */
2243 static bool amd64_nb_mce_bank_enabled_on_node(unsigned nid)
2244 {
2245 cpumask_var_t mask;
2246 int cpu, nbe;
2247 bool ret = false;
2248
2249 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
2250 amd64_warn("%s: Error allocating mask\n", __func__);
2251 return false;
2252 }
2253
2254 get_cpus_on_this_dct_cpumask(mask, nid);
2255
2256 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
2257
2258 for_each_cpu(cpu, mask) {
2259 struct msr *reg = per_cpu_ptr(msrs, cpu);
2260 nbe = reg->l & MSR_MCGCTL_NBE;
2261
2262 debugf0("core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
2263 cpu, reg->q,
2264 (nbe ? "enabled" : "disabled"));
2265
2266 if (!nbe)
2267 goto out;
2268 }
2269 ret = true;
2270
2271 out:
2272 free_cpumask_var(mask);
2273 return ret;
2274 }
2275
2276 static int toggle_ecc_err_reporting(struct ecc_settings *s, u8 nid, bool on)
2277 {
2278 cpumask_var_t cmask;
2279 int cpu;
2280
2281 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
2282 amd64_warn("%s: error allocating mask\n", __func__);
2283 return false;
2284 }
2285
2286 get_cpus_on_this_dct_cpumask(cmask, nid);
2287
2288 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2289
2290 for_each_cpu(cpu, cmask) {
2291
2292 struct msr *reg = per_cpu_ptr(msrs, cpu);
2293
2294 if (on) {
2295 if (reg->l & MSR_MCGCTL_NBE)
2296 s->flags.nb_mce_enable = 1;
2297
2298 reg->l |= MSR_MCGCTL_NBE;
2299 } else {
2300 /*
2301 * Turn off NB MCE reporting only when it was off before
2302 */
2303 if (!s->flags.nb_mce_enable)
2304 reg->l &= ~MSR_MCGCTL_NBE;
2305 }
2306 }
2307 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2308
2309 free_cpumask_var(cmask);
2310
2311 return 0;
2312 }
2313
2314 static bool enable_ecc_error_reporting(struct ecc_settings *s, u8 nid,
2315 struct pci_dev *F3)
2316 {
2317 bool ret = true;
2318 u32 value, mask = 0x3; /* UECC/CECC enable */
2319
2320 if (toggle_ecc_err_reporting(s, nid, ON)) {
2321 amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
2322 return false;
2323 }
2324
2325 amd64_read_pci_cfg(F3, NBCTL, &value);
2326
2327 s->old_nbctl = value & mask;
2328 s->nbctl_valid = true;
2329
2330 value |= mask;
2331 amd64_write_pci_cfg(F3, NBCTL, value);
2332
2333 amd64_read_pci_cfg(F3, NBCFG, &value);
2334
2335 debugf0("1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2336 nid, value, !!(value & NBCFG_ECC_ENABLE));
2337
2338 if (!(value & NBCFG_ECC_ENABLE)) {
2339 amd64_warn("DRAM ECC disabled on this node, enabling...\n");
2340
2341 s->flags.nb_ecc_prev = 0;
2342
2343 /* Attempt to turn on DRAM ECC Enable */
2344 value |= NBCFG_ECC_ENABLE;
2345 amd64_write_pci_cfg(F3, NBCFG, value);
2346
2347 amd64_read_pci_cfg(F3, NBCFG, &value);
2348
2349 if (!(value & NBCFG_ECC_ENABLE)) {
2350 amd64_warn("Hardware rejected DRAM ECC enable,"
2351 "check memory DIMM configuration.\n");
2352 ret = false;
2353 } else {
2354 amd64_info("Hardware accepted DRAM ECC Enable\n");
2355 }
2356 } else {
2357 s->flags.nb_ecc_prev = 1;
2358 }
2359
2360 debugf0("2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2361 nid, value, !!(value & NBCFG_ECC_ENABLE));
2362
2363 return ret;
2364 }
2365
2366 static void restore_ecc_error_reporting(struct ecc_settings *s, u8 nid,
2367 struct pci_dev *F3)
2368 {
2369 u32 value, mask = 0x3; /* UECC/CECC enable */
2370
2371
2372 if (!s->nbctl_valid)
2373 return;
2374
2375 amd64_read_pci_cfg(F3, NBCTL, &value);
2376 value &= ~mask;
2377 value |= s->old_nbctl;
2378
2379 amd64_write_pci_cfg(F3, NBCTL, value);
2380
2381 /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
2382 if (!s->flags.nb_ecc_prev) {
2383 amd64_read_pci_cfg(F3, NBCFG, &value);
2384 value &= ~NBCFG_ECC_ENABLE;
2385 amd64_write_pci_cfg(F3, NBCFG, value);
2386 }
2387
2388 /* restore the NB Enable MCGCTL bit */
2389 if (toggle_ecc_err_reporting(s, nid, OFF))
2390 amd64_warn("Error restoring NB MCGCTL settings!\n");
2391 }
2392
2393 /*
2394 * EDAC requires that the BIOS have ECC enabled before
2395 * taking over the processing of ECC errors. A command line
2396 * option allows to force-enable hardware ECC later in
2397 * enable_ecc_error_reporting().
2398 */
2399 static const char *ecc_msg =
2400 "ECC disabled in the BIOS or no ECC capability, module will not load.\n"
2401 " Either enable ECC checking or force module loading by setting "
2402 "'ecc_enable_override'.\n"
2403 " (Note that use of the override may cause unknown side effects.)\n";
2404
2405 static bool ecc_enabled(struct pci_dev *F3, u8 nid)
2406 {
2407 u32 value;
2408 u8 ecc_en = 0;
2409 bool nb_mce_en = false;
2410
2411 amd64_read_pci_cfg(F3, NBCFG, &value);
2412
2413 ecc_en = !!(value & NBCFG_ECC_ENABLE);
2414 amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled"));
2415
2416 nb_mce_en = amd64_nb_mce_bank_enabled_on_node(nid);
2417 if (!nb_mce_en)
2418 amd64_notice("NB MCE bank disabled, set MSR "
2419 "0x%08x[4] on node %d to enable.\n",
2420 MSR_IA32_MCG_CTL, nid);
2421
2422 if (!ecc_en || !nb_mce_en) {
2423 amd64_notice("%s", ecc_msg);
2424 return false;
2425 }
2426 return true;
2427 }
2428
2429 struct mcidev_sysfs_attribute sysfs_attrs[ARRAY_SIZE(amd64_dbg_attrs) +
2430 ARRAY_SIZE(amd64_inj_attrs) +
2431 1];
2432
2433 struct mcidev_sysfs_attribute terminator = { .attr = { .name = NULL } };
2434
2435 static void set_mc_sysfs_attrs(struct mem_ctl_info *mci)
2436 {
2437 unsigned int i = 0, j = 0;
2438
2439 for (; i < ARRAY_SIZE(amd64_dbg_attrs); i++)
2440 sysfs_attrs[i] = amd64_dbg_attrs[i];
2441
2442 if (boot_cpu_data.x86 >= 0x10)
2443 for (j = 0; j < ARRAY_SIZE(amd64_inj_attrs); j++, i++)
2444 sysfs_attrs[i] = amd64_inj_attrs[j];
2445
2446 sysfs_attrs[i] = terminator;
2447
2448 mci->mc_driver_sysfs_attributes = sysfs_attrs;
2449 }
2450
2451 static void setup_mci_misc_attrs(struct mem_ctl_info *mci,
2452 struct amd64_family_type *fam)
2453 {
2454 struct amd64_pvt *pvt = mci->pvt_info;
2455
2456 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
2457 mci->edac_ctl_cap = EDAC_FLAG_NONE;
2458
2459 if (pvt->nbcap & NBCAP_SECDED)
2460 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
2461
2462 if (pvt->nbcap & NBCAP_CHIPKILL)
2463 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
2464
2465 mci->edac_cap = amd64_determine_edac_cap(pvt);
2466 mci->mod_name = EDAC_MOD_STR;
2467 mci->mod_ver = EDAC_AMD64_VERSION;
2468 mci->ctl_name = fam->ctl_name;
2469 mci->dev_name = pci_name(pvt->F2);
2470 mci->ctl_page_to_phys = NULL;
2471
2472 /* memory scrubber interface */
2473 mci->set_sdram_scrub_rate = amd64_set_scrub_rate;
2474 mci->get_sdram_scrub_rate = amd64_get_scrub_rate;
2475 }
2476
2477 /*
2478 * returns a pointer to the family descriptor on success, NULL otherwise.
2479 */
2480 static struct amd64_family_type *amd64_per_family_init(struct amd64_pvt *pvt)
2481 {
2482 u8 fam = boot_cpu_data.x86;
2483 struct amd64_family_type *fam_type = NULL;
2484
2485 switch (fam) {
2486 case 0xf:
2487 fam_type = &amd64_family_types[K8_CPUS];
2488 pvt->ops = &amd64_family_types[K8_CPUS].ops;
2489 break;
2490
2491 case 0x10:
2492 fam_type = &amd64_family_types[F10_CPUS];
2493 pvt->ops = &amd64_family_types[F10_CPUS].ops;
2494 break;
2495
2496 case 0x15:
2497 fam_type = &amd64_family_types[F15_CPUS];
2498 pvt->ops = &amd64_family_types[F15_CPUS].ops;
2499 break;
2500
2501 default:
2502 amd64_err("Unsupported family!\n");
2503 return NULL;
2504 }
2505
2506 pvt->ext_model = boot_cpu_data.x86_model >> 4;
2507
2508 amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name,
2509 (fam == 0xf ?
2510 (pvt->ext_model >= K8_REV_F ? "revF or later "
2511 : "revE or earlier ")
2512 : ""), pvt->mc_node_id);
2513 return fam_type;
2514 }
2515
2516 static int amd64_init_one_instance(struct pci_dev *F2)
2517 {
2518 struct amd64_pvt *pvt = NULL;
2519 struct amd64_family_type *fam_type = NULL;
2520 struct mem_ctl_info *mci = NULL;
2521 int err = 0, ret;
2522 u8 nid = get_node_id(F2);
2523
2524 ret = -ENOMEM;
2525 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
2526 if (!pvt)
2527 goto err_ret;
2528
2529 pvt->mc_node_id = nid;
2530 pvt->F2 = F2;
2531
2532 ret = -EINVAL;
2533 fam_type = amd64_per_family_init(pvt);
2534 if (!fam_type)
2535 goto err_free;
2536
2537 ret = -ENODEV;
2538 err = reserve_mc_sibling_devs(pvt, fam_type->f1_id, fam_type->f3_id);
2539 if (err)
2540 goto err_free;
2541
2542 read_mc_regs(pvt);
2543
2544 /*
2545 * We need to determine how many memory channels there are. Then use
2546 * that information for calculating the size of the dynamic instance
2547 * tables in the 'mci' structure.
2548 */
2549 ret = -EINVAL;
2550 pvt->channel_count = pvt->ops->early_channel_count(pvt);
2551 if (pvt->channel_count < 0)
2552 goto err_siblings;
2553
2554 ret = -ENOMEM;
2555 mci = edac_mc_alloc(0, pvt->csels[0].b_cnt, pvt->channel_count, nid);
2556 if (!mci)
2557 goto err_siblings;
2558
2559 mci->pvt_info = pvt;
2560 mci->dev = &pvt->F2->dev;
2561
2562 setup_mci_misc_attrs(mci, fam_type);
2563
2564 if (init_csrows(mci))
2565 mci->edac_cap = EDAC_FLAG_NONE;
2566
2567 set_mc_sysfs_attrs(mci);
2568
2569 ret = -ENODEV;
2570 if (edac_mc_add_mc(mci)) {
2571 debugf1("failed edac_mc_add_mc()\n");
2572 goto err_add_mc;
2573 }
2574
2575 /* register stuff with EDAC MCE */
2576 if (report_gart_errors)
2577 amd_report_gart_errors(true);
2578
2579 amd_register_ecc_decoder(amd64_decode_bus_error);
2580
2581 mcis[nid] = mci;
2582
2583 atomic_inc(&drv_instances);
2584
2585 return 0;
2586
2587 err_add_mc:
2588 edac_mc_free(mci);
2589
2590 err_siblings:
2591 free_mc_sibling_devs(pvt);
2592
2593 err_free:
2594 kfree(pvt);
2595
2596 err_ret:
2597 return ret;
2598 }
2599
2600 static int __devinit amd64_probe_one_instance(struct pci_dev *pdev,
2601 const struct pci_device_id *mc_type)
2602 {
2603 u8 nid = get_node_id(pdev);
2604 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2605 struct ecc_settings *s;
2606 int ret = 0;
2607
2608 ret = pci_enable_device(pdev);
2609 if (ret < 0) {
2610 debugf0("ret=%d\n", ret);
2611 return -EIO;
2612 }
2613
2614 ret = -ENOMEM;
2615 s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
2616 if (!s)
2617 goto err_out;
2618
2619 ecc_stngs[nid] = s;
2620
2621 if (!ecc_enabled(F3, nid)) {
2622 ret = -ENODEV;
2623
2624 if (!ecc_enable_override)
2625 goto err_enable;
2626
2627 amd64_warn("Forcing ECC on!\n");
2628
2629 if (!enable_ecc_error_reporting(s, nid, F3))
2630 goto err_enable;
2631 }
2632
2633 ret = amd64_init_one_instance(pdev);
2634 if (ret < 0) {
2635 amd64_err("Error probing instance: %d\n", nid);
2636 restore_ecc_error_reporting(s, nid, F3);
2637 }
2638
2639 return ret;
2640
2641 err_enable:
2642 kfree(s);
2643 ecc_stngs[nid] = NULL;
2644
2645 err_out:
2646 return ret;
2647 }
2648
2649 static void __devexit amd64_remove_one_instance(struct pci_dev *pdev)
2650 {
2651 struct mem_ctl_info *mci;
2652 struct amd64_pvt *pvt;
2653 u8 nid = get_node_id(pdev);
2654 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2655 struct ecc_settings *s = ecc_stngs[nid];
2656
2657 /* Remove from EDAC CORE tracking list */
2658 mci = edac_mc_del_mc(&pdev->dev);
2659 if (!mci)
2660 return;
2661
2662 pvt = mci->pvt_info;
2663
2664 restore_ecc_error_reporting(s, nid, F3);
2665
2666 free_mc_sibling_devs(pvt);
2667
2668 /* unregister from EDAC MCE */
2669 amd_report_gart_errors(false);
2670 amd_unregister_ecc_decoder(amd64_decode_bus_error);
2671
2672 kfree(ecc_stngs[nid]);
2673 ecc_stngs[nid] = NULL;
2674
2675 /* Free the EDAC CORE resources */
2676 mci->pvt_info = NULL;
2677 mcis[nid] = NULL;
2678
2679 kfree(pvt);
2680 edac_mc_free(mci);
2681 }
2682
2683 /*
2684 * This table is part of the interface for loading drivers for PCI devices. The
2685 * PCI core identifies what devices are on a system during boot, and then
2686 * inquiry this table to see if this driver is for a given device found.
2687 */
2688 static const struct pci_device_id amd64_pci_table[] __devinitdata = {
2689 {
2690 .vendor = PCI_VENDOR_ID_AMD,
2691 .device = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
2692 .subvendor = PCI_ANY_ID,
2693 .subdevice = PCI_ANY_ID,
2694 .class = 0,
2695 .class_mask = 0,
2696 },
2697 {
2698 .vendor = PCI_VENDOR_ID_AMD,
2699 .device = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
2700 .subvendor = PCI_ANY_ID,
2701 .subdevice = PCI_ANY_ID,
2702 .class = 0,
2703 .class_mask = 0,
2704 },
2705 {
2706 .vendor = PCI_VENDOR_ID_AMD,
2707 .device = PCI_DEVICE_ID_AMD_15H_NB_F2,
2708 .subvendor = PCI_ANY_ID,
2709 .subdevice = PCI_ANY_ID,
2710 .class = 0,
2711 .class_mask = 0,
2712 },
2713
2714 {0, }
2715 };
2716 MODULE_DEVICE_TABLE(pci, amd64_pci_table);
2717
2718 static struct pci_driver amd64_pci_driver = {
2719 .name = EDAC_MOD_STR,
2720 .probe = amd64_probe_one_instance,
2721 .remove = __devexit_p(amd64_remove_one_instance),
2722 .id_table = amd64_pci_table,
2723 };
2724
2725 static void setup_pci_device(void)
2726 {
2727 struct mem_ctl_info *mci;
2728 struct amd64_pvt *pvt;
2729
2730 if (amd64_ctl_pci)
2731 return;
2732
2733 mci = mcis[0];
2734 if (mci) {
2735
2736 pvt = mci->pvt_info;
2737 amd64_ctl_pci =
2738 edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR);
2739
2740 if (!amd64_ctl_pci) {
2741 pr_warning("%s(): Unable to create PCI control\n",
2742 __func__);
2743
2744 pr_warning("%s(): PCI error report via EDAC not set\n",
2745 __func__);
2746 }
2747 }
2748 }
2749
2750 static int __init amd64_edac_init(void)
2751 {
2752 int err = -ENODEV;
2753
2754 printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION);
2755
2756 opstate_init();
2757
2758 if (amd_cache_northbridges() < 0)
2759 goto err_ret;
2760
2761 err = -ENOMEM;
2762 mcis = kzalloc(amd_nb_num() * sizeof(mcis[0]), GFP_KERNEL);
2763 ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL);
2764 if (!(mcis && ecc_stngs))
2765 goto err_free;
2766
2767 msrs = msrs_alloc();
2768 if (!msrs)
2769 goto err_free;
2770
2771 err = pci_register_driver(&amd64_pci_driver);
2772 if (err)
2773 goto err_pci;
2774
2775 err = -ENODEV;
2776 if (!atomic_read(&drv_instances))
2777 goto err_no_instances;
2778
2779 setup_pci_device();
2780 return 0;
2781
2782 err_no_instances:
2783 pci_unregister_driver(&amd64_pci_driver);
2784
2785 err_pci:
2786 msrs_free(msrs);
2787 msrs = NULL;
2788
2789 err_free:
2790 kfree(mcis);
2791 mcis = NULL;
2792
2793 kfree(ecc_stngs);
2794 ecc_stngs = NULL;
2795
2796 err_ret:
2797 return err;
2798 }
2799
2800 static void __exit amd64_edac_exit(void)
2801 {
2802 if (amd64_ctl_pci)
2803 edac_pci_release_generic_ctl(amd64_ctl_pci);
2804
2805 pci_unregister_driver(&amd64_pci_driver);
2806
2807 kfree(ecc_stngs);
2808 ecc_stngs = NULL;
2809
2810 kfree(mcis);
2811 mcis = NULL;
2812
2813 msrs_free(msrs);
2814 msrs = NULL;
2815 }
2816
2817 module_init(amd64_edac_init);
2818 module_exit(amd64_edac_exit);
2819
2820 MODULE_LICENSE("GPL");
2821 MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
2822 "Dave Peterson, Thayne Harbaugh");
2823 MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
2824 EDAC_AMD64_VERSION);
2825
2826 module_param(edac_op_state, int, 0444);
2827 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");