1 // SPDX-License-Identifier: GPL-2.0-only
2 /* chmc.c: Driver for UltraSPARC-III memory controller.
4 * Copyright (C) 2001, 2007, 2008 David S. Miller (davem@davemloft.net)
7 #include <linux/module.h>
8 #include <linux/kernel.h>
9 #include <linux/types.h>
10 #include <linux/slab.h>
11 #include <linux/list.h>
12 #include <linux/string.h>
13 #include <linux/sched.h>
14 #include <linux/smp.h>
15 #include <linux/errno.h>
16 #include <linux/init.h>
18 #include <linux/of_device.h>
19 #include <asm/spitfire.h>
20 #include <asm/chmctrl.h>
21 #include <asm/cpudata.h>
22 #include <asm/oplib.h>
26 #include <asm/memctrl.h>
28 #define DRV_MODULE_NAME "chmc"
29 #define PFX DRV_MODULE_NAME ": "
30 #define DRV_MODULE_VERSION "0.2"
32 MODULE_AUTHOR("David S. Miller (davem@davemloft.net)");
33 MODULE_DESCRIPTION("UltraSPARC-III memory controller driver");
34 MODULE_LICENSE("GPL");
35 MODULE_VERSION(DRV_MODULE_VERSION
);
38 #define MC_TYPE_SAFARI 1
39 #define MC_TYPE_JBUS 2
41 static dimm_printer_t us3mc_dimm_printer
;
43 #define CHMCTRL_NDGRPS 2
44 #define CHMCTRL_NDIMMS 4
46 #define CHMC_DIMMS_PER_MC (CHMCTRL_NDGRPS * CHMCTRL_NDIMMS)
48 /* OBP memory-layout property format. */
50 unsigned char dimm_map
[144];
51 unsigned char pin_map
[576];
54 #define DIMM_LABEL_SZ 8
56 struct chmc_obp_mem_layout
{
57 /* One max 8-byte string label per DIMM. Usually
58 * this matches the label on the motherboard where
61 char dimm_labels
[CHMC_DIMMS_PER_MC
][DIMM_LABEL_SZ
];
63 /* If symmetric use map[0], else it is
64 * asymmetric and map[1] should be used.
68 struct chmc_obp_map map
[2];
71 #define CHMCTRL_NBANKS 4
73 struct chmc_bank_info
{
89 struct list_head list
;
92 struct chmc_obp_mem_layout layout_prop
;
103 struct chmc_bank_info logical_banks
[CHMCTRL_NBANKS
];
106 #define JBUSMC_REGS_SIZE 8
108 #define JB_MC_REG1_DIMM2_BANK3 0x8000000000000000UL
109 #define JB_MC_REG1_DIMM1_BANK1 0x4000000000000000UL
110 #define JB_MC_REG1_DIMM2_BANK2 0x2000000000000000UL
111 #define JB_MC_REG1_DIMM1_BANK0 0x1000000000000000UL
112 #define JB_MC_REG1_XOR 0x0000010000000000UL
113 #define JB_MC_REG1_ADDR_GEN_2 0x000000e000000000UL
114 #define JB_MC_REG1_ADDR_GEN_2_SHIFT 37
115 #define JB_MC_REG1_ADDR_GEN_1 0x0000001c00000000UL
116 #define JB_MC_REG1_ADDR_GEN_1_SHIFT 34
117 #define JB_MC_REG1_INTERLEAVE 0x0000000001800000UL
118 #define JB_MC_REG1_INTERLEAVE_SHIFT 23
119 #define JB_MC_REG1_DIMM2_PTYPE 0x0000000000200000UL
120 #define JB_MC_REG1_DIMM2_PTYPE_SHIFT 21
121 #define JB_MC_REG1_DIMM1_PTYPE 0x0000000000100000UL
122 #define JB_MC_REG1_DIMM1_PTYPE_SHIFT 20
124 #define PART_TYPE_X8 0
125 #define PART_TYPE_X4 1
127 #define INTERLEAVE_NONE 0
128 #define INTERLEAVE_SAME 1
129 #define INTERLEAVE_INTERNAL 2
130 #define INTERLEAVE_BOTH 3
132 #define ADDR_GEN_128MB 0
133 #define ADDR_GEN_256MB 1
134 #define ADDR_GEN_512MB 2
135 #define ADDR_GEN_1GB 3
137 #define JB_NUM_DIMM_GROUPS 2
138 #define JB_NUM_DIMMS_PER_GROUP 2
139 #define JB_NUM_DIMMS (JB_NUM_DIMM_GROUPS * JB_NUM_DIMMS_PER_GROUP)
141 struct jbusmc_obp_map
{
142 unsigned char dimm_map
[18];
143 unsigned char pin_map
[144];
146 struct jbusmc_obp_mem_layout
{
147 /* One max 8-byte string label per DIMM. Usually
148 * this matches the label on the motherboard where
151 char dimm_labels
[JB_NUM_DIMMS
][DIMM_LABEL_SZ
];
153 /* If symmetric use map[0], else it is
154 * asymmetric and map[1] should be used.
158 struct jbusmc_obp_map map
;
163 struct jbusmc_dimm_group
{
164 struct jbusmc
*controller
;
174 struct jbusmc_obp_mem_layout layout
;
177 struct jbusmc_dimm_group dimm_groups
[JB_NUM_DIMM_GROUPS
];
178 struct list_head list
;
181 static DEFINE_SPINLOCK(mctrl_list_lock
);
182 static LIST_HEAD(mctrl_list
);
184 static void mc_list_add(struct list_head
*list
)
186 spin_lock(&mctrl_list_lock
);
187 list_add(list
, &mctrl_list
);
188 spin_unlock(&mctrl_list_lock
);
191 static void mc_list_del(struct list_head
*list
)
193 spin_lock(&mctrl_list_lock
);
195 spin_unlock(&mctrl_list_lock
);
198 #define SYNDROME_MIN -1
199 #define SYNDROME_MAX 144
201 /* Covert syndrome code into the way the bits are positioned
204 static int syndrome_to_qword_code(int syndrome_code
)
206 if (syndrome_code
< 128)
208 else if (syndrome_code
< 128 + 9)
209 syndrome_code
-= (128 - 7);
210 else if (syndrome_code
< (128 + 9 + 3))
211 syndrome_code
-= (128 + 9 - 4);
213 syndrome_code
-= (128 + 9 + 3);
214 return syndrome_code
;
217 /* All this magic has to do with how a cache line comes over the wire
218 * on Safari and JBUS. A 64-bit line comes over in 1 or more quadword
219 * cycles, each of which transmit ECC/MTAG info as well as the actual
222 #define L2_LINE_SIZE 64
223 #define L2_LINE_ADDR_MSK (L2_LINE_SIZE - 1)
224 #define QW_PER_LINE 4
225 #define QW_BYTES (L2_LINE_SIZE / QW_PER_LINE)
227 #define SAFARI_LAST_BIT (576 - 1)
228 #define JBUS_LAST_BIT (144 - 1)
230 static void get_pin_and_dimm_str(int syndrome_code
, unsigned long paddr
,
231 int *pin_p
, char **dimm_str_p
, void *_prop
,
232 int base_dimm_offset
)
234 int qword_code
= syndrome_to_qword_code(syndrome_code
);
235 int cache_line_offset
;
240 if (mc_type
== MC_TYPE_JBUS
) {
241 struct jbusmc_obp_mem_layout
*p
= _prop
;
244 cache_line_offset
= qword_code
;
245 offset_inverse
= (JBUS_LAST_BIT
- cache_line_offset
);
246 dimm_map_index
= offset_inverse
/ 8;
247 map_val
= p
->map
.dimm_map
[dimm_map_index
];
248 map_val
= ((map_val
>> ((7 - (offset_inverse
& 7)))) & 1);
249 *dimm_str_p
= p
->dimm_labels
[base_dimm_offset
+ map_val
];
250 *pin_p
= p
->map
.pin_map
[cache_line_offset
];
252 struct chmc_obp_mem_layout
*p
= _prop
;
253 struct chmc_obp_map
*mp
;
262 qword
= (paddr
& L2_LINE_ADDR_MSK
) / QW_BYTES
;
263 cache_line_offset
= ((3 - qword
) * QW_BITS
) + qword_code
;
264 offset_inverse
= (SAFARI_LAST_BIT
- cache_line_offset
);
265 dimm_map_index
= offset_inverse
>> 2;
266 map_val
= mp
->dimm_map
[dimm_map_index
];
267 map_val
= ((map_val
>> ((3 - (offset_inverse
& 3)) << 1)) & 0x3);
268 *dimm_str_p
= p
->dimm_labels
[base_dimm_offset
+ map_val
];
269 *pin_p
= mp
->pin_map
[cache_line_offset
];
273 static struct jbusmc_dimm_group
*jbusmc_find_dimm_group(unsigned long phys_addr
)
277 list_for_each_entry(p
, &mctrl_list
, list
) {
280 for (i
= 0; i
< p
->num_dimm_groups
; i
++) {
281 struct jbusmc_dimm_group
*dp
= &p
->dimm_groups
[i
];
283 if (phys_addr
< dp
->base_addr
||
284 (dp
->base_addr
+ dp
->size
) <= phys_addr
)
293 static int jbusmc_print_dimm(int syndrome_code
,
294 unsigned long phys_addr
,
295 char *buf
, int buflen
)
297 struct jbusmc_obp_mem_layout
*prop
;
298 struct jbusmc_dimm_group
*dp
;
302 dp
= jbusmc_find_dimm_group(phys_addr
);
304 syndrome_code
< SYNDROME_MIN
||
305 syndrome_code
> SYNDROME_MAX
) {
315 first_dimm
= dp
->index
* JB_NUM_DIMMS_PER_GROUP
;
317 if (syndrome_code
!= SYNDROME_MIN
) {
321 get_pin_and_dimm_str(syndrome_code
, phys_addr
, &pin
,
322 &dimm_str
, prop
, first_dimm
);
323 sprintf(buf
, "%s, pin %3d", dimm_str
, pin
);
327 /* Multi-bit error, we just dump out all the
328 * dimm labels associated with this dimm group.
330 for (dimm
= 0; dimm
< JB_NUM_DIMMS_PER_GROUP
; dimm
++) {
332 prop
->dimm_labels
[first_dimm
+ dimm
]);
340 static u64
jbusmc_dimm_group_size(u64 base
,
341 const struct linux_prom64_registers
*mem_regs
,
344 u64 max
= base
+ (8UL * 1024 * 1024 * 1024);
348 for (i
= 0; i
< num_mem_regs
; i
++) {
349 const struct linux_prom64_registers
*ent
;
354 this_base
= ent
->phys_addr
;
355 this_end
= this_base
+ ent
->reg_size
;
356 if (base
< this_base
|| base
>= this_end
)
360 if (this_end
> max_seen
)
364 return max_seen
- base
;
367 static void jbusmc_construct_one_dimm_group(struct jbusmc
*p
,
369 const struct linux_prom64_registers
*mem_regs
,
372 struct jbusmc_dimm_group
*dp
= &p
->dimm_groups
[index
];
377 dp
->base_addr
= (p
->portid
* (64UL * 1024 * 1024 * 1024));
378 dp
->base_addr
+= (index
* (8UL * 1024 * 1024 * 1024));
379 dp
->size
= jbusmc_dimm_group_size(dp
->base_addr
, mem_regs
, num_mem_regs
);
382 static void jbusmc_construct_dimm_groups(struct jbusmc
*p
,
383 const struct linux_prom64_registers
*mem_regs
,
386 if (p
->mc_reg_1
& JB_MC_REG1_DIMM1_BANK0
) {
387 jbusmc_construct_one_dimm_group(p
, 0, mem_regs
, num_mem_regs
);
388 p
->num_dimm_groups
++;
390 if (p
->mc_reg_1
& JB_MC_REG1_DIMM2_BANK2
) {
391 jbusmc_construct_one_dimm_group(p
, 1, mem_regs
, num_mem_regs
);
392 p
->num_dimm_groups
++;
396 static int jbusmc_probe(struct platform_device
*op
)
398 const struct linux_prom64_registers
*mem_regs
;
399 struct device_node
*mem_node
;
400 int err
, len
, num_mem_regs
;
406 mem_node
= of_find_node_by_path("/memory");
408 printk(KERN_ERR PFX
"Cannot find /memory node.\n");
411 mem_regs
= of_get_property(mem_node
, "reg", &len
);
413 printk(KERN_ERR PFX
"Cannot get reg property of /memory node.\n");
416 num_mem_regs
= len
/ sizeof(*mem_regs
);
419 p
= kzalloc(sizeof(*p
), GFP_KERNEL
);
421 printk(KERN_ERR PFX
"Cannot allocate struct jbusmc.\n");
425 INIT_LIST_HEAD(&p
->list
);
428 prop
= of_get_property(op
->dev
.of_node
, "portid", &len
);
429 if (!prop
|| len
!= 4) {
430 printk(KERN_ERR PFX
"Cannot find portid.\n");
436 prop
= of_get_property(op
->dev
.of_node
, "memory-control-register-1", &len
);
437 if (!prop
|| len
!= 8) {
438 printk(KERN_ERR PFX
"Cannot get memory control register 1.\n");
442 p
->mc_reg_1
= ((u64
)prop
[0] << 32) | (u64
) prop
[1];
445 p
->regs
= of_ioremap(&op
->resource
[0], 0, JBUSMC_REGS_SIZE
, "jbusmc");
447 printk(KERN_ERR PFX
"Cannot map jbusmc regs.\n");
452 ml
= of_get_property(op
->dev
.of_node
, "memory-layout", &p
->layout_len
);
454 printk(KERN_ERR PFX
"Cannot get memory layout property.\n");
457 if (p
->layout_len
> sizeof(p
->layout
)) {
458 printk(KERN_ERR PFX
"Unexpected memory-layout size %d\n",
462 memcpy(&p
->layout
, ml
, p
->layout_len
);
464 jbusmc_construct_dimm_groups(p
, mem_regs
, num_mem_regs
);
466 mc_list_add(&p
->list
);
468 printk(KERN_INFO PFX
"UltraSPARC-IIIi memory controller at %pOF\n",
471 dev_set_drvdata(&op
->dev
, p
);
479 of_iounmap(&op
->resource
[0], p
->regs
, JBUSMC_REGS_SIZE
);
486 /* Does BANK decode PHYS_ADDR? */
487 static int chmc_bank_match(struct chmc_bank_info
*bp
, unsigned long phys_addr
)
489 unsigned long upper_bits
= (phys_addr
& PA_UPPER_BITS
) >> PA_UPPER_BITS_SHIFT
;
490 unsigned long lower_bits
= (phys_addr
& PA_LOWER_BITS
) >> PA_LOWER_BITS_SHIFT
;
492 /* Bank must be enabled to match. */
496 /* Would BANK match upper bits? */
497 upper_bits
^= bp
->um
; /* What bits are different? */
498 upper_bits
= ~upper_bits
; /* Invert. */
499 upper_bits
|= bp
->uk
; /* What bits don't matter for matching? */
500 upper_bits
= ~upper_bits
; /* Invert. */
505 /* Would BANK match lower bits? */
506 lower_bits
^= bp
->lm
; /* What bits are different? */
507 lower_bits
= ~lower_bits
; /* Invert. */
508 lower_bits
|= bp
->lk
; /* What bits don't matter for matching? */
509 lower_bits
= ~lower_bits
; /* Invert. */
514 /* I always knew you'd be the one. */
518 /* Given PHYS_ADDR, search memory controller banks for a match. */
519 static struct chmc_bank_info
*chmc_find_bank(unsigned long phys_addr
)
523 list_for_each_entry(p
, &mctrl_list
, list
) {
526 for (bank_no
= 0; bank_no
< CHMCTRL_NBANKS
; bank_no
++) {
527 struct chmc_bank_info
*bp
;
529 bp
= &p
->logical_banks
[bank_no
];
530 if (chmc_bank_match(bp
, phys_addr
))
538 /* This is the main purpose of this driver. */
539 static int chmc_print_dimm(int syndrome_code
,
540 unsigned long phys_addr
,
541 char *buf
, int buflen
)
543 struct chmc_bank_info
*bp
;
544 struct chmc_obp_mem_layout
*prop
;
545 int bank_in_controller
, first_dimm
;
547 bp
= chmc_find_bank(phys_addr
);
549 syndrome_code
< SYNDROME_MIN
||
550 syndrome_code
> SYNDROME_MAX
) {
558 prop
= &bp
->p
->layout_prop
;
559 bank_in_controller
= bp
->bank_id
& (CHMCTRL_NBANKS
- 1);
560 first_dimm
= (bank_in_controller
& (CHMCTRL_NDGRPS
- 1));
561 first_dimm
*= CHMCTRL_NDIMMS
;
563 if (syndrome_code
!= SYNDROME_MIN
) {
567 get_pin_and_dimm_str(syndrome_code
, phys_addr
, &pin
,
568 &dimm_str
, prop
, first_dimm
);
569 sprintf(buf
, "%s, pin %3d", dimm_str
, pin
);
573 /* Multi-bit error, we just dump out all the
574 * dimm labels associated with this bank.
576 for (dimm
= 0; dimm
< CHMCTRL_NDIMMS
; dimm
++) {
578 prop
->dimm_labels
[first_dimm
+ dimm
]);
585 /* Accessing the registers is slightly complicated. If you want
586 * to get at the memory controller which is on the same processor
587 * the code is executing, you must use special ASI load/store else
588 * you go through the global mapping.
590 static u64
chmc_read_mcreg(struct chmc
*p
, unsigned long offset
)
592 unsigned long ret
, this_cpu
;
596 this_cpu
= real_hard_smp_processor_id();
598 if (p
->portid
== this_cpu
) {
599 __asm__
__volatile__("ldxa [%1] %2, %0"
601 : "r" (offset
), "i" (ASI_MCU_CTRL_REG
));
603 __asm__
__volatile__("ldxa [%1] %2, %0"
605 : "r" (p
->regs
+ offset
),
606 "i" (ASI_PHYS_BYPASS_EC_E
));
614 #if 0 /* currently unused */
615 static void chmc_write_mcreg(struct chmc
*p
, unsigned long offset
, u64 val
)
617 if (p
->portid
== smp_processor_id()) {
618 __asm__
__volatile__("stxa %0, [%1] %2"
620 "r" (offset
), "i" (ASI_MCU_CTRL_REG
));
622 __asm__
__volatile__("ldxa %0, [%1] %2"
624 "r" (p
->regs
+ offset
),
625 "i" (ASI_PHYS_BYPASS_EC_E
));
630 static void chmc_interpret_one_decode_reg(struct chmc
*p
, int which_bank
, u64 val
)
632 struct chmc_bank_info
*bp
= &p
->logical_banks
[which_bank
];
635 bp
->bank_id
= (CHMCTRL_NBANKS
* p
->portid
) + which_bank
;
637 bp
->valid
= (val
& MEM_DECODE_VALID
) >> MEM_DECODE_VALID_SHIFT
;
638 bp
->uk
= (val
& MEM_DECODE_UK
) >> MEM_DECODE_UK_SHIFT
;
639 bp
->um
= (val
& MEM_DECODE_UM
) >> MEM_DECODE_UM_SHIFT
;
640 bp
->lk
= (val
& MEM_DECODE_LK
) >> MEM_DECODE_LK_SHIFT
;
641 bp
->lm
= (val
& MEM_DECODE_LM
) >> MEM_DECODE_LM_SHIFT
;
644 bp
->base
&= ~(bp
->uk
);
645 bp
->base
<<= PA_UPPER_BITS_SHIFT
;
670 /* UK[10] is reserved, and UK[11] is not set for the SDRAM
671 * bank size definition.
673 bp
->size
= (((unsigned long)bp
->uk
&
674 ((1UL << 10UL) - 1UL)) + 1UL) << PA_UPPER_BITS_SHIFT
;
675 bp
->size
/= bp
->interleave
;
678 static void chmc_fetch_decode_regs(struct chmc
*p
)
680 if (p
->layout_size
== 0)
683 chmc_interpret_one_decode_reg(p
, 0,
684 chmc_read_mcreg(p
, CHMCTRL_DECODE1
));
685 chmc_interpret_one_decode_reg(p
, 1,
686 chmc_read_mcreg(p
, CHMCTRL_DECODE2
));
687 chmc_interpret_one_decode_reg(p
, 2,
688 chmc_read_mcreg(p
, CHMCTRL_DECODE3
));
689 chmc_interpret_one_decode_reg(p
, 3,
690 chmc_read_mcreg(p
, CHMCTRL_DECODE4
));
693 static int chmc_probe(struct platform_device
*op
)
695 struct device_node
*dp
= op
->dev
.of_node
;
703 __asm__ ("rdpr %%ver, %0" : "=r" (ver
));
704 if ((ver
>> 32UL) == __JALAPENO_ID
||
705 (ver
>> 32UL) == __SERRANO_ID
)
708 portid
= of_getintprop_default(dp
, "portid", -1);
712 pval
= of_get_property(dp
, "memory-layout", &len
);
713 if (pval
&& len
> sizeof(p
->layout_prop
)) {
714 printk(KERN_ERR PFX
"Unexpected memory-layout property "
720 p
= kzalloc(sizeof(*p
), GFP_KERNEL
);
722 printk(KERN_ERR PFX
"Could not allocate struct chmc.\n");
727 p
->layout_size
= len
;
731 memcpy(&p
->layout_prop
, pval
, len
);
733 p
->regs
= of_ioremap(&op
->resource
[0], 0, 0x48, "chmc");
735 printk(KERN_ERR PFX
"Could not map registers.\n");
739 if (p
->layout_size
!= 0UL) {
740 p
->timing_control1
= chmc_read_mcreg(p
, CHMCTRL_TCTRL1
);
741 p
->timing_control2
= chmc_read_mcreg(p
, CHMCTRL_TCTRL2
);
742 p
->timing_control3
= chmc_read_mcreg(p
, CHMCTRL_TCTRL3
);
743 p
->timing_control4
= chmc_read_mcreg(p
, CHMCTRL_TCTRL4
);
744 p
->memaddr_control
= chmc_read_mcreg(p
, CHMCTRL_MACTRL
);
747 chmc_fetch_decode_regs(p
);
749 mc_list_add(&p
->list
);
751 printk(KERN_INFO PFX
"UltraSPARC-III memory controller at %pOF [%s]\n",
753 (p
->layout_size
? "ACTIVE" : "INACTIVE"));
755 dev_set_drvdata(&op
->dev
, p
);
767 static int us3mc_probe(struct platform_device
*op
)
769 if (mc_type
== MC_TYPE_SAFARI
)
770 return chmc_probe(op
);
771 else if (mc_type
== MC_TYPE_JBUS
)
772 return jbusmc_probe(op
);
776 static void chmc_destroy(struct platform_device
*op
, struct chmc
*p
)
779 of_iounmap(&op
->resource
[0], p
->regs
, 0x48);
783 static void jbusmc_destroy(struct platform_device
*op
, struct jbusmc
*p
)
785 mc_list_del(&p
->list
);
786 of_iounmap(&op
->resource
[0], p
->regs
, JBUSMC_REGS_SIZE
);
790 static int us3mc_remove(struct platform_device
*op
)
792 void *p
= dev_get_drvdata(&op
->dev
);
795 if (mc_type
== MC_TYPE_SAFARI
)
797 else if (mc_type
== MC_TYPE_JBUS
)
798 jbusmc_destroy(op
, p
);
803 static const struct of_device_id us3mc_match
[] = {
805 .name
= "memory-controller",
809 MODULE_DEVICE_TABLE(of
, us3mc_match
);
811 static struct platform_driver us3mc_driver
= {
814 .of_match_table
= us3mc_match
,
816 .probe
= us3mc_probe
,
817 .remove
= us3mc_remove
,
820 static inline bool us3mc_platform(void)
822 if (tlb_type
== cheetah
|| tlb_type
== cheetah_plus
)
827 static int __init
us3mc_init(void)
832 if (!us3mc_platform())
835 __asm__
__volatile__("rdpr %%ver, %0" : "=r" (ver
));
836 if ((ver
>> 32UL) == __JALAPENO_ID
||
837 (ver
>> 32UL) == __SERRANO_ID
) {
838 mc_type
= MC_TYPE_JBUS
;
839 us3mc_dimm_printer
= jbusmc_print_dimm
;
841 mc_type
= MC_TYPE_SAFARI
;
842 us3mc_dimm_printer
= chmc_print_dimm
;
845 ret
= register_dimm_printer(us3mc_dimm_printer
);
848 ret
= platform_driver_register(&us3mc_driver
);
850 unregister_dimm_printer(us3mc_dimm_printer
);
855 static void __exit
us3mc_cleanup(void)
857 if (us3mc_platform()) {
858 unregister_dimm_printer(us3mc_dimm_printer
);
859 platform_driver_unregister(&us3mc_driver
);
863 module_init(us3mc_init
);
864 module_exit(us3mc_cleanup
);