3 * Copyright 2017 Free Electrons
5 * Author: Boris Brezillon <boris.brezillon@free-electrons.com>
7 * Derived from the atmel_nand.c driver which contained the following
10 * Copyright 2003 Rick Bronson
12 * Derived from drivers/mtd/nand/autcpu12.c (removed in v3.8)
13 * Copyright 2001 Thomas Gleixner (gleixner@autronix.de)
15 * Derived from drivers/mtd/spia.c (removed in v3.8)
16 * Copyright 2000 Steven J. Hill (sjhill@cotw.com)
18 * Add Hardware ECC support for AT91SAM9260 / AT91SAM9263
19 * Richard Genoud (richard.genoud@gmail.com), Adeneo Copyright 2007
21 * Derived from Das U-Boot source code
22 * (u-boot-1.1.5/board/atmel/at91sam9263ek/nand.c)
23 * Copyright 2006 ATMEL Rousset, Lacressonniere Nicolas
25 * Add Programmable Multibit ECC support for various AT91 SoC
26 * Copyright 2012 ATMEL, Hong Xu
28 * Add Nand Flash Controller support for SAMA5 SoC
29 * Copyright 2013 ATMEL, Josh Wu (josh.wu@atmel.com)
31 * This program is free software; you can redistribute it and/or modify
32 * it under the terms of the GNU General Public License version 2 as
33 * published by the Free Software Foundation.
35 * The PMECC is an hardware assisted BCH engine, which means part of the
36 * ECC algorithm is left to the software. The hardware/software repartition
37 * is explained in the "PMECC Controller Functional Description" chapter in
38 * Atmel datasheets, and some of the functions in this file are directly
39 * implementing the algorithms described in the "Software Implementation"
42 * TODO: it seems that the software BCH implementation in lib/bch.c is already
43 * providing some of the logic we are implementing here. It would be smart
44 * to expose the needed lib/bch.c helpers/functions and re-use them here.
47 #include <linux/genalloc.h>
48 #include <linux/iopoll.h>
49 #include <linux/module.h>
50 #include <linux/mtd/rawnand.h>
51 #include <linux/of_irq.h>
52 #include <linux/of_platform.h>
53 #include <linux/platform_device.h>
54 #include <linux/slab.h>
58 /* Galois field dimension */
59 #define PMECC_GF_DIMENSION_13 13
60 #define PMECC_GF_DIMENSION_14 14
62 /* Primitive Polynomial used by PMECC */
63 #define PMECC_GF_13_PRIMITIVE_POLY 0x201b
64 #define PMECC_GF_14_PRIMITIVE_POLY 0x4443
66 #define PMECC_LOOKUP_TABLE_SIZE_512 0x2000
67 #define PMECC_LOOKUP_TABLE_SIZE_1024 0x4000
69 /* Time out value for reading PMECC status register */
70 #define PMECC_MAX_TIMEOUT_MS 100
72 /* PMECC Register Definitions */
73 #define ATMEL_PMECC_CFG 0x0
74 #define PMECC_CFG_BCH_STRENGTH(x) (x)
75 #define PMECC_CFG_BCH_STRENGTH_MASK GENMASK(2, 0)
76 #define PMECC_CFG_SECTOR512 (0 << 4)
77 #define PMECC_CFG_SECTOR1024 (1 << 4)
78 #define PMECC_CFG_NSECTORS(x) ((fls(x) - 1) << 8)
79 #define PMECC_CFG_READ_OP (0 << 12)
80 #define PMECC_CFG_WRITE_OP (1 << 12)
81 #define PMECC_CFG_SPARE_ENABLE BIT(16)
82 #define PMECC_CFG_AUTO_ENABLE BIT(20)
84 #define ATMEL_PMECC_SAREA 0x4
85 #define ATMEL_PMECC_SADDR 0x8
86 #define ATMEL_PMECC_EADDR 0xc
88 #define ATMEL_PMECC_CLK 0x10
89 #define PMECC_CLK_133MHZ (2 << 0)
91 #define ATMEL_PMECC_CTRL 0x14
92 #define PMECC_CTRL_RST BIT(0)
93 #define PMECC_CTRL_DATA BIT(1)
94 #define PMECC_CTRL_USER BIT(2)
95 #define PMECC_CTRL_ENABLE BIT(4)
96 #define PMECC_CTRL_DISABLE BIT(5)
98 #define ATMEL_PMECC_SR 0x18
99 #define PMECC_SR_BUSY BIT(0)
100 #define PMECC_SR_ENABLE BIT(4)
102 #define ATMEL_PMECC_IER 0x1c
103 #define ATMEL_PMECC_IDR 0x20
104 #define ATMEL_PMECC_IMR 0x24
105 #define ATMEL_PMECC_ISR 0x28
106 #define PMECC_ERROR_INT BIT(0)
108 #define ATMEL_PMECC_ECC(sector, n) \
109 ((((sector) + 1) * 0x40) + (n))
111 #define ATMEL_PMECC_REM(sector, n) \
112 ((((sector) + 1) * 0x40) + ((n) * 4) + 0x200)
114 /* PMERRLOC Register Definitions */
115 #define ATMEL_PMERRLOC_ELCFG 0x0
116 #define PMERRLOC_ELCFG_SECTOR_512 (0 << 0)
117 #define PMERRLOC_ELCFG_SECTOR_1024 (1 << 0)
118 #define PMERRLOC_ELCFG_NUM_ERRORS(n) ((n) << 16)
120 #define ATMEL_PMERRLOC_ELPRIM 0x4
121 #define ATMEL_PMERRLOC_ELEN 0x8
122 #define ATMEL_PMERRLOC_ELDIS 0xc
123 #define PMERRLOC_DISABLE BIT(0)
125 #define ATMEL_PMERRLOC_ELSR 0x10
126 #define PMERRLOC_ELSR_BUSY BIT(0)
128 #define ATMEL_PMERRLOC_ELIER 0x14
129 #define ATMEL_PMERRLOC_ELIDR 0x18
130 #define ATMEL_PMERRLOC_ELIMR 0x1c
131 #define ATMEL_PMERRLOC_ELISR 0x20
132 #define PMERRLOC_ERR_NUM_MASK GENMASK(12, 8)
133 #define PMERRLOC_CALC_DONE BIT(0)
135 #define ATMEL_PMERRLOC_SIGMA(x) (((x) * 0x4) + 0x28)
137 #define ATMEL_PMERRLOC_EL(offs, x) (((x) * 0x4) + (offs))
139 struct atmel_pmecc_gf_tables
{
144 struct atmel_pmecc_caps
{
145 const int *strengths
;
148 bool correct_erased_chunks
;
153 const struct atmel_pmecc_caps
*caps
;
157 void __iomem
*errloc
;
163 struct atmel_pmecc_user_conf_cache
{
170 struct atmel_pmecc_user
{
171 struct atmel_pmecc_user_conf_cache cache
;
172 struct atmel_pmecc
*pmecc
;
173 const struct atmel_pmecc_gf_tables
*gf_tables
;
185 static DEFINE_MUTEX(pmecc_gf_tables_lock
);
186 static const struct atmel_pmecc_gf_tables
*pmecc_gf_tables_512
;
187 static const struct atmel_pmecc_gf_tables
*pmecc_gf_tables_1024
;
189 static inline int deg(unsigned int poly
)
191 /* polynomial degree is the most-significant bit index */
192 return fls(poly
) - 1;
195 static int atmel_pmecc_build_gf_tables(int mm
, unsigned int poly
,
196 struct atmel_pmecc_gf_tables
*gf_tables
)
198 unsigned int i
, x
= 1;
199 const unsigned int k
= BIT(deg(poly
));
200 unsigned int nn
= BIT(mm
) - 1;
202 /* primitive polynomial must be of degree m */
206 for (i
= 0; i
< nn
; i
++) {
207 gf_tables
->alpha_to
[i
] = x
;
208 gf_tables
->index_of
[x
] = i
;
210 /* polynomial is not primitive (a^i=1 with 0<i<2^m-1) */
216 gf_tables
->alpha_to
[nn
] = 1;
217 gf_tables
->index_of
[0] = 0;
222 static const struct atmel_pmecc_gf_tables
*
223 atmel_pmecc_create_gf_tables(const struct atmel_pmecc_user_req
*req
)
225 struct atmel_pmecc_gf_tables
*gf_tables
;
226 unsigned int poly
, degree
, table_size
;
229 if (req
->ecc
.sectorsize
== 512) {
230 degree
= PMECC_GF_DIMENSION_13
;
231 poly
= PMECC_GF_13_PRIMITIVE_POLY
;
232 table_size
= PMECC_LOOKUP_TABLE_SIZE_512
;
234 degree
= PMECC_GF_DIMENSION_14
;
235 poly
= PMECC_GF_14_PRIMITIVE_POLY
;
236 table_size
= PMECC_LOOKUP_TABLE_SIZE_1024
;
239 gf_tables
= kzalloc(sizeof(*gf_tables
) +
240 (2 * table_size
* sizeof(u16
)),
243 return ERR_PTR(-ENOMEM
);
245 gf_tables
->alpha_to
= (void *)(gf_tables
+ 1);
246 gf_tables
->index_of
= gf_tables
->alpha_to
+ table_size
;
248 ret
= atmel_pmecc_build_gf_tables(degree
, poly
, gf_tables
);
257 static const struct atmel_pmecc_gf_tables
*
258 atmel_pmecc_get_gf_tables(const struct atmel_pmecc_user_req
*req
)
260 const struct atmel_pmecc_gf_tables
**gf_tables
, *ret
;
262 mutex_lock(&pmecc_gf_tables_lock
);
263 if (req
->ecc
.sectorsize
== 512)
264 gf_tables
= &pmecc_gf_tables_512
;
266 gf_tables
= &pmecc_gf_tables_1024
;
271 ret
= atmel_pmecc_create_gf_tables(req
);
275 mutex_unlock(&pmecc_gf_tables_lock
);
280 static int atmel_pmecc_prepare_user_req(struct atmel_pmecc
*pmecc
,
281 struct atmel_pmecc_user_req
*req
)
283 int i
, max_eccbytes
, eccbytes
= 0, eccstrength
= 0;
285 if (req
->pagesize
<= 0 || req
->oobsize
<= 0 || req
->ecc
.bytes
<= 0)
288 if (req
->ecc
.ooboffset
>= 0 &&
289 req
->ecc
.ooboffset
+ req
->ecc
.bytes
> req
->oobsize
)
292 if (req
->ecc
.sectorsize
== ATMEL_PMECC_SECTOR_SIZE_AUTO
) {
293 if (req
->ecc
.strength
!= ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH
)
296 if (req
->pagesize
> 512)
297 req
->ecc
.sectorsize
= 1024;
299 req
->ecc
.sectorsize
= 512;
302 if (req
->ecc
.sectorsize
!= 512 && req
->ecc
.sectorsize
!= 1024)
305 if (req
->pagesize
% req
->ecc
.sectorsize
)
308 req
->ecc
.nsectors
= req
->pagesize
/ req
->ecc
.sectorsize
;
310 max_eccbytes
= req
->ecc
.bytes
;
312 for (i
= 0; i
< pmecc
->caps
->nstrengths
; i
++) {
313 int nbytes
, strength
= pmecc
->caps
->strengths
[i
];
315 if (req
->ecc
.strength
!= ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH
&&
316 strength
< req
->ecc
.strength
)
319 nbytes
= DIV_ROUND_UP(strength
* fls(8 * req
->ecc
.sectorsize
),
321 nbytes
*= req
->ecc
.nsectors
;
323 if (nbytes
> max_eccbytes
)
326 eccstrength
= strength
;
329 if (req
->ecc
.strength
!= ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH
)
336 req
->ecc
.bytes
= eccbytes
;
337 req
->ecc
.strength
= eccstrength
;
339 if (req
->ecc
.ooboffset
< 0)
340 req
->ecc
.ooboffset
= req
->oobsize
- eccbytes
;
345 struct atmel_pmecc_user
*
346 atmel_pmecc_create_user(struct atmel_pmecc
*pmecc
,
347 struct atmel_pmecc_user_req
*req
)
349 struct atmel_pmecc_user
*user
;
350 const struct atmel_pmecc_gf_tables
*gf_tables
;
351 int strength
, size
, ret
;
353 ret
= atmel_pmecc_prepare_user_req(pmecc
, req
);
357 size
= sizeof(*user
);
358 size
= ALIGN(size
, sizeof(u16
));
359 /* Reserve space for partial_syn, si and smu */
360 size
+= ((2 * req
->ecc
.strength
) + 1) * sizeof(u16
) *
361 (2 + req
->ecc
.strength
+ 2);
362 /* Reserve space for lmu. */
363 size
+= (req
->ecc
.strength
+ 1) * sizeof(u16
);
364 /* Reserve space for mu, dmu and delta. */
365 size
= ALIGN(size
, sizeof(s32
));
366 size
+= (req
->ecc
.strength
+ 1) * sizeof(s32
) * 3;
368 user
= kzalloc(size
, GFP_KERNEL
);
370 return ERR_PTR(-ENOMEM
);
374 user
->partial_syn
= (s16
*)PTR_ALIGN(user
+ 1, sizeof(u16
));
375 user
->si
= user
->partial_syn
+ ((2 * req
->ecc
.strength
) + 1);
376 user
->lmu
= user
->si
+ ((2 * req
->ecc
.strength
) + 1);
377 user
->smu
= user
->lmu
+ (req
->ecc
.strength
+ 1);
378 user
->mu
= (s32
*)PTR_ALIGN(user
->smu
+
379 (((2 * req
->ecc
.strength
) + 1) *
380 (req
->ecc
.strength
+ 2)),
382 user
->dmu
= user
->mu
+ req
->ecc
.strength
+ 1;
383 user
->delta
= user
->dmu
+ req
->ecc
.strength
+ 1;
385 gf_tables
= atmel_pmecc_get_gf_tables(req
);
386 if (IS_ERR(gf_tables
)) {
388 return ERR_CAST(gf_tables
);
391 user
->gf_tables
= gf_tables
;
393 user
->eccbytes
= req
->ecc
.bytes
/ req
->ecc
.nsectors
;
395 for (strength
= 0; strength
< pmecc
->caps
->nstrengths
; strength
++) {
396 if (pmecc
->caps
->strengths
[strength
] == req
->ecc
.strength
)
400 user
->cache
.cfg
= PMECC_CFG_BCH_STRENGTH(strength
) |
401 PMECC_CFG_NSECTORS(req
->ecc
.nsectors
);
403 if (req
->ecc
.sectorsize
== 1024)
404 user
->cache
.cfg
|= PMECC_CFG_SECTOR1024
;
406 user
->cache
.sarea
= req
->oobsize
- 1;
407 user
->cache
.saddr
= req
->ecc
.ooboffset
;
408 user
->cache
.eaddr
= req
->ecc
.ooboffset
+ req
->ecc
.bytes
- 1;
412 EXPORT_SYMBOL_GPL(atmel_pmecc_create_user
);
414 void atmel_pmecc_destroy_user(struct atmel_pmecc_user
*user
)
418 EXPORT_SYMBOL_GPL(atmel_pmecc_destroy_user
);
420 static int get_strength(struct atmel_pmecc_user
*user
)
422 const int *strengths
= user
->pmecc
->caps
->strengths
;
424 return strengths
[user
->cache
.cfg
& PMECC_CFG_BCH_STRENGTH_MASK
];
427 static int get_sectorsize(struct atmel_pmecc_user
*user
)
429 return user
->cache
.cfg
& PMECC_CFG_SECTOR1024
? 1024 : 512;
432 static void atmel_pmecc_gen_syndrome(struct atmel_pmecc_user
*user
, int sector
)
434 int strength
= get_strength(user
);
438 /* Fill odd syndromes */
439 for (i
= 0; i
< strength
; i
++) {
440 value
= readl_relaxed(user
->pmecc
->regs
.base
+
441 ATMEL_PMECC_REM(sector
, i
/ 2));
445 user
->partial_syn
[(2 * i
) + 1] = value
;
449 static void atmel_pmecc_substitute(struct atmel_pmecc_user
*user
)
451 int degree
= get_sectorsize(user
) == 512 ? 13 : 14;
452 int cw_len
= BIT(degree
) - 1;
453 int strength
= get_strength(user
);
454 s16
*alpha_to
= user
->gf_tables
->alpha_to
;
455 s16
*index_of
= user
->gf_tables
->index_of
;
456 s16
*partial_syn
= user
->partial_syn
;
461 * si[] is a table that holds the current syndrome value,
462 * an element of that table belongs to the field
466 memset(&si
[1], 0, sizeof(s16
) * ((2 * strength
) - 1));
468 /* Computation 2t syndromes based on S(x) */
470 for (i
= 1; i
< 2 * strength
; i
+= 2) {
471 for (j
= 0; j
< degree
; j
++) {
472 if (partial_syn
[i
] & BIT(j
))
473 si
[i
] = alpha_to
[i
* j
] ^ si
[i
];
476 /* Even syndrome = (Odd syndrome) ** 2 */
477 for (i
= 2, j
= 1; j
<= strength
; i
= ++j
<< 1) {
483 tmp
= index_of
[si
[j
]];
484 tmp
= (tmp
* 2) % cw_len
;
485 si
[i
] = alpha_to
[tmp
];
490 static void atmel_pmecc_get_sigma(struct atmel_pmecc_user
*user
)
492 s16
*lmu
= user
->lmu
;
495 s32
*dmu
= user
->dmu
;
496 s32
*delta
= user
->delta
;
497 int degree
= get_sectorsize(user
) == 512 ? 13 : 14;
498 int cw_len
= BIT(degree
) - 1;
499 int strength
= get_strength(user
);
500 int num
= 2 * strength
+ 1;
501 s16
*index_of
= user
->gf_tables
->index_of
;
502 s16
*alpha_to
= user
->gf_tables
->alpha_to
;
504 u32 dmu_0_count
, tmp
;
505 s16
*smu
= user
->smu
;
507 /* index of largest delta */
519 memset(smu
, 0, sizeof(s16
) * num
);
522 /* discrepancy set to 1 */
524 /* polynom order set to 0 */
526 delta
[0] = (mu
[0] * 2 - lmu
[0]) >> 1;
532 /* Sigma(x) set to 1 */
533 memset(&smu
[num
], 0, sizeof(s16
) * num
);
536 /* discrepancy set to S1 */
539 /* polynom order set to 0 */
542 delta
[1] = (mu
[1] * 2 - lmu
[1]) >> 1;
544 /* Init the Sigma(x) last row */
545 memset(&smu
[(strength
+ 1) * num
], 0, sizeof(s16
) * num
);
547 for (i
= 1; i
<= strength
; i
++) {
549 /* Begin Computing Sigma (Mu+1) and L(mu) */
550 /* check if discrepancy is set to 0 */
554 tmp
= ((strength
- (lmu
[i
] >> 1) - 1) / 2);
555 if ((strength
- (lmu
[i
] >> 1) - 1) & 0x1)
560 if (dmu_0_count
== tmp
) {
561 for (j
= 0; j
<= (lmu
[i
] >> 1) + 1; j
++)
562 smu
[(strength
+ 1) * num
+ j
] =
565 lmu
[strength
+ 1] = lmu
[i
];
570 for (j
= 0; j
<= lmu
[i
] >> 1; j
++)
571 smu
[(i
+ 1) * num
+ j
] = smu
[i
* num
+ j
];
573 /* copy previous polynom order to the next */
578 /* find largest delta with dmu != 0 */
579 for (j
= 0; j
< i
; j
++) {
580 if ((dmu
[j
]) && (delta
[j
] > largest
)) {
586 /* compute difference */
587 diff
= (mu
[i
] - mu
[ro
]);
589 /* Compute degree of the new smu polynomial */
590 if ((lmu
[i
] >> 1) > ((lmu
[ro
] >> 1) + diff
))
593 lmu
[i
+ 1] = ((lmu
[ro
] >> 1) + diff
) * 2;
595 /* Init smu[i+1] with 0 */
596 for (k
= 0; k
< num
; k
++)
597 smu
[(i
+ 1) * num
+ k
] = 0;
599 /* Compute smu[i+1] */
600 for (k
= 0; k
<= lmu
[ro
] >> 1; k
++) {
603 if (!(smu
[ro
* num
+ k
] && dmu
[i
]))
606 a
= index_of
[dmu
[i
]];
607 b
= index_of
[dmu
[ro
]];
608 c
= index_of
[smu
[ro
* num
+ k
]];
609 tmp
= a
+ (cw_len
- b
) + c
;
610 a
= alpha_to
[tmp
% cw_len
];
611 smu
[(i
+ 1) * num
+ (k
+ diff
)] = a
;
614 for (k
= 0; k
<= lmu
[i
] >> 1; k
++)
615 smu
[(i
+ 1) * num
+ k
] ^= smu
[i
* num
+ k
];
618 /* End Computing Sigma (Mu+1) and L(mu) */
619 /* In either case compute delta */
620 delta
[i
+ 1] = (mu
[i
+ 1] * 2 - lmu
[i
+ 1]) >> 1;
622 /* Do not compute discrepancy for the last iteration */
626 for (k
= 0; k
<= (lmu
[i
+ 1] >> 1); k
++) {
629 dmu
[i
+ 1] = si
[tmp
+ 3];
630 } else if (smu
[(i
+ 1) * num
+ k
] && si
[tmp
+ 3 - k
]) {
633 a
= index_of
[smu
[(i
+ 1) * num
+ k
]];
634 b
= si
[2 * (i
- 1) + 3 - k
];
638 dmu
[i
+ 1] = alpha_to
[tmp
] ^ dmu
[i
+ 1];
644 static int atmel_pmecc_err_location(struct atmel_pmecc_user
*user
)
646 int sector_size
= get_sectorsize(user
);
647 int degree
= sector_size
== 512 ? 13 : 14;
648 struct atmel_pmecc
*pmecc
= user
->pmecc
;
649 int strength
= get_strength(user
);
650 int ret
, roots_nbr
, i
, err_nbr
= 0;
651 int num
= (2 * strength
) + 1;
652 s16
*smu
= user
->smu
;
655 writel(PMERRLOC_DISABLE
, pmecc
->regs
.errloc
+ ATMEL_PMERRLOC_ELDIS
);
657 for (i
= 0; i
<= user
->lmu
[strength
+ 1] >> 1; i
++) {
658 writel_relaxed(smu
[(strength
+ 1) * num
+ i
],
659 pmecc
->regs
.errloc
+ ATMEL_PMERRLOC_SIGMA(i
));
663 val
= (err_nbr
- 1) << 16;
664 if (sector_size
== 1024)
667 writel(val
, pmecc
->regs
.errloc
+ ATMEL_PMERRLOC_ELCFG
);
668 writel((sector_size
* 8) + (degree
* strength
),
669 pmecc
->regs
.errloc
+ ATMEL_PMERRLOC_ELEN
);
671 ret
= readl_relaxed_poll_timeout(pmecc
->regs
.errloc
+
672 ATMEL_PMERRLOC_ELISR
,
673 val
, val
& PMERRLOC_CALC_DONE
, 0,
674 PMECC_MAX_TIMEOUT_MS
* 1000);
677 "PMECC: Timeout to calculate error location.\n");
681 roots_nbr
= (val
& PMERRLOC_ERR_NUM_MASK
) >> 8;
682 /* Number of roots == degree of smu hence <= cap */
683 if (roots_nbr
== user
->lmu
[strength
+ 1] >> 1)
687 * Number of roots does not match the degree of smu
688 * unable to correct error.
693 int atmel_pmecc_correct_sector(struct atmel_pmecc_user
*user
, int sector
,
694 void *data
, void *ecc
)
696 struct atmel_pmecc
*pmecc
= user
->pmecc
;
697 int sectorsize
= get_sectorsize(user
);
698 int eccbytes
= user
->eccbytes
;
701 if (!(user
->isr
& BIT(sector
)))
704 atmel_pmecc_gen_syndrome(user
, sector
);
705 atmel_pmecc_substitute(user
);
706 atmel_pmecc_get_sigma(user
);
708 nerrors
= atmel_pmecc_err_location(user
);
712 for (i
= 0; i
< nerrors
; i
++) {
718 errpos
= readl_relaxed(pmecc
->regs
.errloc
+
719 ATMEL_PMERRLOC_EL(pmecc
->caps
->el_offset
, i
));
725 if (byte
< sectorsize
) {
728 } else if (byte
< sectorsize
+ eccbytes
) {
729 ptr
= ecc
+ byte
- sectorsize
;
733 "Invalid errpos value (%d, max is %d)\n",
734 errpos
, (sectorsize
+ eccbytes
) * 8);
739 "Bit flip in %s area, byte %d: 0x%02x -> 0x%02x\n",
740 area
, byte
, *ptr
, (unsigned int)(*ptr
^ BIT(bit
)));
747 EXPORT_SYMBOL_GPL(atmel_pmecc_correct_sector
);
749 bool atmel_pmecc_correct_erased_chunks(struct atmel_pmecc_user
*user
)
751 return user
->pmecc
->caps
->correct_erased_chunks
;
753 EXPORT_SYMBOL_GPL(atmel_pmecc_correct_erased_chunks
);
755 void atmel_pmecc_get_generated_eccbytes(struct atmel_pmecc_user
*user
,
756 int sector
, void *ecc
)
758 struct atmel_pmecc
*pmecc
= user
->pmecc
;
762 for (i
= 0; i
< user
->eccbytes
; i
++)
763 ptr
[i
] = readb_relaxed(pmecc
->regs
.base
+
764 ATMEL_PMECC_ECC(sector
, i
));
766 EXPORT_SYMBOL_GPL(atmel_pmecc_get_generated_eccbytes
);
768 void atmel_pmecc_reset(struct atmel_pmecc
*pmecc
)
770 writel(PMECC_CTRL_RST
, pmecc
->regs
.base
+ ATMEL_PMECC_CTRL
);
771 writel(PMECC_CTRL_DISABLE
, pmecc
->regs
.base
+ ATMEL_PMECC_CTRL
);
773 EXPORT_SYMBOL_GPL(atmel_pmecc_reset
);
775 int atmel_pmecc_enable(struct atmel_pmecc_user
*user
, int op
)
777 struct atmel_pmecc
*pmecc
= user
->pmecc
;
780 if (op
!= NAND_ECC_READ
&& op
!= NAND_ECC_WRITE
) {
781 dev_err(pmecc
->dev
, "Bad ECC operation!");
785 mutex_lock(&user
->pmecc
->lock
);
787 cfg
= user
->cache
.cfg
;
788 if (op
== NAND_ECC_WRITE
)
789 cfg
|= PMECC_CFG_WRITE_OP
;
791 cfg
|= PMECC_CFG_AUTO_ENABLE
;
793 writel(cfg
, pmecc
->regs
.base
+ ATMEL_PMECC_CFG
);
794 writel(user
->cache
.sarea
, pmecc
->regs
.base
+ ATMEL_PMECC_SAREA
);
795 writel(user
->cache
.saddr
, pmecc
->regs
.base
+ ATMEL_PMECC_SADDR
);
796 writel(user
->cache
.eaddr
, pmecc
->regs
.base
+ ATMEL_PMECC_EADDR
);
798 writel(PMECC_CTRL_ENABLE
, pmecc
->regs
.base
+ ATMEL_PMECC_CTRL
);
799 writel(PMECC_CTRL_DATA
, pmecc
->regs
.base
+ ATMEL_PMECC_CTRL
);
803 EXPORT_SYMBOL_GPL(atmel_pmecc_enable
);
805 void atmel_pmecc_disable(struct atmel_pmecc_user
*user
)
807 atmel_pmecc_reset(user
->pmecc
);
808 mutex_unlock(&user
->pmecc
->lock
);
810 EXPORT_SYMBOL_GPL(atmel_pmecc_disable
);
812 int atmel_pmecc_wait_rdy(struct atmel_pmecc_user
*user
)
814 struct atmel_pmecc
*pmecc
= user
->pmecc
;
818 ret
= readl_relaxed_poll_timeout(pmecc
->regs
.base
+
820 status
, !(status
& PMECC_SR_BUSY
), 0,
821 PMECC_MAX_TIMEOUT_MS
* 1000);
824 "Timeout while waiting for PMECC ready.\n");
828 user
->isr
= readl_relaxed(pmecc
->regs
.base
+ ATMEL_PMECC_ISR
);
832 EXPORT_SYMBOL_GPL(atmel_pmecc_wait_rdy
);
834 static struct atmel_pmecc
*atmel_pmecc_create(struct platform_device
*pdev
,
835 const struct atmel_pmecc_caps
*caps
,
836 int pmecc_res_idx
, int errloc_res_idx
)
838 struct device
*dev
= &pdev
->dev
;
839 struct atmel_pmecc
*pmecc
;
840 struct resource
*res
;
842 pmecc
= devm_kzalloc(dev
, sizeof(*pmecc
), GFP_KERNEL
);
844 return ERR_PTR(-ENOMEM
);
848 mutex_init(&pmecc
->lock
);
850 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, pmecc_res_idx
);
851 pmecc
->regs
.base
= devm_ioremap_resource(dev
, res
);
852 if (IS_ERR(pmecc
->regs
.base
))
853 return ERR_CAST(pmecc
->regs
.base
);
855 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, errloc_res_idx
);
856 pmecc
->regs
.errloc
= devm_ioremap_resource(dev
, res
);
857 if (IS_ERR(pmecc
->regs
.errloc
))
858 return ERR_CAST(pmecc
->regs
.errloc
);
860 /* Disable all interrupts before registering the PMECC handler. */
861 writel(0xffffffff, pmecc
->regs
.base
+ ATMEL_PMECC_IDR
);
862 atmel_pmecc_reset(pmecc
);
867 static void devm_atmel_pmecc_put(struct device
*dev
, void *res
)
869 struct atmel_pmecc
**pmecc
= res
;
871 put_device((*pmecc
)->dev
);
874 static struct atmel_pmecc
*atmel_pmecc_get_by_node(struct device
*userdev
,
875 struct device_node
*np
)
877 struct platform_device
*pdev
;
878 struct atmel_pmecc
*pmecc
, **ptr
;
881 pdev
= of_find_device_by_node(np
);
883 return ERR_PTR(-EPROBE_DEFER
);
884 pmecc
= platform_get_drvdata(pdev
);
890 ptr
= devres_alloc(devm_atmel_pmecc_put
, sizeof(*ptr
), GFP_KERNEL
);
898 devres_add(userdev
, ptr
);
903 put_device(&pdev
->dev
);
907 static const int atmel_pmecc_strengths
[] = { 2, 4, 8, 12, 24, 32 };
909 static struct atmel_pmecc_caps at91sam9g45_caps
= {
910 .strengths
= atmel_pmecc_strengths
,
915 static struct atmel_pmecc_caps sama5d4_caps
= {
916 .strengths
= atmel_pmecc_strengths
,
919 .correct_erased_chunks
= true,
922 static struct atmel_pmecc_caps sama5d2_caps
= {
923 .strengths
= atmel_pmecc_strengths
,
926 .correct_erased_chunks
= true,
929 static const struct of_device_id atmel_pmecc_legacy_match
[] = {
930 { .compatible
= "atmel,sama5d4-nand", &sama5d4_caps
},
931 { .compatible
= "atmel,sama5d2-nand", &sama5d2_caps
},
935 struct atmel_pmecc
*devm_atmel_pmecc_get(struct device
*userdev
)
937 struct atmel_pmecc
*pmecc
;
938 struct device_node
*np
;
941 return ERR_PTR(-EINVAL
);
943 if (!userdev
->of_node
)
946 np
= of_parse_phandle(userdev
->of_node
, "ecc-engine", 0);
948 pmecc
= atmel_pmecc_get_by_node(userdev
, np
);
952 * Support old DT bindings: in this case the PMECC iomem
953 * resources are directly defined in the user pdev at position
954 * 1 and 2. Extract all relevant information from there.
956 struct platform_device
*pdev
= to_platform_device(userdev
);
957 const struct atmel_pmecc_caps
*caps
;
958 const struct of_device_id
*match
;
960 /* No PMECC engine available. */
961 if (!of_property_read_bool(userdev
->of_node
,
965 caps
= &at91sam9g45_caps
;
967 /* Find the caps associated to the NAND dev node. */
968 match
= of_match_node(atmel_pmecc_legacy_match
,
970 if (match
&& match
->data
)
973 pmecc
= atmel_pmecc_create(pdev
, caps
, 1, 2);
978 EXPORT_SYMBOL(devm_atmel_pmecc_get
);
980 static const struct of_device_id atmel_pmecc_match
[] = {
981 { .compatible
= "atmel,at91sam9g45-pmecc", &at91sam9g45_caps
},
982 { .compatible
= "atmel,sama5d4-pmecc", &sama5d4_caps
},
983 { .compatible
= "atmel,sama5d2-pmecc", &sama5d2_caps
},
986 MODULE_DEVICE_TABLE(of
, atmel_pmecc_match
);
988 static int atmel_pmecc_probe(struct platform_device
*pdev
)
990 struct device
*dev
= &pdev
->dev
;
991 const struct atmel_pmecc_caps
*caps
;
992 struct atmel_pmecc
*pmecc
;
994 caps
= of_device_get_match_data(&pdev
->dev
);
996 dev_err(dev
, "Invalid caps\n");
1000 pmecc
= atmel_pmecc_create(pdev
, caps
, 0, 1);
1002 return PTR_ERR(pmecc
);
1004 platform_set_drvdata(pdev
, pmecc
);
1009 static struct platform_driver atmel_pmecc_driver
= {
1011 .name
= "atmel-pmecc",
1012 .of_match_table
= of_match_ptr(atmel_pmecc_match
),
1014 .probe
= atmel_pmecc_probe
,
1016 module_platform_driver(atmel_pmecc_driver
);
1018 MODULE_LICENSE("GPL");
1019 MODULE_AUTHOR("Boris Brezillon <boris.brezillon@free-electrons.com>");
1020 MODULE_DESCRIPTION("PMECC engine driver");
1021 MODULE_ALIAS("platform:atmel_pmecc");