drivers/mtd/nand/raw/atmel/pmecc.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright 2017 ATMEL
   4  * Copyright 2017 Free Electrons
   5  *
   6  * Author: Boris Brezillon <boris.brezillon@free-electrons.com>
   7  *
   8  * Derived from the atmel_nand.c driver which contained the following
   9  * copyrights:
  10  *
  11  *   Copyright 2003 Rick Bronson
  12  *
  13  *   Derived from drivers/mtd/nand/autcpu12.c (removed in v3.8)
  14  *      Copyright 2001 Thomas Gleixner (gleixner@autronix.de)
  15  *
  16  *   Derived from drivers/mtd/spia.c (removed in v3.8)
  17  *      Copyright 2000 Steven J. Hill (sjhill@cotw.com)
  18  *
  19  *   Add Hardware ECC support for AT91SAM9260 / AT91SAM9263
  20  *      Richard Genoud (richard.genoud@gmail.com), Adeneo Copyright 2007
  21  *
  22  *   Derived from Das U-Boot source code
  23  *      (u-boot-1.1.5/board/atmel/at91sam9263ek/nand.c)
  24  *      Copyright 2006 ATMEL Rousset, Lacressonniere Nicolas
  25  *
  26  *   Add Programmable Multibit ECC support for various AT91 SoC
  27  *      Copyright 2012 ATMEL, Hong Xu
  28  *
  29  *   Add Nand Flash Controller support for SAMA5 SoC
  30  *      Copyright 2013 ATMEL, Josh Wu (josh.wu@atmel.com)
  31  *
  32  * The PMECC is an hardware assisted BCH engine, which means part of the
  33  * ECC algorithm is left to the software. The hardware/software repartition
  34  * is explained in the "PMECC Controller Functional Description" chapter in
  35  * Atmel datasheets, and some of the functions in this file are directly
  36  * implementing the algorithms described in the "Software Implementation"
  37  * sub-section.
  38  *
  39  * TODO: it seems that the software BCH implementation in lib/bch.c is already
  40  * providing some of the logic we are implementing here. It would be smart
  41  * to expose the needed lib/bch.c helpers/functions and re-use them here.
  42  */
  43
  44 #include <linux/genalloc.h>
  45 #include <linux/iopoll.h>
  46 #include <linux/module.h>
  47 #include <linux/mtd/rawnand.h>
  48 #include <linux/of_irq.h>
  49 #include <linux/of_platform.h>
  50 #include <linux/platform_device.h>
  51 #include <linux/slab.h>
  52
  53 #include "pmecc.h"
  54
  55 /* Galois field dimension */
  56 #define PMECC_GF_DIMENSION_13                   13
  57 #define PMECC_GF_DIMENSION_14                   14
  58
  59 /* Primitive Polynomial used by PMECC */
  60 #define PMECC_GF_13_PRIMITIVE_POLY              0x201b
  61 #define PMECC_GF_14_PRIMITIVE_POLY              0x4443
  62
  63 #define PMECC_LOOKUP_TABLE_SIZE_512             0x2000
  64 #define PMECC_LOOKUP_TABLE_SIZE_1024            0x4000
  65
  66 /* Time out value for reading PMECC status register */
  67 #define PMECC_MAX_TIMEOUT_MS                    100
  68
  69 /* PMECC Register Definitions */
  70 #define ATMEL_PMECC_CFG                         0x0
  71 #define PMECC_CFG_BCH_STRENGTH(x)               (x)
  72 #define PMECC_CFG_BCH_STRENGTH_MASK             GENMASK(2, 0)
  73 #define PMECC_CFG_SECTOR512                     (0 << 4)
  74 #define PMECC_CFG_SECTOR1024                    (1 << 4)
  75 #define PMECC_CFG_NSECTORS(x)                   ((fls(x) - 1) << 8)
  76 #define PMECC_CFG_READ_OP                       (0 << 12)
  77 #define PMECC_CFG_WRITE_OP                      (1 << 12)
  78 #define PMECC_CFG_SPARE_ENABLE                  BIT(16)
  79 #define PMECC_CFG_AUTO_ENABLE                   BIT(20)
  80
  81 #define ATMEL_PMECC_SAREA                       0x4
  82 #define ATMEL_PMECC_SADDR                       0x8
  83 #define ATMEL_PMECC_EADDR                       0xc
  84
  85 #define ATMEL_PMECC_CLK                         0x10
  86 #define PMECC_CLK_133MHZ                        (2 << 0)
  87
  88 #define ATMEL_PMECC_CTRL                        0x14
  89 #define PMECC_CTRL_RST                          BIT(0)
  90 #define PMECC_CTRL_DATA                         BIT(1)
  91 #define PMECC_CTRL_USER                         BIT(2)
  92 #define PMECC_CTRL_ENABLE                       BIT(4)
  93 #define PMECC_CTRL_DISABLE                      BIT(5)
  94
  95 #define ATMEL_PMECC_SR                          0x18
  96 #define PMECC_SR_BUSY                           BIT(0)
  97 #define PMECC_SR_ENABLE                         BIT(4)
  98
  99 #define ATMEL_PMECC_IER                         0x1c
 100 #define ATMEL_PMECC_IDR                         0x20
 101 #define ATMEL_PMECC_IMR                         0x24
 102 #define ATMEL_PMECC_ISR                         0x28
 103 #define PMECC_ERROR_INT                         BIT(0)
 104
 105 #define ATMEL_PMECC_ECC(sector, n)              \
 106         ((((sector) + 1) * 0x40) + (n))
 107
 108 #define ATMEL_PMECC_REM(sector, n)              \
 109         ((((sector) + 1) * 0x40) + ((n) * 4) + 0x200)
 110
 111 /* PMERRLOC Register Definitions */
 112 #define ATMEL_PMERRLOC_ELCFG                    0x0
 113 #define PMERRLOC_ELCFG_SECTOR_512               (0 << 0)
 114 #define PMERRLOC_ELCFG_SECTOR_1024              (1 << 0)
 115 #define PMERRLOC_ELCFG_NUM_ERRORS(n)            ((n) << 16)
 116
 117 #define ATMEL_PMERRLOC_ELPRIM                   0x4
 118 #define ATMEL_PMERRLOC_ELEN                     0x8
 119 #define ATMEL_PMERRLOC_ELDIS                    0xc
 120 #define PMERRLOC_DISABLE                        BIT(0)
 121
 122 #define ATMEL_PMERRLOC_ELSR                     0x10
 123 #define PMERRLOC_ELSR_BUSY                      BIT(0)
 124
 125 #define ATMEL_PMERRLOC_ELIER                    0x14
 126 #define ATMEL_PMERRLOC_ELIDR                    0x18
 127 #define ATMEL_PMERRLOC_ELIMR                    0x1c
 128 #define ATMEL_PMERRLOC_ELISR                    0x20
 129 #define PMERRLOC_ERR_NUM_MASK                   GENMASK(12, 8)
 130 #define PMERRLOC_CALC_DONE                      BIT(0)
 131
 132 #define ATMEL_PMERRLOC_SIGMA(x)                 (((x) * 0x4) + 0x28)
 133
 134 #define ATMEL_PMERRLOC_EL(offs, x)              (((x) * 0x4) + (offs))
 135
 136 struct atmel_pmecc_gf_tables {
 137         u16 *alpha_to;
 138         u16 *index_of;
 139 };
 140
 141 struct atmel_pmecc_caps {
 142         const int *strengths;
 143         int nstrengths;
 144         int el_offset;
 145         bool correct_erased_chunks;
 146 };
 147
 148 struct atmel_pmecc {
 149         struct device *dev;
 150         const struct atmel_pmecc_caps *caps;
 151
 152         struct {
 153                 void __iomem *base;
 154                 void __iomem *errloc;
 155         } regs;
 156
 157         struct mutex lock;
 158 };
 159
 160 struct atmel_pmecc_user_conf_cache {
 161         u32 cfg;
 162         u32 sarea;
 163         u32 saddr;
 164         u32 eaddr;
 165 };
 166
 167 struct atmel_pmecc_user {
 168         struct atmel_pmecc_user_conf_cache cache;
 169         struct atmel_pmecc *pmecc;
 170         const struct atmel_pmecc_gf_tables *gf_tables;
 171         int eccbytes;
 172         s16 *partial_syn;
 173         s16 *si;
 174         s16 *lmu;
 175         s16 *smu;
 176         s32 *mu;
 177         s32 *dmu;
 178         s32 *delta;
 179         u32 isr;
 180 };
 181
 182 static DEFINE_MUTEX(pmecc_gf_tables_lock);
 183 static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_512;
 184 static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_1024;
 185
 186 static inline int deg(unsigned int poly)
 187 {
 188         /* polynomial degree is the most-significant bit index */
 189         return fls(poly) - 1;
 190 }
 191
 192 static int atmel_pmecc_build_gf_tables(int mm, unsigned int poly,
 193                                        struct atmel_pmecc_gf_tables *gf_tables)
 194 {
 195         unsigned int i, x = 1;
 196         const unsigned int k = BIT(deg(poly));
 197         unsigned int nn = BIT(mm) - 1;
 198
 199         /* primitive polynomial must be of degree m */
 200         if (k != (1u << mm))
 201                 return -EINVAL;
 202
 203         for (i = 0; i < nn; i++) {
 204                 gf_tables->alpha_to[i] = x;
 205                 gf_tables->index_of[x] = i;
 206                 if (i && (x == 1))
 207                         /* polynomial is not primitive (a^i=1 with 0<i<2^m-1) */
 208                         return -EINVAL;
 209                 x <<= 1;
 210                 if (x & k)
 211                         x ^= poly;
 212         }
 213         gf_tables->alpha_to[nn] = 1;
 214         gf_tables->index_of[0] = 0;
 215
 216         return 0;
 217 }
 218
 219 static const struct atmel_pmecc_gf_tables *
 220 atmel_pmecc_create_gf_tables(const struct atmel_pmecc_user_req *req)
 221 {
 222         struct atmel_pmecc_gf_tables *gf_tables;
 223         unsigned int poly, degree, table_size;
 224         int ret;
 225
 226         if (req->ecc.sectorsize == 512) {
 227                 degree = PMECC_GF_DIMENSION_13;
 228                 poly = PMECC_GF_13_PRIMITIVE_POLY;
 229                 table_size = PMECC_LOOKUP_TABLE_SIZE_512;
 230         } else {
 231                 degree = PMECC_GF_DIMENSION_14;
 232                 poly = PMECC_GF_14_PRIMITIVE_POLY;
 233                 table_size = PMECC_LOOKUP_TABLE_SIZE_1024;
 234         }
 235
 236         gf_tables = kzalloc(sizeof(*gf_tables) +
 237                             (2 * table_size * sizeof(u16)),
 238                             GFP_KERNEL);
 239         if (!gf_tables)
 240                 return ERR_PTR(-ENOMEM);
 241
 242         gf_tables->alpha_to = (void *)(gf_tables + 1);
 243         gf_tables->index_of = gf_tables->alpha_to + table_size;
 244
 245         ret = atmel_pmecc_build_gf_tables(degree, poly, gf_tables);
 246         if (ret) {
 247                 kfree(gf_tables);
 248                 return ERR_PTR(ret);
 249         }
 250
 251         return gf_tables;
 252 }
 253
 254 static const struct atmel_pmecc_gf_tables *
 255 atmel_pmecc_get_gf_tables(const struct atmel_pmecc_user_req *req)
 256 {
 257         const struct atmel_pmecc_gf_tables **gf_tables, *ret;
 258
 259         mutex_lock(&pmecc_gf_tables_lock);
 260         if (req->ecc.sectorsize == 512)
 261                 gf_tables = &pmecc_gf_tables_512;
 262         else
 263                 gf_tables = &pmecc_gf_tables_1024;
 264
 265         ret = *gf_tables;
 266
 267         if (!ret) {
 268                 ret = atmel_pmecc_create_gf_tables(req);
 269                 if (!IS_ERR(ret))
 270                         *gf_tables = ret;
 271         }
 272         mutex_unlock(&pmecc_gf_tables_lock);
 273
 274         return ret;
 275 }
 276
 277 static int atmel_pmecc_prepare_user_req(struct atmel_pmecc *pmecc,
 278                                         struct atmel_pmecc_user_req *req)
 279 {
 280         int i, max_eccbytes, eccbytes = 0, eccstrength = 0;
 281
 282         if (req->pagesize <= 0 || req->oobsize <= 0 || req->ecc.bytes <= 0)
 283                 return -EINVAL;
 284
 285         if (req->ecc.ooboffset >= 0 &&
 286             req->ecc.ooboffset + req->ecc.bytes > req->oobsize)
 287                 return -EINVAL;
 288
 289         if (req->ecc.sectorsize == ATMEL_PMECC_SECTOR_SIZE_AUTO) {
 290                 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH)
 291                         return -EINVAL;
 292
 293                 if (req->pagesize > 512)
 294                         req->ecc.sectorsize = 1024;
 295                 else
 296                         req->ecc.sectorsize = 512;
 297         }
 298
 299         if (req->ecc.sectorsize != 512 && req->ecc.sectorsize != 1024)
 300                 return -EINVAL;
 301
 302         if (req->pagesize % req->ecc.sectorsize)
 303                 return -EINVAL;
 304
 305         req->ecc.nsectors = req->pagesize / req->ecc.sectorsize;
 306
 307         max_eccbytes = req->ecc.bytes;
 308
 309         for (i = 0; i < pmecc->caps->nstrengths; i++) {
 310                 int nbytes, strength = pmecc->caps->strengths[i];
 311
 312                 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH &&
 313                     strength < req->ecc.strength)
 314                         continue;
 315
 316                 nbytes = DIV_ROUND_UP(strength * fls(8 * req->ecc.sectorsize),
 317                                       8);
 318                 nbytes *= req->ecc.nsectors;
 319
 320                 if (nbytes > max_eccbytes)
 321                         break;
 322
 323                 eccstrength = strength;
 324                 eccbytes = nbytes;
 325
 326                 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH)
 327                         break;
 328         }
 329
 330         if (!eccstrength)
 331                 return -EINVAL;
 332
 333         req->ecc.bytes = eccbytes;
 334         req->ecc.strength = eccstrength;
 335
 336         if (req->ecc.ooboffset < 0)
 337                 req->ecc.ooboffset = req->oobsize - eccbytes;
 338
 339         return 0;
 340 }
 341
 342 struct atmel_pmecc_user *
 343 atmel_pmecc_create_user(struct atmel_pmecc *pmecc,
 344                         struct atmel_pmecc_user_req *req)
 345 {
 346         struct atmel_pmecc_user *user;
 347         const struct atmel_pmecc_gf_tables *gf_tables;
 348         int strength, size, ret;
 349
 350         ret = atmel_pmecc_prepare_user_req(pmecc, req);
 351         if (ret)
 352                 return ERR_PTR(ret);
 353
 354         size = sizeof(*user);
 355         size = ALIGN(size, sizeof(u16));
 356         /* Reserve space for partial_syn, si and smu */
 357         size += ((2 * req->ecc.strength) + 1) * sizeof(u16) *
 358                 (2 + req->ecc.strength + 2);
 359         /* Reserve space for lmu. */
 360         size += (req->ecc.strength + 1) * sizeof(u16);
 361         /* Reserve space for mu, dmu and delta. */
 362         size = ALIGN(size, sizeof(s32));
 363         size += (req->ecc.strength + 1) * sizeof(s32) * 3;
 364
 365         user = kzalloc(size, GFP_KERNEL);
 366         if (!user)
 367                 return ERR_PTR(-ENOMEM);
 368
 369         user->pmecc = pmecc;
 370
 371         user->partial_syn = (s16 *)PTR_ALIGN(user + 1, sizeof(u16));
 372         user->si = user->partial_syn + ((2 * req->ecc.strength) + 1);
 373         user->lmu = user->si + ((2 * req->ecc.strength) + 1);
 374         user->smu = user->lmu + (req->ecc.strength + 1);
 375         user->mu = (s32 *)PTR_ALIGN(user->smu +
 376                                     (((2 * req->ecc.strength) + 1) *
 377                                      (req->ecc.strength + 2)),
 378                                     sizeof(s32));
 379         user->dmu = user->mu + req->ecc.strength + 1;
 380         user->delta = user->dmu + req->ecc.strength + 1;
 381
 382         gf_tables = atmel_pmecc_get_gf_tables(req);
 383         if (IS_ERR(gf_tables)) {
 384                 kfree(user);
 385                 return ERR_CAST(gf_tables);
 386         }
 387
 388         user->gf_tables = gf_tables;
 389
 390         user->eccbytes = req->ecc.bytes / req->ecc.nsectors;
 391
 392         for (strength = 0; strength < pmecc->caps->nstrengths; strength++) {
 393                 if (pmecc->caps->strengths[strength] == req->ecc.strength)
 394                         break;
 395         }
 396
 397         user->cache.cfg = PMECC_CFG_BCH_STRENGTH(strength) |
 398                           PMECC_CFG_NSECTORS(req->ecc.nsectors);
 399
 400         if (req->ecc.sectorsize == 1024)
 401                 user->cache.cfg |= PMECC_CFG_SECTOR1024;
 402
 403         user->cache.sarea = req->oobsize - 1;
 404         user->cache.saddr = req->ecc.ooboffset;
 405         user->cache.eaddr = req->ecc.ooboffset + req->ecc.bytes - 1;
 406
 407         return user;
 408 }
 409 EXPORT_SYMBOL_GPL(atmel_pmecc_create_user);
 410
 411 void atmel_pmecc_destroy_user(struct atmel_pmecc_user *user)
 412 {
 413         kfree(user);
 414 }
 415 EXPORT_SYMBOL_GPL(atmel_pmecc_destroy_user);
 416
 417 static int get_strength(struct atmel_pmecc_user *user)
 418 {
 419         const int *strengths = user->pmecc->caps->strengths;
 420
 421         return strengths[user->cache.cfg & PMECC_CFG_BCH_STRENGTH_MASK];
 422 }
 423
 424 static int get_sectorsize(struct atmel_pmecc_user *user)
 425 {
 426         return user->cache.cfg & PMECC_CFG_SECTOR1024 ? 1024 : 512;
 427 }
 428
 429 static void atmel_pmecc_gen_syndrome(struct atmel_pmecc_user *user, int sector)
 430 {
 431         int strength = get_strength(user);
 432         u32 value;
 433         int i;
 434
 435         /* Fill odd syndromes */
 436         for (i = 0; i < strength; i++) {
 437                 value = readl_relaxed(user->pmecc->regs.base +
 438                                       ATMEL_PMECC_REM(sector, i / 2));
 439                 if (i & 1)
 440                         value >>= 16;
 441
 442                 user->partial_syn[(2 * i) + 1] = value;
 443         }
 444 }
 445
 446 static void atmel_pmecc_substitute(struct atmel_pmecc_user *user)
 447 {
 448         int degree = get_sectorsize(user) == 512 ? 13 : 14;
 449         int cw_len = BIT(degree) - 1;
 450         int strength = get_strength(user);
 451         s16 *alpha_to = user->gf_tables->alpha_to;
 452         s16 *index_of = user->gf_tables->index_of;
 453         s16 *partial_syn = user->partial_syn;
 454         s16 *si;
 455         int i, j;
 456
 457         /*
 458          * si[] is a table that holds the current syndrome value,
 459          * an element of that table belongs to the field
 460          */
 461         si = user->si;
 462
 463         memset(&si[1], 0, sizeof(s16) * ((2 * strength) - 1));
 464
 465         /* Computation 2t syndromes based on S(x) */
 466         /* Odd syndromes */
 467         for (i = 1; i < 2 * strength; i += 2) {
 468                 for (j = 0; j < degree; j++) {
 469                         if (partial_syn[i] & BIT(j))
 470                                 si[i] = alpha_to[i * j] ^ si[i];
 471                 }
 472         }
 473         /* Even syndrome = (Odd syndrome) ** 2 */
 474         for (i = 2, j = 1; j <= strength; i = ++j << 1) {
 475                 if (si[j] == 0) {
 476                         si[i] = 0;
 477                 } else {
 478                         s16 tmp;
 479
 480                         tmp = index_of[si[j]];
 481                         tmp = (tmp * 2) % cw_len;
 482                         si[i] = alpha_to[tmp];
 483                 }
 484         }
 485 }
 486
 487 static void atmel_pmecc_get_sigma(struct atmel_pmecc_user *user)
 488 {
 489         s16 *lmu = user->lmu;
 490         s16 *si = user->si;
 491         s32 *mu = user->mu;
 492         s32 *dmu = user->dmu;
 493         s32 *delta = user->delta;
 494         int degree = get_sectorsize(user) == 512 ? 13 : 14;
 495         int cw_len = BIT(degree) - 1;
 496         int strength = get_strength(user);
 497         int num = 2 * strength + 1;
 498         s16 *index_of = user->gf_tables->index_of;
 499         s16 *alpha_to = user->gf_tables->alpha_to;
 500         int i, j, k;
 501         u32 dmu_0_count, tmp;
 502         s16 *smu = user->smu;
 503
 504         /* index of largest delta */
 505         int ro;
 506         int largest;
 507         int diff;
 508
 509         dmu_0_count = 0;
 510
 511         /* First Row */
 512
 513         /* Mu */
 514         mu[0] = -1;
 515
 516         memset(smu, 0, sizeof(s16) * num);
 517         smu[0] = 1;
 518
 519         /* discrepancy set to 1 */
 520         dmu[0] = 1;
 521         /* polynom order set to 0 */
 522         lmu[0] = 0;
 523         delta[0] = (mu[0] * 2 - lmu[0]) >> 1;
 524
 525         /* Second Row */
 526
 527         /* Mu */
 528         mu[1] = 0;
 529         /* Sigma(x) set to 1 */
 530         memset(&smu[num], 0, sizeof(s16) * num);
 531         smu[num] = 1;
 532
 533         /* discrepancy set to S1 */
 534         dmu[1] = si[1];
 535
 536         /* polynom order set to 0 */
 537         lmu[1] = 0;
 538
 539         delta[1] = (mu[1] * 2 - lmu[1]) >> 1;
 540
 541         /* Init the Sigma(x) last row */
 542         memset(&smu[(strength + 1) * num], 0, sizeof(s16) * num);
 543
 544         for (i = 1; i <= strength; i++) {
 545                 mu[i + 1] = i << 1;
 546                 /* Begin Computing Sigma (Mu+1) and L(mu) */
 547                 /* check if discrepancy is set to 0 */
 548                 if (dmu[i] == 0) {
 549                         dmu_0_count++;
 550
 551                         tmp = ((strength - (lmu[i] >> 1) - 1) / 2);
 552                         if ((strength - (lmu[i] >> 1) - 1) & 0x1)
 553                                 tmp += 2;
 554                         else
 555                                 tmp += 1;
 556
 557                         if (dmu_0_count == tmp) {
 558                                 for (j = 0; j <= (lmu[i] >> 1) + 1; j++)
 559                                         smu[(strength + 1) * num + j] =
 560                                                         smu[i * num + j];
 561
 562                                 lmu[strength + 1] = lmu[i];
 563                                 return;
 564                         }
 565
 566                         /* copy polynom */
 567                         for (j = 0; j <= lmu[i] >> 1; j++)
 568                                 smu[(i + 1) * num + j] = smu[i * num + j];
 569
 570                         /* copy previous polynom order to the next */
 571                         lmu[i + 1] = lmu[i];
 572                 } else {
 573                         ro = 0;
 574                         largest = -1;
 575                         /* find largest delta with dmu != 0 */
 576                         for (j = 0; j < i; j++) {
 577                                 if ((dmu[j]) && (delta[j] > largest)) {
 578                                         largest = delta[j];
 579                                         ro = j;
 580                                 }
 581                         }
 582
 583                         /* compute difference */
 584                         diff = (mu[i] - mu[ro]);
 585
 586                         /* Compute degree of the new smu polynomial */
 587                         if ((lmu[i] >> 1) > ((lmu[ro] >> 1) + diff))
 588                                 lmu[i + 1] = lmu[i];
 589                         else
 590                                 lmu[i + 1] = ((lmu[ro] >> 1) + diff) * 2;
 591
 592                         /* Init smu[i+1] with 0 */
 593                         for (k = 0; k < num; k++)
 594                                 smu[(i + 1) * num + k] = 0;
 595
 596                         /* Compute smu[i+1] */
 597                         for (k = 0; k <= lmu[ro] >> 1; k++) {
 598                                 s16 a, b, c;
 599
 600                                 if (!(smu[ro * num + k] && dmu[i]))
 601                                         continue;
 602
 603                                 a = index_of[dmu[i]];
 604                                 b = index_of[dmu[ro]];
 605                                 c = index_of[smu[ro * num + k]];
 606                                 tmp = a + (cw_len - b) + c;
 607                                 a = alpha_to[tmp % cw_len];
 608                                 smu[(i + 1) * num + (k + diff)] = a;
 609                         }
 610
 611                         for (k = 0; k <= lmu[i] >> 1; k++)
 612                                 smu[(i + 1) * num + k] ^= smu[i * num + k];
 613                 }
 614
 615                 /* End Computing Sigma (Mu+1) and L(mu) */
 616                 /* In either case compute delta */
 617                 delta[i + 1] = (mu[i + 1] * 2 - lmu[i + 1]) >> 1;
 618
 619                 /* Do not compute discrepancy for the last iteration */
 620                 if (i >= strength)
 621                         continue;
 622
 623                 for (k = 0; k <= (lmu[i + 1] >> 1); k++) {
 624                         tmp = 2 * (i - 1);
 625                         if (k == 0) {
 626                                 dmu[i + 1] = si[tmp + 3];
 627                         } else if (smu[(i + 1) * num + k] && si[tmp + 3 - k]) {
 628                                 s16 a, b, c;
 629
 630                                 a = index_of[smu[(i + 1) * num + k]];
 631                                 b = si[2 * (i - 1) + 3 - k];
 632                                 c = index_of[b];
 633                                 tmp = a + c;
 634                                 tmp %= cw_len;
 635                                 dmu[i + 1] = alpha_to[tmp] ^ dmu[i + 1];
 636                         }
 637                 }
 638         }
 639 }
 640
 641 static int atmel_pmecc_err_location(struct atmel_pmecc_user *user)
 642 {
 643         int sector_size = get_sectorsize(user);
 644         int degree = sector_size == 512 ? 13 : 14;
 645         struct atmel_pmecc *pmecc = user->pmecc;
 646         int strength = get_strength(user);
 647         int ret, roots_nbr, i, err_nbr = 0;
 648         int num = (2 * strength) + 1;
 649         s16 *smu = user->smu;
 650         u32 val;
 651
 652         writel(PMERRLOC_DISABLE, pmecc->regs.errloc + ATMEL_PMERRLOC_ELDIS);
 653
 654         for (i = 0; i <= user->lmu[strength + 1] >> 1; i++) {
 655                 writel_relaxed(smu[(strength + 1) * num + i],
 656                                pmecc->regs.errloc + ATMEL_PMERRLOC_SIGMA(i));
 657                 err_nbr++;
 658         }
 659
 660         val = (err_nbr - 1) << 16;
 661         if (sector_size == 1024)
 662                 val |= 1;
 663
 664         writel(val, pmecc->regs.errloc + ATMEL_PMERRLOC_ELCFG);
 665         writel((sector_size * 8) + (degree * strength),
 666                pmecc->regs.errloc + ATMEL_PMERRLOC_ELEN);
 667
 668         ret = readl_relaxed_poll_timeout(pmecc->regs.errloc +
 669                                          ATMEL_PMERRLOC_ELISR,
 670                                          val, val & PMERRLOC_CALC_DONE, 0,
 671                                          PMECC_MAX_TIMEOUT_MS * 1000);
 672         if (ret) {
 673                 dev_err(pmecc->dev,
 674                         "PMECC: Timeout to calculate error location.\n");
 675                 return ret;
 676         }
 677
 678         roots_nbr = (val & PMERRLOC_ERR_NUM_MASK) >> 8;
 679         /* Number of roots == degree of smu hence <= cap */
 680         if (roots_nbr == user->lmu[strength + 1] >> 1)
 681                 return err_nbr - 1;
 682
 683         /*
 684          * Number of roots does not match the degree of smu
 685          * unable to correct error.
 686          */
 687         return -EBADMSG;
 688 }
 689
 690 int atmel_pmecc_correct_sector(struct atmel_pmecc_user *user, int sector,
 691                                void *data, void *ecc)
 692 {
 693         struct atmel_pmecc *pmecc = user->pmecc;
 694         int sectorsize = get_sectorsize(user);
 695         int eccbytes = user->eccbytes;
 696         int i, nerrors;
 697
 698         if (!(user->isr & BIT(sector)))
 699                 return 0;
 700
 701         atmel_pmecc_gen_syndrome(user, sector);
 702         atmel_pmecc_substitute(user);
 703         atmel_pmecc_get_sigma(user);
 704
 705         nerrors = atmel_pmecc_err_location(user);
 706         if (nerrors < 0)
 707                 return nerrors;
 708
 709         for (i = 0; i < nerrors; i++) {
 710                 const char *area;
 711                 int byte, bit;
 712                 u32 errpos;
 713                 u8 *ptr;
 714
 715                 errpos = readl_relaxed(pmecc->regs.errloc +
 716                                 ATMEL_PMERRLOC_EL(pmecc->caps->el_offset, i));
 717                 errpos--;
 718
 719                 byte = errpos / 8;
 720                 bit = errpos % 8;
 721
 722                 if (byte < sectorsize) {
 723                         ptr = data + byte;
 724                         area = "data";
 725                 } else if (byte < sectorsize + eccbytes) {
 726                         ptr = ecc + byte - sectorsize;
 727                         area = "ECC";
 728                 } else {
 729                         dev_dbg(pmecc->dev,
 730                                 "Invalid errpos value (%d, max is %d)\n",
 731                                 errpos, (sectorsize + eccbytes) * 8);
 732                         return -EINVAL;
 733                 }
 734
 735                 dev_dbg(pmecc->dev,
 736                         "Bit flip in %s area, byte %d: 0x%02x -> 0x%02x\n",
 737                         area, byte, *ptr, (unsigned int)(*ptr ^ BIT(bit)));
 738
 739                 *ptr ^= BIT(bit);
 740         }
 741
 742         return nerrors;
 743 }
 744 EXPORT_SYMBOL_GPL(atmel_pmecc_correct_sector);
 745
 746 bool atmel_pmecc_correct_erased_chunks(struct atmel_pmecc_user *user)
 747 {
 748         return user->pmecc->caps->correct_erased_chunks;
 749 }
 750 EXPORT_SYMBOL_GPL(atmel_pmecc_correct_erased_chunks);
 751
 752 void atmel_pmecc_get_generated_eccbytes(struct atmel_pmecc_user *user,
 753                                         int sector, void *ecc)
 754 {
 755         struct atmel_pmecc *pmecc = user->pmecc;
 756         u8 *ptr = ecc;
 757         int i;
 758
 759         for (i = 0; i < user->eccbytes; i++)
 760                 ptr[i] = readb_relaxed(pmecc->regs.base +
 761                                        ATMEL_PMECC_ECC(sector, i));
 762 }
 763 EXPORT_SYMBOL_GPL(atmel_pmecc_get_generated_eccbytes);
 764
 765 void atmel_pmecc_reset(struct atmel_pmecc *pmecc)
 766 {
 767         writel(PMECC_CTRL_RST, pmecc->regs.base + ATMEL_PMECC_CTRL);
 768         writel(PMECC_CTRL_DISABLE, pmecc->regs.base + ATMEL_PMECC_CTRL);
 769 }
 770 EXPORT_SYMBOL_GPL(atmel_pmecc_reset);
 771
 772 int atmel_pmecc_enable(struct atmel_pmecc_user *user, int op)
 773 {
 774         struct atmel_pmecc *pmecc = user->pmecc;
 775         u32 cfg;
 776
 777         if (op != NAND_ECC_READ && op != NAND_ECC_WRITE) {
 778                 dev_err(pmecc->dev, "Bad ECC operation!");
 779                 return -EINVAL;
 780         }
 781
 782         mutex_lock(&user->pmecc->lock);
 783
 784         cfg = user->cache.cfg;
 785         if (op == NAND_ECC_WRITE)
 786                 cfg |= PMECC_CFG_WRITE_OP;
 787         else
 788                 cfg |= PMECC_CFG_AUTO_ENABLE;
 789
 790         writel(cfg, pmecc->regs.base + ATMEL_PMECC_CFG);
 791         writel(user->cache.sarea, pmecc->regs.base + ATMEL_PMECC_SAREA);
 792         writel(user->cache.saddr, pmecc->regs.base + ATMEL_PMECC_SADDR);
 793         writel(user->cache.eaddr, pmecc->regs.base + ATMEL_PMECC_EADDR);
 794
 795         writel(PMECC_CTRL_ENABLE, pmecc->regs.base + ATMEL_PMECC_CTRL);
 796         writel(PMECC_CTRL_DATA, pmecc->regs.base + ATMEL_PMECC_CTRL);
 797
 798         return 0;
 799 }
 800 EXPORT_SYMBOL_GPL(atmel_pmecc_enable);
 801
 802 void atmel_pmecc_disable(struct atmel_pmecc_user *user)
 803 {
 804         atmel_pmecc_reset(user->pmecc);
 805         mutex_unlock(&user->pmecc->lock);
 806 }
 807 EXPORT_SYMBOL_GPL(atmel_pmecc_disable);
 808
 809 int atmel_pmecc_wait_rdy(struct atmel_pmecc_user *user)
 810 {
 811         struct atmel_pmecc *pmecc = user->pmecc;
 812         u32 status;
 813         int ret;
 814
 815         ret = readl_relaxed_poll_timeout(pmecc->regs.base +
 816                                          ATMEL_PMECC_SR,
 817                                          status, !(status & PMECC_SR_BUSY), 0,
 818                                          PMECC_MAX_TIMEOUT_MS * 1000);
 819         if (ret) {
 820                 dev_err(pmecc->dev,
 821                         "Timeout while waiting for PMECC ready.\n");
 822                 return ret;
 823         }
 824
 825         user->isr = readl_relaxed(pmecc->regs.base + ATMEL_PMECC_ISR);
 826
 827         return 0;
 828 }
 829 EXPORT_SYMBOL_GPL(atmel_pmecc_wait_rdy);
 830
 831 static struct atmel_pmecc *atmel_pmecc_create(struct platform_device *pdev,
 832                                         const struct atmel_pmecc_caps *caps,
 833                                         int pmecc_res_idx, int errloc_res_idx)
 834 {
 835         struct device *dev = &pdev->dev;
 836         struct atmel_pmecc *pmecc;
 837         struct resource *res;
 838
 839         pmecc = devm_kzalloc(dev, sizeof(*pmecc), GFP_KERNEL);
 840         if (!pmecc)
 841                 return ERR_PTR(-ENOMEM);
 842
 843         pmecc->caps = caps;
 844         pmecc->dev = dev;
 845         mutex_init(&pmecc->lock);
 846
 847         res = platform_get_resource(pdev, IORESOURCE_MEM, pmecc_res_idx);
 848         pmecc->regs.base = devm_ioremap_resource(dev, res);
 849         if (IS_ERR(pmecc->regs.base))
 850                 return ERR_CAST(pmecc->regs.base);
 851
 852         res = platform_get_resource(pdev, IORESOURCE_MEM, errloc_res_idx);
 853         pmecc->regs.errloc = devm_ioremap_resource(dev, res);
 854         if (IS_ERR(pmecc->regs.errloc))
 855                 return ERR_CAST(pmecc->regs.errloc);
 856
 857         /* Disable all interrupts before registering the PMECC handler. */
 858         writel(0xffffffff, pmecc->regs.base + ATMEL_PMECC_IDR);
 859         atmel_pmecc_reset(pmecc);
 860
 861         return pmecc;
 862 }
 863
 864 static void devm_atmel_pmecc_put(struct device *dev, void *res)
 865 {
 866         struct atmel_pmecc **pmecc = res;
 867
 868         put_device((*pmecc)->dev);
 869 }
 870
 871 static struct atmel_pmecc *atmel_pmecc_get_by_node(struct device *userdev,
 872                                                    struct device_node *np)
 873 {
 874         struct platform_device *pdev;
 875         struct atmel_pmecc *pmecc, **ptr;
 876         int ret;
 877
 878         pdev = of_find_device_by_node(np);
 879         if (!pdev)
 880                 return ERR_PTR(-EPROBE_DEFER);
 881         pmecc = platform_get_drvdata(pdev);
 882         if (!pmecc) {
 883                 ret = -EPROBE_DEFER;
 884                 goto err_put_device;
 885         }
 886
 887         ptr = devres_alloc(devm_atmel_pmecc_put, sizeof(*ptr), GFP_KERNEL);
 888         if (!ptr) {
 889                 ret = -ENOMEM;
 890                 goto err_put_device;
 891         }
 892
 893         *ptr = pmecc;
 894
 895         devres_add(userdev, ptr);
 896
 897         return pmecc;
 898
 899 err_put_device:
 900         put_device(&pdev->dev);
 901         return ERR_PTR(ret);
 902 }
 903
 904 static const int atmel_pmecc_strengths[] = { 2, 4, 8, 12, 24, 32 };
 905
 906 static struct atmel_pmecc_caps at91sam9g45_caps = {
 907         .strengths = atmel_pmecc_strengths,
 908         .nstrengths = 5,
 909         .el_offset = 0x8c,
 910 };
 911
 912 static struct atmel_pmecc_caps sama5d4_caps = {
 913         .strengths = atmel_pmecc_strengths,
 914         .nstrengths = 5,
 915         .el_offset = 0x8c,
 916         .correct_erased_chunks = true,
 917 };
 918
 919 static struct atmel_pmecc_caps sama5d2_caps = {
 920         .strengths = atmel_pmecc_strengths,
 921         .nstrengths = 6,
 922         .el_offset = 0xac,
 923         .correct_erased_chunks = true,
 924 };
 925
 926 static const struct of_device_id atmel_pmecc_legacy_match[] = {
 927         { .compatible = "atmel,sama5d4-nand", &sama5d4_caps },
 928         { .compatible = "atmel,sama5d2-nand", &sama5d2_caps },
 929         { /* sentinel */ }
 930 };
 931
 932 struct atmel_pmecc *devm_atmel_pmecc_get(struct device *userdev)
 933 {
 934         struct atmel_pmecc *pmecc;
 935         struct device_node *np;
 936
 937         if (!userdev)
 938                 return ERR_PTR(-EINVAL);
 939
 940         if (!userdev->of_node)
 941                 return NULL;
 942
 943         np = of_parse_phandle(userdev->of_node, "ecc-engine", 0);
 944         if (np) {
 945                 pmecc = atmel_pmecc_get_by_node(userdev, np);
 946                 of_node_put(np);
 947         } else {
 948                 /*
 949                  * Support old DT bindings: in this case the PMECC iomem
 950                  * resources are directly defined in the user pdev at position
 951                  * 1 and 2. Extract all relevant information from there.
 952                  */
 953                 struct platform_device *pdev = to_platform_device(userdev);
 954                 const struct atmel_pmecc_caps *caps;
 955                 const struct of_device_id *match;
 956
 957                 /* No PMECC engine available. */
 958                 if (!of_property_read_bool(userdev->of_node,
 959                                            "atmel,has-pmecc"))
 960                         return NULL;
 961
 962                 caps = &at91sam9g45_caps;
 963
 964                 /* Find the caps associated to the NAND dev node. */
 965                 match = of_match_node(atmel_pmecc_legacy_match,
 966                                       userdev->of_node);
 967                 if (match && match->data)
 968                         caps = match->data;
 969
 970                 pmecc = atmel_pmecc_create(pdev, caps, 1, 2);
 971         }
 972
 973         return pmecc;
 974 }
 975 EXPORT_SYMBOL(devm_atmel_pmecc_get);
 976
 977 static const struct of_device_id atmel_pmecc_match[] = {
 978         { .compatible = "atmel,at91sam9g45-pmecc", &at91sam9g45_caps },
 979         { .compatible = "atmel,sama5d4-pmecc", &sama5d4_caps },
 980         { .compatible = "atmel,sama5d2-pmecc", &sama5d2_caps },
 981         { /* sentinel */ }
 982 };
 983 MODULE_DEVICE_TABLE(of, atmel_pmecc_match);
 984
 985 static int atmel_pmecc_probe(struct platform_device *pdev)
 986 {
 987         struct device *dev = &pdev->dev;
 988         const struct atmel_pmecc_caps *caps;
 989         struct atmel_pmecc *pmecc;
 990
 991         caps = of_device_get_match_data(&pdev->dev);
 992         if (!caps) {
 993                 dev_err(dev, "Invalid caps\n");
 994                 return -EINVAL;
 995         }
 996
 997         pmecc = atmel_pmecc_create(pdev, caps, 0, 1);
 998         if (IS_ERR(pmecc))
 999                 return PTR_ERR(pmecc);
1000
1001         platform_set_drvdata(pdev, pmecc);
1002
1003         return 0;
1004 }
1005
1006 static struct platform_driver atmel_pmecc_driver = {
1007         .driver = {
1008                 .name = "atmel-pmecc",
1009                 .of_match_table = of_match_ptr(atmel_pmecc_match),
1010         },
1011         .probe = atmel_pmecc_probe,
1012 };
1013 module_platform_driver(atmel_pmecc_driver);
1014
1015 MODULE_LICENSE("GPL");
1016 MODULE_AUTHOR("Boris Brezillon <boris.brezillon@free-electrons.com>");
1017 MODULE_DESCRIPTION("PMECC engine driver");
1018 MODULE_ALIAS("platform:atmel_pmecc");