?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 4.19.133
[linux/fpc-iii.git] / drivers / mtd / nand / raw / omap2.c
blob7a4af5f3e3d36e9aac47532a84f8e7b1cc15d35c
1 /*
2 * Copyright © 2004 Texas Instruments, Jian Zhang <jzhang@ti.com>
3 * Copyright © 2004 Micron Technology Inc.
4 * Copyright © 2004 David Brownell
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11 #include <linux/platform_device.h>
12 #include <linux/dmaengine.h>
13 #include <linux/dma-mapping.h>
14 #include <linux/delay.h>
15 #include <linux/gpio/consumer.h>
16 #include <linux/module.h>
17 #include <linux/interrupt.h>
18 #include <linux/jiffies.h>
19 #include <linux/sched.h>
20 #include <linux/mtd/mtd.h>
21 #include <linux/mtd/rawnand.h>
22 #include <linux/mtd/partitions.h>
23 #include <linux/omap-dma.h>
24 #include <linux/io.h>
25 #include <linux/slab.h>
26 #include <linux/of.h>
27 #include <linux/of_device.h>
28
29 #include <linux/mtd/nand_bch.h>
30 #include <linux/platform_data/elm.h>
31
32 #include <linux/omap-gpmc.h>
33 #include <linux/platform_data/mtd-nand-omap2.h>
34
35 #define DRIVER_NAME "omap2-nand"
36 #define OMAP_NAND_TIMEOUT_MS 5000
37
38 #define NAND_Ecc_P1e (1 << 0)
39 #define NAND_Ecc_P2e (1 << 1)
40 #define NAND_Ecc_P4e (1 << 2)
41 #define NAND_Ecc_P8e (1 << 3)
42 #define NAND_Ecc_P16e (1 << 4)
43 #define NAND_Ecc_P32e (1 << 5)
44 #define NAND_Ecc_P64e (1 << 6)
45 #define NAND_Ecc_P128e (1 << 7)
46 #define NAND_Ecc_P256e (1 << 8)
47 #define NAND_Ecc_P512e (1 << 9)
48 #define NAND_Ecc_P1024e (1 << 10)
49 #define NAND_Ecc_P2048e (1 << 11)
50
51 #define NAND_Ecc_P1o (1 << 16)
52 #define NAND_Ecc_P2o (1 << 17)
53 #define NAND_Ecc_P4o (1 << 18)
54 #define NAND_Ecc_P8o (1 << 19)
55 #define NAND_Ecc_P16o (1 << 20)
56 #define NAND_Ecc_P32o (1 << 21)
57 #define NAND_Ecc_P64o (1 << 22)
58 #define NAND_Ecc_P128o (1 << 23)
59 #define NAND_Ecc_P256o (1 << 24)
60 #define NAND_Ecc_P512o (1 << 25)
61 #define NAND_Ecc_P1024o (1 << 26)
62 #define NAND_Ecc_P2048o (1 << 27)
63
64 #define TF(value) (value ? 1 : 0)
65
66 #define P2048e(a) (TF(a & NAND_Ecc_P2048e) << 0)
67 #define P2048o(a) (TF(a & NAND_Ecc_P2048o) << 1)
68 #define P1e(a) (TF(a & NAND_Ecc_P1e) << 2)
69 #define P1o(a) (TF(a & NAND_Ecc_P1o) << 3)
70 #define P2e(a) (TF(a & NAND_Ecc_P2e) << 4)
71 #define P2o(a) (TF(a & NAND_Ecc_P2o) << 5)
72 #define P4e(a) (TF(a & NAND_Ecc_P4e) << 6)
73 #define P4o(a) (TF(a & NAND_Ecc_P4o) << 7)
74
75 #define P8e(a) (TF(a & NAND_Ecc_P8e) << 0)
76 #define P8o(a) (TF(a & NAND_Ecc_P8o) << 1)
77 #define P16e(a) (TF(a & NAND_Ecc_P16e) << 2)
78 #define P16o(a) (TF(a & NAND_Ecc_P16o) << 3)
79 #define P32e(a) (TF(a & NAND_Ecc_P32e) << 4)
80 #define P32o(a) (TF(a & NAND_Ecc_P32o) << 5)
81 #define P64e(a) (TF(a & NAND_Ecc_P64e) << 6)
82 #define P64o(a) (TF(a & NAND_Ecc_P64o) << 7)
83
84 #define P128e(a) (TF(a & NAND_Ecc_P128e) << 0)
85 #define P128o(a) (TF(a & NAND_Ecc_P128o) << 1)
86 #define P256e(a) (TF(a & NAND_Ecc_P256e) << 2)
87 #define P256o(a) (TF(a & NAND_Ecc_P256o) << 3)
88 #define P512e(a) (TF(a & NAND_Ecc_P512e) << 4)
89 #define P512o(a) (TF(a & NAND_Ecc_P512o) << 5)
90 #define P1024e(a) (TF(a & NAND_Ecc_P1024e) << 6)
91 #define P1024o(a) (TF(a & NAND_Ecc_P1024o) << 7)
92
93 #define P8e_s(a) (TF(a & NAND_Ecc_P8e) << 0)
94 #define P8o_s(a) (TF(a & NAND_Ecc_P8o) << 1)
95 #define P16e_s(a) (TF(a & NAND_Ecc_P16e) << 2)
96 #define P16o_s(a) (TF(a & NAND_Ecc_P16o) << 3)
97 #define P1e_s(a) (TF(a & NAND_Ecc_P1e) << 4)
98 #define P1o_s(a) (TF(a & NAND_Ecc_P1o) << 5)
99 #define P2e_s(a) (TF(a & NAND_Ecc_P2e) << 6)
100 #define P2o_s(a) (TF(a & NAND_Ecc_P2o) << 7)
101
102 #define P4e_s(a) (TF(a & NAND_Ecc_P4e) << 0)
103 #define P4o_s(a) (TF(a & NAND_Ecc_P4o) << 1)
104
105 #define PREFETCH_CONFIG1_CS_SHIFT 24
106 #define ECC_CONFIG_CS_SHIFT 1
107 #define CS_MASK 0x7
108 #define ENABLE_PREFETCH (0x1 << 7)
109 #define DMA_MPU_MODE_SHIFT 2
110 #define ECCSIZE0_SHIFT 12
111 #define ECCSIZE1_SHIFT 22
112 #define ECC1RESULTSIZE 0x1
113 #define ECCCLEAR 0x100
114 #define ECC1 0x1
115 #define PREFETCH_FIFOTHRESHOLD_MAX 0x40
116 #define PREFETCH_FIFOTHRESHOLD(val) ((val) << 8)
117 #define PREFETCH_STATUS_COUNT(val) (val & 0x00003fff)
118 #define PREFETCH_STATUS_FIFO_CNT(val) ((val >> 24) & 0x7F)
119 #define STATUS_BUFF_EMPTY 0x00000001
120
121 #define SECTOR_BYTES 512
122 /* 4 bit padding to make byte aligned, 56 = 52 + 4 */
123 #define BCH4_BIT_PAD 4
124
125 /* GPMC ecc engine settings for read */
126 #define BCH_WRAPMODE_1 1 /* BCH wrap mode 1 */
127 #define BCH8R_ECC_SIZE0 0x1a /* ecc_size0 = 26 */
128 #define BCH8R_ECC_SIZE1 0x2 /* ecc_size1 = 2 */
129 #define BCH4R_ECC_SIZE0 0xd /* ecc_size0 = 13 */
130 #define BCH4R_ECC_SIZE1 0x3 /* ecc_size1 = 3 */
131
132 /* GPMC ecc engine settings for write */
133 #define BCH_WRAPMODE_6 6 /* BCH wrap mode 6 */
134 #define BCH_ECC_SIZE0 0x0 /* ecc_size0 = 0, no oob protection */
135 #define BCH_ECC_SIZE1 0x20 /* ecc_size1 = 32 */
136
137 #define BADBLOCK_MARKER_LENGTH 2
138
139 static u_char bch16_vector[] = {0xf5, 0x24, 0x1c, 0xd0, 0x61, 0xb3, 0xf1, 0x55,
140 0x2e, 0x2c, 0x86, 0xa3, 0xed, 0x36, 0x1b, 0x78,
141 0x48, 0x76, 0xa9, 0x3b, 0x97, 0xd1, 0x7a, 0x93,
142 0x07, 0x0e};
143 static u_char bch8_vector[] = {0xf3, 0xdb, 0x14, 0x16, 0x8b, 0xd2, 0xbe, 0xcc,
144 0xac, 0x6b, 0xff, 0x99, 0x7b};
145 static u_char bch4_vector[] = {0x00, 0x6b, 0x31, 0xdd, 0x41, 0xbc, 0x10};
146
147 struct omap_nand_info {
148 struct nand_chip nand;
149 struct platform_device *pdev;
150
151 int gpmc_cs;
152 bool dev_ready;
153 enum nand_io xfer_type;
154 int devsize;
155 enum omap_ecc ecc_opt;
156 struct device_node *elm_of_node;
157
158 unsigned long phys_base;
159 struct completion comp;
160 struct dma_chan *dma;
161 int gpmc_irq_fifo;
162 int gpmc_irq_count;
163 enum {
164 OMAP_NAND_IO_READ = 0, /* read */
165 OMAP_NAND_IO_WRITE, /* write */
166 } iomode;
167 u_char *buf;
168 int buf_len;
169 /* Interface to GPMC */
170 struct gpmc_nand_regs reg;
171 struct gpmc_nand_ops *ops;
172 bool flash_bbt;
173 /* fields specific for BCHx_HW ECC scheme */
174 struct device *elm_dev;
175 /* NAND ready gpio */
176 struct gpio_desc *ready_gpiod;
177 };
178
179 static inline struct omap_nand_info *mtd_to_omap(struct mtd_info *mtd)
180 {
181 return container_of(mtd_to_nand(mtd), struct omap_nand_info, nand);
182 }
183
184 /**
185 * omap_prefetch_enable - configures and starts prefetch transfer
186 * @cs: cs (chip select) number
187 * @fifo_th: fifo threshold to be used for read/ write
188 * @dma_mode: dma mode enable (1) or disable (0)
189 * @u32_count: number of bytes to be transferred
190 * @is_write: prefetch read(0) or write post(1) mode
191 */
192 static int omap_prefetch_enable(int cs, int fifo_th, int dma_mode,
193 unsigned int u32_count, int is_write, struct omap_nand_info *info)
194 {
195 u32 val;
196
197 if (fifo_th > PREFETCH_FIFOTHRESHOLD_MAX)
198 return -1;
199
200 if (readl(info->reg.gpmc_prefetch_control))
201 return -EBUSY;
202
203 /* Set the amount of bytes to be prefetched */
204 writel(u32_count, info->reg.gpmc_prefetch_config2);
205
206 /* Set dma/mpu mode, the prefetch read / post write and
207 * enable the engine. Set which cs is has requested for.
208 */
209 val = ((cs << PREFETCH_CONFIG1_CS_SHIFT) |
210 PREFETCH_FIFOTHRESHOLD(fifo_th) | ENABLE_PREFETCH |
211 (dma_mode << DMA_MPU_MODE_SHIFT) | (is_write & 0x1));
212 writel(val, info->reg.gpmc_prefetch_config1);
213
214 /* Start the prefetch engine */
215 writel(0x1, info->reg.gpmc_prefetch_control);
216
217 return 0;
218 }
219
220 /**
221 * omap_prefetch_reset - disables and stops the prefetch engine
222 */
223 static int omap_prefetch_reset(int cs, struct omap_nand_info *info)
224 {
225 u32 config1;
226
227 /* check if the same module/cs is trying to reset */
228 config1 = readl(info->reg.gpmc_prefetch_config1);
229 if (((config1 >> PREFETCH_CONFIG1_CS_SHIFT) & CS_MASK) != cs)
230 return -EINVAL;
231
232 /* Stop the PFPW engine */
233 writel(0x0, info->reg.gpmc_prefetch_control);
234
235 /* Reset/disable the PFPW engine */
236 writel(0x0, info->reg.gpmc_prefetch_config1);
237
238 return 0;
239 }
240
241 /**
242 * omap_hwcontrol - hardware specific access to control-lines
243 * @mtd: MTD device structure
244 * @cmd: command to device
245 * @ctrl:
246 * NAND_NCE: bit 0 -> don't care
247 * NAND_CLE: bit 1 -> Command Latch
248 * NAND_ALE: bit 2 -> Address Latch
249 *
250 * NOTE: boards may use different bits for these!!
251 */
252 static void omap_hwcontrol(struct mtd_info *mtd, int cmd, unsigned int ctrl)
253 {
254 struct omap_nand_info *info = mtd_to_omap(mtd);
255
256 if (cmd != NAND_CMD_NONE) {
257 if (ctrl & NAND_CLE)
258 writeb(cmd, info->reg.gpmc_nand_command);
259
260 else if (ctrl & NAND_ALE)
261 writeb(cmd, info->reg.gpmc_nand_address);
262
263 else /* NAND_NCE */
264 writeb(cmd, info->reg.gpmc_nand_data);
265 }
266 }
267
268 /**
269 * omap_read_buf8 - read data from NAND controller into buffer
270 * @mtd: MTD device structure
271 * @buf: buffer to store date
272 * @len: number of bytes to read
273 */
274 static void omap_read_buf8(struct mtd_info *mtd, u_char *buf, int len)
275 {
276 struct nand_chip *nand = mtd_to_nand(mtd);
277
278 ioread8_rep(nand->IO_ADDR_R, buf, len);
279 }
280
281 /**
282 * omap_write_buf8 - write buffer to NAND controller
283 * @mtd: MTD device structure
284 * @buf: data buffer
285 * @len: number of bytes to write
286 */
287 static void omap_write_buf8(struct mtd_info *mtd, const u_char *buf, int len)
288 {
289 struct omap_nand_info *info = mtd_to_omap(mtd);
290 u_char *p = (u_char *)buf;
291 bool status;
292
293 while (len--) {
294 iowrite8(*p++, info->nand.IO_ADDR_W);
295 /* wait until buffer is available for write */
296 do {
297 status = info->ops->nand_writebuffer_empty();
298 } while (!status);
299 }
300 }
301
302 /**
303 * omap_read_buf16 - read data from NAND controller into buffer
304 * @mtd: MTD device structure
305 * @buf: buffer to store date
306 * @len: number of bytes to read
307 */
308 static void omap_read_buf16(struct mtd_info *mtd, u_char *buf, int len)
309 {
310 struct nand_chip *nand = mtd_to_nand(mtd);
311
312 ioread16_rep(nand->IO_ADDR_R, buf, len / 2);
313 }
314
315 /**
316 * omap_write_buf16 - write buffer to NAND controller
317 * @mtd: MTD device structure
318 * @buf: data buffer
319 * @len: number of bytes to write
320 */
321 static void omap_write_buf16(struct mtd_info *mtd, const u_char * buf, int len)
322 {
323 struct omap_nand_info *info = mtd_to_omap(mtd);
324 u16 *p = (u16 *) buf;
325 bool status;
326 /* FIXME try bursts of writesw() or DMA ... */
327 len >>= 1;
328
329 while (len--) {
330 iowrite16(*p++, info->nand.IO_ADDR_W);
331 /* wait until buffer is available for write */
332 do {
333 status = info->ops->nand_writebuffer_empty();
334 } while (!status);
335 }
336 }
337
338 /**
339 * omap_read_buf_pref - read data from NAND controller into buffer
340 * @mtd: MTD device structure
341 * @buf: buffer to store date
342 * @len: number of bytes to read
343 */
344 static void omap_read_buf_pref(struct mtd_info *mtd, u_char *buf, int len)
345 {
346 struct omap_nand_info *info = mtd_to_omap(mtd);
347 uint32_t r_count = 0;
348 int ret = 0;
349 u32 *p = (u32 *)buf;
350
351 /* take care of subpage reads */
352 if (len % 4) {
353 if (info->nand.options & NAND_BUSWIDTH_16)
354 omap_read_buf16(mtd, buf, len % 4);
355 else
356 omap_read_buf8(mtd, buf, len % 4);
357 p = (u32 *) (buf + len % 4);
358 len -= len % 4;
359 }
360
361 /* configure and start prefetch transfer */
362 ret = omap_prefetch_enable(info->gpmc_cs,
363 PREFETCH_FIFOTHRESHOLD_MAX, 0x0, len, 0x0, info);
364 if (ret) {
365 /* PFPW engine is busy, use cpu copy method */
366 if (info->nand.options & NAND_BUSWIDTH_16)
367 omap_read_buf16(mtd, (u_char *)p, len);
368 else
369 omap_read_buf8(mtd, (u_char *)p, len);
370 } else {
371 do {
372 r_count = readl(info->reg.gpmc_prefetch_status);
373 r_count = PREFETCH_STATUS_FIFO_CNT(r_count);
374 r_count = r_count >> 2;
375 ioread32_rep(info->nand.IO_ADDR_R, p, r_count);
376 p += r_count;
377 len -= r_count << 2;
378 } while (len);
379 /* disable and stop the PFPW engine */
380 omap_prefetch_reset(info->gpmc_cs, info);
381 }
382 }
383
384 /**
385 * omap_write_buf_pref - write buffer to NAND controller
386 * @mtd: MTD device structure
387 * @buf: data buffer
388 * @len: number of bytes to write
389 */
390 static void omap_write_buf_pref(struct mtd_info *mtd,
391 const u_char *buf, int len)
392 {
393 struct omap_nand_info *info = mtd_to_omap(mtd);
394 uint32_t w_count = 0;
395 int i = 0, ret = 0;
396 u16 *p = (u16 *)buf;
397 unsigned long tim, limit;
398 u32 val;
399
400 /* take care of subpage writes */
401 if (len % 2 != 0) {
402 writeb(*buf, info->nand.IO_ADDR_W);
403 p = (u16 *)(buf + 1);
404 len--;
405 }
406
407 /* configure and start prefetch transfer */
408 ret = omap_prefetch_enable(info->gpmc_cs,
409 PREFETCH_FIFOTHRESHOLD_MAX, 0x0, len, 0x1, info);
410 if (ret) {
411 /* PFPW engine is busy, use cpu copy method */
412 if (info->nand.options & NAND_BUSWIDTH_16)
413 omap_write_buf16(mtd, (u_char *)p, len);
414 else
415 omap_write_buf8(mtd, (u_char *)p, len);
416 } else {
417 while (len) {
418 w_count = readl(info->reg.gpmc_prefetch_status);
419 w_count = PREFETCH_STATUS_FIFO_CNT(w_count);
420 w_count = w_count >> 1;
421 for (i = 0; (i < w_count) && len; i++, len -= 2)
422 iowrite16(*p++, info->nand.IO_ADDR_W);
423 }
424 /* wait for data to flushed-out before reset the prefetch */
425 tim = 0;
426 limit = (loops_per_jiffy *
427 msecs_to_jiffies(OMAP_NAND_TIMEOUT_MS));
428 do {
429 cpu_relax();
430 val = readl(info->reg.gpmc_prefetch_status);
431 val = PREFETCH_STATUS_COUNT(val);
432 } while (val && (tim++ < limit));
433
434 /* disable and stop the PFPW engine */
435 omap_prefetch_reset(info->gpmc_cs, info);
436 }
437 }
438
439 /*
440 * omap_nand_dma_callback: callback on the completion of dma transfer
441 * @data: pointer to completion data structure
442 */
443 static void omap_nand_dma_callback(void *data)
444 {
445 complete((struct completion *) data);
446 }
447
448 /*
449 * omap_nand_dma_transfer: configure and start dma transfer
450 * @mtd: MTD device structure
451 * @addr: virtual address in RAM of source/destination
452 * @len: number of data bytes to be transferred
453 * @is_write: flag for read/write operation
454 */
455 static inline int omap_nand_dma_transfer(struct mtd_info *mtd, void *addr,
456 unsigned int len, int is_write)
457 {
458 struct omap_nand_info *info = mtd_to_omap(mtd);
459 struct dma_async_tx_descriptor *tx;
460 enum dma_data_direction dir = is_write ? DMA_TO_DEVICE :
461 DMA_FROM_DEVICE;
462 struct scatterlist sg;
463 unsigned long tim, limit;
464 unsigned n;
465 int ret;
466 u32 val;
467
468 if (!virt_addr_valid(addr))
469 goto out_copy;
470
471 sg_init_one(&sg, addr, len);
472 n = dma_map_sg(info->dma->device->dev, &sg, 1, dir);
473 if (n == 0) {
474 dev_err(&info->pdev->dev,
475 "Couldn't DMA map a %d byte buffer\n", len);
476 goto out_copy;
477 }
478
479 tx = dmaengine_prep_slave_sg(info->dma, &sg, n,
480 is_write ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM,
481 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
482 if (!tx)
483 goto out_copy_unmap;
484
485 tx->callback = omap_nand_dma_callback;
486 tx->callback_param = &info->comp;
487 dmaengine_submit(tx);
488
489 init_completion(&info->comp);
490
491 /* setup and start DMA using dma_addr */
492 dma_async_issue_pending(info->dma);
493
494 /* configure and start prefetch transfer */
495 ret = omap_prefetch_enable(info->gpmc_cs,
496 PREFETCH_FIFOTHRESHOLD_MAX, 0x1, len, is_write, info);
497 if (ret)
498 /* PFPW engine is busy, use cpu copy method */
499 goto out_copy_unmap;
500
501 wait_for_completion(&info->comp);
502 tim = 0;
503 limit = (loops_per_jiffy * msecs_to_jiffies(OMAP_NAND_TIMEOUT_MS));
504
505 do {
506 cpu_relax();
507 val = readl(info->reg.gpmc_prefetch_status);
508 val = PREFETCH_STATUS_COUNT(val);
509 } while (val && (tim++ < limit));
510
511 /* disable and stop the PFPW engine */
512 omap_prefetch_reset(info->gpmc_cs, info);
513
514 dma_unmap_sg(info->dma->device->dev, &sg, 1, dir);
515 return 0;
516
517 out_copy_unmap:
518 dma_unmap_sg(info->dma->device->dev, &sg, 1, dir);
519 out_copy:
520 if (info->nand.options & NAND_BUSWIDTH_16)
521 is_write == 0 ? omap_read_buf16(mtd, (u_char *) addr, len)
522 : omap_write_buf16(mtd, (u_char *) addr, len);
523 else
524 is_write == 0 ? omap_read_buf8(mtd, (u_char *) addr, len)
525 : omap_write_buf8(mtd, (u_char *) addr, len);
526 return 0;
527 }
528
529 /**
530 * omap_read_buf_dma_pref - read data from NAND controller into buffer
531 * @mtd: MTD device structure
532 * @buf: buffer to store date
533 * @len: number of bytes to read
534 */
535 static void omap_read_buf_dma_pref(struct mtd_info *mtd, u_char *buf, int len)
536 {
537 if (len <= mtd->oobsize)
538 omap_read_buf_pref(mtd, buf, len);
539 else
540 /* start transfer in DMA mode */
541 omap_nand_dma_transfer(mtd, buf, len, 0x0);
542 }
543
544 /**
545 * omap_write_buf_dma_pref - write buffer to NAND controller
546 * @mtd: MTD device structure
547 * @buf: data buffer
548 * @len: number of bytes to write
549 */
550 static void omap_write_buf_dma_pref(struct mtd_info *mtd,
551 const u_char *buf, int len)
552 {
553 if (len <= mtd->oobsize)
554 omap_write_buf_pref(mtd, buf, len);
555 else
556 /* start transfer in DMA mode */
557 omap_nand_dma_transfer(mtd, (u_char *) buf, len, 0x1);
558 }
559
560 /*
561 * omap_nand_irq - GPMC irq handler
562 * @this_irq: gpmc irq number
563 * @dev: omap_nand_info structure pointer is passed here
564 */
565 static irqreturn_t omap_nand_irq(int this_irq, void *dev)
566 {
567 struct omap_nand_info *info = (struct omap_nand_info *) dev;
568 u32 bytes;
569
570 bytes = readl(info->reg.gpmc_prefetch_status);
571 bytes = PREFETCH_STATUS_FIFO_CNT(bytes);
572 bytes = bytes & 0xFFFC; /* io in multiple of 4 bytes */
573 if (info->iomode == OMAP_NAND_IO_WRITE) { /* checks for write io */
574 if (this_irq == info->gpmc_irq_count)
575 goto done;
576
577 if (info->buf_len && (info->buf_len < bytes))
578 bytes = info->buf_len;
579 else if (!info->buf_len)
580 bytes = 0;
581 iowrite32_rep(info->nand.IO_ADDR_W,
582 (u32 *)info->buf, bytes >> 2);
583 info->buf = info->buf + bytes;
584 info->buf_len -= bytes;
585
586 } else {
587 ioread32_rep(info->nand.IO_ADDR_R,
588 (u32 *)info->buf, bytes >> 2);
589 info->buf = info->buf + bytes;
590
591 if (this_irq == info->gpmc_irq_count)
592 goto done;
593 }
594
595 return IRQ_HANDLED;
596
597 done:
598 complete(&info->comp);
599
600 disable_irq_nosync(info->gpmc_irq_fifo);
601 disable_irq_nosync(info->gpmc_irq_count);
602
603 return IRQ_HANDLED;
604 }
605
606 /*
607 * omap_read_buf_irq_pref - read data from NAND controller into buffer
608 * @mtd: MTD device structure
609 * @buf: buffer to store date
610 * @len: number of bytes to read
611 */
612 static void omap_read_buf_irq_pref(struct mtd_info *mtd, u_char *buf, int len)
613 {
614 struct omap_nand_info *info = mtd_to_omap(mtd);
615 int ret = 0;
616
617 if (len <= mtd->oobsize) {
618 omap_read_buf_pref(mtd, buf, len);
619 return;
620 }
621
622 info->iomode = OMAP_NAND_IO_READ;
623 info->buf = buf;
624 init_completion(&info->comp);
625
626 /* configure and start prefetch transfer */
627 ret = omap_prefetch_enable(info->gpmc_cs,
628 PREFETCH_FIFOTHRESHOLD_MAX/2, 0x0, len, 0x0, info);
629 if (ret)
630 /* PFPW engine is busy, use cpu copy method */
631 goto out_copy;
632
633 info->buf_len = len;
634
635 enable_irq(info->gpmc_irq_count);
636 enable_irq(info->gpmc_irq_fifo);
637
638 /* waiting for read to complete */
639 wait_for_completion(&info->comp);
640
641 /* disable and stop the PFPW engine */
642 omap_prefetch_reset(info->gpmc_cs, info);
643 return;
644
645 out_copy:
646 if (info->nand.options & NAND_BUSWIDTH_16)
647 omap_read_buf16(mtd, buf, len);
648 else
649 omap_read_buf8(mtd, buf, len);
650 }
651
652 /*
653 * omap_write_buf_irq_pref - write buffer to NAND controller
654 * @mtd: MTD device structure
655 * @buf: data buffer
656 * @len: number of bytes to write
657 */
658 static void omap_write_buf_irq_pref(struct mtd_info *mtd,
659 const u_char *buf, int len)
660 {
661 struct omap_nand_info *info = mtd_to_omap(mtd);
662 int ret = 0;
663 unsigned long tim, limit;
664 u32 val;
665
666 if (len <= mtd->oobsize) {
667 omap_write_buf_pref(mtd, buf, len);
668 return;
669 }
670
671 info->iomode = OMAP_NAND_IO_WRITE;
672 info->buf = (u_char *) buf;
673 init_completion(&info->comp);
674
675 /* configure and start prefetch transfer : size=24 */
676 ret = omap_prefetch_enable(info->gpmc_cs,
677 (PREFETCH_FIFOTHRESHOLD_MAX * 3) / 8, 0x0, len, 0x1, info);
678 if (ret)
679 /* PFPW engine is busy, use cpu copy method */
680 goto out_copy;
681
682 info->buf_len = len;
683
684 enable_irq(info->gpmc_irq_count);
685 enable_irq(info->gpmc_irq_fifo);
686
687 /* waiting for write to complete */
688 wait_for_completion(&info->comp);
689
690 /* wait for data to flushed-out before reset the prefetch */
691 tim = 0;
692 limit = (loops_per_jiffy * msecs_to_jiffies(OMAP_NAND_TIMEOUT_MS));
693 do {
694 val = readl(info->reg.gpmc_prefetch_status);
695 val = PREFETCH_STATUS_COUNT(val);
696 cpu_relax();
697 } while (val && (tim++ < limit));
698
699 /* disable and stop the PFPW engine */
700 omap_prefetch_reset(info->gpmc_cs, info);
701 return;
702
703 out_copy:
704 if (info->nand.options & NAND_BUSWIDTH_16)
705 omap_write_buf16(mtd, buf, len);
706 else
707 omap_write_buf8(mtd, buf, len);
708 }
709
710 /**
711 * gen_true_ecc - This function will generate true ECC value
712 * @ecc_buf: buffer to store ecc code
713 *
714 * This generated true ECC value can be used when correcting
715 * data read from NAND flash memory core
716 */
717 static void gen_true_ecc(u8 *ecc_buf)
718 {
719 u32 tmp = ecc_buf[0] | (ecc_buf[1] << 16) |
720 ((ecc_buf[2] & 0xF0) << 20) | ((ecc_buf[2] & 0x0F) << 8);
721
722 ecc_buf[0] = ~(P64o(tmp) | P64e(tmp) | P32o(tmp) | P32e(tmp) |
723 P16o(tmp) | P16e(tmp) | P8o(tmp) | P8e(tmp));
724 ecc_buf[1] = ~(P1024o(tmp) | P1024e(tmp) | P512o(tmp) | P512e(tmp) |
725 P256o(tmp) | P256e(tmp) | P128o(tmp) | P128e(tmp));
726 ecc_buf[2] = ~(P4o(tmp) | P4e(tmp) | P2o(tmp) | P2e(tmp) | P1o(tmp) |
727 P1e(tmp) | P2048o(tmp) | P2048e(tmp));
728 }
729
730 /**
731 * omap_compare_ecc - Detect (2 bits) and correct (1 bit) error in data
732 * @ecc_data1: ecc code from nand spare area
733 * @ecc_data2: ecc code from hardware register obtained from hardware ecc
734 * @page_data: page data
735 *
736 * This function compares two ECC's and indicates if there is an error.
737 * If the error can be corrected it will be corrected to the buffer.
738 * If there is no error, %0 is returned. If there is an error but it
739 * was corrected, %1 is returned. Otherwise, %-1 is returned.
740 */
741 static int omap_compare_ecc(u8 *ecc_data1, /* read from NAND memory */
742 u8 *ecc_data2, /* read from register */
743 u8 *page_data)
744 {
745 uint i;
746 u8 tmp0_bit[8], tmp1_bit[8], tmp2_bit[8];
747 u8 comp0_bit[8], comp1_bit[8], comp2_bit[8];
748 u8 ecc_bit[24];
749 u8 ecc_sum = 0;
750 u8 find_bit = 0;
751 uint find_byte = 0;
752 int isEccFF;
753
754 isEccFF = ((*(u32 *)ecc_data1 & 0xFFFFFF) == 0xFFFFFF);
755
756 gen_true_ecc(ecc_data1);
757 gen_true_ecc(ecc_data2);
758
759 for (i = 0; i <= 2; i++) {
760 *(ecc_data1 + i) = ~(*(ecc_data1 + i));
761 *(ecc_data2 + i) = ~(*(ecc_data2 + i));
762 }
763
764 for (i = 0; i < 8; i++) {
765 tmp0_bit[i] = *ecc_data1 % 2;
766 *ecc_data1 = *ecc_data1 / 2;
767 }
768
769 for (i = 0; i < 8; i++) {
770 tmp1_bit[i] = *(ecc_data1 + 1) % 2;
771 *(ecc_data1 + 1) = *(ecc_data1 + 1) / 2;
772 }
773
774 for (i = 0; i < 8; i++) {
775 tmp2_bit[i] = *(ecc_data1 + 2) % 2;
776 *(ecc_data1 + 2) = *(ecc_data1 + 2) / 2;
777 }
778
779 for (i = 0; i < 8; i++) {
780 comp0_bit[i] = *ecc_data2 % 2;
781 *ecc_data2 = *ecc_data2 / 2;
782 }
783
784 for (i = 0; i < 8; i++) {
785 comp1_bit[i] = *(ecc_data2 + 1) % 2;
786 *(ecc_data2 + 1) = *(ecc_data2 + 1) / 2;
787 }
788
789 for (i = 0; i < 8; i++) {
790 comp2_bit[i] = *(ecc_data2 + 2) % 2;
791 *(ecc_data2 + 2) = *(ecc_data2 + 2) / 2;
792 }
793
794 for (i = 0; i < 6; i++)
795 ecc_bit[i] = tmp2_bit[i + 2] ^ comp2_bit[i + 2];
796
797 for (i = 0; i < 8; i++)
798 ecc_bit[i + 6] = tmp0_bit[i] ^ comp0_bit[i];
799
800 for (i = 0; i < 8; i++)
801 ecc_bit[i + 14] = tmp1_bit[i] ^ comp1_bit[i];
802
803 ecc_bit[22] = tmp2_bit[0] ^ comp2_bit[0];
804 ecc_bit[23] = tmp2_bit[1] ^ comp2_bit[1];
805
806 for (i = 0; i < 24; i++)
807 ecc_sum += ecc_bit[i];
808
809 switch (ecc_sum) {
810 case 0:
811 /* Not reached because this function is not called if
812 * ECC values are equal
813 */
814 return 0;
815
816 case 1:
817 /* Uncorrectable error */
818 pr_debug("ECC UNCORRECTED_ERROR 1\n");
819 return -EBADMSG;
820
821 case 11:
822 /* UN-Correctable error */
823 pr_debug("ECC UNCORRECTED_ERROR B\n");
824 return -EBADMSG;
825
826 case 12:
827 /* Correctable error */
828 find_byte = (ecc_bit[23] << 8) +
829 (ecc_bit[21] << 7) +
830 (ecc_bit[19] << 6) +
831 (ecc_bit[17] << 5) +
832 (ecc_bit[15] << 4) +
833 (ecc_bit[13] << 3) +
834 (ecc_bit[11] << 2) +
835 (ecc_bit[9] << 1) +
836 ecc_bit[7];
837
838 find_bit = (ecc_bit[5] << 2) + (ecc_bit[3] << 1) + ecc_bit[1];
839
840 pr_debug("Correcting single bit ECC error at offset: "
841 "%d, bit: %d\n", find_byte, find_bit);
842
843 page_data[find_byte] ^= (1 << find_bit);
844
845 return 1;
846 default:
847 if (isEccFF) {
848 if (ecc_data2[0] == 0 &&
849 ecc_data2[1] == 0 &&
850 ecc_data2[2] == 0)
851 return 0;
852 }
853 pr_debug("UNCORRECTED_ERROR default\n");
854 return -EBADMSG;
855 }
856 }
857
858 /**
859 * omap_correct_data - Compares the ECC read with HW generated ECC
860 * @mtd: MTD device structure
861 * @dat: page data
862 * @read_ecc: ecc read from nand flash
863 * @calc_ecc: ecc read from HW ECC registers
864 *
865 * Compares the ecc read from nand spare area with ECC registers values
866 * and if ECC's mismatched, it will call 'omap_compare_ecc' for error
867 * detection and correction. If there are no errors, %0 is returned. If
868 * there were errors and all of the errors were corrected, the number of
869 * corrected errors is returned. If uncorrectable errors exist, %-1 is
870 * returned.
871 */
872 static int omap_correct_data(struct mtd_info *mtd, u_char *dat,
873 u_char *read_ecc, u_char *calc_ecc)
874 {
875 struct omap_nand_info *info = mtd_to_omap(mtd);
876 int blockCnt = 0, i = 0, ret = 0;
877 int stat = 0;
878
879 /* Ex NAND_ECC_HW12_2048 */
880 if ((info->nand.ecc.mode == NAND_ECC_HW) &&
881 (info->nand.ecc.size == 2048))
882 blockCnt = 4;
883 else
884 blockCnt = 1;
885
886 for (i = 0; i < blockCnt; i++) {
887 if (memcmp(read_ecc, calc_ecc, 3) != 0) {
888 ret = omap_compare_ecc(read_ecc, calc_ecc, dat);
889 if (ret < 0)
890 return ret;
891 /* keep track of the number of corrected errors */
892 stat += ret;
893 }
894 read_ecc += 3;
895 calc_ecc += 3;
896 dat += 512;
897 }
898 return stat;
899 }
900
901 /**
902 * omap_calcuate_ecc - Generate non-inverted ECC bytes.
903 * @mtd: MTD device structure
904 * @dat: The pointer to data on which ecc is computed
905 * @ecc_code: The ecc_code buffer
906 *
907 * Using noninverted ECC can be considered ugly since writing a blank
908 * page ie. padding will clear the ECC bytes. This is no problem as long
909 * nobody is trying to write data on the seemingly unused page. Reading
910 * an erased page will produce an ECC mismatch between generated and read
911 * ECC bytes that has to be dealt with separately.
912 */
913 static int omap_calculate_ecc(struct mtd_info *mtd, const u_char *dat,
914 u_char *ecc_code)
915 {
916 struct omap_nand_info *info = mtd_to_omap(mtd);
917 u32 val;
918
919 val = readl(info->reg.gpmc_ecc_config);
920 if (((val >> ECC_CONFIG_CS_SHIFT) & CS_MASK) != info->gpmc_cs)
921 return -EINVAL;
922
923 /* read ecc result */
924 val = readl(info->reg.gpmc_ecc1_result);
925 *ecc_code++ = val; /* P128e, ..., P1e */
926 *ecc_code++ = val >> 16; /* P128o, ..., P1o */
927 /* P2048o, P1024o, P512o, P256o, P2048e, P1024e, P512e, P256e */
928 *ecc_code++ = ((val >> 8) & 0x0f) | ((val >> 20) & 0xf0);
929
930 return 0;
931 }
932
933 /**
934 * omap_enable_hwecc - This function enables the hardware ecc functionality
935 * @mtd: MTD device structure
936 * @mode: Read/Write mode
937 */
938 static void omap_enable_hwecc(struct mtd_info *mtd, int mode)
939 {
940 struct omap_nand_info *info = mtd_to_omap(mtd);
941 struct nand_chip *chip = mtd_to_nand(mtd);
942 unsigned int dev_width = (chip->options & NAND_BUSWIDTH_16) ? 1 : 0;
943 u32 val;
944
945 /* clear ecc and enable bits */
946 val = ECCCLEAR | ECC1;
947 writel(val, info->reg.gpmc_ecc_control);
948
949 /* program ecc and result sizes */
950 val = ((((info->nand.ecc.size >> 1) - 1) << ECCSIZE1_SHIFT) |
951 ECC1RESULTSIZE);
952 writel(val, info->reg.gpmc_ecc_size_config);
953
954 switch (mode) {
955 case NAND_ECC_READ:
956 case NAND_ECC_WRITE:
957 writel(ECCCLEAR | ECC1, info->reg.gpmc_ecc_control);
958 break;
959 case NAND_ECC_READSYN:
960 writel(ECCCLEAR, info->reg.gpmc_ecc_control);
961 break;
962 default:
963 dev_info(&info->pdev->dev,
964 "error: unrecognized Mode[%d]!\n", mode);
965 break;
966 }
967
968 /* (ECC 16 or 8 bit col) | ( CS ) | ECC Enable */
969 val = (dev_width << 7) | (info->gpmc_cs << 1) | (0x1);
970 writel(val, info->reg.gpmc_ecc_config);
971 }
972
973 /**
974 * omap_wait - wait until the command is done
975 * @mtd: MTD device structure
976 * @chip: NAND Chip structure
977 *
978 * Wait function is called during Program and erase operations and
979 * the way it is called from MTD layer, we should wait till the NAND
980 * chip is ready after the programming/erase operation has completed.
981 *
982 * Erase can take up to 400ms and program up to 20ms according to
983 * general NAND and SmartMedia specs
984 */
985 static int omap_wait(struct mtd_info *mtd, struct nand_chip *chip)
986 {
987 struct nand_chip *this = mtd_to_nand(mtd);
988 struct omap_nand_info *info = mtd_to_omap(mtd);
989 unsigned long timeo = jiffies;
990 int status, state = this->state;
991
992 if (state == FL_ERASING)
993 timeo += msecs_to_jiffies(400);
994 else
995 timeo += msecs_to_jiffies(20);
996
997 writeb(NAND_CMD_STATUS & 0xFF, info->reg.gpmc_nand_command);
998 while (time_before(jiffies, timeo)) {
999 status = readb(info->reg.gpmc_nand_data);
1000 if (status & NAND_STATUS_READY)
1001 break;
1002 cond_resched();
1003 }
1004
1005 status = readb(info->reg.gpmc_nand_data);
1006 return status;
1007 }
1008
1009 /**
1010 * omap_dev_ready - checks the NAND Ready GPIO line
1011 * @mtd: MTD device structure
1012 *
1013 * Returns true if ready and false if busy.
1014 */
1015 static int omap_dev_ready(struct mtd_info *mtd)
1016 {
1017 struct omap_nand_info *info = mtd_to_omap(mtd);
1018
1019 return gpiod_get_value(info->ready_gpiod);
1020 }
1021
1022 /**
1023 * omap_enable_hwecc_bch - Program GPMC to perform BCH ECC calculation
1024 * @mtd: MTD device structure
1025 * @mode: Read/Write mode
1026 *
1027 * When using BCH with SW correction (i.e. no ELM), sector size is set
1028 * to 512 bytes and we use BCH_WRAPMODE_6 wrapping mode
1029 * for both reading and writing with:
1030 * eccsize0 = 0 (no additional protected byte in spare area)
1031 * eccsize1 = 32 (skip 32 nibbles = 16 bytes per sector in spare area)
1032 */
1033 static void __maybe_unused omap_enable_hwecc_bch(struct mtd_info *mtd, int mode)
1034 {
1035 unsigned int bch_type;
1036 unsigned int dev_width, nsectors;
1037 struct omap_nand_info *info = mtd_to_omap(mtd);
1038 enum omap_ecc ecc_opt = info->ecc_opt;
1039 struct nand_chip *chip = mtd_to_nand(mtd);
1040 u32 val, wr_mode;
1041 unsigned int ecc_size1, ecc_size0;
1042
1043 /* GPMC configurations for calculating ECC */
1044 switch (ecc_opt) {
1045 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW:
1046 bch_type = 0;
1047 nsectors = 1;
1048 wr_mode = BCH_WRAPMODE_6;
1049 ecc_size0 = BCH_ECC_SIZE0;
1050 ecc_size1 = BCH_ECC_SIZE1;
1051 break;
1052 case OMAP_ECC_BCH4_CODE_HW:
1053 bch_type = 0;
1054 nsectors = chip->ecc.steps;
1055 if (mode == NAND_ECC_READ) {
1056 wr_mode = BCH_WRAPMODE_1;
1057 ecc_size0 = BCH4R_ECC_SIZE0;
1058 ecc_size1 = BCH4R_ECC_SIZE1;
1059 } else {
1060 wr_mode = BCH_WRAPMODE_6;
1061 ecc_size0 = BCH_ECC_SIZE0;
1062 ecc_size1 = BCH_ECC_SIZE1;
1063 }
1064 break;
1065 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
1066 bch_type = 1;
1067 nsectors = 1;
1068 wr_mode = BCH_WRAPMODE_6;
1069 ecc_size0 = BCH_ECC_SIZE0;
1070 ecc_size1 = BCH_ECC_SIZE1;
1071 break;
1072 case OMAP_ECC_BCH8_CODE_HW:
1073 bch_type = 1;
1074 nsectors = chip->ecc.steps;
1075 if (mode == NAND_ECC_READ) {
1076 wr_mode = BCH_WRAPMODE_1;
1077 ecc_size0 = BCH8R_ECC_SIZE0;
1078 ecc_size1 = BCH8R_ECC_SIZE1;
1079 } else {
1080 wr_mode = BCH_WRAPMODE_6;
1081 ecc_size0 = BCH_ECC_SIZE0;
1082 ecc_size1 = BCH_ECC_SIZE1;
1083 }
1084 break;
1085 case OMAP_ECC_BCH16_CODE_HW:
1086 bch_type = 0x2;
1087 nsectors = chip->ecc.steps;
1088 if (mode == NAND_ECC_READ) {
1089 wr_mode = 0x01;
1090 ecc_size0 = 52; /* ECC bits in nibbles per sector */
1091 ecc_size1 = 0; /* non-ECC bits in nibbles per sector */
1092 } else {
1093 wr_mode = 0x01;
1094 ecc_size0 = 0; /* extra bits in nibbles per sector */
1095 ecc_size1 = 52; /* OOB bits in nibbles per sector */
1096 }
1097 break;
1098 default:
1099 return;
1100 }
1101
1102 writel(ECC1, info->reg.gpmc_ecc_control);
1103
1104 /* Configure ecc size for BCH */
1105 val = (ecc_size1 << ECCSIZE1_SHIFT) | (ecc_size0 << ECCSIZE0_SHIFT);
1106 writel(val, info->reg.gpmc_ecc_size_config);
1107
1108 dev_width = (chip->options & NAND_BUSWIDTH_16) ? 1 : 0;
1109
1110 /* BCH configuration */
1111 val = ((1 << 16) | /* enable BCH */
1112 (bch_type << 12) | /* BCH4/BCH8/BCH16 */
1113 (wr_mode << 8) | /* wrap mode */
1114 (dev_width << 7) | /* bus width */
1115 (((nsectors-1) & 0x7) << 4) | /* number of sectors */
1116 (info->gpmc_cs << 1) | /* ECC CS */
1117 (0x1)); /* enable ECC */
1118
1119 writel(val, info->reg.gpmc_ecc_config);
1120
1121 /* Clear ecc and enable bits */
1122 writel(ECCCLEAR | ECC1, info->reg.gpmc_ecc_control);
1123 }
1124
1125 static u8 bch4_polynomial[] = {0x28, 0x13, 0xcc, 0x39, 0x96, 0xac, 0x7f};
1126 static u8 bch8_polynomial[] = {0xef, 0x51, 0x2e, 0x09, 0xed, 0x93, 0x9a, 0xc2,
1127 0x97, 0x79, 0xe5, 0x24, 0xb5};
1128
1129 /**
1130 * _omap_calculate_ecc_bch - Generate ECC bytes for one sector
1131 * @mtd: MTD device structure
1132 * @dat: The pointer to data on which ecc is computed
1133 * @ecc_code: The ecc_code buffer
1134 * @i: The sector number (for a multi sector page)
1135 *
1136 * Support calculating of BCH4/8/16 ECC vectors for one sector
1137 * within a page. Sector number is in @i.
1138 */
1139 static int _omap_calculate_ecc_bch(struct mtd_info *mtd,
1140 const u_char *dat, u_char *ecc_calc, int i)
1141 {
1142 struct omap_nand_info *info = mtd_to_omap(mtd);
1143 int eccbytes = info->nand.ecc.bytes;
1144 struct gpmc_nand_regs *gpmc_regs = &info->reg;
1145 u8 *ecc_code;
1146 unsigned long bch_val1, bch_val2, bch_val3, bch_val4;
1147 u32 val;
1148 int j;
1149
1150 ecc_code = ecc_calc;
1151 switch (info->ecc_opt) {
1152 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
1153 case OMAP_ECC_BCH8_CODE_HW:
1154 bch_val1 = readl(gpmc_regs->gpmc_bch_result0[i]);
1155 bch_val2 = readl(gpmc_regs->gpmc_bch_result1[i]);
1156 bch_val3 = readl(gpmc_regs->gpmc_bch_result2[i]);
1157 bch_val4 = readl(gpmc_regs->gpmc_bch_result3[i]);
1158 *ecc_code++ = (bch_val4 & 0xFF);
1159 *ecc_code++ = ((bch_val3 >> 24) & 0xFF);
1160 *ecc_code++ = ((bch_val3 >> 16) & 0xFF);
1161 *ecc_code++ = ((bch_val3 >> 8) & 0xFF);
1162 *ecc_code++ = (bch_val3 & 0xFF);
1163 *ecc_code++ = ((bch_val2 >> 24) & 0xFF);
1164 *ecc_code++ = ((bch_val2 >> 16) & 0xFF);
1165 *ecc_code++ = ((bch_val2 >> 8) & 0xFF);
1166 *ecc_code++ = (bch_val2 & 0xFF);
1167 *ecc_code++ = ((bch_val1 >> 24) & 0xFF);
1168 *ecc_code++ = ((bch_val1 >> 16) & 0xFF);
1169 *ecc_code++ = ((bch_val1 >> 8) & 0xFF);
1170 *ecc_code++ = (bch_val1 & 0xFF);
1171 break;
1172 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW:
1173 case OMAP_ECC_BCH4_CODE_HW:
1174 bch_val1 = readl(gpmc_regs->gpmc_bch_result0[i]);
1175 bch_val2 = readl(gpmc_regs->gpmc_bch_result1[i]);
1176 *ecc_code++ = ((bch_val2 >> 12) & 0xFF);
1177 *ecc_code++ = ((bch_val2 >> 4) & 0xFF);
1178 *ecc_code++ = ((bch_val2 & 0xF) << 4) |
1179 ((bch_val1 >> 28) & 0xF);
1180 *ecc_code++ = ((bch_val1 >> 20) & 0xFF);
1181 *ecc_code++ = ((bch_val1 >> 12) & 0xFF);
1182 *ecc_code++ = ((bch_val1 >> 4) & 0xFF);
1183 *ecc_code++ = ((bch_val1 & 0xF) << 4);
1184 break;
1185 case OMAP_ECC_BCH16_CODE_HW:
1186 val = readl(gpmc_regs->gpmc_bch_result6[i]);
1187 ecc_code[0] = ((val >> 8) & 0xFF);
1188 ecc_code[1] = ((val >> 0) & 0xFF);
1189 val = readl(gpmc_regs->gpmc_bch_result5[i]);
1190 ecc_code[2] = ((val >> 24) & 0xFF);
1191 ecc_code[3] = ((val >> 16) & 0xFF);
1192 ecc_code[4] = ((val >> 8) & 0xFF);
1193 ecc_code[5] = ((val >> 0) & 0xFF);
1194 val = readl(gpmc_regs->gpmc_bch_result4[i]);
1195 ecc_code[6] = ((val >> 24) & 0xFF);
1196 ecc_code[7] = ((val >> 16) & 0xFF);
1197 ecc_code[8] = ((val >> 8) & 0xFF);
1198 ecc_code[9] = ((val >> 0) & 0xFF);
1199 val = readl(gpmc_regs->gpmc_bch_result3[i]);
1200 ecc_code[10] = ((val >> 24) & 0xFF);
1201 ecc_code[11] = ((val >> 16) & 0xFF);
1202 ecc_code[12] = ((val >> 8) & 0xFF);
1203 ecc_code[13] = ((val >> 0) & 0xFF);
1204 val = readl(gpmc_regs->gpmc_bch_result2[i]);
1205 ecc_code[14] = ((val >> 24) & 0xFF);
1206 ecc_code[15] = ((val >> 16) & 0xFF);
1207 ecc_code[16] = ((val >> 8) & 0xFF);
1208 ecc_code[17] = ((val >> 0) & 0xFF);
1209 val = readl(gpmc_regs->gpmc_bch_result1[i]);
1210 ecc_code[18] = ((val >> 24) & 0xFF);
1211 ecc_code[19] = ((val >> 16) & 0xFF);
1212 ecc_code[20] = ((val >> 8) & 0xFF);
1213 ecc_code[21] = ((val >> 0) & 0xFF);
1214 val = readl(gpmc_regs->gpmc_bch_result0[i]);
1215 ecc_code[22] = ((val >> 24) & 0xFF);
1216 ecc_code[23] = ((val >> 16) & 0xFF);
1217 ecc_code[24] = ((val >> 8) & 0xFF);
1218 ecc_code[25] = ((val >> 0) & 0xFF);
1219 break;
1220 default:
1221 return -EINVAL;
1222 }
1223
1224 /* ECC scheme specific syndrome customizations */
1225 switch (info->ecc_opt) {
1226 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW:
1227 /* Add constant polynomial to remainder, so that
1228 * ECC of blank pages results in 0x0 on reading back
1229 */
1230 for (j = 0; j < eccbytes; j++)
1231 ecc_calc[j] ^= bch4_polynomial[j];
1232 break;
1233 case OMAP_ECC_BCH4_CODE_HW:
1234 /* Set 8th ECC byte as 0x0 for ROM compatibility */
1235 ecc_calc[eccbytes - 1] = 0x0;
1236 break;
1237 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
1238 /* Add constant polynomial to remainder, so that
1239 * ECC of blank pages results in 0x0 on reading back
1240 */
1241 for (j = 0; j < eccbytes; j++)
1242 ecc_calc[j] ^= bch8_polynomial[j];
1243 break;
1244 case OMAP_ECC_BCH8_CODE_HW:
1245 /* Set 14th ECC byte as 0x0 for ROM compatibility */
1246 ecc_calc[eccbytes - 1] = 0x0;
1247 break;
1248 case OMAP_ECC_BCH16_CODE_HW:
1249 break;
1250 default:
1251 return -EINVAL;
1252 }
1253
1254 return 0;
1255 }
1256
1257 /**
1258 * omap_calculate_ecc_bch_sw - ECC generator for sector for SW based correction
1259 * @mtd: MTD device structure
1260 * @dat: The pointer to data on which ecc is computed
1261 * @ecc_code: The ecc_code buffer
1262 *
1263 * Support calculating of BCH4/8/16 ECC vectors for one sector. This is used
1264 * when SW based correction is required as ECC is required for one sector
1265 * at a time.
1266 */
1267 static int omap_calculate_ecc_bch_sw(struct mtd_info *mtd,
1268 const u_char *dat, u_char *ecc_calc)
1269 {
1270 return _omap_calculate_ecc_bch(mtd, dat, ecc_calc, 0);
1271 }
1272
1273 /**
1274 * omap_calculate_ecc_bch_multi - Generate ECC for multiple sectors
1275 * @mtd: MTD device structure
1276 * @dat: The pointer to data on which ecc is computed
1277 * @ecc_code: The ecc_code buffer
1278 *
1279 * Support calculating of BCH4/8/16 ecc vectors for the entire page in one go.
1280 */
1281 static int omap_calculate_ecc_bch_multi(struct mtd_info *mtd,
1282 const u_char *dat, u_char *ecc_calc)
1283 {
1284 struct omap_nand_info *info = mtd_to_omap(mtd);
1285 int eccbytes = info->nand.ecc.bytes;
1286 unsigned long nsectors;
1287 int i, ret;
1288
1289 nsectors = ((readl(info->reg.gpmc_ecc_config) >> 4) & 0x7) + 1;
1290 for (i = 0; i < nsectors; i++) {
1291 ret = _omap_calculate_ecc_bch(mtd, dat, ecc_calc, i);
1292 if (ret)
1293 return ret;
1294
1295 ecc_calc += eccbytes;
1296 }
1297
1298 return 0;
1299 }
1300
1301 /**
1302 * erased_sector_bitflips - count bit flips
1303 * @data: data sector buffer
1304 * @oob: oob buffer
1305 * @info: omap_nand_info
1306 *
1307 * Check the bit flips in erased page falls below correctable level.
1308 * If falls below, report the page as erased with correctable bit
1309 * flip, else report as uncorrectable page.
1310 */
1311 static int erased_sector_bitflips(u_char *data, u_char *oob,
1312 struct omap_nand_info *info)
1313 {
1314 int flip_bits = 0, i;
1315
1316 for (i = 0; i < info->nand.ecc.size; i++) {
1317 flip_bits += hweight8(~data[i]);
1318 if (flip_bits > info->nand.ecc.strength)
1319 return 0;
1320 }
1321
1322 for (i = 0; i < info->nand.ecc.bytes - 1; i++) {
1323 flip_bits += hweight8(~oob[i]);
1324 if (flip_bits > info->nand.ecc.strength)
1325 return 0;
1326 }
1327
1328 /*
1329 * Bit flips falls in correctable level.
1330 * Fill data area with 0xFF
1331 */
1332 if (flip_bits) {
1333 memset(data, 0xFF, info->nand.ecc.size);
1334 memset(oob, 0xFF, info->nand.ecc.bytes);
1335 }
1336
1337 return flip_bits;
1338 }
1339
1340 /**
1341 * omap_elm_correct_data - corrects page data area in case error reported
1342 * @mtd: MTD device structure
1343 * @data: page data
1344 * @read_ecc: ecc read from nand flash
1345 * @calc_ecc: ecc read from HW ECC registers
1346 *
1347 * Calculated ecc vector reported as zero in case of non-error pages.
1348 * In case of non-zero ecc vector, first filter out erased-pages, and
1349 * then process data via ELM to detect bit-flips.
1350 */
1351 static int omap_elm_correct_data(struct mtd_info *mtd, u_char *data,
1352 u_char *read_ecc, u_char *calc_ecc)
1353 {
1354 struct omap_nand_info *info = mtd_to_omap(mtd);
1355 struct nand_ecc_ctrl *ecc = &info->nand.ecc;
1356 int eccsteps = info->nand.ecc.steps;
1357 int i , j, stat = 0;
1358 int eccflag, actual_eccbytes;
1359 struct elm_errorvec err_vec[ERROR_VECTOR_MAX];
1360 u_char *ecc_vec = calc_ecc;
1361 u_char *spare_ecc = read_ecc;
1362 u_char *erased_ecc_vec;
1363 u_char *buf;
1364 int bitflip_count;
1365 bool is_error_reported = false;
1366 u32 bit_pos, byte_pos, error_max, pos;
1367 int err;
1368
1369 switch (info->ecc_opt) {
1370 case OMAP_ECC_BCH4_CODE_HW:
1371 /* omit 7th ECC byte reserved for ROM code compatibility */
1372 actual_eccbytes = ecc->bytes - 1;
1373 erased_ecc_vec = bch4_vector;
1374 break;
1375 case OMAP_ECC_BCH8_CODE_HW:
1376 /* omit 14th ECC byte reserved for ROM code compatibility */
1377 actual_eccbytes = ecc->bytes - 1;
1378 erased_ecc_vec = bch8_vector;
1379 break;
1380 case OMAP_ECC_BCH16_CODE_HW:
1381 actual_eccbytes = ecc->bytes;
1382 erased_ecc_vec = bch16_vector;
1383 break;
1384 default:
1385 dev_err(&info->pdev->dev, "invalid driver configuration\n");
1386 return -EINVAL;
1387 }
1388
1389 /* Initialize elm error vector to zero */
1390 memset(err_vec, 0, sizeof(err_vec));
1391
1392 for (i = 0; i < eccsteps ; i++) {
1393 eccflag = 0; /* initialize eccflag */
1394
1395 /*
1396 * Check any error reported,
1397 * In case of error, non zero ecc reported.
1398 */
1399 for (j = 0; j < actual_eccbytes; j++) {
1400 if (calc_ecc[j] != 0) {
1401 eccflag = 1; /* non zero ecc, error present */
1402 break;
1403 }
1404 }
1405
1406 if (eccflag == 1) {
1407 if (memcmp(calc_ecc, erased_ecc_vec,
1408 actual_eccbytes) == 0) {
1409 /*
1410 * calc_ecc[] matches pattern for ECC(all 0xff)
1411 * so this is definitely an erased-page
1412 */
1413 } else {
1414 buf = &data[info->nand.ecc.size * i];
1415 /*
1416 * count number of 0-bits in read_buf.
1417 * This check can be removed once a similar
1418 * check is introduced in generic NAND driver
1419 */
1420 bitflip_count = erased_sector_bitflips(
1421 buf, read_ecc, info);
1422 if (bitflip_count) {
1423 /*
1424 * number of 0-bits within ECC limits
1425 * So this may be an erased-page
1426 */
1427 stat += bitflip_count;
1428 } else {
1429 /*
1430 * Too many 0-bits. It may be a
1431 * - programmed-page, OR
1432 * - erased-page with many bit-flips
1433 * So this page requires check by ELM
1434 */
1435 err_vec[i].error_reported = true;
1436 is_error_reported = true;
1437 }
1438 }
1439 }
1440
1441 /* Update the ecc vector */
1442 calc_ecc += ecc->bytes;
1443 read_ecc += ecc->bytes;
1444 }
1445
1446 /* Check if any error reported */
1447 if (!is_error_reported)
1448 return stat;
1449
1450 /* Decode BCH error using ELM module */
1451 elm_decode_bch_error_page(info->elm_dev, ecc_vec, err_vec);
1452
1453 err = 0;
1454 for (i = 0; i < eccsteps; i++) {
1455 if (err_vec[i].error_uncorrectable) {
1456 dev_err(&info->pdev->dev,
1457 "uncorrectable bit-flips found\n");
1458 err = -EBADMSG;
1459 } else if (err_vec[i].error_reported) {
1460 for (j = 0; j < err_vec[i].error_count; j++) {
1461 switch (info->ecc_opt) {
1462 case OMAP_ECC_BCH4_CODE_HW:
1463 /* Add 4 bits to take care of padding */
1464 pos = err_vec[i].error_loc[j] +
1465 BCH4_BIT_PAD;
1466 break;
1467 case OMAP_ECC_BCH8_CODE_HW:
1468 case OMAP_ECC_BCH16_CODE_HW:
1469 pos = err_vec[i].error_loc[j];
1470 break;
1471 default:
1472 return -EINVAL;
1473 }
1474 error_max = (ecc->size + actual_eccbytes) * 8;
1475 /* Calculate bit position of error */
1476 bit_pos = pos % 8;
1477
1478 /* Calculate byte position of error */
1479 byte_pos = (error_max - pos - 1) / 8;
1480
1481 if (pos < error_max) {
1482 if (byte_pos < 512) {
1483 pr_debug("bitflip@dat[%d]=%x\n",
1484 byte_pos, data[byte_pos]);
1485 data[byte_pos] ^= 1 << bit_pos;
1486 } else {
1487 pr_debug("bitflip@oob[%d]=%x\n",
1488 (byte_pos - 512),
1489 spare_ecc[byte_pos - 512]);
1490 spare_ecc[byte_pos - 512] ^=
1491 1 << bit_pos;
1492 }
1493 } else {
1494 dev_err(&info->pdev->dev,
1495 "invalid bit-flip @ %d:%d\n",
1496 byte_pos, bit_pos);
1497 err = -EBADMSG;
1498 }
1499 }
1500 }
1501
1502 /* Update number of correctable errors */
1503 stat += err_vec[i].error_count;
1504
1505 /* Update page data with sector size */
1506 data += ecc->size;
1507 spare_ecc += ecc->bytes;
1508 }
1509
1510 return (err) ? err : stat;
1511 }
1512
1513 /**
1514 * omap_write_page_bch - BCH ecc based write page function for entire page
1515 * @mtd: mtd info structure
1516 * @chip: nand chip info structure
1517 * @buf: data buffer
1518 * @oob_required: must write chip->oob_poi to OOB
1519 * @page: page
1520 *
1521 * Custom write page method evolved to support multi sector writing in one shot
1522 */
1523 static int omap_write_page_bch(struct mtd_info *mtd, struct nand_chip *chip,
1524 const uint8_t *buf, int oob_required, int page)
1525 {
1526 int ret;
1527 uint8_t *ecc_calc = chip->ecc.calc_buf;
1528
1529 nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1530
1531 /* Enable GPMC ecc engine */
1532 chip->ecc.hwctl(mtd, NAND_ECC_WRITE);
1533
1534 /* Write data */
1535 chip->write_buf(mtd, buf, mtd->writesize);
1536
1537 /* Update ecc vector from GPMC result registers */
1538 omap_calculate_ecc_bch_multi(mtd, buf, &ecc_calc[0]);
1539
1540 ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0,
1541 chip->ecc.total);
1542 if (ret)
1543 return ret;
1544
1545 /* Write ecc vector to OOB area */
1546 chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
1547
1548 return nand_prog_page_end_op(chip);
1549 }
1550
1551 /**
1552 * omap_write_subpage_bch - BCH hardware ECC based subpage write
1553 * @mtd: mtd info structure
1554 * @chip: nand chip info structure
1555 * @offset: column address of subpage within the page
1556 * @data_len: data length
1557 * @buf: data buffer
1558 * @oob_required: must write chip->oob_poi to OOB
1559 * @page: page number to write
1560 *
1561 * OMAP optimized subpage write method.
1562 */
1563 static int omap_write_subpage_bch(struct mtd_info *mtd,
1564 struct nand_chip *chip, u32 offset,
1565 u32 data_len, const u8 *buf,
1566 int oob_required, int page)
1567 {
1568 u8 *ecc_calc = chip->ecc.calc_buf;
1569 int ecc_size = chip->ecc.size;
1570 int ecc_bytes = chip->ecc.bytes;
1571 int ecc_steps = chip->ecc.steps;
1572 u32 start_step = offset / ecc_size;
1573 u32 end_step = (offset + data_len - 1) / ecc_size;
1574 int step, ret = 0;
1575
1576 /*
1577 * Write entire page at one go as it would be optimal
1578 * as ECC is calculated by hardware.
1579 * ECC is calculated for all subpages but we choose
1580 * only what we want.
1581 */
1582 nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1583
1584 /* Enable GPMC ECC engine */
1585 chip->ecc.hwctl(mtd, NAND_ECC_WRITE);
1586
1587 /* Write data */
1588 chip->write_buf(mtd, buf, mtd->writesize);
1589
1590 for (step = 0; step < ecc_steps; step++) {
1591 /* mask ECC of un-touched subpages by padding 0xFF */
1592 if (step < start_step || step > end_step)
1593 memset(ecc_calc, 0xff, ecc_bytes);
1594 else
1595 ret = _omap_calculate_ecc_bch(mtd, buf, ecc_calc, step);
1596
1597 if (ret)
1598 return ret;
1599
1600 buf += ecc_size;
1601 ecc_calc += ecc_bytes;
1602 }
1603
1604 /* copy calculated ECC for whole page to chip->buffer->oob */
1605 /* this include masked-value(0xFF) for unwritten subpages */
1606 ecc_calc = chip->ecc.calc_buf;
1607 ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0,
1608 chip->ecc.total);
1609 if (ret)
1610 return ret;
1611
1612 /* write OOB buffer to NAND device */
1613 chip->write_buf(mtd, chip->oob_poi, mtd->oobsize);
1614
1615 return nand_prog_page_end_op(chip);
1616 }
1617
1618 /**
1619 * omap_read_page_bch - BCH ecc based page read function for entire page
1620 * @mtd: mtd info structure
1621 * @chip: nand chip info structure
1622 * @buf: buffer to store read data
1623 * @oob_required: caller requires OOB data read to chip->oob_poi
1624 * @page: page number to read
1625 *
1626 * For BCH ecc scheme, GPMC used for syndrome calculation and ELM module
1627 * used for error correction.
1628 * Custom method evolved to support ELM error correction & multi sector
1629 * reading. On reading page data area is read along with OOB data with
1630 * ecc engine enabled. ecc vector updated after read of OOB data.
1631 * For non error pages ecc vector reported as zero.
1632 */
1633 static int omap_read_page_bch(struct mtd_info *mtd, struct nand_chip *chip,
1634 uint8_t *buf, int oob_required, int page)
1635 {
1636 uint8_t *ecc_calc = chip->ecc.calc_buf;
1637 uint8_t *ecc_code = chip->ecc.code_buf;
1638 int stat, ret;
1639 unsigned int max_bitflips = 0;
1640
1641 nand_read_page_op(chip, page, 0, NULL, 0);
1642
1643 /* Enable GPMC ecc engine */
1644 chip->ecc.hwctl(mtd, NAND_ECC_READ);
1645
1646 /* Read data */
1647 chip->read_buf(mtd, buf, mtd->writesize);
1648
1649 /* Read oob bytes */
1650 nand_change_read_column_op(chip,
1651 mtd->writesize + BADBLOCK_MARKER_LENGTH,
1652 chip->oob_poi + BADBLOCK_MARKER_LENGTH,
1653 chip->ecc.total, false);
1654
1655 /* Calculate ecc bytes */
1656 omap_calculate_ecc_bch_multi(mtd, buf, ecc_calc);
1657
1658 ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0,
1659 chip->ecc.total);
1660 if (ret)
1661 return ret;
1662
1663 stat = chip->ecc.correct(mtd, buf, ecc_code, ecc_calc);
1664
1665 if (stat < 0) {
1666 mtd->ecc_stats.failed++;
1667 } else {
1668 mtd->ecc_stats.corrected += stat;
1669 max_bitflips = max_t(unsigned int, max_bitflips, stat);
1670 }
1671
1672 return max_bitflips;
1673 }
1674
1675 /**
1676 * is_elm_present - checks for presence of ELM module by scanning DT nodes
1677 * @omap_nand_info: NAND device structure containing platform data
1678 */
1679 static bool is_elm_present(struct omap_nand_info *info,
1680 struct device_node *elm_node)
1681 {
1682 struct platform_device *pdev;
1683
1684 /* check whether elm-id is passed via DT */
1685 if (!elm_node) {
1686 dev_err(&info->pdev->dev, "ELM devicetree node not found\n");
1687 return false;
1688 }
1689 pdev = of_find_device_by_node(elm_node);
1690 /* check whether ELM device is registered */
1691 if (!pdev) {
1692 dev_err(&info->pdev->dev, "ELM device not found\n");
1693 return false;
1694 }
1695 /* ELM module available, now configure it */
1696 info->elm_dev = &pdev->dev;
1697 return true;
1698 }
1699
1700 static bool omap2_nand_ecc_check(struct omap_nand_info *info)
1701 {
1702 bool ecc_needs_bch, ecc_needs_omap_bch, ecc_needs_elm;
1703
1704 switch (info->ecc_opt) {
1705 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW:
1706 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
1707 ecc_needs_omap_bch = false;
1708 ecc_needs_bch = true;
1709 ecc_needs_elm = false;
1710 break;
1711 case OMAP_ECC_BCH4_CODE_HW:
1712 case OMAP_ECC_BCH8_CODE_HW:
1713 case OMAP_ECC_BCH16_CODE_HW:
1714 ecc_needs_omap_bch = true;
1715 ecc_needs_bch = false;
1716 ecc_needs_elm = true;
1717 break;
1718 default:
1719 ecc_needs_omap_bch = false;
1720 ecc_needs_bch = false;
1721 ecc_needs_elm = false;
1722 break;
1723 }
1724
1725 if (ecc_needs_bch && !IS_ENABLED(CONFIG_MTD_NAND_ECC_BCH)) {
1726 dev_err(&info->pdev->dev,
1727 "CONFIG_MTD_NAND_ECC_BCH not enabled\n");
1728 return false;
1729 }
1730 if (ecc_needs_omap_bch && !IS_ENABLED(CONFIG_MTD_NAND_OMAP_BCH)) {
1731 dev_err(&info->pdev->dev,
1732 "CONFIG_MTD_NAND_OMAP_BCH not enabled\n");
1733 return false;
1734 }
1735 if (ecc_needs_elm && !is_elm_present(info, info->elm_of_node)) {
1736 dev_err(&info->pdev->dev, "ELM not available\n");
1737 return false;
1738 }
1739
1740 return true;
1741 }
1742
1743 static const char * const nand_xfer_types[] = {
1744 [NAND_OMAP_PREFETCH_POLLED] = "prefetch-polled",
1745 [NAND_OMAP_POLLED] = "polled",
1746 [NAND_OMAP_PREFETCH_DMA] = "prefetch-dma",
1747 [NAND_OMAP_PREFETCH_IRQ] = "prefetch-irq",
1748 };
1749
1750 static int omap_get_dt_info(struct device *dev, struct omap_nand_info *info)
1751 {
1752 struct device_node *child = dev->of_node;
1753 int i;
1754 const char *s;
1755 u32 cs;
1756
1757 if (of_property_read_u32(child, "reg", &cs) < 0) {
1758 dev_err(dev, "reg not found in DT\n");
1759 return -EINVAL;
1760 }
1761
1762 info->gpmc_cs = cs;
1763
1764 /* detect availability of ELM module. Won't be present pre-OMAP4 */
1765 info->elm_of_node = of_parse_phandle(child, "ti,elm-id", 0);
1766 if (!info->elm_of_node) {
1767 info->elm_of_node = of_parse_phandle(child, "elm_id", 0);
1768 if (!info->elm_of_node)
1769 dev_dbg(dev, "ti,elm-id not in DT\n");
1770 }
1771
1772 /* select ecc-scheme for NAND */
1773 if (of_property_read_string(child, "ti,nand-ecc-opt", &s)) {
1774 dev_err(dev, "ti,nand-ecc-opt not found\n");
1775 return -EINVAL;
1776 }
1777
1778 if (!strcmp(s, "sw")) {
1779 info->ecc_opt = OMAP_ECC_HAM1_CODE_SW;
1780 } else if (!strcmp(s, "ham1") ||
1781 !strcmp(s, "hw") || !strcmp(s, "hw-romcode")) {
1782 info->ecc_opt = OMAP_ECC_HAM1_CODE_HW;
1783 } else if (!strcmp(s, "bch4")) {
1784 if (info->elm_of_node)
1785 info->ecc_opt = OMAP_ECC_BCH4_CODE_HW;
1786 else
1787 info->ecc_opt = OMAP_ECC_BCH4_CODE_HW_DETECTION_SW;
1788 } else if (!strcmp(s, "bch8")) {
1789 if (info->elm_of_node)
1790 info->ecc_opt = OMAP_ECC_BCH8_CODE_HW;
1791 else
1792 info->ecc_opt = OMAP_ECC_BCH8_CODE_HW_DETECTION_SW;
1793 } else if (!strcmp(s, "bch16")) {
1794 info->ecc_opt = OMAP_ECC_BCH16_CODE_HW;
1795 } else {
1796 dev_err(dev, "unrecognized value for ti,nand-ecc-opt\n");
1797 return -EINVAL;
1798 }
1799
1800 /* select data transfer mode */
1801 if (!of_property_read_string(child, "ti,nand-xfer-type", &s)) {
1802 for (i = 0; i < ARRAY_SIZE(nand_xfer_types); i++) {
1803 if (!strcasecmp(s, nand_xfer_types[i])) {
1804 info->xfer_type = i;
1805 return 0;
1806 }
1807 }
1808
1809 dev_err(dev, "unrecognized value for ti,nand-xfer-type\n");
1810 return -EINVAL;
1811 }
1812
1813 return 0;
1814 }
1815
1816 static int omap_ooblayout_ecc(struct mtd_info *mtd, int section,
1817 struct mtd_oob_region *oobregion)
1818 {
1819 struct omap_nand_info *info = mtd_to_omap(mtd);
1820 struct nand_chip *chip = &info->nand;
1821 int off = BADBLOCK_MARKER_LENGTH;
1822
1823 if (info->ecc_opt == OMAP_ECC_HAM1_CODE_HW &&
1824 !(chip->options & NAND_BUSWIDTH_16))
1825 off = 1;
1826
1827 if (section)
1828 return -ERANGE;
1829
1830 oobregion->offset = off;
1831 oobregion->length = chip->ecc.total;
1832
1833 return 0;
1834 }
1835
1836 static int omap_ooblayout_free(struct mtd_info *mtd, int section,
1837 struct mtd_oob_region *oobregion)
1838 {
1839 struct omap_nand_info *info = mtd_to_omap(mtd);
1840 struct nand_chip *chip = &info->nand;
1841 int off = BADBLOCK_MARKER_LENGTH;
1842
1843 if (info->ecc_opt == OMAP_ECC_HAM1_CODE_HW &&
1844 !(chip->options & NAND_BUSWIDTH_16))
1845 off = 1;
1846
1847 if (section)
1848 return -ERANGE;
1849
1850 off += chip->ecc.total;
1851 if (off >= mtd->oobsize)
1852 return -ERANGE;
1853
1854 oobregion->offset = off;
1855 oobregion->length = mtd->oobsize - off;
1856
1857 return 0;
1858 }
1859
1860 static const struct mtd_ooblayout_ops omap_ooblayout_ops = {
1861 .ecc = omap_ooblayout_ecc,
1862 .free = omap_ooblayout_free,
1863 };
1864
1865 static int omap_sw_ooblayout_ecc(struct mtd_info *mtd, int section,
1866 struct mtd_oob_region *oobregion)
1867 {
1868 struct nand_chip *chip = mtd_to_nand(mtd);
1869 int off = BADBLOCK_MARKER_LENGTH;
1870
1871 if (section >= chip->ecc.steps)
1872 return -ERANGE;
1873
1874 /*
1875 * When SW correction is employed, one OMAP specific marker byte is
1876 * reserved after each ECC step.
1877 */
1878 oobregion->offset = off + (section * (chip->ecc.bytes + 1));
1879 oobregion->length = chip->ecc.bytes;
1880
1881 return 0;
1882 }
1883
1884 static int omap_sw_ooblayout_free(struct mtd_info *mtd, int section,
1885 struct mtd_oob_region *oobregion)
1886 {
1887 struct nand_chip *chip = mtd_to_nand(mtd);
1888 int off = BADBLOCK_MARKER_LENGTH;
1889
1890 if (section)
1891 return -ERANGE;
1892
1893 /*
1894 * When SW correction is employed, one OMAP specific marker byte is
1895 * reserved after each ECC step.
1896 */
1897 off += ((chip->ecc.bytes + 1) * chip->ecc.steps);
1898 if (off >= mtd->oobsize)
1899 return -ERANGE;
1900
1901 oobregion->offset = off;
1902 oobregion->length = mtd->oobsize - off;
1903
1904 return 0;
1905 }
1906
1907 static const struct mtd_ooblayout_ops omap_sw_ooblayout_ops = {
1908 .ecc = omap_sw_ooblayout_ecc,
1909 .free = omap_sw_ooblayout_free,
1910 };
1911
1912 static int omap_nand_attach_chip(struct nand_chip *chip)
1913 {
1914 struct mtd_info *mtd = nand_to_mtd(chip);
1915 struct omap_nand_info *info = mtd_to_omap(mtd);
1916 struct device *dev = &info->pdev->dev;
1917 int min_oobbytes = BADBLOCK_MARKER_LENGTH;
1918 int oobbytes_per_step;
1919 dma_cap_mask_t mask;
1920 int err;
1921
1922 if (chip->bbt_options & NAND_BBT_USE_FLASH)
1923 chip->bbt_options |= NAND_BBT_NO_OOB;
1924 else
1925 chip->options |= NAND_SKIP_BBTSCAN;
1926
1927 /* Re-populate low-level callbacks based on xfer modes */
1928 switch (info->xfer_type) {
1929 case NAND_OMAP_PREFETCH_POLLED:
1930 chip->read_buf = omap_read_buf_pref;
1931 chip->write_buf = omap_write_buf_pref;
1932 break;
1933
1934 case NAND_OMAP_POLLED:
1935 /* Use nand_base defaults for {read,write}_buf */
1936 break;
1937
1938 case NAND_OMAP_PREFETCH_DMA:
1939 dma_cap_zero(mask);
1940 dma_cap_set(DMA_SLAVE, mask);
1941 info->dma = dma_request_chan(dev->parent, "rxtx");
1942
1943 if (IS_ERR(info->dma)) {
1944 dev_err(dev, "DMA engine request failed\n");
1945 return PTR_ERR(info->dma);
1946 } else {
1947 struct dma_slave_config cfg;
1948
1949 memset(&cfg, 0, sizeof(cfg));
1950 cfg.src_addr = info->phys_base;
1951 cfg.dst_addr = info->phys_base;
1952 cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
1953 cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
1954 cfg.src_maxburst = 16;
1955 cfg.dst_maxburst = 16;
1956 err = dmaengine_slave_config(info->dma, &cfg);
1957 if (err) {
1958 dev_err(dev,
1959 "DMA engine slave config failed: %d\n",
1960 err);
1961 return err;
1962 }
1963 chip->read_buf = omap_read_buf_dma_pref;
1964 chip->write_buf = omap_write_buf_dma_pref;
1965 }
1966 break;
1967
1968 case NAND_OMAP_PREFETCH_IRQ:
1969 info->gpmc_irq_fifo = platform_get_irq(info->pdev, 0);
1970 if (info->gpmc_irq_fifo <= 0) {
1971 dev_err(dev, "Error getting fifo IRQ\n");
1972 return -ENODEV;
1973 }
1974 err = devm_request_irq(dev, info->gpmc_irq_fifo,
1975 omap_nand_irq, IRQF_SHARED,
1976 "gpmc-nand-fifo", info);
1977 if (err) {
1978 dev_err(dev, "Requesting IRQ %d, error %d\n",
1979 info->gpmc_irq_fifo, err);
1980 info->gpmc_irq_fifo = 0;
1981 return err;
1982 }
1983
1984 info->gpmc_irq_count = platform_get_irq(info->pdev, 1);
1985 if (info->gpmc_irq_count <= 0) {
1986 dev_err(dev, "Error getting IRQ count\n");
1987 return -ENODEV;
1988 }
1989 err = devm_request_irq(dev, info->gpmc_irq_count,
1990 omap_nand_irq, IRQF_SHARED,
1991 "gpmc-nand-count", info);
1992 if (err) {
1993 dev_err(dev, "Requesting IRQ %d, error %d\n",
1994 info->gpmc_irq_count, err);
1995 info->gpmc_irq_count = 0;
1996 return err;
1997 }
1998
1999 chip->read_buf = omap_read_buf_irq_pref;
2000 chip->write_buf = omap_write_buf_irq_pref;
2001
2002 break;
2003
2004 default:
2005 dev_err(dev, "xfer_type %d not supported!\n", info->xfer_type);
2006 return -EINVAL;
2007 }
2008
2009 if (!omap2_nand_ecc_check(info))
2010 return -EINVAL;
2011
2012 /*
2013 * Bail out earlier to let NAND_ECC_SOFT code create its own
2014 * ooblayout instead of using ours.
2015 */
2016 if (info->ecc_opt == OMAP_ECC_HAM1_CODE_SW) {
2017 chip->ecc.mode = NAND_ECC_SOFT;
2018 chip->ecc.algo = NAND_ECC_HAMMING;
2019 return 0;
2020 }
2021
2022 /* Populate MTD interface based on ECC scheme */
2023 switch (info->ecc_opt) {
2024 case OMAP_ECC_HAM1_CODE_HW:
2025 dev_info(dev, "nand: using OMAP_ECC_HAM1_CODE_HW\n");
2026 chip->ecc.mode = NAND_ECC_HW;
2027 chip->ecc.bytes = 3;
2028 chip->ecc.size = 512;
2029 chip->ecc.strength = 1;
2030 chip->ecc.calculate = omap_calculate_ecc;
2031 chip->ecc.hwctl = omap_enable_hwecc;
2032 chip->ecc.correct = omap_correct_data;
2033 mtd_set_ooblayout(mtd, &omap_ooblayout_ops);
2034 oobbytes_per_step = chip->ecc.bytes;
2035
2036 if (!(chip->options & NAND_BUSWIDTH_16))
2037 min_oobbytes = 1;
2038
2039 break;
2040
2041 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW:
2042 pr_info("nand: using OMAP_ECC_BCH4_CODE_HW_DETECTION_SW\n");
2043 chip->ecc.mode = NAND_ECC_HW;
2044 chip->ecc.size = 512;
2045 chip->ecc.bytes = 7;
2046 chip->ecc.strength = 4;
2047 chip->ecc.hwctl = omap_enable_hwecc_bch;
2048 chip->ecc.correct = nand_bch_correct_data;
2049 chip->ecc.calculate = omap_calculate_ecc_bch_sw;
2050 mtd_set_ooblayout(mtd, &omap_sw_ooblayout_ops);
2051 /* Reserve one byte for the OMAP marker */
2052 oobbytes_per_step = chip->ecc.bytes + 1;
2053 /* Software BCH library is used for locating errors */
2054 chip->ecc.priv = nand_bch_init(mtd);
2055 if (!chip->ecc.priv) {
2056 dev_err(dev, "Unable to use BCH library\n");
2057 return -EINVAL;
2058 }
2059 break;
2060
2061 case OMAP_ECC_BCH4_CODE_HW:
2062 pr_info("nand: using OMAP_ECC_BCH4_CODE_HW ECC scheme\n");
2063 chip->ecc.mode = NAND_ECC_HW;
2064 chip->ecc.size = 512;
2065 /* 14th bit is kept reserved for ROM-code compatibility */
2066 chip->ecc.bytes = 7 + 1;
2067 chip->ecc.strength = 4;
2068 chip->ecc.hwctl = omap_enable_hwecc_bch;
2069 chip->ecc.correct = omap_elm_correct_data;
2070 chip->ecc.read_page = omap_read_page_bch;
2071 chip->ecc.write_page = omap_write_page_bch;
2072 chip->ecc.write_subpage = omap_write_subpage_bch;
2073 mtd_set_ooblayout(mtd, &omap_ooblayout_ops);
2074 oobbytes_per_step = chip->ecc.bytes;
2075
2076 err = elm_config(info->elm_dev, BCH4_ECC,
2077 mtd->writesize / chip->ecc.size,
2078 chip->ecc.size, chip->ecc.bytes);
2079 if (err < 0)
2080 return err;
2081 break;
2082
2083 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
2084 pr_info("nand: using OMAP_ECC_BCH8_CODE_HW_DETECTION_SW\n");
2085 chip->ecc.mode = NAND_ECC_HW;
2086 chip->ecc.size = 512;
2087 chip->ecc.bytes = 13;
2088 chip->ecc.strength = 8;
2089 chip->ecc.hwctl = omap_enable_hwecc_bch;
2090 chip->ecc.correct = nand_bch_correct_data;
2091 chip->ecc.calculate = omap_calculate_ecc_bch_sw;
2092 mtd_set_ooblayout(mtd, &omap_sw_ooblayout_ops);
2093 /* Reserve one byte for the OMAP marker */
2094 oobbytes_per_step = chip->ecc.bytes + 1;
2095 /* Software BCH library is used for locating errors */
2096 chip->ecc.priv = nand_bch_init(mtd);
2097 if (!chip->ecc.priv) {
2098 dev_err(dev, "unable to use BCH library\n");
2099 return -EINVAL;
2100 }
2101 break;
2102
2103 case OMAP_ECC_BCH8_CODE_HW:
2104 pr_info("nand: using OMAP_ECC_BCH8_CODE_HW ECC scheme\n");
2105 chip->ecc.mode = NAND_ECC_HW;
2106 chip->ecc.size = 512;
2107 /* 14th bit is kept reserved for ROM-code compatibility */
2108 chip->ecc.bytes = 13 + 1;
2109 chip->ecc.strength = 8;
2110 chip->ecc.hwctl = omap_enable_hwecc_bch;
2111 chip->ecc.correct = omap_elm_correct_data;
2112 chip->ecc.read_page = omap_read_page_bch;
2113 chip->ecc.write_page = omap_write_page_bch;
2114 chip->ecc.write_subpage = omap_write_subpage_bch;
2115 mtd_set_ooblayout(mtd, &omap_ooblayout_ops);
2116 oobbytes_per_step = chip->ecc.bytes;
2117
2118 err = elm_config(info->elm_dev, BCH8_ECC,
2119 mtd->writesize / chip->ecc.size,
2120 chip->ecc.size, chip->ecc.bytes);
2121 if (err < 0)
2122 return err;
2123
2124 break;
2125
2126 case OMAP_ECC_BCH16_CODE_HW:
2127 pr_info("Using OMAP_ECC_BCH16_CODE_HW ECC scheme\n");
2128 chip->ecc.mode = NAND_ECC_HW;
2129 chip->ecc.size = 512;
2130 chip->ecc.bytes = 26;
2131 chip->ecc.strength = 16;
2132 chip->ecc.hwctl = omap_enable_hwecc_bch;
2133 chip->ecc.correct = omap_elm_correct_data;
2134 chip->ecc.read_page = omap_read_page_bch;
2135 chip->ecc.write_page = omap_write_page_bch;
2136 chip->ecc.write_subpage = omap_write_subpage_bch;
2137 mtd_set_ooblayout(mtd, &omap_ooblayout_ops);
2138 oobbytes_per_step = chip->ecc.bytes;
2139
2140 err = elm_config(info->elm_dev, BCH16_ECC,
2141 mtd->writesize / chip->ecc.size,
2142 chip->ecc.size, chip->ecc.bytes);
2143 if (err < 0)
2144 return err;
2145
2146 break;
2147 default:
2148 dev_err(dev, "Invalid or unsupported ECC scheme\n");
2149 return -EINVAL;
2150 }
2151
2152 /* Check if NAND device's OOB is enough to store ECC signatures */
2153 min_oobbytes += (oobbytes_per_step *
2154 (mtd->writesize / chip->ecc.size));
2155 if (mtd->oobsize < min_oobbytes) {
2156 dev_err(dev,
2157 "Not enough OOB bytes: required = %d, available=%d\n",
2158 min_oobbytes, mtd->oobsize);
2159 return -EINVAL;
2160 }
2161
2162 return 0;
2163 }
2164
2165 static const struct nand_controller_ops omap_nand_controller_ops = {
2166 .attach_chip = omap_nand_attach_chip,
2167 };
2168
2169 /* Shared among all NAND instances to synchronize access to the ECC Engine */
2170 static struct nand_controller omap_gpmc_controller = {
2171 .lock = __SPIN_LOCK_UNLOCKED(omap_gpmc_controller.lock),
2172 .wq = __WAIT_QUEUE_HEAD_INITIALIZER(omap_gpmc_controller.wq),
2173 .ops = &omap_nand_controller_ops,
2174 };
2175
2176 static int omap_nand_probe(struct platform_device *pdev)
2177 {
2178 struct omap_nand_info *info;
2179 struct mtd_info *mtd;
2180 struct nand_chip *nand_chip;
2181 int err;
2182 struct resource *res;
2183 struct device *dev = &pdev->dev;
2184
2185 info = devm_kzalloc(&pdev->dev, sizeof(struct omap_nand_info),
2186 GFP_KERNEL);
2187 if (!info)
2188 return -ENOMEM;
2189
2190 info->pdev = pdev;
2191
2192 err = omap_get_dt_info(dev, info);
2193 if (err)
2194 return err;
2195
2196 info->ops = gpmc_omap_get_nand_ops(&info->reg, info->gpmc_cs);
2197 if (!info->ops) {
2198 dev_err(&pdev->dev, "Failed to get GPMC->NAND interface\n");
2199 return -ENODEV;
2200 }
2201
2202 nand_chip = &info->nand;
2203 mtd = nand_to_mtd(nand_chip);
2204 mtd->dev.parent = &pdev->dev;
2205 nand_chip->ecc.priv = NULL;
2206 nand_set_flash_node(nand_chip, dev->of_node);
2207
2208 if (!mtd->name) {
2209 mtd->name = devm_kasprintf(&pdev->dev, GFP_KERNEL,
2210 "omap2-nand.%d", info->gpmc_cs);
2211 if (!mtd->name) {
2212 dev_err(&pdev->dev, "Failed to set MTD name\n");
2213 return -ENOMEM;
2214 }
2215 }
2216
2217 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2218 nand_chip->IO_ADDR_R = devm_ioremap_resource(&pdev->dev, res);
2219 if (IS_ERR(nand_chip->IO_ADDR_R))
2220 return PTR_ERR(nand_chip->IO_ADDR_R);
2221
2222 info->phys_base = res->start;
2223
2224 nand_chip->controller = &omap_gpmc_controller;
2225
2226 nand_chip->IO_ADDR_W = nand_chip->IO_ADDR_R;
2227 nand_chip->cmd_ctrl = omap_hwcontrol;
2228
2229 info->ready_gpiod = devm_gpiod_get_optional(&pdev->dev, "rb",
2230 GPIOD_IN);
2231 if (IS_ERR(info->ready_gpiod)) {
2232 dev_err(dev, "failed to get ready gpio\n");
2233 return PTR_ERR(info->ready_gpiod);
2234 }
2235
2236 /*
2237 * If RDY/BSY line is connected to OMAP then use the omap ready
2238 * function and the generic nand_wait function which reads the status
2239 * register after monitoring the RDY/BSY line. Otherwise use a standard
2240 * chip delay which is slightly more than tR (AC Timing) of the NAND
2241 * device and read status register until you get a failure or success
2242 */
2243 if (info->ready_gpiod) {
2244 nand_chip->dev_ready = omap_dev_ready;
2245 nand_chip->chip_delay = 0;
2246 } else {
2247 nand_chip->waitfunc = omap_wait;
2248 nand_chip->chip_delay = 50;
2249 }
2250
2251 if (info->flash_bbt)
2252 nand_chip->bbt_options |= NAND_BBT_USE_FLASH;
2253
2254 /* scan NAND device connected to chip controller */
2255 nand_chip->options |= info->devsize & NAND_BUSWIDTH_16;
2256
2257 err = nand_scan(nand_chip, 1);
2258 if (err)
2259 goto return_error;
2260
2261 err = mtd_device_register(mtd, NULL, 0);
2262 if (err)
2263 goto cleanup_nand;
2264
2265 platform_set_drvdata(pdev, mtd);
2266
2267 return 0;
2268
2269 cleanup_nand:
2270 nand_cleanup(nand_chip);
2271
2272 return_error:
2273 if (!IS_ERR_OR_NULL(info->dma))
2274 dma_release_channel(info->dma);
2275 if (nand_chip->ecc.priv) {
2276 nand_bch_free(nand_chip->ecc.priv);
2277 nand_chip->ecc.priv = NULL;
2278 }
2279 return err;
2280 }
2281
2282 static int omap_nand_remove(struct platform_device *pdev)
2283 {
2284 struct mtd_info *mtd = platform_get_drvdata(pdev);
2285 struct nand_chip *nand_chip = mtd_to_nand(mtd);
2286 struct omap_nand_info *info = mtd_to_omap(mtd);
2287 if (nand_chip->ecc.priv) {
2288 nand_bch_free(nand_chip->ecc.priv);
2289 nand_chip->ecc.priv = NULL;
2290 }
2291 if (info->dma)
2292 dma_release_channel(info->dma);
2293 nand_release(nand_chip);
2294 return 0;
2295 }
2296
2297 static const struct of_device_id omap_nand_ids[] = {
2298 { .compatible = "ti,omap2-nand", },
2299 {},
2300 };
2301 MODULE_DEVICE_TABLE(of, omap_nand_ids);
2302
2303 static struct platform_driver omap_nand_driver = {
2304 .probe = omap_nand_probe,
2305 .remove = omap_nand_remove,
2306 .driver = {
2307 .name = DRIVER_NAME,
2308 .of_match_table = of_match_ptr(omap_nand_ids),
2309 },
2310 };
2311
2312 module_platform_driver(omap_nand_driver);
2313
2314 MODULE_ALIAS("platform:" DRIVER_NAME);
2315 MODULE_LICENSE("GPL");
2316 MODULE_DESCRIPTION("Glue layer for NAND flash on TI OMAP boards");
Linux with added FPC-III support