?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
treewide: remove redundant IS_ERR() before error code check
[linux/fpc-iii.git] / drivers / mtd / nand / raw / omap2.c
blobad77c112a78a51cf761bee00c0fdcf717b5815a4
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright © 2004 Texas Instruments, Jian Zhang <jzhang@ti.com>
4 * Copyright © 2004 Micron Technology Inc.
5 * Copyright © 2004 David Brownell
6 */
7
8 #include <linux/platform_device.h>
9 #include <linux/dmaengine.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/delay.h>
12 #include <linux/gpio/consumer.h>
13 #include <linux/module.h>
14 #include <linux/interrupt.h>
15 #include <linux/jiffies.h>
16 #include <linux/sched.h>
17 #include <linux/mtd/mtd.h>
18 #include <linux/mtd/rawnand.h>
19 #include <linux/mtd/partitions.h>
20 #include <linux/omap-dma.h>
21 #include <linux/io.h>
22 #include <linux/slab.h>
23 #include <linux/of.h>
24 #include <linux/of_device.h>
25
26 #include <linux/mtd/nand_bch.h>
27 #include <linux/platform_data/elm.h>
28
29 #include <linux/omap-gpmc.h>
30 #include <linux/platform_data/mtd-nand-omap2.h>
31
32 #define DRIVER_NAME "omap2-nand"
33 #define OMAP_NAND_TIMEOUT_MS 5000
34
35 #define NAND_Ecc_P1e (1 << 0)
36 #define NAND_Ecc_P2e (1 << 1)
37 #define NAND_Ecc_P4e (1 << 2)
38 #define NAND_Ecc_P8e (1 << 3)
39 #define NAND_Ecc_P16e (1 << 4)
40 #define NAND_Ecc_P32e (1 << 5)
41 #define NAND_Ecc_P64e (1 << 6)
42 #define NAND_Ecc_P128e (1 << 7)
43 #define NAND_Ecc_P256e (1 << 8)
44 #define NAND_Ecc_P512e (1 << 9)
45 #define NAND_Ecc_P1024e (1 << 10)
46 #define NAND_Ecc_P2048e (1 << 11)
47
48 #define NAND_Ecc_P1o (1 << 16)
49 #define NAND_Ecc_P2o (1 << 17)
50 #define NAND_Ecc_P4o (1 << 18)
51 #define NAND_Ecc_P8o (1 << 19)
52 #define NAND_Ecc_P16o (1 << 20)
53 #define NAND_Ecc_P32o (1 << 21)
54 #define NAND_Ecc_P64o (1 << 22)
55 #define NAND_Ecc_P128o (1 << 23)
56 #define NAND_Ecc_P256o (1 << 24)
57 #define NAND_Ecc_P512o (1 << 25)
58 #define NAND_Ecc_P1024o (1 << 26)
59 #define NAND_Ecc_P2048o (1 << 27)
60
61 #define TF(value) (value ? 1 : 0)
62
63 #define P2048e(a) (TF(a & NAND_Ecc_P2048e) << 0)
64 #define P2048o(a) (TF(a & NAND_Ecc_P2048o) << 1)
65 #define P1e(a) (TF(a & NAND_Ecc_P1e) << 2)
66 #define P1o(a) (TF(a & NAND_Ecc_P1o) << 3)
67 #define P2e(a) (TF(a & NAND_Ecc_P2e) << 4)
68 #define P2o(a) (TF(a & NAND_Ecc_P2o) << 5)
69 #define P4e(a) (TF(a & NAND_Ecc_P4e) << 6)
70 #define P4o(a) (TF(a & NAND_Ecc_P4o) << 7)
71
72 #define P8e(a) (TF(a & NAND_Ecc_P8e) << 0)
73 #define P8o(a) (TF(a & NAND_Ecc_P8o) << 1)
74 #define P16e(a) (TF(a & NAND_Ecc_P16e) << 2)
75 #define P16o(a) (TF(a & NAND_Ecc_P16o) << 3)
76 #define P32e(a) (TF(a & NAND_Ecc_P32e) << 4)
77 #define P32o(a) (TF(a & NAND_Ecc_P32o) << 5)
78 #define P64e(a) (TF(a & NAND_Ecc_P64e) << 6)
79 #define P64o(a) (TF(a & NAND_Ecc_P64o) << 7)
80
81 #define P128e(a) (TF(a & NAND_Ecc_P128e) << 0)
82 #define P128o(a) (TF(a & NAND_Ecc_P128o) << 1)
83 #define P256e(a) (TF(a & NAND_Ecc_P256e) << 2)
84 #define P256o(a) (TF(a & NAND_Ecc_P256o) << 3)
85 #define P512e(a) (TF(a & NAND_Ecc_P512e) << 4)
86 #define P512o(a) (TF(a & NAND_Ecc_P512o) << 5)
87 #define P1024e(a) (TF(a & NAND_Ecc_P1024e) << 6)
88 #define P1024o(a) (TF(a & NAND_Ecc_P1024o) << 7)
89
90 #define P8e_s(a) (TF(a & NAND_Ecc_P8e) << 0)
91 #define P8o_s(a) (TF(a & NAND_Ecc_P8o) << 1)
92 #define P16e_s(a) (TF(a & NAND_Ecc_P16e) << 2)
93 #define P16o_s(a) (TF(a & NAND_Ecc_P16o) << 3)
94 #define P1e_s(a) (TF(a & NAND_Ecc_P1e) << 4)
95 #define P1o_s(a) (TF(a & NAND_Ecc_P1o) << 5)
96 #define P2e_s(a) (TF(a & NAND_Ecc_P2e) << 6)
97 #define P2o_s(a) (TF(a & NAND_Ecc_P2o) << 7)
98
99 #define P4e_s(a) (TF(a & NAND_Ecc_P4e) << 0)
100 #define P4o_s(a) (TF(a & NAND_Ecc_P4o) << 1)
101
102 #define PREFETCH_CONFIG1_CS_SHIFT 24
103 #define ECC_CONFIG_CS_SHIFT 1
104 #define CS_MASK 0x7
105 #define ENABLE_PREFETCH (0x1 << 7)
106 #define DMA_MPU_MODE_SHIFT 2
107 #define ECCSIZE0_SHIFT 12
108 #define ECCSIZE1_SHIFT 22
109 #define ECC1RESULTSIZE 0x1
110 #define ECCCLEAR 0x100
111 #define ECC1 0x1
112 #define PREFETCH_FIFOTHRESHOLD_MAX 0x40
113 #define PREFETCH_FIFOTHRESHOLD(val) ((val) << 8)
114 #define PREFETCH_STATUS_COUNT(val) (val & 0x00003fff)
115 #define PREFETCH_STATUS_FIFO_CNT(val) ((val >> 24) & 0x7F)
116 #define STATUS_BUFF_EMPTY 0x00000001
117
118 #define SECTOR_BYTES 512
119 /* 4 bit padding to make byte aligned, 56 = 52 + 4 */
120 #define BCH4_BIT_PAD 4
121
122 /* GPMC ecc engine settings for read */
123 #define BCH_WRAPMODE_1 1 /* BCH wrap mode 1 */
124 #define BCH8R_ECC_SIZE0 0x1a /* ecc_size0 = 26 */
125 #define BCH8R_ECC_SIZE1 0x2 /* ecc_size1 = 2 */
126 #define BCH4R_ECC_SIZE0 0xd /* ecc_size0 = 13 */
127 #define BCH4R_ECC_SIZE1 0x3 /* ecc_size1 = 3 */
128
129 /* GPMC ecc engine settings for write */
130 #define BCH_WRAPMODE_6 6 /* BCH wrap mode 6 */
131 #define BCH_ECC_SIZE0 0x0 /* ecc_size0 = 0, no oob protection */
132 #define BCH_ECC_SIZE1 0x20 /* ecc_size1 = 32 */
133
134 #define BADBLOCK_MARKER_LENGTH 2
135
136 static u_char bch16_vector[] = {0xf5, 0x24, 0x1c, 0xd0, 0x61, 0xb3, 0xf1, 0x55,
137 0x2e, 0x2c, 0x86, 0xa3, 0xed, 0x36, 0x1b, 0x78,
138 0x48, 0x76, 0xa9, 0x3b, 0x97, 0xd1, 0x7a, 0x93,
139 0x07, 0x0e};
140 static u_char bch8_vector[] = {0xf3, 0xdb, 0x14, 0x16, 0x8b, 0xd2, 0xbe, 0xcc,
141 0xac, 0x6b, 0xff, 0x99, 0x7b};
142 static u_char bch4_vector[] = {0x00, 0x6b, 0x31, 0xdd, 0x41, 0xbc, 0x10};
143
144 struct omap_nand_info {
145 struct nand_chip nand;
146 struct platform_device *pdev;
147
148 int gpmc_cs;
149 bool dev_ready;
150 enum nand_io xfer_type;
151 int devsize;
152 enum omap_ecc ecc_opt;
153 struct device_node *elm_of_node;
154
155 unsigned long phys_base;
156 struct completion comp;
157 struct dma_chan *dma;
158 int gpmc_irq_fifo;
159 int gpmc_irq_count;
160 enum {
161 OMAP_NAND_IO_READ = 0, /* read */
162 OMAP_NAND_IO_WRITE, /* write */
163 } iomode;
164 u_char *buf;
165 int buf_len;
166 /* Interface to GPMC */
167 struct gpmc_nand_regs reg;
168 struct gpmc_nand_ops *ops;
169 bool flash_bbt;
170 /* fields specific for BCHx_HW ECC scheme */
171 struct device *elm_dev;
172 /* NAND ready gpio */
173 struct gpio_desc *ready_gpiod;
174 };
175
176 static inline struct omap_nand_info *mtd_to_omap(struct mtd_info *mtd)
177 {
178 return container_of(mtd_to_nand(mtd), struct omap_nand_info, nand);
179 }
180
181 /**
182 * omap_prefetch_enable - configures and starts prefetch transfer
183 * @cs: cs (chip select) number
184 * @fifo_th: fifo threshold to be used for read/ write
185 * @dma_mode: dma mode enable (1) or disable (0)
186 * @u32_count: number of bytes to be transferred
187 * @is_write: prefetch read(0) or write post(1) mode
188 */
189 static int omap_prefetch_enable(int cs, int fifo_th, int dma_mode,
190 unsigned int u32_count, int is_write, struct omap_nand_info *info)
191 {
192 u32 val;
193
194 if (fifo_th > PREFETCH_FIFOTHRESHOLD_MAX)
195 return -1;
196
197 if (readl(info->reg.gpmc_prefetch_control))
198 return -EBUSY;
199
200 /* Set the amount of bytes to be prefetched */
201 writel(u32_count, info->reg.gpmc_prefetch_config2);
202
203 /* Set dma/mpu mode, the prefetch read / post write and
204 * enable the engine. Set which cs is has requested for.
205 */
206 val = ((cs << PREFETCH_CONFIG1_CS_SHIFT) |
207 PREFETCH_FIFOTHRESHOLD(fifo_th) | ENABLE_PREFETCH |
208 (dma_mode << DMA_MPU_MODE_SHIFT) | (is_write & 0x1));
209 writel(val, info->reg.gpmc_prefetch_config1);
210
211 /* Start the prefetch engine */
212 writel(0x1, info->reg.gpmc_prefetch_control);
213
214 return 0;
215 }
216
217 /**
218 * omap_prefetch_reset - disables and stops the prefetch engine
219 */
220 static int omap_prefetch_reset(int cs, struct omap_nand_info *info)
221 {
222 u32 config1;
223
224 /* check if the same module/cs is trying to reset */
225 config1 = readl(info->reg.gpmc_prefetch_config1);
226 if (((config1 >> PREFETCH_CONFIG1_CS_SHIFT) & CS_MASK) != cs)
227 return -EINVAL;
228
229 /* Stop the PFPW engine */
230 writel(0x0, info->reg.gpmc_prefetch_control);
231
232 /* Reset/disable the PFPW engine */
233 writel(0x0, info->reg.gpmc_prefetch_config1);
234
235 return 0;
236 }
237
238 /**
239 * omap_hwcontrol - hardware specific access to control-lines
240 * @chip: NAND chip object
241 * @cmd: command to device
242 * @ctrl:
243 * NAND_NCE: bit 0 -> don't care
244 * NAND_CLE: bit 1 -> Command Latch
245 * NAND_ALE: bit 2 -> Address Latch
246 *
247 * NOTE: boards may use different bits for these!!
248 */
249 static void omap_hwcontrol(struct nand_chip *chip, int cmd, unsigned int ctrl)
250 {
251 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip));
252
253 if (cmd != NAND_CMD_NONE) {
254 if (ctrl & NAND_CLE)
255 writeb(cmd, info->reg.gpmc_nand_command);
256
257 else if (ctrl & NAND_ALE)
258 writeb(cmd, info->reg.gpmc_nand_address);
259
260 else /* NAND_NCE */
261 writeb(cmd, info->reg.gpmc_nand_data);
262 }
263 }
264
265 /**
266 * omap_read_buf8 - read data from NAND controller into buffer
267 * @mtd: MTD device structure
268 * @buf: buffer to store date
269 * @len: number of bytes to read
270 */
271 static void omap_read_buf8(struct mtd_info *mtd, u_char *buf, int len)
272 {
273 struct nand_chip *nand = mtd_to_nand(mtd);
274
275 ioread8_rep(nand->legacy.IO_ADDR_R, buf, len);
276 }
277
278 /**
279 * omap_write_buf8 - write buffer to NAND controller
280 * @mtd: MTD device structure
281 * @buf: data buffer
282 * @len: number of bytes to write
283 */
284 static void omap_write_buf8(struct mtd_info *mtd, const u_char *buf, int len)
285 {
286 struct omap_nand_info *info = mtd_to_omap(mtd);
287 u_char *p = (u_char *)buf;
288 bool status;
289
290 while (len--) {
291 iowrite8(*p++, info->nand.legacy.IO_ADDR_W);
292 /* wait until buffer is available for write */
293 do {
294 status = info->ops->nand_writebuffer_empty();
295 } while (!status);
296 }
297 }
298
299 /**
300 * omap_read_buf16 - read data from NAND controller into buffer
301 * @mtd: MTD device structure
302 * @buf: buffer to store date
303 * @len: number of bytes to read
304 */
305 static void omap_read_buf16(struct mtd_info *mtd, u_char *buf, int len)
306 {
307 struct nand_chip *nand = mtd_to_nand(mtd);
308
309 ioread16_rep(nand->legacy.IO_ADDR_R, buf, len / 2);
310 }
311
312 /**
313 * omap_write_buf16 - write buffer to NAND controller
314 * @mtd: MTD device structure
315 * @buf: data buffer
316 * @len: number of bytes to write
317 */
318 static void omap_write_buf16(struct mtd_info *mtd, const u_char * buf, int len)
319 {
320 struct omap_nand_info *info = mtd_to_omap(mtd);
321 u16 *p = (u16 *) buf;
322 bool status;
323 /* FIXME try bursts of writesw() or DMA ... */
324 len >>= 1;
325
326 while (len--) {
327 iowrite16(*p++, info->nand.legacy.IO_ADDR_W);
328 /* wait until buffer is available for write */
329 do {
330 status = info->ops->nand_writebuffer_empty();
331 } while (!status);
332 }
333 }
334
335 /**
336 * omap_read_buf_pref - read data from NAND controller into buffer
337 * @chip: NAND chip object
338 * @buf: buffer to store date
339 * @len: number of bytes to read
340 */
341 static void omap_read_buf_pref(struct nand_chip *chip, u_char *buf, int len)
342 {
343 struct mtd_info *mtd = nand_to_mtd(chip);
344 struct omap_nand_info *info = mtd_to_omap(mtd);
345 uint32_t r_count = 0;
346 int ret = 0;
347 u32 *p = (u32 *)buf;
348
349 /* take care of subpage reads */
350 if (len % 4) {
351 if (info->nand.options & NAND_BUSWIDTH_16)
352 omap_read_buf16(mtd, buf, len % 4);
353 else
354 omap_read_buf8(mtd, buf, len % 4);
355 p = (u32 *) (buf + len % 4);
356 len -= len % 4;
357 }
358
359 /* configure and start prefetch transfer */
360 ret = omap_prefetch_enable(info->gpmc_cs,
361 PREFETCH_FIFOTHRESHOLD_MAX, 0x0, len, 0x0, info);
362 if (ret) {
363 /* PFPW engine is busy, use cpu copy method */
364 if (info->nand.options & NAND_BUSWIDTH_16)
365 omap_read_buf16(mtd, (u_char *)p, len);
366 else
367 omap_read_buf8(mtd, (u_char *)p, len);
368 } else {
369 do {
370 r_count = readl(info->reg.gpmc_prefetch_status);
371 r_count = PREFETCH_STATUS_FIFO_CNT(r_count);
372 r_count = r_count >> 2;
373 ioread32_rep(info->nand.legacy.IO_ADDR_R, p, r_count);
374 p += r_count;
375 len -= r_count << 2;
376 } while (len);
377 /* disable and stop the PFPW engine */
378 omap_prefetch_reset(info->gpmc_cs, info);
379 }
380 }
381
382 /**
383 * omap_write_buf_pref - write buffer to NAND controller
384 * @chip: NAND chip object
385 * @buf: data buffer
386 * @len: number of bytes to write
387 */
388 static void omap_write_buf_pref(struct nand_chip *chip, const u_char *buf,
389 int len)
390 {
391 struct mtd_info *mtd = nand_to_mtd(chip);
392 struct omap_nand_info *info = mtd_to_omap(mtd);
393 uint32_t w_count = 0;
394 int i = 0, ret = 0;
395 u16 *p = (u16 *)buf;
396 unsigned long tim, limit;
397 u32 val;
398
399 /* take care of subpage writes */
400 if (len % 2 != 0) {
401 writeb(*buf, info->nand.legacy.IO_ADDR_W);
402 p = (u16 *)(buf + 1);
403 len--;
404 }
405
406 /* configure and start prefetch transfer */
407 ret = omap_prefetch_enable(info->gpmc_cs,
408 PREFETCH_FIFOTHRESHOLD_MAX, 0x0, len, 0x1, info);
409 if (ret) {
410 /* PFPW engine is busy, use cpu copy method */
411 if (info->nand.options & NAND_BUSWIDTH_16)
412 omap_write_buf16(mtd, (u_char *)p, len);
413 else
414 omap_write_buf8(mtd, (u_char *)p, len);
415 } else {
416 while (len) {
417 w_count = readl(info->reg.gpmc_prefetch_status);
418 w_count = PREFETCH_STATUS_FIFO_CNT(w_count);
419 w_count = w_count >> 1;
420 for (i = 0; (i < w_count) && len; i++, len -= 2)
421 iowrite16(*p++, info->nand.legacy.IO_ADDR_W);
422 }
423 /* wait for data to flushed-out before reset the prefetch */
424 tim = 0;
425 limit = (loops_per_jiffy *
426 msecs_to_jiffies(OMAP_NAND_TIMEOUT_MS));
427 do {
428 cpu_relax();
429 val = readl(info->reg.gpmc_prefetch_status);
430 val = PREFETCH_STATUS_COUNT(val);
431 } while (val && (tim++ < limit));
432
433 /* disable and stop the PFPW engine */
434 omap_prefetch_reset(info->gpmc_cs, info);
435 }
436 }
437
438 /*
439 * omap_nand_dma_callback: callback on the completion of dma transfer
440 * @data: pointer to completion data structure
441 */
442 static void omap_nand_dma_callback(void *data)
443 {
444 complete((struct completion *) data);
445 }
446
447 /*
448 * omap_nand_dma_transfer: configure and start dma transfer
449 * @mtd: MTD device structure
450 * @addr: virtual address in RAM of source/destination
451 * @len: number of data bytes to be transferred
452 * @is_write: flag for read/write operation
453 */
454 static inline int omap_nand_dma_transfer(struct mtd_info *mtd, void *addr,
455 unsigned int len, int is_write)
456 {
457 struct omap_nand_info *info = mtd_to_omap(mtd);
458 struct dma_async_tx_descriptor *tx;
459 enum dma_data_direction dir = is_write ? DMA_TO_DEVICE :
460 DMA_FROM_DEVICE;
461 struct scatterlist sg;
462 unsigned long tim, limit;
463 unsigned n;
464 int ret;
465 u32 val;
466
467 if (!virt_addr_valid(addr))
468 goto out_copy;
469
470 sg_init_one(&sg, addr, len);
471 n = dma_map_sg(info->dma->device->dev, &sg, 1, dir);
472 if (n == 0) {
473 dev_err(&info->pdev->dev,
474 "Couldn't DMA map a %d byte buffer\n", len);
475 goto out_copy;
476 }
477
478 tx = dmaengine_prep_slave_sg(info->dma, &sg, n,
479 is_write ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM,
480 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
481 if (!tx)
482 goto out_copy_unmap;
483
484 tx->callback = omap_nand_dma_callback;
485 tx->callback_param = &info->comp;
486 dmaengine_submit(tx);
487
488 init_completion(&info->comp);
489
490 /* setup and start DMA using dma_addr */
491 dma_async_issue_pending(info->dma);
492
493 /* configure and start prefetch transfer */
494 ret = omap_prefetch_enable(info->gpmc_cs,
495 PREFETCH_FIFOTHRESHOLD_MAX, 0x1, len, is_write, info);
496 if (ret)
497 /* PFPW engine is busy, use cpu copy method */
498 goto out_copy_unmap;
499
500 wait_for_completion(&info->comp);
501 tim = 0;
502 limit = (loops_per_jiffy * msecs_to_jiffies(OMAP_NAND_TIMEOUT_MS));
503
504 do {
505 cpu_relax();
506 val = readl(info->reg.gpmc_prefetch_status);
507 val = PREFETCH_STATUS_COUNT(val);
508 } while (val && (tim++ < limit));
509
510 /* disable and stop the PFPW engine */
511 omap_prefetch_reset(info->gpmc_cs, info);
512
513 dma_unmap_sg(info->dma->device->dev, &sg, 1, dir);
514 return 0;
515
516 out_copy_unmap:
517 dma_unmap_sg(info->dma->device->dev, &sg, 1, dir);
518 out_copy:
519 if (info->nand.options & NAND_BUSWIDTH_16)
520 is_write == 0 ? omap_read_buf16(mtd, (u_char *) addr, len)
521 : omap_write_buf16(mtd, (u_char *) addr, len);
522 else
523 is_write == 0 ? omap_read_buf8(mtd, (u_char *) addr, len)
524 : omap_write_buf8(mtd, (u_char *) addr, len);
525 return 0;
526 }
527
528 /**
529 * omap_read_buf_dma_pref - read data from NAND controller into buffer
530 * @chip: NAND chip object
531 * @buf: buffer to store date
532 * @len: number of bytes to read
533 */
534 static void omap_read_buf_dma_pref(struct nand_chip *chip, u_char *buf,
535 int len)
536 {
537 struct mtd_info *mtd = nand_to_mtd(chip);
538
539 if (len <= mtd->oobsize)
540 omap_read_buf_pref(chip, buf, len);
541 else
542 /* start transfer in DMA mode */
543 omap_nand_dma_transfer(mtd, buf, len, 0x0);
544 }
545
546 /**
547 * omap_write_buf_dma_pref - write buffer to NAND controller
548 * @chip: NAND chip object
549 * @buf: data buffer
550 * @len: number of bytes to write
551 */
552 static void omap_write_buf_dma_pref(struct nand_chip *chip, const u_char *buf,
553 int len)
554 {
555 struct mtd_info *mtd = nand_to_mtd(chip);
556
557 if (len <= mtd->oobsize)
558 omap_write_buf_pref(chip, buf, len);
559 else
560 /* start transfer in DMA mode */
561 omap_nand_dma_transfer(mtd, (u_char *)buf, len, 0x1);
562 }
563
564 /*
565 * omap_nand_irq - GPMC irq handler
566 * @this_irq: gpmc irq number
567 * @dev: omap_nand_info structure pointer is passed here
568 */
569 static irqreturn_t omap_nand_irq(int this_irq, void *dev)
570 {
571 struct omap_nand_info *info = (struct omap_nand_info *) dev;
572 u32 bytes;
573
574 bytes = readl(info->reg.gpmc_prefetch_status);
575 bytes = PREFETCH_STATUS_FIFO_CNT(bytes);
576 bytes = bytes & 0xFFFC; /* io in multiple of 4 bytes */
577 if (info->iomode == OMAP_NAND_IO_WRITE) { /* checks for write io */
578 if (this_irq == info->gpmc_irq_count)
579 goto done;
580
581 if (info->buf_len && (info->buf_len < bytes))
582 bytes = info->buf_len;
583 else if (!info->buf_len)
584 bytes = 0;
585 iowrite32_rep(info->nand.legacy.IO_ADDR_W, (u32 *)info->buf,
586 bytes >> 2);
587 info->buf = info->buf + bytes;
588 info->buf_len -= bytes;
589
590 } else {
591 ioread32_rep(info->nand.legacy.IO_ADDR_R, (u32 *)info->buf,
592 bytes >> 2);
593 info->buf = info->buf + bytes;
594
595 if (this_irq == info->gpmc_irq_count)
596 goto done;
597 }
598
599 return IRQ_HANDLED;
600
601 done:
602 complete(&info->comp);
603
604 disable_irq_nosync(info->gpmc_irq_fifo);
605 disable_irq_nosync(info->gpmc_irq_count);
606
607 return IRQ_HANDLED;
608 }
609
610 /*
611 * omap_read_buf_irq_pref - read data from NAND controller into buffer
612 * @chip: NAND chip object
613 * @buf: buffer to store date
614 * @len: number of bytes to read
615 */
616 static void omap_read_buf_irq_pref(struct nand_chip *chip, u_char *buf,
617 int len)
618 {
619 struct mtd_info *mtd = nand_to_mtd(chip);
620 struct omap_nand_info *info = mtd_to_omap(mtd);
621 int ret = 0;
622
623 if (len <= mtd->oobsize) {
624 omap_read_buf_pref(chip, buf, len);
625 return;
626 }
627
628 info->iomode = OMAP_NAND_IO_READ;
629 info->buf = buf;
630 init_completion(&info->comp);
631
632 /* configure and start prefetch transfer */
633 ret = omap_prefetch_enable(info->gpmc_cs,
634 PREFETCH_FIFOTHRESHOLD_MAX/2, 0x0, len, 0x0, info);
635 if (ret)
636 /* PFPW engine is busy, use cpu copy method */
637 goto out_copy;
638
639 info->buf_len = len;
640
641 enable_irq(info->gpmc_irq_count);
642 enable_irq(info->gpmc_irq_fifo);
643
644 /* waiting for read to complete */
645 wait_for_completion(&info->comp);
646
647 /* disable and stop the PFPW engine */
648 omap_prefetch_reset(info->gpmc_cs, info);
649 return;
650
651 out_copy:
652 if (info->nand.options & NAND_BUSWIDTH_16)
653 omap_read_buf16(mtd, buf, len);
654 else
655 omap_read_buf8(mtd, buf, len);
656 }
657
658 /*
659 * omap_write_buf_irq_pref - write buffer to NAND controller
660 * @chip: NAND chip object
661 * @buf: data buffer
662 * @len: number of bytes to write
663 */
664 static void omap_write_buf_irq_pref(struct nand_chip *chip, const u_char *buf,
665 int len)
666 {
667 struct mtd_info *mtd = nand_to_mtd(chip);
668 struct omap_nand_info *info = mtd_to_omap(mtd);
669 int ret = 0;
670 unsigned long tim, limit;
671 u32 val;
672
673 if (len <= mtd->oobsize) {
674 omap_write_buf_pref(chip, buf, len);
675 return;
676 }
677
678 info->iomode = OMAP_NAND_IO_WRITE;
679 info->buf = (u_char *) buf;
680 init_completion(&info->comp);
681
682 /* configure and start prefetch transfer : size=24 */
683 ret = omap_prefetch_enable(info->gpmc_cs,
684 (PREFETCH_FIFOTHRESHOLD_MAX * 3) / 8, 0x0, len, 0x1, info);
685 if (ret)
686 /* PFPW engine is busy, use cpu copy method */
687 goto out_copy;
688
689 info->buf_len = len;
690
691 enable_irq(info->gpmc_irq_count);
692 enable_irq(info->gpmc_irq_fifo);
693
694 /* waiting for write to complete */
695 wait_for_completion(&info->comp);
696
697 /* wait for data to flushed-out before reset the prefetch */
698 tim = 0;
699 limit = (loops_per_jiffy * msecs_to_jiffies(OMAP_NAND_TIMEOUT_MS));
700 do {
701 val = readl(info->reg.gpmc_prefetch_status);
702 val = PREFETCH_STATUS_COUNT(val);
703 cpu_relax();
704 } while (val && (tim++ < limit));
705
706 /* disable and stop the PFPW engine */
707 omap_prefetch_reset(info->gpmc_cs, info);
708 return;
709
710 out_copy:
711 if (info->nand.options & NAND_BUSWIDTH_16)
712 omap_write_buf16(mtd, buf, len);
713 else
714 omap_write_buf8(mtd, buf, len);
715 }
716
717 /**
718 * gen_true_ecc - This function will generate true ECC value
719 * @ecc_buf: buffer to store ecc code
720 *
721 * This generated true ECC value can be used when correcting
722 * data read from NAND flash memory core
723 */
724 static void gen_true_ecc(u8 *ecc_buf)
725 {
726 u32 tmp = ecc_buf[0] | (ecc_buf[1] << 16) |
727 ((ecc_buf[2] & 0xF0) << 20) | ((ecc_buf[2] & 0x0F) << 8);
728
729 ecc_buf[0] = ~(P64o(tmp) | P64e(tmp) | P32o(tmp) | P32e(tmp) |
730 P16o(tmp) | P16e(tmp) | P8o(tmp) | P8e(tmp));
731 ecc_buf[1] = ~(P1024o(tmp) | P1024e(tmp) | P512o(tmp) | P512e(tmp) |
732 P256o(tmp) | P256e(tmp) | P128o(tmp) | P128e(tmp));
733 ecc_buf[2] = ~(P4o(tmp) | P4e(tmp) | P2o(tmp) | P2e(tmp) | P1o(tmp) |
734 P1e(tmp) | P2048o(tmp) | P2048e(tmp));
735 }
736
737 /**
738 * omap_compare_ecc - Detect (2 bits) and correct (1 bit) error in data
739 * @ecc_data1: ecc code from nand spare area
740 * @ecc_data2: ecc code from hardware register obtained from hardware ecc
741 * @page_data: page data
742 *
743 * This function compares two ECC's and indicates if there is an error.
744 * If the error can be corrected it will be corrected to the buffer.
745 * If there is no error, %0 is returned. If there is an error but it
746 * was corrected, %1 is returned. Otherwise, %-1 is returned.
747 */
748 static int omap_compare_ecc(u8 *ecc_data1, /* read from NAND memory */
749 u8 *ecc_data2, /* read from register */
750 u8 *page_data)
751 {
752 uint i;
753 u8 tmp0_bit[8], tmp1_bit[8], tmp2_bit[8];
754 u8 comp0_bit[8], comp1_bit[8], comp2_bit[8];
755 u8 ecc_bit[24];
756 u8 ecc_sum = 0;
757 u8 find_bit = 0;
758 uint find_byte = 0;
759 int isEccFF;
760
761 isEccFF = ((*(u32 *)ecc_data1 & 0xFFFFFF) == 0xFFFFFF);
762
763 gen_true_ecc(ecc_data1);
764 gen_true_ecc(ecc_data2);
765
766 for (i = 0; i <= 2; i++) {
767 *(ecc_data1 + i) = ~(*(ecc_data1 + i));
768 *(ecc_data2 + i) = ~(*(ecc_data2 + i));
769 }
770
771 for (i = 0; i < 8; i++) {
772 tmp0_bit[i] = *ecc_data1 % 2;
773 *ecc_data1 = *ecc_data1 / 2;
774 }
775
776 for (i = 0; i < 8; i++) {
777 tmp1_bit[i] = *(ecc_data1 + 1) % 2;
778 *(ecc_data1 + 1) = *(ecc_data1 + 1) / 2;
779 }
780
781 for (i = 0; i < 8; i++) {
782 tmp2_bit[i] = *(ecc_data1 + 2) % 2;
783 *(ecc_data1 + 2) = *(ecc_data1 + 2) / 2;
784 }
785
786 for (i = 0; i < 8; i++) {
787 comp0_bit[i] = *ecc_data2 % 2;
788 *ecc_data2 = *ecc_data2 / 2;
789 }
790
791 for (i = 0; i < 8; i++) {
792 comp1_bit[i] = *(ecc_data2 + 1) % 2;
793 *(ecc_data2 + 1) = *(ecc_data2 + 1) / 2;
794 }
795
796 for (i = 0; i < 8; i++) {
797 comp2_bit[i] = *(ecc_data2 + 2) % 2;
798 *(ecc_data2 + 2) = *(ecc_data2 + 2) / 2;
799 }
800
801 for (i = 0; i < 6; i++)
802 ecc_bit[i] = tmp2_bit[i + 2] ^ comp2_bit[i + 2];
803
804 for (i = 0; i < 8; i++)
805 ecc_bit[i + 6] = tmp0_bit[i] ^ comp0_bit[i];
806
807 for (i = 0; i < 8; i++)
808 ecc_bit[i + 14] = tmp1_bit[i] ^ comp1_bit[i];
809
810 ecc_bit[22] = tmp2_bit[0] ^ comp2_bit[0];
811 ecc_bit[23] = tmp2_bit[1] ^ comp2_bit[1];
812
813 for (i = 0; i < 24; i++)
814 ecc_sum += ecc_bit[i];
815
816 switch (ecc_sum) {
817 case 0:
818 /* Not reached because this function is not called if
819 * ECC values are equal
820 */
821 return 0;
822
823 case 1:
824 /* Uncorrectable error */
825 pr_debug("ECC UNCORRECTED_ERROR 1\n");
826 return -EBADMSG;
827
828 case 11:
829 /* UN-Correctable error */
830 pr_debug("ECC UNCORRECTED_ERROR B\n");
831 return -EBADMSG;
832
833 case 12:
834 /* Correctable error */
835 find_byte = (ecc_bit[23] << 8) +
836 (ecc_bit[21] << 7) +
837 (ecc_bit[19] << 6) +
838 (ecc_bit[17] << 5) +
839 (ecc_bit[15] << 4) +
840 (ecc_bit[13] << 3) +
841 (ecc_bit[11] << 2) +
842 (ecc_bit[9] << 1) +
843 ecc_bit[7];
844
845 find_bit = (ecc_bit[5] << 2) + (ecc_bit[3] << 1) + ecc_bit[1];
846
847 pr_debug("Correcting single bit ECC error at offset: "
848 "%d, bit: %d\n", find_byte, find_bit);
849
850 page_data[find_byte] ^= (1 << find_bit);
851
852 return 1;
853 default:
854 if (isEccFF) {
855 if (ecc_data2[0] == 0 &&
856 ecc_data2[1] == 0 &&
857 ecc_data2[2] == 0)
858 return 0;
859 }
860 pr_debug("UNCORRECTED_ERROR default\n");
861 return -EBADMSG;
862 }
863 }
864
865 /**
866 * omap_correct_data - Compares the ECC read with HW generated ECC
867 * @chip: NAND chip object
868 * @dat: page data
869 * @read_ecc: ecc read from nand flash
870 * @calc_ecc: ecc read from HW ECC registers
871 *
872 * Compares the ecc read from nand spare area with ECC registers values
873 * and if ECC's mismatched, it will call 'omap_compare_ecc' for error
874 * detection and correction. If there are no errors, %0 is returned. If
875 * there were errors and all of the errors were corrected, the number of
876 * corrected errors is returned. If uncorrectable errors exist, %-1 is
877 * returned.
878 */
879 static int omap_correct_data(struct nand_chip *chip, u_char *dat,
880 u_char *read_ecc, u_char *calc_ecc)
881 {
882 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip));
883 int blockCnt = 0, i = 0, ret = 0;
884 int stat = 0;
885
886 /* Ex NAND_ECC_HW12_2048 */
887 if ((info->nand.ecc.mode == NAND_ECC_HW) &&
888 (info->nand.ecc.size == 2048))
889 blockCnt = 4;
890 else
891 blockCnt = 1;
892
893 for (i = 0; i < blockCnt; i++) {
894 if (memcmp(read_ecc, calc_ecc, 3) != 0) {
895 ret = omap_compare_ecc(read_ecc, calc_ecc, dat);
896 if (ret < 0)
897 return ret;
898 /* keep track of the number of corrected errors */
899 stat += ret;
900 }
901 read_ecc += 3;
902 calc_ecc += 3;
903 dat += 512;
904 }
905 return stat;
906 }
907
908 /**
909 * omap_calcuate_ecc - Generate non-inverted ECC bytes.
910 * @chip: NAND chip object
911 * @dat: The pointer to data on which ecc is computed
912 * @ecc_code: The ecc_code buffer
913 *
914 * Using noninverted ECC can be considered ugly since writing a blank
915 * page ie. padding will clear the ECC bytes. This is no problem as long
916 * nobody is trying to write data on the seemingly unused page. Reading
917 * an erased page will produce an ECC mismatch between generated and read
918 * ECC bytes that has to be dealt with separately.
919 */
920 static int omap_calculate_ecc(struct nand_chip *chip, const u_char *dat,
921 u_char *ecc_code)
922 {
923 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip));
924 u32 val;
925
926 val = readl(info->reg.gpmc_ecc_config);
927 if (((val >> ECC_CONFIG_CS_SHIFT) & CS_MASK) != info->gpmc_cs)
928 return -EINVAL;
929
930 /* read ecc result */
931 val = readl(info->reg.gpmc_ecc1_result);
932 *ecc_code++ = val; /* P128e, ..., P1e */
933 *ecc_code++ = val >> 16; /* P128o, ..., P1o */
934 /* P2048o, P1024o, P512o, P256o, P2048e, P1024e, P512e, P256e */
935 *ecc_code++ = ((val >> 8) & 0x0f) | ((val >> 20) & 0xf0);
936
937 return 0;
938 }
939
940 /**
941 * omap_enable_hwecc - This function enables the hardware ecc functionality
942 * @mtd: MTD device structure
943 * @mode: Read/Write mode
944 */
945 static void omap_enable_hwecc(struct nand_chip *chip, int mode)
946 {
947 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip));
948 unsigned int dev_width = (chip->options & NAND_BUSWIDTH_16) ? 1 : 0;
949 u32 val;
950
951 /* clear ecc and enable bits */
952 val = ECCCLEAR | ECC1;
953 writel(val, info->reg.gpmc_ecc_control);
954
955 /* program ecc and result sizes */
956 val = ((((info->nand.ecc.size >> 1) - 1) << ECCSIZE1_SHIFT) |
957 ECC1RESULTSIZE);
958 writel(val, info->reg.gpmc_ecc_size_config);
959
960 switch (mode) {
961 case NAND_ECC_READ:
962 case NAND_ECC_WRITE:
963 writel(ECCCLEAR | ECC1, info->reg.gpmc_ecc_control);
964 break;
965 case NAND_ECC_READSYN:
966 writel(ECCCLEAR, info->reg.gpmc_ecc_control);
967 break;
968 default:
969 dev_info(&info->pdev->dev,
970 "error: unrecognized Mode[%d]!\n", mode);
971 break;
972 }
973
974 /* (ECC 16 or 8 bit col) | ( CS ) | ECC Enable */
975 val = (dev_width << 7) | (info->gpmc_cs << 1) | (0x1);
976 writel(val, info->reg.gpmc_ecc_config);
977 }
978
979 /**
980 * omap_wait - wait until the command is done
981 * @this: NAND Chip structure
982 *
983 * Wait function is called during Program and erase operations and
984 * the way it is called from MTD layer, we should wait till the NAND
985 * chip is ready after the programming/erase operation has completed.
986 *
987 * Erase can take up to 400ms and program up to 20ms according to
988 * general NAND and SmartMedia specs
989 */
990 static int omap_wait(struct nand_chip *this)
991 {
992 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(this));
993 unsigned long timeo = jiffies;
994 int status;
995
996 timeo += msecs_to_jiffies(400);
997
998 writeb(NAND_CMD_STATUS & 0xFF, info->reg.gpmc_nand_command);
999 while (time_before(jiffies, timeo)) {
1000 status = readb(info->reg.gpmc_nand_data);
1001 if (status & NAND_STATUS_READY)
1002 break;
1003 cond_resched();
1004 }
1005
1006 status = readb(info->reg.gpmc_nand_data);
1007 return status;
1008 }
1009
1010 /**
1011 * omap_dev_ready - checks the NAND Ready GPIO line
1012 * @mtd: MTD device structure
1013 *
1014 * Returns true if ready and false if busy.
1015 */
1016 static int omap_dev_ready(struct nand_chip *chip)
1017 {
1018 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip));
1019
1020 return gpiod_get_value(info->ready_gpiod);
1021 }
1022
1023 /**
1024 * omap_enable_hwecc_bch - Program GPMC to perform BCH ECC calculation
1025 * @mtd: MTD device structure
1026 * @mode: Read/Write mode
1027 *
1028 * When using BCH with SW correction (i.e. no ELM), sector size is set
1029 * to 512 bytes and we use BCH_WRAPMODE_6 wrapping mode
1030 * for both reading and writing with:
1031 * eccsize0 = 0 (no additional protected byte in spare area)
1032 * eccsize1 = 32 (skip 32 nibbles = 16 bytes per sector in spare area)
1033 */
1034 static void __maybe_unused omap_enable_hwecc_bch(struct nand_chip *chip,
1035 int mode)
1036 {
1037 unsigned int bch_type;
1038 unsigned int dev_width, nsectors;
1039 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip));
1040 enum omap_ecc ecc_opt = info->ecc_opt;
1041 u32 val, wr_mode;
1042 unsigned int ecc_size1, ecc_size0;
1043
1044 /* GPMC configurations for calculating ECC */
1045 switch (ecc_opt) {
1046 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW:
1047 bch_type = 0;
1048 nsectors = 1;
1049 wr_mode = BCH_WRAPMODE_6;
1050 ecc_size0 = BCH_ECC_SIZE0;
1051 ecc_size1 = BCH_ECC_SIZE1;
1052 break;
1053 case OMAP_ECC_BCH4_CODE_HW:
1054 bch_type = 0;
1055 nsectors = chip->ecc.steps;
1056 if (mode == NAND_ECC_READ) {
1057 wr_mode = BCH_WRAPMODE_1;
1058 ecc_size0 = BCH4R_ECC_SIZE0;
1059 ecc_size1 = BCH4R_ECC_SIZE1;
1060 } else {
1061 wr_mode = BCH_WRAPMODE_6;
1062 ecc_size0 = BCH_ECC_SIZE0;
1063 ecc_size1 = BCH_ECC_SIZE1;
1064 }
1065 break;
1066 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
1067 bch_type = 1;
1068 nsectors = 1;
1069 wr_mode = BCH_WRAPMODE_6;
1070 ecc_size0 = BCH_ECC_SIZE0;
1071 ecc_size1 = BCH_ECC_SIZE1;
1072 break;
1073 case OMAP_ECC_BCH8_CODE_HW:
1074 bch_type = 1;
1075 nsectors = chip->ecc.steps;
1076 if (mode == NAND_ECC_READ) {
1077 wr_mode = BCH_WRAPMODE_1;
1078 ecc_size0 = BCH8R_ECC_SIZE0;
1079 ecc_size1 = BCH8R_ECC_SIZE1;
1080 } else {
1081 wr_mode = BCH_WRAPMODE_6;
1082 ecc_size0 = BCH_ECC_SIZE0;
1083 ecc_size1 = BCH_ECC_SIZE1;
1084 }
1085 break;
1086 case OMAP_ECC_BCH16_CODE_HW:
1087 bch_type = 0x2;
1088 nsectors = chip->ecc.steps;
1089 if (mode == NAND_ECC_READ) {
1090 wr_mode = 0x01;
1091 ecc_size0 = 52; /* ECC bits in nibbles per sector */
1092 ecc_size1 = 0; /* non-ECC bits in nibbles per sector */
1093 } else {
1094 wr_mode = 0x01;
1095 ecc_size0 = 0; /* extra bits in nibbles per sector */
1096 ecc_size1 = 52; /* OOB bits in nibbles per sector */
1097 }
1098 break;
1099 default:
1100 return;
1101 }
1102
1103 writel(ECC1, info->reg.gpmc_ecc_control);
1104
1105 /* Configure ecc size for BCH */
1106 val = (ecc_size1 << ECCSIZE1_SHIFT) | (ecc_size0 << ECCSIZE0_SHIFT);
1107 writel(val, info->reg.gpmc_ecc_size_config);
1108
1109 dev_width = (chip->options & NAND_BUSWIDTH_16) ? 1 : 0;
1110
1111 /* BCH configuration */
1112 val = ((1 << 16) | /* enable BCH */
1113 (bch_type << 12) | /* BCH4/BCH8/BCH16 */
1114 (wr_mode << 8) | /* wrap mode */
1115 (dev_width << 7) | /* bus width */
1116 (((nsectors-1) & 0x7) << 4) | /* number of sectors */
1117 (info->gpmc_cs << 1) | /* ECC CS */
1118 (0x1)); /* enable ECC */
1119
1120 writel(val, info->reg.gpmc_ecc_config);
1121
1122 /* Clear ecc and enable bits */
1123 writel(ECCCLEAR | ECC1, info->reg.gpmc_ecc_control);
1124 }
1125
1126 static u8 bch4_polynomial[] = {0x28, 0x13, 0xcc, 0x39, 0x96, 0xac, 0x7f};
1127 static u8 bch8_polynomial[] = {0xef, 0x51, 0x2e, 0x09, 0xed, 0x93, 0x9a, 0xc2,
1128 0x97, 0x79, 0xe5, 0x24, 0xb5};
1129
1130 /**
1131 * _omap_calculate_ecc_bch - Generate ECC bytes for one sector
1132 * @mtd: MTD device structure
1133 * @dat: The pointer to data on which ecc is computed
1134 * @ecc_code: The ecc_code buffer
1135 * @i: The sector number (for a multi sector page)
1136 *
1137 * Support calculating of BCH4/8/16 ECC vectors for one sector
1138 * within a page. Sector number is in @i.
1139 */
1140 static int _omap_calculate_ecc_bch(struct mtd_info *mtd,
1141 const u_char *dat, u_char *ecc_calc, int i)
1142 {
1143 struct omap_nand_info *info = mtd_to_omap(mtd);
1144 int eccbytes = info->nand.ecc.bytes;
1145 struct gpmc_nand_regs *gpmc_regs = &info->reg;
1146 u8 *ecc_code;
1147 unsigned long bch_val1, bch_val2, bch_val3, bch_val4;
1148 u32 val;
1149 int j;
1150
1151 ecc_code = ecc_calc;
1152 switch (info->ecc_opt) {
1153 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
1154 case OMAP_ECC_BCH8_CODE_HW:
1155 bch_val1 = readl(gpmc_regs->gpmc_bch_result0[i]);
1156 bch_val2 = readl(gpmc_regs->gpmc_bch_result1[i]);
1157 bch_val3 = readl(gpmc_regs->gpmc_bch_result2[i]);
1158 bch_val4 = readl(gpmc_regs->gpmc_bch_result3[i]);
1159 *ecc_code++ = (bch_val4 & 0xFF);
1160 *ecc_code++ = ((bch_val3 >> 24) & 0xFF);
1161 *ecc_code++ = ((bch_val3 >> 16) & 0xFF);
1162 *ecc_code++ = ((bch_val3 >> 8) & 0xFF);
1163 *ecc_code++ = (bch_val3 & 0xFF);
1164 *ecc_code++ = ((bch_val2 >> 24) & 0xFF);
1165 *ecc_code++ = ((bch_val2 >> 16) & 0xFF);
1166 *ecc_code++ = ((bch_val2 >> 8) & 0xFF);
1167 *ecc_code++ = (bch_val2 & 0xFF);
1168 *ecc_code++ = ((bch_val1 >> 24) & 0xFF);
1169 *ecc_code++ = ((bch_val1 >> 16) & 0xFF);
1170 *ecc_code++ = ((bch_val1 >> 8) & 0xFF);
1171 *ecc_code++ = (bch_val1 & 0xFF);
1172 break;
1173 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW:
1174 case OMAP_ECC_BCH4_CODE_HW:
1175 bch_val1 = readl(gpmc_regs->gpmc_bch_result0[i]);
1176 bch_val2 = readl(gpmc_regs->gpmc_bch_result1[i]);
1177 *ecc_code++ = ((bch_val2 >> 12) & 0xFF);
1178 *ecc_code++ = ((bch_val2 >> 4) & 0xFF);
1179 *ecc_code++ = ((bch_val2 & 0xF) << 4) |
1180 ((bch_val1 >> 28) & 0xF);
1181 *ecc_code++ = ((bch_val1 >> 20) & 0xFF);
1182 *ecc_code++ = ((bch_val1 >> 12) & 0xFF);
1183 *ecc_code++ = ((bch_val1 >> 4) & 0xFF);
1184 *ecc_code++ = ((bch_val1 & 0xF) << 4);
1185 break;
1186 case OMAP_ECC_BCH16_CODE_HW:
1187 val = readl(gpmc_regs->gpmc_bch_result6[i]);
1188 ecc_code[0] = ((val >> 8) & 0xFF);
1189 ecc_code[1] = ((val >> 0) & 0xFF);
1190 val = readl(gpmc_regs->gpmc_bch_result5[i]);
1191 ecc_code[2] = ((val >> 24) & 0xFF);
1192 ecc_code[3] = ((val >> 16) & 0xFF);
1193 ecc_code[4] = ((val >> 8) & 0xFF);
1194 ecc_code[5] = ((val >> 0) & 0xFF);
1195 val = readl(gpmc_regs->gpmc_bch_result4[i]);
1196 ecc_code[6] = ((val >> 24) & 0xFF);
1197 ecc_code[7] = ((val >> 16) & 0xFF);
1198 ecc_code[8] = ((val >> 8) & 0xFF);
1199 ecc_code[9] = ((val >> 0) & 0xFF);
1200 val = readl(gpmc_regs->gpmc_bch_result3[i]);
1201 ecc_code[10] = ((val >> 24) & 0xFF);
1202 ecc_code[11] = ((val >> 16) & 0xFF);
1203 ecc_code[12] = ((val >> 8) & 0xFF);
1204 ecc_code[13] = ((val >> 0) & 0xFF);
1205 val = readl(gpmc_regs->gpmc_bch_result2[i]);
1206 ecc_code[14] = ((val >> 24) & 0xFF);
1207 ecc_code[15] = ((val >> 16) & 0xFF);
1208 ecc_code[16] = ((val >> 8) & 0xFF);
1209 ecc_code[17] = ((val >> 0) & 0xFF);
1210 val = readl(gpmc_regs->gpmc_bch_result1[i]);
1211 ecc_code[18] = ((val >> 24) & 0xFF);
1212 ecc_code[19] = ((val >> 16) & 0xFF);
1213 ecc_code[20] = ((val >> 8) & 0xFF);
1214 ecc_code[21] = ((val >> 0) & 0xFF);
1215 val = readl(gpmc_regs->gpmc_bch_result0[i]);
1216 ecc_code[22] = ((val >> 24) & 0xFF);
1217 ecc_code[23] = ((val >> 16) & 0xFF);
1218 ecc_code[24] = ((val >> 8) & 0xFF);
1219 ecc_code[25] = ((val >> 0) & 0xFF);
1220 break;
1221 default:
1222 return -EINVAL;
1223 }
1224
1225 /* ECC scheme specific syndrome customizations */
1226 switch (info->ecc_opt) {
1227 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW:
1228 /* Add constant polynomial to remainder, so that
1229 * ECC of blank pages results in 0x0 on reading back
1230 */
1231 for (j = 0; j < eccbytes; j++)
1232 ecc_calc[j] ^= bch4_polynomial[j];
1233 break;
1234 case OMAP_ECC_BCH4_CODE_HW:
1235 /* Set 8th ECC byte as 0x0 for ROM compatibility */
1236 ecc_calc[eccbytes - 1] = 0x0;
1237 break;
1238 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
1239 /* Add constant polynomial to remainder, so that
1240 * ECC of blank pages results in 0x0 on reading back
1241 */
1242 for (j = 0; j < eccbytes; j++)
1243 ecc_calc[j] ^= bch8_polynomial[j];
1244 break;
1245 case OMAP_ECC_BCH8_CODE_HW:
1246 /* Set 14th ECC byte as 0x0 for ROM compatibility */
1247 ecc_calc[eccbytes - 1] = 0x0;
1248 break;
1249 case OMAP_ECC_BCH16_CODE_HW:
1250 break;
1251 default:
1252 return -EINVAL;
1253 }
1254
1255 return 0;
1256 }
1257
1258 /**
1259 * omap_calculate_ecc_bch_sw - ECC generator for sector for SW based correction
1260 * @chip: NAND chip object
1261 * @dat: The pointer to data on which ecc is computed
1262 * @ecc_code: The ecc_code buffer
1263 *
1264 * Support calculating of BCH4/8/16 ECC vectors for one sector. This is used
1265 * when SW based correction is required as ECC is required for one sector
1266 * at a time.
1267 */
1268 static int omap_calculate_ecc_bch_sw(struct nand_chip *chip,
1269 const u_char *dat, u_char *ecc_calc)
1270 {
1271 return _omap_calculate_ecc_bch(nand_to_mtd(chip), dat, ecc_calc, 0);
1272 }
1273
1274 /**
1275 * omap_calculate_ecc_bch_multi - Generate ECC for multiple sectors
1276 * @mtd: MTD device structure
1277 * @dat: The pointer to data on which ecc is computed
1278 * @ecc_code: The ecc_code buffer
1279 *
1280 * Support calculating of BCH4/8/16 ecc vectors for the entire page in one go.
1281 */
1282 static int omap_calculate_ecc_bch_multi(struct mtd_info *mtd,
1283 const u_char *dat, u_char *ecc_calc)
1284 {
1285 struct omap_nand_info *info = mtd_to_omap(mtd);
1286 int eccbytes = info->nand.ecc.bytes;
1287 unsigned long nsectors;
1288 int i, ret;
1289
1290 nsectors = ((readl(info->reg.gpmc_ecc_config) >> 4) & 0x7) + 1;
1291 for (i = 0; i < nsectors; i++) {
1292 ret = _omap_calculate_ecc_bch(mtd, dat, ecc_calc, i);
1293 if (ret)
1294 return ret;
1295
1296 ecc_calc += eccbytes;
1297 }
1298
1299 return 0;
1300 }
1301
1302 /**
1303 * erased_sector_bitflips - count bit flips
1304 * @data: data sector buffer
1305 * @oob: oob buffer
1306 * @info: omap_nand_info
1307 *
1308 * Check the bit flips in erased page falls below correctable level.
1309 * If falls below, report the page as erased with correctable bit
1310 * flip, else report as uncorrectable page.
1311 */
1312 static int erased_sector_bitflips(u_char *data, u_char *oob,
1313 struct omap_nand_info *info)
1314 {
1315 int flip_bits = 0, i;
1316
1317 for (i = 0; i < info->nand.ecc.size; i++) {
1318 flip_bits += hweight8(~data[i]);
1319 if (flip_bits > info->nand.ecc.strength)
1320 return 0;
1321 }
1322
1323 for (i = 0; i < info->nand.ecc.bytes - 1; i++) {
1324 flip_bits += hweight8(~oob[i]);
1325 if (flip_bits > info->nand.ecc.strength)
1326 return 0;
1327 }
1328
1329 /*
1330 * Bit flips falls in correctable level.
1331 * Fill data area with 0xFF
1332 */
1333 if (flip_bits) {
1334 memset(data, 0xFF, info->nand.ecc.size);
1335 memset(oob, 0xFF, info->nand.ecc.bytes);
1336 }
1337
1338 return flip_bits;
1339 }
1340
1341 /**
1342 * omap_elm_correct_data - corrects page data area in case error reported
1343 * @chip: NAND chip object
1344 * @data: page data
1345 * @read_ecc: ecc read from nand flash
1346 * @calc_ecc: ecc read from HW ECC registers
1347 *
1348 * Calculated ecc vector reported as zero in case of non-error pages.
1349 * In case of non-zero ecc vector, first filter out erased-pages, and
1350 * then process data via ELM to detect bit-flips.
1351 */
1352 static int omap_elm_correct_data(struct nand_chip *chip, u_char *data,
1353 u_char *read_ecc, u_char *calc_ecc)
1354 {
1355 struct omap_nand_info *info = mtd_to_omap(nand_to_mtd(chip));
1356 struct nand_ecc_ctrl *ecc = &info->nand.ecc;
1357 int eccsteps = info->nand.ecc.steps;
1358 int i , j, stat = 0;
1359 int eccflag, actual_eccbytes;
1360 struct elm_errorvec err_vec[ERROR_VECTOR_MAX];
1361 u_char *ecc_vec = calc_ecc;
1362 u_char *spare_ecc = read_ecc;
1363 u_char *erased_ecc_vec;
1364 u_char *buf;
1365 int bitflip_count;
1366 bool is_error_reported = false;
1367 u32 bit_pos, byte_pos, error_max, pos;
1368 int err;
1369
1370 switch (info->ecc_opt) {
1371 case OMAP_ECC_BCH4_CODE_HW:
1372 /* omit 7th ECC byte reserved for ROM code compatibility */
1373 actual_eccbytes = ecc->bytes - 1;
1374 erased_ecc_vec = bch4_vector;
1375 break;
1376 case OMAP_ECC_BCH8_CODE_HW:
1377 /* omit 14th ECC byte reserved for ROM code compatibility */
1378 actual_eccbytes = ecc->bytes - 1;
1379 erased_ecc_vec = bch8_vector;
1380 break;
1381 case OMAP_ECC_BCH16_CODE_HW:
1382 actual_eccbytes = ecc->bytes;
1383 erased_ecc_vec = bch16_vector;
1384 break;
1385 default:
1386 dev_err(&info->pdev->dev, "invalid driver configuration\n");
1387 return -EINVAL;
1388 }
1389
1390 /* Initialize elm error vector to zero */
1391 memset(err_vec, 0, sizeof(err_vec));
1392
1393 for (i = 0; i < eccsteps ; i++) {
1394 eccflag = 0; /* initialize eccflag */
1395
1396 /*
1397 * Check any error reported,
1398 * In case of error, non zero ecc reported.
1399 */
1400 for (j = 0; j < actual_eccbytes; j++) {
1401 if (calc_ecc[j] != 0) {
1402 eccflag = 1; /* non zero ecc, error present */
1403 break;
1404 }
1405 }
1406
1407 if (eccflag == 1) {
1408 if (memcmp(calc_ecc, erased_ecc_vec,
1409 actual_eccbytes) == 0) {
1410 /*
1411 * calc_ecc[] matches pattern for ECC(all 0xff)
1412 * so this is definitely an erased-page
1413 */
1414 } else {
1415 buf = &data[info->nand.ecc.size * i];
1416 /*
1417 * count number of 0-bits in read_buf.
1418 * This check can be removed once a similar
1419 * check is introduced in generic NAND driver
1420 */
1421 bitflip_count = erased_sector_bitflips(
1422 buf, read_ecc, info);
1423 if (bitflip_count) {
1424 /*
1425 * number of 0-bits within ECC limits
1426 * So this may be an erased-page
1427 */
1428 stat += bitflip_count;
1429 } else {
1430 /*
1431 * Too many 0-bits. It may be a
1432 * - programmed-page, OR
1433 * - erased-page with many bit-flips
1434 * So this page requires check by ELM
1435 */
1436 err_vec[i].error_reported = true;
1437 is_error_reported = true;
1438 }
1439 }
1440 }
1441
1442 /* Update the ecc vector */
1443 calc_ecc += ecc->bytes;
1444 read_ecc += ecc->bytes;
1445 }
1446
1447 /* Check if any error reported */
1448 if (!is_error_reported)
1449 return stat;
1450
1451 /* Decode BCH error using ELM module */
1452 elm_decode_bch_error_page(info->elm_dev, ecc_vec, err_vec);
1453
1454 err = 0;
1455 for (i = 0; i < eccsteps; i++) {
1456 if (err_vec[i].error_uncorrectable) {
1457 dev_err(&info->pdev->dev,
1458 "uncorrectable bit-flips found\n");
1459 err = -EBADMSG;
1460 } else if (err_vec[i].error_reported) {
1461 for (j = 0; j < err_vec[i].error_count; j++) {
1462 switch (info->ecc_opt) {
1463 case OMAP_ECC_BCH4_CODE_HW:
1464 /* Add 4 bits to take care of padding */
1465 pos = err_vec[i].error_loc[j] +
1466 BCH4_BIT_PAD;
1467 break;
1468 case OMAP_ECC_BCH8_CODE_HW:
1469 case OMAP_ECC_BCH16_CODE_HW:
1470 pos = err_vec[i].error_loc[j];
1471 break;
1472 default:
1473 return -EINVAL;
1474 }
1475 error_max = (ecc->size + actual_eccbytes) * 8;
1476 /* Calculate bit position of error */
1477 bit_pos = pos % 8;
1478
1479 /* Calculate byte position of error */
1480 byte_pos = (error_max - pos - 1) / 8;
1481
1482 if (pos < error_max) {
1483 if (byte_pos < 512) {
1484 pr_debug("bitflip@dat[%d]=%x\n",
1485 byte_pos, data[byte_pos]);
1486 data[byte_pos] ^= 1 << bit_pos;
1487 } else {
1488 pr_debug("bitflip@oob[%d]=%x\n",
1489 (byte_pos - 512),
1490 spare_ecc[byte_pos - 512]);
1491 spare_ecc[byte_pos - 512] ^=
1492 1 << bit_pos;
1493 }
1494 } else {
1495 dev_err(&info->pdev->dev,
1496 "invalid bit-flip @ %d:%d\n",
1497 byte_pos, bit_pos);
1498 err = -EBADMSG;
1499 }
1500 }
1501 }
1502
1503 /* Update number of correctable errors */
1504 stat = max_t(unsigned int, stat, err_vec[i].error_count);
1505
1506 /* Update page data with sector size */
1507 data += ecc->size;
1508 spare_ecc += ecc->bytes;
1509 }
1510
1511 return (err) ? err : stat;
1512 }
1513
1514 /**
1515 * omap_write_page_bch - BCH ecc based write page function for entire page
1516 * @chip: nand chip info structure
1517 * @buf: data buffer
1518 * @oob_required: must write chip->oob_poi to OOB
1519 * @page: page
1520 *
1521 * Custom write page method evolved to support multi sector writing in one shot
1522 */
1523 static int omap_write_page_bch(struct nand_chip *chip, const uint8_t *buf,
1524 int oob_required, int page)
1525 {
1526 struct mtd_info *mtd = nand_to_mtd(chip);
1527 int ret;
1528 uint8_t *ecc_calc = chip->ecc.calc_buf;
1529
1530 nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1531
1532 /* Enable GPMC ecc engine */
1533 chip->ecc.hwctl(chip, NAND_ECC_WRITE);
1534
1535 /* Write data */
1536 chip->legacy.write_buf(chip, buf, mtd->writesize);
1537
1538 /* Update ecc vector from GPMC result registers */
1539 omap_calculate_ecc_bch_multi(mtd, buf, &ecc_calc[0]);
1540
1541 ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0,
1542 chip->ecc.total);
1543 if (ret)
1544 return ret;
1545
1546 /* Write ecc vector to OOB area */
1547 chip->legacy.write_buf(chip, chip->oob_poi, mtd->oobsize);
1548
1549 return nand_prog_page_end_op(chip);
1550 }
1551
1552 /**
1553 * omap_write_subpage_bch - BCH hardware ECC based subpage write
1554 * @chip: nand chip info structure
1555 * @offset: column address of subpage within the page
1556 * @data_len: data length
1557 * @buf: data buffer
1558 * @oob_required: must write chip->oob_poi to OOB
1559 * @page: page number to write
1560 *
1561 * OMAP optimized subpage write method.
1562 */
1563 static int omap_write_subpage_bch(struct nand_chip *chip, u32 offset,
1564 u32 data_len, const u8 *buf,
1565 int oob_required, int page)
1566 {
1567 struct mtd_info *mtd = nand_to_mtd(chip);
1568 u8 *ecc_calc = chip->ecc.calc_buf;
1569 int ecc_size = chip->ecc.size;
1570 int ecc_bytes = chip->ecc.bytes;
1571 int ecc_steps = chip->ecc.steps;
1572 u32 start_step = offset / ecc_size;
1573 u32 end_step = (offset + data_len - 1) / ecc_size;
1574 int step, ret = 0;
1575
1576 /*
1577 * Write entire page at one go as it would be optimal
1578 * as ECC is calculated by hardware.
1579 * ECC is calculated for all subpages but we choose
1580 * only what we want.
1581 */
1582 nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1583
1584 /* Enable GPMC ECC engine */
1585 chip->ecc.hwctl(chip, NAND_ECC_WRITE);
1586
1587 /* Write data */
1588 chip->legacy.write_buf(chip, buf, mtd->writesize);
1589
1590 for (step = 0; step < ecc_steps; step++) {
1591 /* mask ECC of un-touched subpages by padding 0xFF */
1592 if (step < start_step || step > end_step)
1593 memset(ecc_calc, 0xff, ecc_bytes);
1594 else
1595 ret = _omap_calculate_ecc_bch(mtd, buf, ecc_calc, step);
1596
1597 if (ret)
1598 return ret;
1599
1600 buf += ecc_size;
1601 ecc_calc += ecc_bytes;
1602 }
1603
1604 /* copy calculated ECC for whole page to chip->buffer->oob */
1605 /* this include masked-value(0xFF) for unwritten subpages */
1606 ecc_calc = chip->ecc.calc_buf;
1607 ret = mtd_ooblayout_set_eccbytes(mtd, ecc_calc, chip->oob_poi, 0,
1608 chip->ecc.total);
1609 if (ret)
1610 return ret;
1611
1612 /* write OOB buffer to NAND device */
1613 chip->legacy.write_buf(chip, chip->oob_poi, mtd->oobsize);
1614
1615 return nand_prog_page_end_op(chip);
1616 }
1617
1618 /**
1619 * omap_read_page_bch - BCH ecc based page read function for entire page
1620 * @chip: nand chip info structure
1621 * @buf: buffer to store read data
1622 * @oob_required: caller requires OOB data read to chip->oob_poi
1623 * @page: page number to read
1624 *
1625 * For BCH ecc scheme, GPMC used for syndrome calculation and ELM module
1626 * used for error correction.
1627 * Custom method evolved to support ELM error correction & multi sector
1628 * reading. On reading page data area is read along with OOB data with
1629 * ecc engine enabled. ecc vector updated after read of OOB data.
1630 * For non error pages ecc vector reported as zero.
1631 */
1632 static int omap_read_page_bch(struct nand_chip *chip, uint8_t *buf,
1633 int oob_required, int page)
1634 {
1635 struct mtd_info *mtd = nand_to_mtd(chip);
1636 uint8_t *ecc_calc = chip->ecc.calc_buf;
1637 uint8_t *ecc_code = chip->ecc.code_buf;
1638 int stat, ret;
1639 unsigned int max_bitflips = 0;
1640
1641 nand_read_page_op(chip, page, 0, NULL, 0);
1642
1643 /* Enable GPMC ecc engine */
1644 chip->ecc.hwctl(chip, NAND_ECC_READ);
1645
1646 /* Read data */
1647 chip->legacy.read_buf(chip, buf, mtd->writesize);
1648
1649 /* Read oob bytes */
1650 nand_change_read_column_op(chip,
1651 mtd->writesize + BADBLOCK_MARKER_LENGTH,
1652 chip->oob_poi + BADBLOCK_MARKER_LENGTH,
1653 chip->ecc.total, false);
1654
1655 /* Calculate ecc bytes */
1656 omap_calculate_ecc_bch_multi(mtd, buf, ecc_calc);
1657
1658 ret = mtd_ooblayout_get_eccbytes(mtd, ecc_code, chip->oob_poi, 0,
1659 chip->ecc.total);
1660 if (ret)
1661 return ret;
1662
1663 stat = chip->ecc.correct(chip, buf, ecc_code, ecc_calc);
1664
1665 if (stat < 0) {
1666 mtd->ecc_stats.failed++;
1667 } else {
1668 mtd->ecc_stats.corrected += stat;
1669 max_bitflips = max_t(unsigned int, max_bitflips, stat);
1670 }
1671
1672 return max_bitflips;
1673 }
1674
1675 /**
1676 * is_elm_present - checks for presence of ELM module by scanning DT nodes
1677 * @omap_nand_info: NAND device structure containing platform data
1678 */
1679 static bool is_elm_present(struct omap_nand_info *info,
1680 struct device_node *elm_node)
1681 {
1682 struct platform_device *pdev;
1683
1684 /* check whether elm-id is passed via DT */
1685 if (!elm_node) {
1686 dev_err(&info->pdev->dev, "ELM devicetree node not found\n");
1687 return false;
1688 }
1689 pdev = of_find_device_by_node(elm_node);
1690 /* check whether ELM device is registered */
1691 if (!pdev) {
1692 dev_err(&info->pdev->dev, "ELM device not found\n");
1693 return false;
1694 }
1695 /* ELM module available, now configure it */
1696 info->elm_dev = &pdev->dev;
1697 return true;
1698 }
1699
1700 static bool omap2_nand_ecc_check(struct omap_nand_info *info)
1701 {
1702 bool ecc_needs_bch, ecc_needs_omap_bch, ecc_needs_elm;
1703
1704 switch (info->ecc_opt) {
1705 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW:
1706 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
1707 ecc_needs_omap_bch = false;
1708 ecc_needs_bch = true;
1709 ecc_needs_elm = false;
1710 break;
1711 case OMAP_ECC_BCH4_CODE_HW:
1712 case OMAP_ECC_BCH8_CODE_HW:
1713 case OMAP_ECC_BCH16_CODE_HW:
1714 ecc_needs_omap_bch = true;
1715 ecc_needs_bch = false;
1716 ecc_needs_elm = true;
1717 break;
1718 default:
1719 ecc_needs_omap_bch = false;
1720 ecc_needs_bch = false;
1721 ecc_needs_elm = false;
1722 break;
1723 }
1724
1725 if (ecc_needs_bch && !IS_ENABLED(CONFIG_MTD_NAND_ECC_SW_BCH)) {
1726 dev_err(&info->pdev->dev,
1727 "CONFIG_MTD_NAND_ECC_SW_BCH not enabled\n");
1728 return false;
1729 }
1730 if (ecc_needs_omap_bch && !IS_ENABLED(CONFIG_MTD_NAND_OMAP_BCH)) {
1731 dev_err(&info->pdev->dev,
1732 "CONFIG_MTD_NAND_OMAP_BCH not enabled\n");
1733 return false;
1734 }
1735 if (ecc_needs_elm && !is_elm_present(info, info->elm_of_node)) {
1736 dev_err(&info->pdev->dev, "ELM not available\n");
1737 return false;
1738 }
1739
1740 return true;
1741 }
1742
1743 static const char * const nand_xfer_types[] = {
1744 [NAND_OMAP_PREFETCH_POLLED] = "prefetch-polled",
1745 [NAND_OMAP_POLLED] = "polled",
1746 [NAND_OMAP_PREFETCH_DMA] = "prefetch-dma",
1747 [NAND_OMAP_PREFETCH_IRQ] = "prefetch-irq",
1748 };
1749
1750 static int omap_get_dt_info(struct device *dev, struct omap_nand_info *info)
1751 {
1752 struct device_node *child = dev->of_node;
1753 int i;
1754 const char *s;
1755 u32 cs;
1756
1757 if (of_property_read_u32(child, "reg", &cs) < 0) {
1758 dev_err(dev, "reg not found in DT\n");
1759 return -EINVAL;
1760 }
1761
1762 info->gpmc_cs = cs;
1763
1764 /* detect availability of ELM module. Won't be present pre-OMAP4 */
1765 info->elm_of_node = of_parse_phandle(child, "ti,elm-id", 0);
1766 if (!info->elm_of_node) {
1767 info->elm_of_node = of_parse_phandle(child, "elm_id", 0);
1768 if (!info->elm_of_node)
1769 dev_dbg(dev, "ti,elm-id not in DT\n");
1770 }
1771
1772 /* select ecc-scheme for NAND */
1773 if (of_property_read_string(child, "ti,nand-ecc-opt", &s)) {
1774 dev_err(dev, "ti,nand-ecc-opt not found\n");
1775 return -EINVAL;
1776 }
1777
1778 if (!strcmp(s, "sw")) {
1779 info->ecc_opt = OMAP_ECC_HAM1_CODE_SW;
1780 } else if (!strcmp(s, "ham1") ||
1781 !strcmp(s, "hw") || !strcmp(s, "hw-romcode")) {
1782 info->ecc_opt = OMAP_ECC_HAM1_CODE_HW;
1783 } else if (!strcmp(s, "bch4")) {
1784 if (info->elm_of_node)
1785 info->ecc_opt = OMAP_ECC_BCH4_CODE_HW;
1786 else
1787 info->ecc_opt = OMAP_ECC_BCH4_CODE_HW_DETECTION_SW;
1788 } else if (!strcmp(s, "bch8")) {
1789 if (info->elm_of_node)
1790 info->ecc_opt = OMAP_ECC_BCH8_CODE_HW;
1791 else
1792 info->ecc_opt = OMAP_ECC_BCH8_CODE_HW_DETECTION_SW;
1793 } else if (!strcmp(s, "bch16")) {
1794 info->ecc_opt = OMAP_ECC_BCH16_CODE_HW;
1795 } else {
1796 dev_err(dev, "unrecognized value for ti,nand-ecc-opt\n");
1797 return -EINVAL;
1798 }
1799
1800 /* select data transfer mode */
1801 if (!of_property_read_string(child, "ti,nand-xfer-type", &s)) {
1802 for (i = 0; i < ARRAY_SIZE(nand_xfer_types); i++) {
1803 if (!strcasecmp(s, nand_xfer_types[i])) {
1804 info->xfer_type = i;
1805 return 0;
1806 }
1807 }
1808
1809 dev_err(dev, "unrecognized value for ti,nand-xfer-type\n");
1810 return -EINVAL;
1811 }
1812
1813 return 0;
1814 }
1815
1816 static int omap_ooblayout_ecc(struct mtd_info *mtd, int section,
1817 struct mtd_oob_region *oobregion)
1818 {
1819 struct omap_nand_info *info = mtd_to_omap(mtd);
1820 struct nand_chip *chip = &info->nand;
1821 int off = BADBLOCK_MARKER_LENGTH;
1822
1823 if (info->ecc_opt == OMAP_ECC_HAM1_CODE_HW &&
1824 !(chip->options & NAND_BUSWIDTH_16))
1825 off = 1;
1826
1827 if (section)
1828 return -ERANGE;
1829
1830 oobregion->offset = off;
1831 oobregion->length = chip->ecc.total;
1832
1833 return 0;
1834 }
1835
1836 static int omap_ooblayout_free(struct mtd_info *mtd, int section,
1837 struct mtd_oob_region *oobregion)
1838 {
1839 struct omap_nand_info *info = mtd_to_omap(mtd);
1840 struct nand_chip *chip = &info->nand;
1841 int off = BADBLOCK_MARKER_LENGTH;
1842
1843 if (info->ecc_opt == OMAP_ECC_HAM1_CODE_HW &&
1844 !(chip->options & NAND_BUSWIDTH_16))
1845 off = 1;
1846
1847 if (section)
1848 return -ERANGE;
1849
1850 off += chip->ecc.total;
1851 if (off >= mtd->oobsize)
1852 return -ERANGE;
1853
1854 oobregion->offset = off;
1855 oobregion->length = mtd->oobsize - off;
1856
1857 return 0;
1858 }
1859
1860 static const struct mtd_ooblayout_ops omap_ooblayout_ops = {
1861 .ecc = omap_ooblayout_ecc,
1862 .free = omap_ooblayout_free,
1863 };
1864
1865 static int omap_sw_ooblayout_ecc(struct mtd_info *mtd, int section,
1866 struct mtd_oob_region *oobregion)
1867 {
1868 struct nand_chip *chip = mtd_to_nand(mtd);
1869 int off = BADBLOCK_MARKER_LENGTH;
1870
1871 if (section >= chip->ecc.steps)
1872 return -ERANGE;
1873
1874 /*
1875 * When SW correction is employed, one OMAP specific marker byte is
1876 * reserved after each ECC step.
1877 */
1878 oobregion->offset = off + (section * (chip->ecc.bytes + 1));
1879 oobregion->length = chip->ecc.bytes;
1880
1881 return 0;
1882 }
1883
1884 static int omap_sw_ooblayout_free(struct mtd_info *mtd, int section,
1885 struct mtd_oob_region *oobregion)
1886 {
1887 struct nand_chip *chip = mtd_to_nand(mtd);
1888 int off = BADBLOCK_MARKER_LENGTH;
1889
1890 if (section)
1891 return -ERANGE;
1892
1893 /*
1894 * When SW correction is employed, one OMAP specific marker byte is
1895 * reserved after each ECC step.
1896 */
1897 off += ((chip->ecc.bytes + 1) * chip->ecc.steps);
1898 if (off >= mtd->oobsize)
1899 return -ERANGE;
1900
1901 oobregion->offset = off;
1902 oobregion->length = mtd->oobsize - off;
1903
1904 return 0;
1905 }
1906
1907 static const struct mtd_ooblayout_ops omap_sw_ooblayout_ops = {
1908 .ecc = omap_sw_ooblayout_ecc,
1909 .free = omap_sw_ooblayout_free,
1910 };
1911
1912 static int omap_nand_attach_chip(struct nand_chip *chip)
1913 {
1914 struct mtd_info *mtd = nand_to_mtd(chip);
1915 struct omap_nand_info *info = mtd_to_omap(mtd);
1916 struct device *dev = &info->pdev->dev;
1917 int min_oobbytes = BADBLOCK_MARKER_LENGTH;
1918 int oobbytes_per_step;
1919 dma_cap_mask_t mask;
1920 int err;
1921
1922 if (chip->bbt_options & NAND_BBT_USE_FLASH)
1923 chip->bbt_options |= NAND_BBT_NO_OOB;
1924 else
1925 chip->options |= NAND_SKIP_BBTSCAN;
1926
1927 /* Re-populate low-level callbacks based on xfer modes */
1928 switch (info->xfer_type) {
1929 case NAND_OMAP_PREFETCH_POLLED:
1930 chip->legacy.read_buf = omap_read_buf_pref;
1931 chip->legacy.write_buf = omap_write_buf_pref;
1932 break;
1933
1934 case NAND_OMAP_POLLED:
1935 /* Use nand_base defaults for {read,write}_buf */
1936 break;
1937
1938 case NAND_OMAP_PREFETCH_DMA:
1939 dma_cap_zero(mask);
1940 dma_cap_set(DMA_SLAVE, mask);
1941 info->dma = dma_request_chan(dev->parent, "rxtx");
1942
1943 if (IS_ERR(info->dma)) {
1944 dev_err(dev, "DMA engine request failed\n");
1945 return PTR_ERR(info->dma);
1946 } else {
1947 struct dma_slave_config cfg;
1948
1949 memset(&cfg, 0, sizeof(cfg));
1950 cfg.src_addr = info->phys_base;
1951 cfg.dst_addr = info->phys_base;
1952 cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
1953 cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
1954 cfg.src_maxburst = 16;
1955 cfg.dst_maxburst = 16;
1956 err = dmaengine_slave_config(info->dma, &cfg);
1957 if (err) {
1958 dev_err(dev,
1959 "DMA engine slave config failed: %d\n",
1960 err);
1961 return err;
1962 }
1963 chip->legacy.read_buf = omap_read_buf_dma_pref;
1964 chip->legacy.write_buf = omap_write_buf_dma_pref;
1965 }
1966 break;
1967
1968 case NAND_OMAP_PREFETCH_IRQ:
1969 info->gpmc_irq_fifo = platform_get_irq(info->pdev, 0);
1970 if (info->gpmc_irq_fifo <= 0)
1971 return -ENODEV;
1972 err = devm_request_irq(dev, info->gpmc_irq_fifo,
1973 omap_nand_irq, IRQF_SHARED,
1974 "gpmc-nand-fifo", info);
1975 if (err) {
1976 dev_err(dev, "Requesting IRQ %d, error %d\n",
1977 info->gpmc_irq_fifo, err);
1978 info->gpmc_irq_fifo = 0;
1979 return err;
1980 }
1981
1982 info->gpmc_irq_count = platform_get_irq(info->pdev, 1);
1983 if (info->gpmc_irq_count <= 0)
1984 return -ENODEV;
1985 err = devm_request_irq(dev, info->gpmc_irq_count,
1986 omap_nand_irq, IRQF_SHARED,
1987 "gpmc-nand-count", info);
1988 if (err) {
1989 dev_err(dev, "Requesting IRQ %d, error %d\n",
1990 info->gpmc_irq_count, err);
1991 info->gpmc_irq_count = 0;
1992 return err;
1993 }
1994
1995 chip->legacy.read_buf = omap_read_buf_irq_pref;
1996 chip->legacy.write_buf = omap_write_buf_irq_pref;
1997
1998 break;
1999
2000 default:
2001 dev_err(dev, "xfer_type %d not supported!\n", info->xfer_type);
2002 return -EINVAL;
2003 }
2004
2005 if (!omap2_nand_ecc_check(info))
2006 return -EINVAL;
2007
2008 /*
2009 * Bail out earlier to let NAND_ECC_SOFT code create its own
2010 * ooblayout instead of using ours.
2011 */
2012 if (info->ecc_opt == OMAP_ECC_HAM1_CODE_SW) {
2013 chip->ecc.mode = NAND_ECC_SOFT;
2014 chip->ecc.algo = NAND_ECC_HAMMING;
2015 return 0;
2016 }
2017
2018 /* Populate MTD interface based on ECC scheme */
2019 switch (info->ecc_opt) {
2020 case OMAP_ECC_HAM1_CODE_HW:
2021 dev_info(dev, "nand: using OMAP_ECC_HAM1_CODE_HW\n");
2022 chip->ecc.mode = NAND_ECC_HW;
2023 chip->ecc.bytes = 3;
2024 chip->ecc.size = 512;
2025 chip->ecc.strength = 1;
2026 chip->ecc.calculate = omap_calculate_ecc;
2027 chip->ecc.hwctl = omap_enable_hwecc;
2028 chip->ecc.correct = omap_correct_data;
2029 mtd_set_ooblayout(mtd, &omap_ooblayout_ops);
2030 oobbytes_per_step = chip->ecc.bytes;
2031
2032 if (!(chip->options & NAND_BUSWIDTH_16))
2033 min_oobbytes = 1;
2034
2035 break;
2036
2037 case OMAP_ECC_BCH4_CODE_HW_DETECTION_SW:
2038 pr_info("nand: using OMAP_ECC_BCH4_CODE_HW_DETECTION_SW\n");
2039 chip->ecc.mode = NAND_ECC_HW;
2040 chip->ecc.size = 512;
2041 chip->ecc.bytes = 7;
2042 chip->ecc.strength = 4;
2043 chip->ecc.hwctl = omap_enable_hwecc_bch;
2044 chip->ecc.correct = nand_bch_correct_data;
2045 chip->ecc.calculate = omap_calculate_ecc_bch_sw;
2046 mtd_set_ooblayout(mtd, &omap_sw_ooblayout_ops);
2047 /* Reserve one byte for the OMAP marker */
2048 oobbytes_per_step = chip->ecc.bytes + 1;
2049 /* Software BCH library is used for locating errors */
2050 chip->ecc.priv = nand_bch_init(mtd);
2051 if (!chip->ecc.priv) {
2052 dev_err(dev, "Unable to use BCH library\n");
2053 return -EINVAL;
2054 }
2055 break;
2056
2057 case OMAP_ECC_BCH4_CODE_HW:
2058 pr_info("nand: using OMAP_ECC_BCH4_CODE_HW ECC scheme\n");
2059 chip->ecc.mode = NAND_ECC_HW;
2060 chip->ecc.size = 512;
2061 /* 14th bit is kept reserved for ROM-code compatibility */
2062 chip->ecc.bytes = 7 + 1;
2063 chip->ecc.strength = 4;
2064 chip->ecc.hwctl = omap_enable_hwecc_bch;
2065 chip->ecc.correct = omap_elm_correct_data;
2066 chip->ecc.read_page = omap_read_page_bch;
2067 chip->ecc.write_page = omap_write_page_bch;
2068 chip->ecc.write_subpage = omap_write_subpage_bch;
2069 mtd_set_ooblayout(mtd, &omap_ooblayout_ops);
2070 oobbytes_per_step = chip->ecc.bytes;
2071
2072 err = elm_config(info->elm_dev, BCH4_ECC,
2073 mtd->writesize / chip->ecc.size,
2074 chip->ecc.size, chip->ecc.bytes);
2075 if (err < 0)
2076 return err;
2077 break;
2078
2079 case OMAP_ECC_BCH8_CODE_HW_DETECTION_SW:
2080 pr_info("nand: using OMAP_ECC_BCH8_CODE_HW_DETECTION_SW\n");
2081 chip->ecc.mode = NAND_ECC_HW;
2082 chip->ecc.size = 512;
2083 chip->ecc.bytes = 13;
2084 chip->ecc.strength = 8;
2085 chip->ecc.hwctl = omap_enable_hwecc_bch;
2086 chip->ecc.correct = nand_bch_correct_data;
2087 chip->ecc.calculate = omap_calculate_ecc_bch_sw;
2088 mtd_set_ooblayout(mtd, &omap_sw_ooblayout_ops);
2089 /* Reserve one byte for the OMAP marker */
2090 oobbytes_per_step = chip->ecc.bytes + 1;
2091 /* Software BCH library is used for locating errors */
2092 chip->ecc.priv = nand_bch_init(mtd);
2093 if (!chip->ecc.priv) {
2094 dev_err(dev, "unable to use BCH library\n");
2095 return -EINVAL;
2096 }
2097 break;
2098
2099 case OMAP_ECC_BCH8_CODE_HW:
2100 pr_info("nand: using OMAP_ECC_BCH8_CODE_HW ECC scheme\n");
2101 chip->ecc.mode = NAND_ECC_HW;
2102 chip->ecc.size = 512;
2103 /* 14th bit is kept reserved for ROM-code compatibility */
2104 chip->ecc.bytes = 13 + 1;
2105 chip->ecc.strength = 8;
2106 chip->ecc.hwctl = omap_enable_hwecc_bch;
2107 chip->ecc.correct = omap_elm_correct_data;
2108 chip->ecc.read_page = omap_read_page_bch;
2109 chip->ecc.write_page = omap_write_page_bch;
2110 chip->ecc.write_subpage = omap_write_subpage_bch;
2111 mtd_set_ooblayout(mtd, &omap_ooblayout_ops);
2112 oobbytes_per_step = chip->ecc.bytes;
2113
2114 err = elm_config(info->elm_dev, BCH8_ECC,
2115 mtd->writesize / chip->ecc.size,
2116 chip->ecc.size, chip->ecc.bytes);
2117 if (err < 0)
2118 return err;
2119
2120 break;
2121
2122 case OMAP_ECC_BCH16_CODE_HW:
2123 pr_info("Using OMAP_ECC_BCH16_CODE_HW ECC scheme\n");
2124 chip->ecc.mode = NAND_ECC_HW;
2125 chip->ecc.size = 512;
2126 chip->ecc.bytes = 26;
2127 chip->ecc.strength = 16;
2128 chip->ecc.hwctl = omap_enable_hwecc_bch;
2129 chip->ecc.correct = omap_elm_correct_data;
2130 chip->ecc.read_page = omap_read_page_bch;
2131 chip->ecc.write_page = omap_write_page_bch;
2132 chip->ecc.write_subpage = omap_write_subpage_bch;
2133 mtd_set_ooblayout(mtd, &omap_ooblayout_ops);
2134 oobbytes_per_step = chip->ecc.bytes;
2135
2136 err = elm_config(info->elm_dev, BCH16_ECC,
2137 mtd->writesize / chip->ecc.size,
2138 chip->ecc.size, chip->ecc.bytes);
2139 if (err < 0)
2140 return err;
2141
2142 break;
2143 default:
2144 dev_err(dev, "Invalid or unsupported ECC scheme\n");
2145 return -EINVAL;
2146 }
2147
2148 /* Check if NAND device's OOB is enough to store ECC signatures */
2149 min_oobbytes += (oobbytes_per_step *
2150 (mtd->writesize / chip->ecc.size));
2151 if (mtd->oobsize < min_oobbytes) {
2152 dev_err(dev,
2153 "Not enough OOB bytes: required = %d, available=%d\n",
2154 min_oobbytes, mtd->oobsize);
2155 return -EINVAL;
2156 }
2157
2158 return 0;
2159 }
2160
2161 static const struct nand_controller_ops omap_nand_controller_ops = {
2162 .attach_chip = omap_nand_attach_chip,
2163 };
2164
2165 /* Shared among all NAND instances to synchronize access to the ECC Engine */
2166 static struct nand_controller omap_gpmc_controller;
2167 static bool omap_gpmc_controller_initialized;
2168
2169 static int omap_nand_probe(struct platform_device *pdev)
2170 {
2171 struct omap_nand_info *info;
2172 struct mtd_info *mtd;
2173 struct nand_chip *nand_chip;
2174 int err;
2175 struct resource *res;
2176 struct device *dev = &pdev->dev;
2177
2178 info = devm_kzalloc(&pdev->dev, sizeof(struct omap_nand_info),
2179 GFP_KERNEL);
2180 if (!info)
2181 return -ENOMEM;
2182
2183 info->pdev = pdev;
2184
2185 err = omap_get_dt_info(dev, info);
2186 if (err)
2187 return err;
2188
2189 info->ops = gpmc_omap_get_nand_ops(&info->reg, info->gpmc_cs);
2190 if (!info->ops) {
2191 dev_err(&pdev->dev, "Failed to get GPMC->NAND interface\n");
2192 return -ENODEV;
2193 }
2194
2195 nand_chip = &info->nand;
2196 mtd = nand_to_mtd(nand_chip);
2197 mtd->dev.parent = &pdev->dev;
2198 nand_chip->ecc.priv = NULL;
2199 nand_set_flash_node(nand_chip, dev->of_node);
2200
2201 if (!mtd->name) {
2202 mtd->name = devm_kasprintf(&pdev->dev, GFP_KERNEL,
2203 "omap2-nand.%d", info->gpmc_cs);
2204 if (!mtd->name) {
2205 dev_err(&pdev->dev, "Failed to set MTD name\n");
2206 return -ENOMEM;
2207 }
2208 }
2209
2210 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2211 nand_chip->legacy.IO_ADDR_R = devm_ioremap_resource(&pdev->dev, res);
2212 if (IS_ERR(nand_chip->legacy.IO_ADDR_R))
2213 return PTR_ERR(nand_chip->legacy.IO_ADDR_R);
2214
2215 info->phys_base = res->start;
2216
2217 if (!omap_gpmc_controller_initialized) {
2218 omap_gpmc_controller.ops = &omap_nand_controller_ops;
2219 nand_controller_init(&omap_gpmc_controller);
2220 omap_gpmc_controller_initialized = true;
2221 }
2222
2223 nand_chip->controller = &omap_gpmc_controller;
2224
2225 nand_chip->legacy.IO_ADDR_W = nand_chip->legacy.IO_ADDR_R;
2226 nand_chip->legacy.cmd_ctrl = omap_hwcontrol;
2227
2228 info->ready_gpiod = devm_gpiod_get_optional(&pdev->dev, "rb",
2229 GPIOD_IN);
2230 if (IS_ERR(info->ready_gpiod)) {
2231 dev_err(dev, "failed to get ready gpio\n");
2232 return PTR_ERR(info->ready_gpiod);
2233 }
2234
2235 /*
2236 * If RDY/BSY line is connected to OMAP then use the omap ready
2237 * function and the generic nand_wait function which reads the status
2238 * register after monitoring the RDY/BSY line. Otherwise use a standard
2239 * chip delay which is slightly more than tR (AC Timing) of the NAND
2240 * device and read status register until you get a failure or success
2241 */
2242 if (info->ready_gpiod) {
2243 nand_chip->legacy.dev_ready = omap_dev_ready;
2244 nand_chip->legacy.chip_delay = 0;
2245 } else {
2246 nand_chip->legacy.waitfunc = omap_wait;
2247 nand_chip->legacy.chip_delay = 50;
2248 }
2249
2250 if (info->flash_bbt)
2251 nand_chip->bbt_options |= NAND_BBT_USE_FLASH;
2252
2253 /* scan NAND device connected to chip controller */
2254 nand_chip->options |= info->devsize & NAND_BUSWIDTH_16;
2255
2256 err = nand_scan(nand_chip, 1);
2257 if (err)
2258 goto return_error;
2259
2260 err = mtd_device_register(mtd, NULL, 0);
2261 if (err)
2262 goto cleanup_nand;
2263
2264 platform_set_drvdata(pdev, mtd);
2265
2266 return 0;
2267
2268 cleanup_nand:
2269 nand_cleanup(nand_chip);
2270
2271 return_error:
2272 if (!IS_ERR_OR_NULL(info->dma))
2273 dma_release_channel(info->dma);
2274 if (nand_chip->ecc.priv) {
2275 nand_bch_free(nand_chip->ecc.priv);
2276 nand_chip->ecc.priv = NULL;
2277 }
2278 return err;
2279 }
2280
2281 static int omap_nand_remove(struct platform_device *pdev)
2282 {
2283 struct mtd_info *mtd = platform_get_drvdata(pdev);
2284 struct nand_chip *nand_chip = mtd_to_nand(mtd);
2285 struct omap_nand_info *info = mtd_to_omap(mtd);
2286 if (nand_chip->ecc.priv) {
2287 nand_bch_free(nand_chip->ecc.priv);
2288 nand_chip->ecc.priv = NULL;
2289 }
2290 if (info->dma)
2291 dma_release_channel(info->dma);
2292 nand_release(nand_chip);
2293 return 0;
2294 }
2295
2296 static const struct of_device_id omap_nand_ids[] = {
2297 { .compatible = "ti,omap2-nand", },
2298 {},
2299 };
2300 MODULE_DEVICE_TABLE(of, omap_nand_ids);
2301
2302 static struct platform_driver omap_nand_driver = {
2303 .probe = omap_nand_probe,
2304 .remove = omap_nand_remove,
2305 .driver = {
2306 .name = DRIVER_NAME,
2307 .of_match_table = of_match_ptr(omap_nand_ids),
2308 },
2309 };
2310
2311 module_platform_driver(omap_nand_driver);
2312
2313 MODULE_ALIAS("platform:" DRIVER_NAME);
2314 MODULE_LICENSE("GPL");
2315 MODULE_DESCRIPTION("Glue layer for NAND flash on TI OMAP boards");
Linux with added FPC-III support