?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
mtd: nand: brcmnand: Check flash #WP pin status before nand erase/program
[linux/fpc-iii.git] / drivers / mtd / nand / sunxi_nand.c
blob8b8470c4e6d02d2058264cd2a74847ce343df129
1 /*
2 * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com>
3 *
4 * Derived from:
5 * https://github.com/yuq/sunxi-nfc-mtd
6 * Copyright (C) 2013 Qiang Yu <yuq825@gmail.com>
7 *
8 * https://github.com/hno/Allwinner-Info
9 * Copyright (C) 2013 Henrik Nordström <Henrik Nordström>
10 *
11 * Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com>
12 * Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org>
13 *
14 * This program is free software; you can redistribute it and/or modify
15 * it under the terms of the GNU General Public License as published by
16 * the Free Software Foundation; either version 2 of the License, or
17 * (at your option) any later version.
18 *
19 * This program is distributed in the hope that it will be useful,
20 * but WITHOUT ANY WARRANTY; without even the implied warranty of
21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 * GNU General Public License for more details.
23 */
24
25 #include <linux/dma-mapping.h>
26 #include <linux/slab.h>
27 #include <linux/module.h>
28 #include <linux/moduleparam.h>
29 #include <linux/platform_device.h>
30 #include <linux/of.h>
31 #include <linux/of_device.h>
32 #include <linux/of_gpio.h>
33 #include <linux/mtd/mtd.h>
34 #include <linux/mtd/nand.h>
35 #include <linux/mtd/partitions.h>
36 #include <linux/clk.h>
37 #include <linux/delay.h>
38 #include <linux/dmaengine.h>
39 #include <linux/gpio.h>
40 #include <linux/interrupt.h>
41 #include <linux/iopoll.h>
42 #include <linux/reset.h>
43
44 #define NFC_REG_CTL 0x0000
45 #define NFC_REG_ST 0x0004
46 #define NFC_REG_INT 0x0008
47 #define NFC_REG_TIMING_CTL 0x000C
48 #define NFC_REG_TIMING_CFG 0x0010
49 #define NFC_REG_ADDR_LOW 0x0014
50 #define NFC_REG_ADDR_HIGH 0x0018
51 #define NFC_REG_SECTOR_NUM 0x001C
52 #define NFC_REG_CNT 0x0020
53 #define NFC_REG_CMD 0x0024
54 #define NFC_REG_RCMD_SET 0x0028
55 #define NFC_REG_WCMD_SET 0x002C
56 #define NFC_REG_IO_DATA 0x0030
57 #define NFC_REG_ECC_CTL 0x0034
58 #define NFC_REG_ECC_ST 0x0038
59 #define NFC_REG_DEBUG 0x003C
60 #define NFC_REG_ECC_ERR_CNT(x) ((0x0040 + (x)) & ~0x3)
61 #define NFC_REG_USER_DATA(x) (0x0050 + ((x) * 4))
62 #define NFC_REG_SPARE_AREA 0x00A0
63 #define NFC_REG_PAT_ID 0x00A4
64 #define NFC_RAM0_BASE 0x0400
65 #define NFC_RAM1_BASE 0x0800
66
67 /* define bit use in NFC_CTL */
68 #define NFC_EN BIT(0)
69 #define NFC_RESET BIT(1)
70 #define NFC_BUS_WIDTH_MSK BIT(2)
71 #define NFC_BUS_WIDTH_8 (0 << 2)
72 #define NFC_BUS_WIDTH_16 (1 << 2)
73 #define NFC_RB_SEL_MSK BIT(3)
74 #define NFC_RB_SEL(x) ((x) << 3)
75 #define NFC_CE_SEL_MSK GENMASK(26, 24)
76 #define NFC_CE_SEL(x) ((x) << 24)
77 #define NFC_CE_CTL BIT(6)
78 #define NFC_PAGE_SHIFT_MSK GENMASK(11, 8)
79 #define NFC_PAGE_SHIFT(x) (((x) < 10 ? 0 : (x) - 10) << 8)
80 #define NFC_SAM BIT(12)
81 #define NFC_RAM_METHOD BIT(14)
82 #define NFC_DEBUG_CTL BIT(31)
83
84 /* define bit use in NFC_ST */
85 #define NFC_RB_B2R BIT(0)
86 #define NFC_CMD_INT_FLAG BIT(1)
87 #define NFC_DMA_INT_FLAG BIT(2)
88 #define NFC_CMD_FIFO_STATUS BIT(3)
89 #define NFC_STA BIT(4)
90 #define NFC_NATCH_INT_FLAG BIT(5)
91 #define NFC_RB_STATE(x) BIT(x + 8)
92
93 /* define bit use in NFC_INT */
94 #define NFC_B2R_INT_ENABLE BIT(0)
95 #define NFC_CMD_INT_ENABLE BIT(1)
96 #define NFC_DMA_INT_ENABLE BIT(2)
97 #define NFC_INT_MASK (NFC_B2R_INT_ENABLE | \
98 NFC_CMD_INT_ENABLE | \
99 NFC_DMA_INT_ENABLE)
100
101 /* define bit use in NFC_TIMING_CTL */
102 #define NFC_TIMING_CTL_EDO BIT(8)
103
104 /* define NFC_TIMING_CFG register layout */
105 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD) \
106 (((tWB) & 0x3) | (((tADL) & 0x3) << 2) | \
107 (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) | \
108 (((tCAD) & 0x7) << 8))
109
110 /* define bit use in NFC_CMD */
111 #define NFC_CMD_LOW_BYTE_MSK GENMASK(7, 0)
112 #define NFC_CMD_HIGH_BYTE_MSK GENMASK(15, 8)
113 #define NFC_CMD(x) (x)
114 #define NFC_ADR_NUM_MSK GENMASK(18, 16)
115 #define NFC_ADR_NUM(x) (((x) - 1) << 16)
116 #define NFC_SEND_ADR BIT(19)
117 #define NFC_ACCESS_DIR BIT(20)
118 #define NFC_DATA_TRANS BIT(21)
119 #define NFC_SEND_CMD1 BIT(22)
120 #define NFC_WAIT_FLAG BIT(23)
121 #define NFC_SEND_CMD2 BIT(24)
122 #define NFC_SEQ BIT(25)
123 #define NFC_DATA_SWAP_METHOD BIT(26)
124 #define NFC_ROW_AUTO_INC BIT(27)
125 #define NFC_SEND_CMD3 BIT(28)
126 #define NFC_SEND_CMD4 BIT(29)
127 #define NFC_CMD_TYPE_MSK GENMASK(31, 30)
128 #define NFC_NORMAL_OP (0 << 30)
129 #define NFC_ECC_OP (1 << 30)
130 #define NFC_PAGE_OP (2 << 30)
131
132 /* define bit use in NFC_RCMD_SET */
133 #define NFC_READ_CMD_MSK GENMASK(7, 0)
134 #define NFC_RND_READ_CMD0_MSK GENMASK(15, 8)
135 #define NFC_RND_READ_CMD1_MSK GENMASK(23, 16)
136
137 /* define bit use in NFC_WCMD_SET */
138 #define NFC_PROGRAM_CMD_MSK GENMASK(7, 0)
139 #define NFC_RND_WRITE_CMD_MSK GENMASK(15, 8)
140 #define NFC_READ_CMD0_MSK GENMASK(23, 16)
141 #define NFC_READ_CMD1_MSK GENMASK(31, 24)
142
143 /* define bit use in NFC_ECC_CTL */
144 #define NFC_ECC_EN BIT(0)
145 #define NFC_ECC_PIPELINE BIT(3)
146 #define NFC_ECC_EXCEPTION BIT(4)
147 #define NFC_ECC_BLOCK_SIZE_MSK BIT(5)
148 #define NFC_RANDOM_EN BIT(9)
149 #define NFC_RANDOM_DIRECTION BIT(10)
150 #define NFC_ECC_MODE_MSK GENMASK(15, 12)
151 #define NFC_ECC_MODE(x) ((x) << 12)
152 #define NFC_RANDOM_SEED_MSK GENMASK(30, 16)
153 #define NFC_RANDOM_SEED(x) ((x) << 16)
154
155 /* define bit use in NFC_ECC_ST */
156 #define NFC_ECC_ERR(x) BIT(x)
157 #define NFC_ECC_ERR_MSK GENMASK(15, 0)
158 #define NFC_ECC_PAT_FOUND(x) BIT(x + 16)
159 #define NFC_ECC_ERR_CNT(b, x) (((x) >> (((b) % 4) * 8)) & 0xff)
160
161 #define NFC_DEFAULT_TIMEOUT_MS 1000
162
163 #define NFC_SRAM_SIZE 1024
164
165 #define NFC_MAX_CS 7
166
167 /*
168 * Ready/Busy detection type: describes the Ready/Busy detection modes
169 *
170 * @RB_NONE: no external detection available, rely on STATUS command
171 * and software timeouts
172 * @RB_NATIVE: use sunxi NAND controller Ready/Busy support. The Ready/Busy
173 * pin of the NAND flash chip must be connected to one of the
174 * native NAND R/B pins (those which can be muxed to the NAND
175 * Controller)
176 * @RB_GPIO: use a simple GPIO to handle Ready/Busy status. The Ready/Busy
177 * pin of the NAND flash chip must be connected to a GPIO capable
178 * pin.
179 */
180 enum sunxi_nand_rb_type {
181 RB_NONE,
182 RB_NATIVE,
183 RB_GPIO,
184 };
185
186 /*
187 * Ready/Busy structure: stores information related to Ready/Busy detection
188 *
189 * @type: the Ready/Busy detection mode
190 * @info: information related to the R/B detection mode. Either a gpio
191 * id or a native R/B id (those supported by the NAND controller).
192 */
193 struct sunxi_nand_rb {
194 enum sunxi_nand_rb_type type;
195 union {
196 int gpio;
197 int nativeid;
198 } info;
199 };
200
201 /*
202 * Chip Select structure: stores information related to NAND Chip Select
203 *
204 * @cs: the NAND CS id used to communicate with a NAND Chip
205 * @rb: the Ready/Busy description
206 */
207 struct sunxi_nand_chip_sel {
208 u8 cs;
209 struct sunxi_nand_rb rb;
210 };
211
212 /*
213 * sunxi HW ECC infos: stores information related to HW ECC support
214 *
215 * @mode: the sunxi ECC mode field deduced from ECC requirements
216 */
217 struct sunxi_nand_hw_ecc {
218 int mode;
219 };
220
221 /*
222 * NAND chip structure: stores NAND chip device related information
223 *
224 * @node: used to store NAND chips into a list
225 * @nand: base NAND chip structure
226 * @mtd: base MTD structure
227 * @clk_rate: clk_rate required for this NAND chip
228 * @timing_cfg TIMING_CFG register value for this NAND chip
229 * @selected: current active CS
230 * @nsels: number of CS lines required by the NAND chip
231 * @sels: array of CS lines descriptions
232 */
233 struct sunxi_nand_chip {
234 struct list_head node;
235 struct nand_chip nand;
236 unsigned long clk_rate;
237 u32 timing_cfg;
238 u32 timing_ctl;
239 int selected;
240 int addr_cycles;
241 u32 addr[2];
242 int cmd_cycles;
243 u8 cmd[2];
244 int nsels;
245 struct sunxi_nand_chip_sel sels[0];
246 };
247
248 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)
249 {
250 return container_of(nand, struct sunxi_nand_chip, nand);
251 }
252
253 /*
254 * NAND Controller structure: stores sunxi NAND controller information
255 *
256 * @controller: base controller structure
257 * @dev: parent device (used to print error messages)
258 * @regs: NAND controller registers
259 * @ahb_clk: NAND Controller AHB clock
260 * @mod_clk: NAND Controller mod clock
261 * @assigned_cs: bitmask describing already assigned CS lines
262 * @clk_rate: NAND controller current clock rate
263 * @chips: a list containing all the NAND chips attached to
264 * this NAND controller
265 * @complete: a completion object used to wait for NAND
266 * controller events
267 */
268 struct sunxi_nfc {
269 struct nand_hw_control controller;
270 struct device *dev;
271 void __iomem *regs;
272 struct clk *ahb_clk;
273 struct clk *mod_clk;
274 struct reset_control *reset;
275 unsigned long assigned_cs;
276 unsigned long clk_rate;
277 struct list_head chips;
278 struct completion complete;
279 struct dma_chan *dmac;
280 };
281
282 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_hw_control *ctrl)
283 {
284 return container_of(ctrl, struct sunxi_nfc, controller);
285 }
286
287 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id)
288 {
289 struct sunxi_nfc *nfc = dev_id;
290 u32 st = readl(nfc->regs + NFC_REG_ST);
291 u32 ien = readl(nfc->regs + NFC_REG_INT);
292
293 if (!(ien & st))
294 return IRQ_NONE;
295
296 if ((ien & st) == ien)
297 complete(&nfc->complete);
298
299 writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
300 writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT);
301
302 return IRQ_HANDLED;
303 }
304
305 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events,
306 bool use_polling, unsigned int timeout_ms)
307 {
308 int ret;
309
310 if (events & ~NFC_INT_MASK)
311 return -EINVAL;
312
313 if (!timeout_ms)
314 timeout_ms = NFC_DEFAULT_TIMEOUT_MS;
315
316 if (!use_polling) {
317 init_completion(&nfc->complete);
318
319 writel(events, nfc->regs + NFC_REG_INT);
320
321 ret = wait_for_completion_timeout(&nfc->complete,
322 msecs_to_jiffies(timeout_ms));
323
324 writel(0, nfc->regs + NFC_REG_INT);
325 } else {
326 u32 status;
327
328 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
329 (status & events) == events, 1,
330 timeout_ms * 1000);
331 }
332
333 writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
334
335 if (ret)
336 dev_err(nfc->dev, "wait interrupt timedout\n");
337
338 return ret;
339 }
340
341 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc)
342 {
343 u32 status;
344 int ret;
345
346 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
347 !(status & NFC_CMD_FIFO_STATUS), 1,
348 NFC_DEFAULT_TIMEOUT_MS * 1000);
349 if (ret)
350 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n");
351
352 return ret;
353 }
354
355 static int sunxi_nfc_rst(struct sunxi_nfc *nfc)
356 {
357 u32 ctl;
358 int ret;
359
360 writel(0, nfc->regs + NFC_REG_ECC_CTL);
361 writel(NFC_RESET, nfc->regs + NFC_REG_CTL);
362
363 ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl,
364 !(ctl & NFC_RESET), 1,
365 NFC_DEFAULT_TIMEOUT_MS * 1000);
366 if (ret)
367 dev_err(nfc->dev, "wait for NAND controller reset timedout\n");
368
369 return ret;
370 }
371
372 static int sunxi_nfc_dma_op_prepare(struct mtd_info *mtd, const void *buf,
373 int chunksize, int nchunks,
374 enum dma_data_direction ddir,
375 struct scatterlist *sg)
376 {
377 struct nand_chip *nand = mtd_to_nand(mtd);
378 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
379 struct dma_async_tx_descriptor *dmad;
380 enum dma_transfer_direction tdir;
381 dma_cookie_t dmat;
382 int ret;
383
384 if (ddir == DMA_FROM_DEVICE)
385 tdir = DMA_DEV_TO_MEM;
386 else
387 tdir = DMA_MEM_TO_DEV;
388
389 sg_init_one(sg, buf, nchunks * chunksize);
390 ret = dma_map_sg(nfc->dev, sg, 1, ddir);
391 if (!ret)
392 return -ENOMEM;
393
394 dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);
395 if (!dmad) {
396 ret = -EINVAL;
397 goto err_unmap_buf;
398 }
399
400 writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,
401 nfc->regs + NFC_REG_CTL);
402 writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);
403 writel(chunksize, nfc->regs + NFC_REG_CNT);
404 dmat = dmaengine_submit(dmad);
405
406 ret = dma_submit_error(dmat);
407 if (ret)
408 goto err_clr_dma_flag;
409
410 return 0;
411
412 err_clr_dma_flag:
413 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
414 nfc->regs + NFC_REG_CTL);
415
416 err_unmap_buf:
417 dma_unmap_sg(nfc->dev, sg, 1, ddir);
418 return ret;
419 }
420
421 static void sunxi_nfc_dma_op_cleanup(struct mtd_info *mtd,
422 enum dma_data_direction ddir,
423 struct scatterlist *sg)
424 {
425 struct nand_chip *nand = mtd_to_nand(mtd);
426 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
427
428 dma_unmap_sg(nfc->dev, sg, 1, ddir);
429 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
430 nfc->regs + NFC_REG_CTL);
431 }
432
433 static int sunxi_nfc_dev_ready(struct mtd_info *mtd)
434 {
435 struct nand_chip *nand = mtd_to_nand(mtd);
436 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
437 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
438 struct sunxi_nand_rb *rb;
439 int ret;
440
441 if (sunxi_nand->selected < 0)
442 return 0;
443
444 rb = &sunxi_nand->sels[sunxi_nand->selected].rb;
445
446 switch (rb->type) {
447 case RB_NATIVE:
448 ret = !!(readl(nfc->regs + NFC_REG_ST) &
449 NFC_RB_STATE(rb->info.nativeid));
450 break;
451 case RB_GPIO:
452 ret = gpio_get_value(rb->info.gpio);
453 break;
454 case RB_NONE:
455 default:
456 ret = 0;
457 dev_err(nfc->dev, "cannot check R/B NAND status!\n");
458 break;
459 }
460
461 return ret;
462 }
463
464 static void sunxi_nfc_select_chip(struct mtd_info *mtd, int chip)
465 {
466 struct nand_chip *nand = mtd_to_nand(mtd);
467 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
468 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
469 struct sunxi_nand_chip_sel *sel;
470 u32 ctl;
471
472 if (chip > 0 && chip >= sunxi_nand->nsels)
473 return;
474
475 if (chip == sunxi_nand->selected)
476 return;
477
478 ctl = readl(nfc->regs + NFC_REG_CTL) &
479 ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN);
480
481 if (chip >= 0) {
482 sel = &sunxi_nand->sels[chip];
483
484 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN |
485 NFC_PAGE_SHIFT(nand->page_shift);
486 if (sel->rb.type == RB_NONE) {
487 nand->dev_ready = NULL;
488 } else {
489 nand->dev_ready = sunxi_nfc_dev_ready;
490 if (sel->rb.type == RB_NATIVE)
491 ctl |= NFC_RB_SEL(sel->rb.info.nativeid);
492 }
493
494 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA);
495
496 if (nfc->clk_rate != sunxi_nand->clk_rate) {
497 clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate);
498 nfc->clk_rate = sunxi_nand->clk_rate;
499 }
500 }
501
502 writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL);
503 writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG);
504 writel(ctl, nfc->regs + NFC_REG_CTL);
505
506 sunxi_nand->selected = chip;
507 }
508
509 static void sunxi_nfc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
510 {
511 struct nand_chip *nand = mtd_to_nand(mtd);
512 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
513 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
514 int ret;
515 int cnt;
516 int offs = 0;
517 u32 tmp;
518
519 while (len > offs) {
520 cnt = min(len - offs, NFC_SRAM_SIZE);
521
522 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
523 if (ret)
524 break;
525
526 writel(cnt, nfc->regs + NFC_REG_CNT);
527 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
528 writel(tmp, nfc->regs + NFC_REG_CMD);
529
530 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
531 if (ret)
532 break;
533
534 if (buf)
535 memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE,
536 cnt);
537 offs += cnt;
538 }
539 }
540
541 static void sunxi_nfc_write_buf(struct mtd_info *mtd, const uint8_t *buf,
542 int len)
543 {
544 struct nand_chip *nand = mtd_to_nand(mtd);
545 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
546 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
547 int ret;
548 int cnt;
549 int offs = 0;
550 u32 tmp;
551
552 while (len > offs) {
553 cnt = min(len - offs, NFC_SRAM_SIZE);
554
555 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
556 if (ret)
557 break;
558
559 writel(cnt, nfc->regs + NFC_REG_CNT);
560 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt);
561 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
562 NFC_ACCESS_DIR;
563 writel(tmp, nfc->regs + NFC_REG_CMD);
564
565 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
566 if (ret)
567 break;
568
569 offs += cnt;
570 }
571 }
572
573 static uint8_t sunxi_nfc_read_byte(struct mtd_info *mtd)
574 {
575 uint8_t ret;
576
577 sunxi_nfc_read_buf(mtd, &ret, 1);
578
579 return ret;
580 }
581
582 static void sunxi_nfc_cmd_ctrl(struct mtd_info *mtd, int dat,
583 unsigned int ctrl)
584 {
585 struct nand_chip *nand = mtd_to_nand(mtd);
586 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
587 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
588 int ret;
589
590 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
591 if (ret)
592 return;
593
594 if (dat == NAND_CMD_NONE && (ctrl & NAND_NCE) &&
595 !(ctrl & (NAND_CLE | NAND_ALE))) {
596 u32 cmd = 0;
597
598 if (!sunxi_nand->addr_cycles && !sunxi_nand->cmd_cycles)
599 return;
600
601 if (sunxi_nand->cmd_cycles--)
602 cmd |= NFC_SEND_CMD1 | sunxi_nand->cmd[0];
603
604 if (sunxi_nand->cmd_cycles--) {
605 cmd |= NFC_SEND_CMD2;
606 writel(sunxi_nand->cmd[1],
607 nfc->regs + NFC_REG_RCMD_SET);
608 }
609
610 sunxi_nand->cmd_cycles = 0;
611
612 if (sunxi_nand->addr_cycles) {
613 cmd |= NFC_SEND_ADR |
614 NFC_ADR_NUM(sunxi_nand->addr_cycles);
615 writel(sunxi_nand->addr[0],
616 nfc->regs + NFC_REG_ADDR_LOW);
617 }
618
619 if (sunxi_nand->addr_cycles > 4)
620 writel(sunxi_nand->addr[1],
621 nfc->regs + NFC_REG_ADDR_HIGH);
622
623 writel(cmd, nfc->regs + NFC_REG_CMD);
624 sunxi_nand->addr[0] = 0;
625 sunxi_nand->addr[1] = 0;
626 sunxi_nand->addr_cycles = 0;
627 sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
628 }
629
630 if (ctrl & NAND_CLE) {
631 sunxi_nand->cmd[sunxi_nand->cmd_cycles++] = dat;
632 } else if (ctrl & NAND_ALE) {
633 sunxi_nand->addr[sunxi_nand->addr_cycles / 4] |=
634 dat << ((sunxi_nand->addr_cycles % 4) * 8);
635 sunxi_nand->addr_cycles++;
636 }
637 }
638
639 /* These seed values have been extracted from Allwinner's BSP */
640 static const u16 sunxi_nfc_randomizer_page_seeds[] = {
641 0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72,
642 0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436,
643 0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d,
644 0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130,
645 0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56,
646 0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55,
647 0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb,
648 0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17,
649 0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62,
650 0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064,
651 0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126,
652 0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e,
653 0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3,
654 0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b,
655 0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d,
656 0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db,
657 };
658
659 /*
660 * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds
661 * have been generated using
662 * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what
663 * the randomizer engine does internally before de/scrambling OOB data.
664 *
665 * Those tables are statically defined to avoid calculating randomizer state
666 * at runtime.
667 */
668 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = {
669 0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64,
670 0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409,
671 0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617,
672 0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d,
673 0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91,
674 0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d,
675 0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab,
676 0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8,
677 0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8,
678 0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b,
679 0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5,
680 0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a,
681 0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891,
682 0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36,
683 0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd,
684 0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0,
685 };
686
687 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = {
688 0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6,
689 0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982,
690 0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9,
691 0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07,
692 0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e,
693 0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2,
694 0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c,
695 0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f,
696 0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc,
697 0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e,
698 0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8,
699 0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68,
700 0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d,
701 0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179,
702 0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601,
703 0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd,
704 };
705
706 static u16 sunxi_nfc_randomizer_step(u16 state, int count)
707 {
708 state &= 0x7fff;
709
710 /*
711 * This loop is just a simple implementation of a Fibonacci LFSR using
712 * the x16 + x15 + 1 polynomial.
713 */
714 while (count--)
715 state = ((state >> 1) |
716 (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff;
717
718 return state;
719 }
720
721 static u16 sunxi_nfc_randomizer_state(struct mtd_info *mtd, int page, bool ecc)
722 {
723 const u16 *seeds = sunxi_nfc_randomizer_page_seeds;
724 int mod = mtd_div_by_ws(mtd->erasesize, mtd);
725
726 if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds))
727 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds);
728
729 if (ecc) {
730 if (mtd->ecc_step_size == 512)
731 seeds = sunxi_nfc_randomizer_ecc512_seeds;
732 else
733 seeds = sunxi_nfc_randomizer_ecc1024_seeds;
734 }
735
736 return seeds[page % mod];
737 }
738
739 static void sunxi_nfc_randomizer_config(struct mtd_info *mtd,
740 int page, bool ecc)
741 {
742 struct nand_chip *nand = mtd_to_nand(mtd);
743 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
744 u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
745 u16 state;
746
747 if (!(nand->options & NAND_NEED_SCRAMBLING))
748 return;
749
750 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
751 state = sunxi_nfc_randomizer_state(mtd, page, ecc);
752 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK;
753 writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL);
754 }
755
756 static void sunxi_nfc_randomizer_enable(struct mtd_info *mtd)
757 {
758 struct nand_chip *nand = mtd_to_nand(mtd);
759 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
760
761 if (!(nand->options & NAND_NEED_SCRAMBLING))
762 return;
763
764 writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN,
765 nfc->regs + NFC_REG_ECC_CTL);
766 }
767
768 static void sunxi_nfc_randomizer_disable(struct mtd_info *mtd)
769 {
770 struct nand_chip *nand = mtd_to_nand(mtd);
771 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
772
773 if (!(nand->options & NAND_NEED_SCRAMBLING))
774 return;
775
776 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN,
777 nfc->regs + NFC_REG_ECC_CTL);
778 }
779
780 static void sunxi_nfc_randomize_bbm(struct mtd_info *mtd, int page, u8 *bbm)
781 {
782 u16 state = sunxi_nfc_randomizer_state(mtd, page, true);
783
784 bbm[0] ^= state;
785 bbm[1] ^= sunxi_nfc_randomizer_step(state, 8);
786 }
787
788 static void sunxi_nfc_randomizer_write_buf(struct mtd_info *mtd,
789 const uint8_t *buf, int len,
790 bool ecc, int page)
791 {
792 sunxi_nfc_randomizer_config(mtd, page, ecc);
793 sunxi_nfc_randomizer_enable(mtd);
794 sunxi_nfc_write_buf(mtd, buf, len);
795 sunxi_nfc_randomizer_disable(mtd);
796 }
797
798 static void sunxi_nfc_randomizer_read_buf(struct mtd_info *mtd, uint8_t *buf,
799 int len, bool ecc, int page)
800 {
801 sunxi_nfc_randomizer_config(mtd, page, ecc);
802 sunxi_nfc_randomizer_enable(mtd);
803 sunxi_nfc_read_buf(mtd, buf, len);
804 sunxi_nfc_randomizer_disable(mtd);
805 }
806
807 static void sunxi_nfc_hw_ecc_enable(struct mtd_info *mtd)
808 {
809 struct nand_chip *nand = mtd_to_nand(mtd);
810 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
811 struct sunxi_nand_hw_ecc *data = nand->ecc.priv;
812 u32 ecc_ctl;
813
814 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
815 ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE |
816 NFC_ECC_BLOCK_SIZE_MSK);
817 ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(data->mode) | NFC_ECC_EXCEPTION |
818 NFC_ECC_PIPELINE;
819
820 writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL);
821 }
822
823 static void sunxi_nfc_hw_ecc_disable(struct mtd_info *mtd)
824 {
825 struct nand_chip *nand = mtd_to_nand(mtd);
826 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
827
828 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN,
829 nfc->regs + NFC_REG_ECC_CTL);
830 }
831
832 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf)
833 {
834 buf[0] = user_data;
835 buf[1] = user_data >> 8;
836 buf[2] = user_data >> 16;
837 buf[3] = user_data >> 24;
838 }
839
840 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf)
841 {
842 return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24);
843 }
844
845 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct mtd_info *mtd, u8 *oob,
846 int step, bool bbm, int page)
847 {
848 struct nand_chip *nand = mtd_to_nand(mtd);
849 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
850
851 sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)),
852 oob);
853
854 /* De-randomize the Bad Block Marker. */
855 if (bbm && (nand->options & NAND_NEED_SCRAMBLING))
856 sunxi_nfc_randomize_bbm(mtd, page, oob);
857 }
858
859 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct mtd_info *mtd,
860 const u8 *oob, int step,
861 bool bbm, int page)
862 {
863 struct nand_chip *nand = mtd_to_nand(mtd);
864 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
865 u8 user_data[4];
866
867 /* Randomize the Bad Block Marker. */
868 if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
869 memcpy(user_data, oob, sizeof(user_data));
870 sunxi_nfc_randomize_bbm(mtd, page, user_data);
871 oob = user_data;
872 }
873
874 writel(sunxi_nfc_buf_to_user_data(oob),
875 nfc->regs + NFC_REG_USER_DATA(step));
876 }
877
878 static void sunxi_nfc_hw_ecc_update_stats(struct mtd_info *mtd,
879 unsigned int *max_bitflips, int ret)
880 {
881 if (ret < 0) {
882 mtd->ecc_stats.failed++;
883 } else {
884 mtd->ecc_stats.corrected += ret;
885 *max_bitflips = max_t(unsigned int, *max_bitflips, ret);
886 }
887 }
888
889 static int sunxi_nfc_hw_ecc_correct(struct mtd_info *mtd, u8 *data, u8 *oob,
890 int step, u32 status, bool *erased)
891 {
892 struct nand_chip *nand = mtd_to_nand(mtd);
893 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
894 struct nand_ecc_ctrl *ecc = &nand->ecc;
895 u32 tmp;
896
897 *erased = false;
898
899 if (status & NFC_ECC_ERR(step))
900 return -EBADMSG;
901
902 if (status & NFC_ECC_PAT_FOUND(step)) {
903 u8 pattern;
904
905 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) {
906 pattern = 0x0;
907 } else {
908 pattern = 0xff;
909 *erased = true;
910 }
911
912 if (data)
913 memset(data, pattern, ecc->size);
914
915 if (oob)
916 memset(oob, pattern, ecc->bytes + 4);
917
918 return 0;
919 }
920
921 tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step));
922
923 return NFC_ECC_ERR_CNT(step, tmp);
924 }
925
926 static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd,
927 u8 *data, int data_off,
928 u8 *oob, int oob_off,
929 int *cur_off,
930 unsigned int *max_bitflips,
931 bool bbm, bool oob_required, int page)
932 {
933 struct nand_chip *nand = mtd_to_nand(mtd);
934 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
935 struct nand_ecc_ctrl *ecc = &nand->ecc;
936 int raw_mode = 0;
937 bool erased;
938 int ret;
939
940 if (*cur_off != data_off)
941 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, data_off, -1);
942
943 sunxi_nfc_randomizer_read_buf(mtd, NULL, ecc->size, false, page);
944
945 if (data_off + ecc->size != oob_off)
946 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
947
948 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
949 if (ret)
950 return ret;
951
952 sunxi_nfc_randomizer_enable(mtd);
953 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP,
954 nfc->regs + NFC_REG_CMD);
955
956 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
957 sunxi_nfc_randomizer_disable(mtd);
958 if (ret)
959 return ret;
960
961 *cur_off = oob_off + ecc->bytes + 4;
962
963 ret = sunxi_nfc_hw_ecc_correct(mtd, data, oob_required ? oob : NULL, 0,
964 readl(nfc->regs + NFC_REG_ECC_ST),
965 &erased);
966 if (erased)
967 return 1;
968
969 if (ret < 0) {
970 /*
971 * Re-read the data with the randomizer disabled to identify
972 * bitflips in erased pages.
973 */
974 if (nand->options & NAND_NEED_SCRAMBLING) {
975 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, data_off, -1);
976 nand->read_buf(mtd, data, ecc->size);
977 } else {
978 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,
979 ecc->size);
980 }
981
982 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
983 nand->read_buf(mtd, oob, ecc->bytes + 4);
984
985 ret = nand_check_erased_ecc_chunk(data, ecc->size,
986 oob, ecc->bytes + 4,
987 NULL, 0, ecc->strength);
988 if (ret >= 0)
989 raw_mode = 1;
990 } else {
991 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);
992
993 if (oob_required) {
994 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
995 sunxi_nfc_randomizer_read_buf(mtd, oob, ecc->bytes + 4,
996 true, page);
997
998 sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, 0,
999 bbm, page);
1000 }
1001 }
1002
1003 sunxi_nfc_hw_ecc_update_stats(mtd, max_bitflips, ret);
1004
1005 return raw_mode;
1006 }
1007
1008 static void sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info *mtd,
1009 u8 *oob, int *cur_off,
1010 bool randomize, int page)
1011 {
1012 struct nand_chip *nand = mtd_to_nand(mtd);
1013 struct nand_ecc_ctrl *ecc = &nand->ecc;
1014 int offset = ((ecc->bytes + 4) * ecc->steps);
1015 int len = mtd->oobsize - offset;
1016
1017 if (len <= 0)
1018 return;
1019
1020 if (!cur_off || *cur_off != offset)
1021 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1022 offset + mtd->writesize, -1);
1023
1024 if (!randomize)
1025 sunxi_nfc_read_buf(mtd, oob + offset, len);
1026 else
1027 sunxi_nfc_randomizer_read_buf(mtd, oob + offset, len,
1028 false, page);
1029
1030 if (cur_off)
1031 *cur_off = mtd->oobsize + mtd->writesize;
1032 }
1033
1034 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf,
1035 int oob_required, int page,
1036 int nchunks)
1037 {
1038 struct nand_chip *nand = mtd_to_nand(mtd);
1039 bool randomized = nand->options & NAND_NEED_SCRAMBLING;
1040 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1041 struct nand_ecc_ctrl *ecc = &nand->ecc;
1042 unsigned int max_bitflips = 0;
1043 int ret, i, raw_mode = 0;
1044 struct scatterlist sg;
1045 u32 status;
1046
1047 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1048 if (ret)
1049 return ret;
1050
1051 ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, nchunks,
1052 DMA_FROM_DEVICE, &sg);
1053 if (ret)
1054 return ret;
1055
1056 sunxi_nfc_hw_ecc_enable(mtd);
1057 sunxi_nfc_randomizer_config(mtd, page, false);
1058 sunxi_nfc_randomizer_enable(mtd);
1059
1060 writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |
1061 NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);
1062
1063 dma_async_issue_pending(nfc->dmac);
1064
1065 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,
1066 nfc->regs + NFC_REG_CMD);
1067
1068 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
1069 if (ret)
1070 dmaengine_terminate_all(nfc->dmac);
1071
1072 sunxi_nfc_randomizer_disable(mtd);
1073 sunxi_nfc_hw_ecc_disable(mtd);
1074
1075 sunxi_nfc_dma_op_cleanup(mtd, DMA_FROM_DEVICE, &sg);
1076
1077 if (ret)
1078 return ret;
1079
1080 status = readl(nfc->regs + NFC_REG_ECC_ST);
1081
1082 for (i = 0; i < nchunks; i++) {
1083 int data_off = i * ecc->size;
1084 int oob_off = i * (ecc->bytes + 4);
1085 u8 *data = buf + data_off;
1086 u8 *oob = nand->oob_poi + oob_off;
1087 bool erased;
1088
1089 ret = sunxi_nfc_hw_ecc_correct(mtd, randomized ? data : NULL,
1090 oob_required ? oob : NULL,
1091 i, status, &erased);
1092
1093 /* ECC errors are handled in the second loop. */
1094 if (ret < 0)
1095 continue;
1096
1097 if (oob_required && !erased) {
1098 /* TODO: use DMA to retrieve OOB */
1099 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1100 mtd->writesize + oob_off, -1);
1101 nand->read_buf(mtd, oob, ecc->bytes + 4);
1102
1103 sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, i,
1104 !i, page);
1105 }
1106
1107 if (erased)
1108 raw_mode = 1;
1109
1110 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1111 }
1112
1113 if (status & NFC_ECC_ERR_MSK) {
1114 for (i = 0; i < nchunks; i++) {
1115 int data_off = i * ecc->size;
1116 int oob_off = i * (ecc->bytes + 4);
1117 u8 *data = buf + data_off;
1118 u8 *oob = nand->oob_poi + oob_off;
1119
1120 if (!(status & NFC_ECC_ERR(i)))
1121 continue;
1122
1123 /*
1124 * Re-read the data with the randomizer disabled to
1125 * identify bitflips in erased pages.
1126 */
1127 if (randomized) {
1128 /* TODO: use DMA to read page in raw mode */
1129 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1130 data_off, -1);
1131 nand->read_buf(mtd, data, ecc->size);
1132 }
1133
1134 /* TODO: use DMA to retrieve OOB */
1135 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1136 mtd->writesize + oob_off, -1);
1137 nand->read_buf(mtd, oob, ecc->bytes + 4);
1138
1139 ret = nand_check_erased_ecc_chunk(data, ecc->size,
1140 oob, ecc->bytes + 4,
1141 NULL, 0,
1142 ecc->strength);
1143 if (ret >= 0)
1144 raw_mode = 1;
1145
1146 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1147 }
1148 }
1149
1150 if (oob_required)
1151 sunxi_nfc_hw_ecc_read_extra_oob(mtd, nand->oob_poi,
1152 NULL, !raw_mode,
1153 page);
1154
1155 return max_bitflips;
1156 }
1157
1158 static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd,
1159 const u8 *data, int data_off,
1160 const u8 *oob, int oob_off,
1161 int *cur_off, bool bbm,
1162 int page)
1163 {
1164 struct nand_chip *nand = mtd_to_nand(mtd);
1165 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1166 struct nand_ecc_ctrl *ecc = &nand->ecc;
1167 int ret;
1168
1169 if (data_off != *cur_off)
1170 nand->cmdfunc(mtd, NAND_CMD_RNDIN, data_off, -1);
1171
1172 sunxi_nfc_randomizer_write_buf(mtd, data, ecc->size, false, page);
1173
1174 if (data_off + ecc->size != oob_off)
1175 nand->cmdfunc(mtd, NAND_CMD_RNDIN, oob_off, -1);
1176
1177 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1178 if (ret)
1179 return ret;
1180
1181 sunxi_nfc_randomizer_enable(mtd);
1182 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, 0, bbm, page);
1183
1184 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
1185 NFC_ACCESS_DIR | NFC_ECC_OP,
1186 nfc->regs + NFC_REG_CMD);
1187
1188 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
1189 sunxi_nfc_randomizer_disable(mtd);
1190 if (ret)
1191 return ret;
1192
1193 *cur_off = oob_off + ecc->bytes + 4;
1194
1195 return 0;
1196 }
1197
1198 static void sunxi_nfc_hw_ecc_write_extra_oob(struct mtd_info *mtd,
1199 u8 *oob, int *cur_off,
1200 int page)
1201 {
1202 struct nand_chip *nand = mtd_to_nand(mtd);
1203 struct nand_ecc_ctrl *ecc = &nand->ecc;
1204 int offset = ((ecc->bytes + 4) * ecc->steps);
1205 int len = mtd->oobsize - offset;
1206
1207 if (len <= 0)
1208 return;
1209
1210 if (!cur_off || *cur_off != offset)
1211 nand->cmdfunc(mtd, NAND_CMD_RNDIN,
1212 offset + mtd->writesize, -1);
1213
1214 sunxi_nfc_randomizer_write_buf(mtd, oob + offset, len, false, page);
1215
1216 if (cur_off)
1217 *cur_off = mtd->oobsize + mtd->writesize;
1218 }
1219
1220 static int sunxi_nfc_hw_ecc_read_page(struct mtd_info *mtd,
1221 struct nand_chip *chip, uint8_t *buf,
1222 int oob_required, int page)
1223 {
1224 struct nand_ecc_ctrl *ecc = &chip->ecc;
1225 unsigned int max_bitflips = 0;
1226 int ret, i, cur_off = 0;
1227 bool raw_mode = false;
1228
1229 sunxi_nfc_hw_ecc_enable(mtd);
1230
1231 for (i = 0; i < ecc->steps; i++) {
1232 int data_off = i * ecc->size;
1233 int oob_off = i * (ecc->bytes + 4);
1234 u8 *data = buf + data_off;
1235 u8 *oob = chip->oob_poi + oob_off;
1236
1237 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1238 oob_off + mtd->writesize,
1239 &cur_off, &max_bitflips,
1240 !i, oob_required, page);
1241 if (ret < 0)
1242 return ret;
1243 else if (ret)
1244 raw_mode = true;
1245 }
1246
1247 if (oob_required)
1248 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1249 !raw_mode, page);
1250
1251 sunxi_nfc_hw_ecc_disable(mtd);
1252
1253 return max_bitflips;
1254 }
1255
1256 static int sunxi_nfc_hw_ecc_read_page_dma(struct mtd_info *mtd,
1257 struct nand_chip *chip, u8 *buf,
1258 int oob_required, int page)
1259 {
1260 int ret;
1261
1262 ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, oob_required, page,
1263 chip->ecc.steps);
1264 if (ret >= 0)
1265 return ret;
1266
1267 /* Fallback to PIO mode */
1268 chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
1269
1270 return sunxi_nfc_hw_ecc_read_page(mtd, chip, buf, oob_required, page);
1271 }
1272
1273 static int sunxi_nfc_hw_ecc_read_subpage(struct mtd_info *mtd,
1274 struct nand_chip *chip,
1275 u32 data_offs, u32 readlen,
1276 u8 *bufpoi, int page)
1277 {
1278 struct nand_ecc_ctrl *ecc = &chip->ecc;
1279 int ret, i, cur_off = 0;
1280 unsigned int max_bitflips = 0;
1281
1282 sunxi_nfc_hw_ecc_enable(mtd);
1283
1284 chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1285 for (i = data_offs / ecc->size;
1286 i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) {
1287 int data_off = i * ecc->size;
1288 int oob_off = i * (ecc->bytes + 4);
1289 u8 *data = bufpoi + data_off;
1290 u8 *oob = chip->oob_poi + oob_off;
1291
1292 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off,
1293 oob,
1294 oob_off + mtd->writesize,
1295 &cur_off, &max_bitflips, !i,
1296 false, page);
1297 if (ret < 0)
1298 return ret;
1299 }
1300
1301 sunxi_nfc_hw_ecc_disable(mtd);
1302
1303 return max_bitflips;
1304 }
1305
1306 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct mtd_info *mtd,
1307 struct nand_chip *chip,
1308 u32 data_offs, u32 readlen,
1309 u8 *buf, int page)
1310 {
1311 int nchunks = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size);
1312 int ret;
1313
1314 ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, false, page, nchunks);
1315 if (ret >= 0)
1316 return ret;
1317
1318 /* Fallback to PIO mode */
1319 chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
1320
1321 return sunxi_nfc_hw_ecc_read_subpage(mtd, chip, data_offs, readlen,
1322 buf, page);
1323 }
1324
1325 static int sunxi_nfc_hw_ecc_write_page(struct mtd_info *mtd,
1326 struct nand_chip *chip,
1327 const uint8_t *buf, int oob_required,
1328 int page)
1329 {
1330 struct nand_ecc_ctrl *ecc = &chip->ecc;
1331 int ret, i, cur_off = 0;
1332
1333 sunxi_nfc_hw_ecc_enable(mtd);
1334
1335 for (i = 0; i < ecc->steps; i++) {
1336 int data_off = i * ecc->size;
1337 int oob_off = i * (ecc->bytes + 4);
1338 const u8 *data = buf + data_off;
1339 const u8 *oob = chip->oob_poi + oob_off;
1340
1341 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1342 oob_off + mtd->writesize,
1343 &cur_off, !i, page);
1344 if (ret)
1345 return ret;
1346 }
1347
1348 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1349 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1350 &cur_off, page);
1351
1352 sunxi_nfc_hw_ecc_disable(mtd);
1353
1354 return 0;
1355 }
1356
1357 static int sunxi_nfc_hw_ecc_write_subpage(struct mtd_info *mtd,
1358 struct nand_chip *chip,
1359 u32 data_offs, u32 data_len,
1360 const u8 *buf, int oob_required,
1361 int page)
1362 {
1363 struct nand_ecc_ctrl *ecc = &chip->ecc;
1364 int ret, i, cur_off = 0;
1365
1366 sunxi_nfc_hw_ecc_enable(mtd);
1367
1368 for (i = data_offs / ecc->size;
1369 i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) {
1370 int data_off = i * ecc->size;
1371 int oob_off = i * (ecc->bytes + 4);
1372 const u8 *data = buf + data_off;
1373 const u8 *oob = chip->oob_poi + oob_off;
1374
1375 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1376 oob_off + mtd->writesize,
1377 &cur_off, !i, page);
1378 if (ret)
1379 return ret;
1380 }
1381
1382 sunxi_nfc_hw_ecc_disable(mtd);
1383
1384 return 0;
1385 }
1386
1387 static int sunxi_nfc_hw_ecc_write_page_dma(struct mtd_info *mtd,
1388 struct nand_chip *chip,
1389 const u8 *buf,
1390 int oob_required,
1391 int page)
1392 {
1393 struct nand_chip *nand = mtd_to_nand(mtd);
1394 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1395 struct nand_ecc_ctrl *ecc = &nand->ecc;
1396 struct scatterlist sg;
1397 int ret, i;
1398
1399 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1400 if (ret)
1401 return ret;
1402
1403 ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, ecc->steps,
1404 DMA_TO_DEVICE, &sg);
1405 if (ret)
1406 goto pio_fallback;
1407
1408 for (i = 0; i < ecc->steps; i++) {
1409 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));
1410
1411 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, i, !i, page);
1412 }
1413
1414 sunxi_nfc_hw_ecc_enable(mtd);
1415 sunxi_nfc_randomizer_config(mtd, page, false);
1416 sunxi_nfc_randomizer_enable(mtd);
1417
1418 writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
1419 nfc->regs + NFC_REG_RCMD_SET);
1420
1421 dma_async_issue_pending(nfc->dmac);
1422
1423 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
1424 NFC_DATA_TRANS | NFC_ACCESS_DIR,
1425 nfc->regs + NFC_REG_CMD);
1426
1427 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
1428 if (ret)
1429 dmaengine_terminate_all(nfc->dmac);
1430
1431 sunxi_nfc_randomizer_disable(mtd);
1432 sunxi_nfc_hw_ecc_disable(mtd);
1433
1434 sunxi_nfc_dma_op_cleanup(mtd, DMA_TO_DEVICE, &sg);
1435
1436 if (ret)
1437 return ret;
1438
1439 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1440 /* TODO: use DMA to transfer extra OOB bytes ? */
1441 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1442 NULL, page);
1443
1444 return 0;
1445
1446 pio_fallback:
1447 return sunxi_nfc_hw_ecc_write_page(mtd, chip, buf, oob_required, page);
1448 }
1449
1450 static int sunxi_nfc_hw_syndrome_ecc_read_page(struct mtd_info *mtd,
1451 struct nand_chip *chip,
1452 uint8_t *buf, int oob_required,
1453 int page)
1454 {
1455 struct nand_ecc_ctrl *ecc = &chip->ecc;
1456 unsigned int max_bitflips = 0;
1457 int ret, i, cur_off = 0;
1458 bool raw_mode = false;
1459
1460 sunxi_nfc_hw_ecc_enable(mtd);
1461
1462 for (i = 0; i < ecc->steps; i++) {
1463 int data_off = i * (ecc->size + ecc->bytes + 4);
1464 int oob_off = data_off + ecc->size;
1465 u8 *data = buf + (i * ecc->size);
1466 u8 *oob = chip->oob_poi + (i * (ecc->bytes + 4));
1467
1468 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1469 oob_off, &cur_off,
1470 &max_bitflips, !i,
1471 oob_required,
1472 page);
1473 if (ret < 0)
1474 return ret;
1475 else if (ret)
1476 raw_mode = true;
1477 }
1478
1479 if (oob_required)
1480 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1481 !raw_mode, page);
1482
1483 sunxi_nfc_hw_ecc_disable(mtd);
1484
1485 return max_bitflips;
1486 }
1487
1488 static int sunxi_nfc_hw_syndrome_ecc_write_page(struct mtd_info *mtd,
1489 struct nand_chip *chip,
1490 const uint8_t *buf,
1491 int oob_required, int page)
1492 {
1493 struct nand_ecc_ctrl *ecc = &chip->ecc;
1494 int ret, i, cur_off = 0;
1495
1496 sunxi_nfc_hw_ecc_enable(mtd);
1497
1498 for (i = 0; i < ecc->steps; i++) {
1499 int data_off = i * (ecc->size + ecc->bytes + 4);
1500 int oob_off = data_off + ecc->size;
1501 const u8 *data = buf + (i * ecc->size);
1502 const u8 *oob = chip->oob_poi + (i * (ecc->bytes + 4));
1503
1504 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off,
1505 oob, oob_off, &cur_off,
1506 false, page);
1507 if (ret)
1508 return ret;
1509 }
1510
1511 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1512 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1513 &cur_off, page);
1514
1515 sunxi_nfc_hw_ecc_disable(mtd);
1516
1517 return 0;
1518 }
1519
1520 static int sunxi_nfc_hw_common_ecc_read_oob(struct mtd_info *mtd,
1521 struct nand_chip *chip,
1522 int page)
1523 {
1524 chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1525
1526 chip->pagebuf = -1;
1527
1528 return chip->ecc.read_page(mtd, chip, chip->buffers->databuf, 1, page);
1529 }
1530
1531 static int sunxi_nfc_hw_common_ecc_write_oob(struct mtd_info *mtd,
1532 struct nand_chip *chip,
1533 int page)
1534 {
1535 int ret, status;
1536
1537 chip->cmdfunc(mtd, NAND_CMD_SEQIN, 0, page);
1538
1539 chip->pagebuf = -1;
1540
1541 memset(chip->buffers->databuf, 0xff, mtd->writesize);
1542 ret = chip->ecc.write_page(mtd, chip, chip->buffers->databuf, 1, page);
1543 if (ret)
1544 return ret;
1545
1546 /* Send command to program the OOB data */
1547 chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1548
1549 status = chip->waitfunc(mtd, chip);
1550
1551 return status & NAND_STATUS_FAIL ? -EIO : 0;
1552 }
1553
1554 static const s32 tWB_lut[] = {6, 12, 16, 20};
1555 static const s32 tRHW_lut[] = {4, 8, 12, 20};
1556
1557 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
1558 u32 clk_period)
1559 {
1560 u32 clk_cycles = DIV_ROUND_UP(duration, clk_period);
1561 int i;
1562
1563 for (i = 0; i < lut_size; i++) {
1564 if (clk_cycles <= lut[i])
1565 return i;
1566 }
1567
1568 /* Doesn't fit */
1569 return -EINVAL;
1570 }
1571
1572 #define sunxi_nand_lookup_timing(l, p, c) \
1573 _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)
1574
1575 static int sunxi_nfc_setup_data_interface(struct mtd_info *mtd,
1576 const struct nand_data_interface *conf,
1577 bool check_only)
1578 {
1579 struct nand_chip *nand = mtd_to_nand(mtd);
1580 struct sunxi_nand_chip *chip = to_sunxi_nand(nand);
1581 struct sunxi_nfc *nfc = to_sunxi_nfc(chip->nand.controller);
1582 const struct nand_sdr_timings *timings;
1583 u32 min_clk_period = 0;
1584 s32 tWB, tADL, tWHR, tRHW, tCAD;
1585 long real_clk_rate;
1586
1587 timings = nand_get_sdr_timings(conf);
1588 if (IS_ERR(timings))
1589 return -ENOTSUPP;
1590
1591 /* T1 <=> tCLS */
1592 if (timings->tCLS_min > min_clk_period)
1593 min_clk_period = timings->tCLS_min;
1594
1595 /* T2 <=> tCLH */
1596 if (timings->tCLH_min > min_clk_period)
1597 min_clk_period = timings->tCLH_min;
1598
1599 /* T3 <=> tCS */
1600 if (timings->tCS_min > min_clk_period)
1601 min_clk_period = timings->tCS_min;
1602
1603 /* T4 <=> tCH */
1604 if (timings->tCH_min > min_clk_period)
1605 min_clk_period = timings->tCH_min;
1606
1607 /* T5 <=> tWP */
1608 if (timings->tWP_min > min_clk_period)
1609 min_clk_period = timings->tWP_min;
1610
1611 /* T6 <=> tWH */
1612 if (timings->tWH_min > min_clk_period)
1613 min_clk_period = timings->tWH_min;
1614
1615 /* T7 <=> tALS */
1616 if (timings->tALS_min > min_clk_period)
1617 min_clk_period = timings->tALS_min;
1618
1619 /* T8 <=> tDS */
1620 if (timings->tDS_min > min_clk_period)
1621 min_clk_period = timings->tDS_min;
1622
1623 /* T9 <=> tDH */
1624 if (timings->tDH_min > min_clk_period)
1625 min_clk_period = timings->tDH_min;
1626
1627 /* T10 <=> tRR */
1628 if (timings->tRR_min > (min_clk_period * 3))
1629 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);
1630
1631 /* T11 <=> tALH */
1632 if (timings->tALH_min > min_clk_period)
1633 min_clk_period = timings->tALH_min;
1634
1635 /* T12 <=> tRP */
1636 if (timings->tRP_min > min_clk_period)
1637 min_clk_period = timings->tRP_min;
1638
1639 /* T13 <=> tREH */
1640 if (timings->tREH_min > min_clk_period)
1641 min_clk_period = timings->tREH_min;
1642
1643 /* T14 <=> tRC */
1644 if (timings->tRC_min > (min_clk_period * 2))
1645 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2);
1646
1647 /* T15 <=> tWC */
1648 if (timings->tWC_min > (min_clk_period * 2))
1649 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);
1650
1651 /* T16 - T19 + tCAD */
1652 if (timings->tWB_max > (min_clk_period * 20))
1653 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);
1654
1655 if (timings->tADL_min > (min_clk_period * 32))
1656 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
1657
1658 if (timings->tWHR_min > (min_clk_period * 32))
1659 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
1660
1661 if (timings->tRHW_min > (min_clk_period * 20))
1662 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);
1663
1664 tWB = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,
1665 min_clk_period);
1666 if (tWB < 0) {
1667 dev_err(nfc->dev, "unsupported tWB\n");
1668 return tWB;
1669 }
1670
1671 tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;
1672 if (tADL > 3) {
1673 dev_err(nfc->dev, "unsupported tADL\n");
1674 return -EINVAL;
1675 }
1676
1677 tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;
1678 if (tWHR > 3) {
1679 dev_err(nfc->dev, "unsupported tWHR\n");
1680 return -EINVAL;
1681 }
1682
1683 tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,
1684 min_clk_period);
1685 if (tRHW < 0) {
1686 dev_err(nfc->dev, "unsupported tRHW\n");
1687 return tRHW;
1688 }
1689
1690 if (check_only)
1691 return 0;
1692
1693 /*
1694 * TODO: according to ONFI specs this value only applies for DDR NAND,
1695 * but Allwinner seems to set this to 0x7. Mimic them for now.
1696 */
1697 tCAD = 0x7;
1698
1699 /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */
1700 chip->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD);
1701
1702 /* Convert min_clk_period from picoseconds to nanoseconds */
1703 min_clk_period = DIV_ROUND_UP(min_clk_period, 1000);
1704
1705 /*
1706 * Unlike what is stated in Allwinner datasheet, the clk_rate should
1707 * be set to (1 / min_clk_period), and not (2 / min_clk_period).
1708 * This new formula was verified with a scope and validated by
1709 * Allwinner engineers.
1710 */
1711 chip->clk_rate = NSEC_PER_SEC / min_clk_period;
1712 real_clk_rate = clk_round_rate(nfc->mod_clk, chip->clk_rate);
1713
1714 /*
1715 * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data
1716 * output cycle timings shall be used if the host drives tRC less than
1717 * 30 ns.
1718 */
1719 min_clk_period = NSEC_PER_SEC / real_clk_rate;
1720 chip->timing_ctl = ((min_clk_period * 2) < 30) ?
1721 NFC_TIMING_CTL_EDO : 0;
1722
1723 return 0;
1724 }
1725
1726 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1727 struct mtd_oob_region *oobregion)
1728 {
1729 struct nand_chip *nand = mtd_to_nand(mtd);
1730 struct nand_ecc_ctrl *ecc = &nand->ecc;
1731
1732 if (section >= ecc->steps)
1733 return -ERANGE;
1734
1735 oobregion->offset = section * (ecc->bytes + 4) + 4;
1736 oobregion->length = ecc->bytes;
1737
1738 return 0;
1739 }
1740
1741 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
1742 struct mtd_oob_region *oobregion)
1743 {
1744 struct nand_chip *nand = mtd_to_nand(mtd);
1745 struct nand_ecc_ctrl *ecc = &nand->ecc;
1746
1747 if (section > ecc->steps)
1748 return -ERANGE;
1749
1750 /*
1751 * The first 2 bytes are used for BB markers, hence we
1752 * only have 2 bytes available in the first user data
1753 * section.
1754 */
1755 if (!section && ecc->mode == NAND_ECC_HW) {
1756 oobregion->offset = 2;
1757 oobregion->length = 2;
1758
1759 return 0;
1760 }
1761
1762 oobregion->offset = section * (ecc->bytes + 4);
1763
1764 if (section < ecc->steps)
1765 oobregion->length = 4;
1766 else
1767 oobregion->offset = mtd->oobsize - oobregion->offset;
1768
1769 return 0;
1770 }
1771
1772 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = {
1773 .ecc = sunxi_nand_ooblayout_ecc,
1774 .free = sunxi_nand_ooblayout_free,
1775 };
1776
1777 static int sunxi_nand_hw_common_ecc_ctrl_init(struct mtd_info *mtd,
1778 struct nand_ecc_ctrl *ecc,
1779 struct device_node *np)
1780 {
1781 static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 };
1782 struct nand_chip *nand = mtd_to_nand(mtd);
1783 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1784 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1785 struct sunxi_nand_hw_ecc *data;
1786 int nsectors;
1787 int ret;
1788 int i;
1789
1790 if (ecc->options & NAND_ECC_MAXIMIZE) {
1791 int bytes;
1792
1793 ecc->size = 1024;
1794 nsectors = mtd->writesize / ecc->size;
1795
1796 /* Reserve 2 bytes for the BBM */
1797 bytes = (mtd->oobsize - 2) / nsectors;
1798
1799 /* 4 non-ECC bytes are added before each ECC bytes section */
1800 bytes -= 4;
1801
1802 /* and bytes has to be even. */
1803 if (bytes % 2)
1804 bytes--;
1805
1806 ecc->strength = bytes * 8 / fls(8 * ecc->size);
1807
1808 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1809 if (strengths[i] > ecc->strength)
1810 break;
1811 }
1812
1813 if (!i)
1814 ecc->strength = 0;
1815 else
1816 ecc->strength = strengths[i - 1];
1817 }
1818
1819 if (ecc->size != 512 && ecc->size != 1024)
1820 return -EINVAL;
1821
1822 data = kzalloc(sizeof(*data), GFP_KERNEL);
1823 if (!data)
1824 return -ENOMEM;
1825
1826 /* Prefer 1k ECC chunk over 512 ones */
1827 if (ecc->size == 512 && mtd->writesize > 512) {
1828 ecc->size = 1024;
1829 ecc->strength *= 2;
1830 }
1831
1832 /* Add ECC info retrieval from DT */
1833 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1834 if (ecc->strength <= strengths[i])
1835 break;
1836 }
1837
1838 if (i >= ARRAY_SIZE(strengths)) {
1839 dev_err(nfc->dev, "unsupported strength\n");
1840 ret = -ENOTSUPP;
1841 goto err;
1842 }
1843
1844 data->mode = i;
1845
1846 /* HW ECC always request ECC bytes for 1024 bytes blocks */
1847 ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8);
1848
1849 /* HW ECC always work with even numbers of ECC bytes */
1850 ecc->bytes = ALIGN(ecc->bytes, 2);
1851
1852 nsectors = mtd->writesize / ecc->size;
1853
1854 if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) {
1855 ret = -EINVAL;
1856 goto err;
1857 }
1858
1859 ecc->read_oob = sunxi_nfc_hw_common_ecc_read_oob;
1860 ecc->write_oob = sunxi_nfc_hw_common_ecc_write_oob;
1861 mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops);
1862 ecc->priv = data;
1863
1864 return 0;
1865
1866 err:
1867 kfree(data);
1868
1869 return ret;
1870 }
1871
1872 static void sunxi_nand_hw_common_ecc_ctrl_cleanup(struct nand_ecc_ctrl *ecc)
1873 {
1874 kfree(ecc->priv);
1875 }
1876
1877 static int sunxi_nand_hw_ecc_ctrl_init(struct mtd_info *mtd,
1878 struct nand_ecc_ctrl *ecc,
1879 struct device_node *np)
1880 {
1881 struct nand_chip *nand = mtd_to_nand(mtd);
1882 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1883 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1884 int ret;
1885
1886 ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
1887 if (ret)
1888 return ret;
1889
1890 if (nfc->dmac) {
1891 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
1892 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
1893 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
1894 nand->options |= NAND_USE_BOUNCE_BUFFER;
1895 } else {
1896 ecc->read_page = sunxi_nfc_hw_ecc_read_page;
1897 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1898 ecc->write_page = sunxi_nfc_hw_ecc_write_page;
1899 }
1900
1901 /* TODO: support DMA for raw accesses and subpage write */
1902 ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage;
1903 ecc->read_oob_raw = nand_read_oob_std;
1904 ecc->write_oob_raw = nand_write_oob_std;
1905 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1906
1907 return 0;
1908 }
1909
1910 static int sunxi_nand_hw_syndrome_ecc_ctrl_init(struct mtd_info *mtd,
1911 struct nand_ecc_ctrl *ecc,
1912 struct device_node *np)
1913 {
1914 int ret;
1915
1916 ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
1917 if (ret)
1918 return ret;
1919
1920 ecc->prepad = 4;
1921 ecc->read_page = sunxi_nfc_hw_syndrome_ecc_read_page;
1922 ecc->write_page = sunxi_nfc_hw_syndrome_ecc_write_page;
1923 ecc->read_oob_raw = nand_read_oob_syndrome;
1924 ecc->write_oob_raw = nand_write_oob_syndrome;
1925
1926 return 0;
1927 }
1928
1929 static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc)
1930 {
1931 switch (ecc->mode) {
1932 case NAND_ECC_HW:
1933 case NAND_ECC_HW_SYNDROME:
1934 sunxi_nand_hw_common_ecc_ctrl_cleanup(ecc);
1935 break;
1936 case NAND_ECC_NONE:
1937 default:
1938 break;
1939 }
1940 }
1941
1942 static int sunxi_nand_ecc_init(struct mtd_info *mtd, struct nand_ecc_ctrl *ecc,
1943 struct device_node *np)
1944 {
1945 struct nand_chip *nand = mtd_to_nand(mtd);
1946 int ret;
1947
1948 if (!ecc->size) {
1949 ecc->size = nand->ecc_step_ds;
1950 ecc->strength = nand->ecc_strength_ds;
1951 }
1952
1953 if (!ecc->size || !ecc->strength)
1954 return -EINVAL;
1955
1956 switch (ecc->mode) {
1957 case NAND_ECC_HW:
1958 ret = sunxi_nand_hw_ecc_ctrl_init(mtd, ecc, np);
1959 if (ret)
1960 return ret;
1961 break;
1962 case NAND_ECC_HW_SYNDROME:
1963 ret = sunxi_nand_hw_syndrome_ecc_ctrl_init(mtd, ecc, np);
1964 if (ret)
1965 return ret;
1966 break;
1967 case NAND_ECC_NONE:
1968 case NAND_ECC_SOFT:
1969 break;
1970 default:
1971 return -EINVAL;
1972 }
1973
1974 return 0;
1975 }
1976
1977 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
1978 struct device_node *np)
1979 {
1980 struct sunxi_nand_chip *chip;
1981 struct mtd_info *mtd;
1982 struct nand_chip *nand;
1983 int nsels;
1984 int ret;
1985 int i;
1986 u32 tmp;
1987
1988 if (!of_get_property(np, "reg", &nsels))
1989 return -EINVAL;
1990
1991 nsels /= sizeof(u32);
1992 if (!nsels) {
1993 dev_err(dev, "invalid reg property size\n");
1994 return -EINVAL;
1995 }
1996
1997 chip = devm_kzalloc(dev,
1998 sizeof(*chip) +
1999 (nsels * sizeof(struct sunxi_nand_chip_sel)),
2000 GFP_KERNEL);
2001 if (!chip) {
2002 dev_err(dev, "could not allocate chip\n");
2003 return -ENOMEM;
2004 }
2005
2006 chip->nsels = nsels;
2007 chip->selected = -1;
2008
2009 for (i = 0; i < nsels; i++) {
2010 ret = of_property_read_u32_index(np, "reg", i, &tmp);
2011 if (ret) {
2012 dev_err(dev, "could not retrieve reg property: %d\n",
2013 ret);
2014 return ret;
2015 }
2016
2017 if (tmp > NFC_MAX_CS) {
2018 dev_err(dev,
2019 "invalid reg value: %u (max CS = 7)\n",
2020 tmp);
2021 return -EINVAL;
2022 }
2023
2024 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
2025 dev_err(dev, "CS %d already assigned\n", tmp);
2026 return -EINVAL;
2027 }
2028
2029 chip->sels[i].cs = tmp;
2030
2031 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) &&
2032 tmp < 2) {
2033 chip->sels[i].rb.type = RB_NATIVE;
2034 chip->sels[i].rb.info.nativeid = tmp;
2035 } else {
2036 ret = of_get_named_gpio(np, "rb-gpios", i);
2037 if (ret >= 0) {
2038 tmp = ret;
2039 chip->sels[i].rb.type = RB_GPIO;
2040 chip->sels[i].rb.info.gpio = tmp;
2041 ret = devm_gpio_request(dev, tmp, "nand-rb");
2042 if (ret)
2043 return ret;
2044
2045 ret = gpio_direction_input(tmp);
2046 if (ret)
2047 return ret;
2048 } else {
2049 chip->sels[i].rb.type = RB_NONE;
2050 }
2051 }
2052 }
2053
2054 nand = &chip->nand;
2055 /* Default tR value specified in the ONFI spec (chapter 4.15.1) */
2056 nand->chip_delay = 200;
2057 nand->controller = &nfc->controller;
2058 /*
2059 * Set the ECC mode to the default value in case nothing is specified
2060 * in the DT.
2061 */
2062 nand->ecc.mode = NAND_ECC_HW;
2063 nand_set_flash_node(nand, np);
2064 nand->select_chip = sunxi_nfc_select_chip;
2065 nand->cmd_ctrl = sunxi_nfc_cmd_ctrl;
2066 nand->read_buf = sunxi_nfc_read_buf;
2067 nand->write_buf = sunxi_nfc_write_buf;
2068 nand->read_byte = sunxi_nfc_read_byte;
2069 nand->setup_data_interface = sunxi_nfc_setup_data_interface;
2070
2071 mtd = nand_to_mtd(nand);
2072 mtd->dev.parent = dev;
2073
2074 ret = nand_scan_ident(mtd, nsels, NULL);
2075 if (ret)
2076 return ret;
2077
2078 if (nand->bbt_options & NAND_BBT_USE_FLASH)
2079 nand->bbt_options |= NAND_BBT_NO_OOB;
2080
2081 if (nand->options & NAND_NEED_SCRAMBLING)
2082 nand->options |= NAND_NO_SUBPAGE_WRITE;
2083
2084 nand->options |= NAND_SUBPAGE_READ;
2085
2086 ret = sunxi_nand_ecc_init(mtd, &nand->ecc, np);
2087 if (ret) {
2088 dev_err(dev, "ECC init failed: %d\n", ret);
2089 return ret;
2090 }
2091
2092 ret = nand_scan_tail(mtd);
2093 if (ret) {
2094 dev_err(dev, "nand_scan_tail failed: %d\n", ret);
2095 return ret;
2096 }
2097
2098 ret = mtd_device_register(mtd, NULL, 0);
2099 if (ret) {
2100 dev_err(dev, "failed to register mtd device: %d\n", ret);
2101 nand_release(mtd);
2102 return ret;
2103 }
2104
2105 list_add_tail(&chip->node, &nfc->chips);
2106
2107 return 0;
2108 }
2109
2110 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)
2111 {
2112 struct device_node *np = dev->of_node;
2113 struct device_node *nand_np;
2114 int nchips = of_get_child_count(np);
2115 int ret;
2116
2117 if (nchips > 8) {
2118 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips);
2119 return -EINVAL;
2120 }
2121
2122 for_each_child_of_node(np, nand_np) {
2123 ret = sunxi_nand_chip_init(dev, nfc, nand_np);
2124 if (ret) {
2125 of_node_put(nand_np);
2126 return ret;
2127 }
2128 }
2129
2130 return 0;
2131 }
2132
2133 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)
2134 {
2135 struct sunxi_nand_chip *chip;
2136
2137 while (!list_empty(&nfc->chips)) {
2138 chip = list_first_entry(&nfc->chips, struct sunxi_nand_chip,
2139 node);
2140 nand_release(nand_to_mtd(&chip->nand));
2141 sunxi_nand_ecc_cleanup(&chip->nand.ecc);
2142 list_del(&chip->node);
2143 }
2144 }
2145
2146 static int sunxi_nfc_probe(struct platform_device *pdev)
2147 {
2148 struct device *dev = &pdev->dev;
2149 struct resource *r;
2150 struct sunxi_nfc *nfc;
2151 int irq;
2152 int ret;
2153
2154 nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
2155 if (!nfc)
2156 return -ENOMEM;
2157
2158 nfc->dev = dev;
2159 nand_hw_control_init(&nfc->controller);
2160 INIT_LIST_HEAD(&nfc->chips);
2161
2162 r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2163 nfc->regs = devm_ioremap_resource(dev, r);
2164 if (IS_ERR(nfc->regs))
2165 return PTR_ERR(nfc->regs);
2166
2167 irq = platform_get_irq(pdev, 0);
2168 if (irq < 0) {
2169 dev_err(dev, "failed to retrieve irq\n");
2170 return irq;
2171 }
2172
2173 nfc->ahb_clk = devm_clk_get(dev, "ahb");
2174 if (IS_ERR(nfc->ahb_clk)) {
2175 dev_err(dev, "failed to retrieve ahb clk\n");
2176 return PTR_ERR(nfc->ahb_clk);
2177 }
2178
2179 ret = clk_prepare_enable(nfc->ahb_clk);
2180 if (ret)
2181 return ret;
2182
2183 nfc->mod_clk = devm_clk_get(dev, "mod");
2184 if (IS_ERR(nfc->mod_clk)) {
2185 dev_err(dev, "failed to retrieve mod clk\n");
2186 ret = PTR_ERR(nfc->mod_clk);
2187 goto out_ahb_clk_unprepare;
2188 }
2189
2190 ret = clk_prepare_enable(nfc->mod_clk);
2191 if (ret)
2192 goto out_ahb_clk_unprepare;
2193
2194 nfc->reset = devm_reset_control_get_optional(dev, "ahb");
2195 if (!IS_ERR(nfc->reset)) {
2196 ret = reset_control_deassert(nfc->reset);
2197 if (ret) {
2198 dev_err(dev, "reset err %d\n", ret);
2199 goto out_mod_clk_unprepare;
2200 }
2201 } else if (PTR_ERR(nfc->reset) != -ENOENT) {
2202 ret = PTR_ERR(nfc->reset);
2203 goto out_mod_clk_unprepare;
2204 }
2205
2206 ret = sunxi_nfc_rst(nfc);
2207 if (ret)
2208 goto out_ahb_reset_reassert;
2209
2210 writel(0, nfc->regs + NFC_REG_INT);
2211 ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt,
2212 0, "sunxi-nand", nfc);
2213 if (ret)
2214 goto out_ahb_reset_reassert;
2215
2216 nfc->dmac = dma_request_slave_channel(dev, "rxtx");
2217 if (nfc->dmac) {
2218 struct dma_slave_config dmac_cfg = { };
2219
2220 dmac_cfg.src_addr = r->start + NFC_REG_IO_DATA;
2221 dmac_cfg.dst_addr = dmac_cfg.src_addr;
2222 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2223 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;
2224 dmac_cfg.src_maxburst = 4;
2225 dmac_cfg.dst_maxburst = 4;
2226 dmaengine_slave_config(nfc->dmac, &dmac_cfg);
2227 } else {
2228 dev_warn(dev, "failed to request rxtx DMA channel\n");
2229 }
2230
2231 platform_set_drvdata(pdev, nfc);
2232
2233 ret = sunxi_nand_chips_init(dev, nfc);
2234 if (ret) {
2235 dev_err(dev, "failed to init nand chips\n");
2236 goto out_release_dmac;
2237 }
2238
2239 return 0;
2240
2241 out_release_dmac:
2242 if (nfc->dmac)
2243 dma_release_channel(nfc->dmac);
2244 out_ahb_reset_reassert:
2245 if (!IS_ERR(nfc->reset))
2246 reset_control_assert(nfc->reset);
2247 out_mod_clk_unprepare:
2248 clk_disable_unprepare(nfc->mod_clk);
2249 out_ahb_clk_unprepare:
2250 clk_disable_unprepare(nfc->ahb_clk);
2251
2252 return ret;
2253 }
2254
2255 static int sunxi_nfc_remove(struct platform_device *pdev)
2256 {
2257 struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
2258
2259 sunxi_nand_chips_cleanup(nfc);
2260
2261 if (!IS_ERR(nfc->reset))
2262 reset_control_assert(nfc->reset);
2263
2264 if (nfc->dmac)
2265 dma_release_channel(nfc->dmac);
2266 clk_disable_unprepare(nfc->mod_clk);
2267 clk_disable_unprepare(nfc->ahb_clk);
2268
2269 return 0;
2270 }
2271
2272 static const struct of_device_id sunxi_nfc_ids[] = {
2273 { .compatible = "allwinner,sun4i-a10-nand" },
2274 { /* sentinel */ }
2275 };
2276 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids);
2277
2278 static struct platform_driver sunxi_nfc_driver = {
2279 .driver = {
2280 .name = "sunxi_nand",
2281 .of_match_table = sunxi_nfc_ids,
2282 },
2283 .probe = sunxi_nfc_probe,
2284 .remove = sunxi_nfc_remove,
2285 };
2286 module_platform_driver(sunxi_nfc_driver);
2287
2288 MODULE_LICENSE("GPL v2");
2289 MODULE_AUTHOR("Boris BREZILLON");
2290 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver");
2291 MODULE_ALIAS("platform:sunxi_nand");
Linux with added FPC-III support