?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge branch 'perf-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...
[cris-mirror.git] / drivers / mtd / nand / sunxi_nand.c
blobe414b31b71c17111e923825ec8c178a3c40e8948
1 /*
2 * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com>
3 *
4 * Derived from:
5 * https://github.com/yuq/sunxi-nfc-mtd
6 * Copyright (C) 2013 Qiang Yu <yuq825@gmail.com>
7 *
8 * https://github.com/hno/Allwinner-Info
9 * Copyright (C) 2013 Henrik Nordström <Henrik Nordström>
10 *
11 * Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com>
12 * Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org>
13 *
14 * This program is free software; you can redistribute it and/or modify
15 * it under the terms of the GNU General Public License as published by
16 * the Free Software Foundation; either version 2 of the License, or
17 * (at your option) any later version.
18 *
19 * This program is distributed in the hope that it will be useful,
20 * but WITHOUT ANY WARRANTY; without even the implied warranty of
21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 * GNU General Public License for more details.
23 */
24
25 #include <linux/dma-mapping.h>
26 #include <linux/slab.h>
27 #include <linux/module.h>
28 #include <linux/moduleparam.h>
29 #include <linux/platform_device.h>
30 #include <linux/of.h>
31 #include <linux/of_device.h>
32 #include <linux/of_gpio.h>
33 #include <linux/mtd/mtd.h>
34 #include <linux/mtd/nand.h>
35 #include <linux/mtd/partitions.h>
36 #include <linux/clk.h>
37 #include <linux/delay.h>
38 #include <linux/dmaengine.h>
39 #include <linux/gpio.h>
40 #include <linux/interrupt.h>
41 #include <linux/iopoll.h>
42 #include <linux/reset.h>
43
44 #define NFC_REG_CTL 0x0000
45 #define NFC_REG_ST 0x0004
46 #define NFC_REG_INT 0x0008
47 #define NFC_REG_TIMING_CTL 0x000C
48 #define NFC_REG_TIMING_CFG 0x0010
49 #define NFC_REG_ADDR_LOW 0x0014
50 #define NFC_REG_ADDR_HIGH 0x0018
51 #define NFC_REG_SECTOR_NUM 0x001C
52 #define NFC_REG_CNT 0x0020
53 #define NFC_REG_CMD 0x0024
54 #define NFC_REG_RCMD_SET 0x0028
55 #define NFC_REG_WCMD_SET 0x002C
56 #define NFC_REG_IO_DATA 0x0030
57 #define NFC_REG_ECC_CTL 0x0034
58 #define NFC_REG_ECC_ST 0x0038
59 #define NFC_REG_DEBUG 0x003C
60 #define NFC_REG_ECC_ERR_CNT(x) ((0x0040 + (x)) & ~0x3)
61 #define NFC_REG_USER_DATA(x) (0x0050 + ((x) * 4))
62 #define NFC_REG_SPARE_AREA 0x00A0
63 #define NFC_REG_PAT_ID 0x00A4
64 #define NFC_RAM0_BASE 0x0400
65 #define NFC_RAM1_BASE 0x0800
66
67 /* define bit use in NFC_CTL */
68 #define NFC_EN BIT(0)
69 #define NFC_RESET BIT(1)
70 #define NFC_BUS_WIDTH_MSK BIT(2)
71 #define NFC_BUS_WIDTH_8 (0 << 2)
72 #define NFC_BUS_WIDTH_16 (1 << 2)
73 #define NFC_RB_SEL_MSK BIT(3)
74 #define NFC_RB_SEL(x) ((x) << 3)
75 #define NFC_CE_SEL_MSK GENMASK(26, 24)
76 #define NFC_CE_SEL(x) ((x) << 24)
77 #define NFC_CE_CTL BIT(6)
78 #define NFC_PAGE_SHIFT_MSK GENMASK(11, 8)
79 #define NFC_PAGE_SHIFT(x) (((x) < 10 ? 0 : (x) - 10) << 8)
80 #define NFC_SAM BIT(12)
81 #define NFC_RAM_METHOD BIT(14)
82 #define NFC_DEBUG_CTL BIT(31)
83
84 /* define bit use in NFC_ST */
85 #define NFC_RB_B2R BIT(0)
86 #define NFC_CMD_INT_FLAG BIT(1)
87 #define NFC_DMA_INT_FLAG BIT(2)
88 #define NFC_CMD_FIFO_STATUS BIT(3)
89 #define NFC_STA BIT(4)
90 #define NFC_NATCH_INT_FLAG BIT(5)
91 #define NFC_RB_STATE(x) BIT(x + 8)
92
93 /* define bit use in NFC_INT */
94 #define NFC_B2R_INT_ENABLE BIT(0)
95 #define NFC_CMD_INT_ENABLE BIT(1)
96 #define NFC_DMA_INT_ENABLE BIT(2)
97 #define NFC_INT_MASK (NFC_B2R_INT_ENABLE | \
98 NFC_CMD_INT_ENABLE | \
99 NFC_DMA_INT_ENABLE)
100
101 /* define bit use in NFC_TIMING_CTL */
102 #define NFC_TIMING_CTL_EDO BIT(8)
103
104 /* define NFC_TIMING_CFG register layout */
105 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD) \
106 (((tWB) & 0x3) | (((tADL) & 0x3) << 2) | \
107 (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) | \
108 (((tCAD) & 0x7) << 8))
109
110 /* define bit use in NFC_CMD */
111 #define NFC_CMD_LOW_BYTE_MSK GENMASK(7, 0)
112 #define NFC_CMD_HIGH_BYTE_MSK GENMASK(15, 8)
113 #define NFC_CMD(x) (x)
114 #define NFC_ADR_NUM_MSK GENMASK(18, 16)
115 #define NFC_ADR_NUM(x) (((x) - 1) << 16)
116 #define NFC_SEND_ADR BIT(19)
117 #define NFC_ACCESS_DIR BIT(20)
118 #define NFC_DATA_TRANS BIT(21)
119 #define NFC_SEND_CMD1 BIT(22)
120 #define NFC_WAIT_FLAG BIT(23)
121 #define NFC_SEND_CMD2 BIT(24)
122 #define NFC_SEQ BIT(25)
123 #define NFC_DATA_SWAP_METHOD BIT(26)
124 #define NFC_ROW_AUTO_INC BIT(27)
125 #define NFC_SEND_CMD3 BIT(28)
126 #define NFC_SEND_CMD4 BIT(29)
127 #define NFC_CMD_TYPE_MSK GENMASK(31, 30)
128 #define NFC_NORMAL_OP (0 << 30)
129 #define NFC_ECC_OP (1 << 30)
130 #define NFC_PAGE_OP (2 << 30)
131
132 /* define bit use in NFC_RCMD_SET */
133 #define NFC_READ_CMD_MSK GENMASK(7, 0)
134 #define NFC_RND_READ_CMD0_MSK GENMASK(15, 8)
135 #define NFC_RND_READ_CMD1_MSK GENMASK(23, 16)
136
137 /* define bit use in NFC_WCMD_SET */
138 #define NFC_PROGRAM_CMD_MSK GENMASK(7, 0)
139 #define NFC_RND_WRITE_CMD_MSK GENMASK(15, 8)
140 #define NFC_READ_CMD0_MSK GENMASK(23, 16)
141 #define NFC_READ_CMD1_MSK GENMASK(31, 24)
142
143 /* define bit use in NFC_ECC_CTL */
144 #define NFC_ECC_EN BIT(0)
145 #define NFC_ECC_PIPELINE BIT(3)
146 #define NFC_ECC_EXCEPTION BIT(4)
147 #define NFC_ECC_BLOCK_SIZE_MSK BIT(5)
148 #define NFC_RANDOM_EN BIT(9)
149 #define NFC_RANDOM_DIRECTION BIT(10)
150 #define NFC_ECC_MODE_MSK GENMASK(15, 12)
151 #define NFC_ECC_MODE(x) ((x) << 12)
152 #define NFC_RANDOM_SEED_MSK GENMASK(30, 16)
153 #define NFC_RANDOM_SEED(x) ((x) << 16)
154
155 /* define bit use in NFC_ECC_ST */
156 #define NFC_ECC_ERR(x) BIT(x)
157 #define NFC_ECC_ERR_MSK GENMASK(15, 0)
158 #define NFC_ECC_PAT_FOUND(x) BIT(x + 16)
159 #define NFC_ECC_ERR_CNT(b, x) (((x) >> (((b) % 4) * 8)) & 0xff)
160
161 #define NFC_DEFAULT_TIMEOUT_MS 1000
162
163 #define NFC_SRAM_SIZE 1024
164
165 #define NFC_MAX_CS 7
166
167 /*
168 * Ready/Busy detection type: describes the Ready/Busy detection modes
169 *
170 * @RB_NONE: no external detection available, rely on STATUS command
171 * and software timeouts
172 * @RB_NATIVE: use sunxi NAND controller Ready/Busy support. The Ready/Busy
173 * pin of the NAND flash chip must be connected to one of the
174 * native NAND R/B pins (those which can be muxed to the NAND
175 * Controller)
176 * @RB_GPIO: use a simple GPIO to handle Ready/Busy status. The Ready/Busy
177 * pin of the NAND flash chip must be connected to a GPIO capable
178 * pin.
179 */
180 enum sunxi_nand_rb_type {
181 RB_NONE,
182 RB_NATIVE,
183 RB_GPIO,
184 };
185
186 /*
187 * Ready/Busy structure: stores information related to Ready/Busy detection
188 *
189 * @type: the Ready/Busy detection mode
190 * @info: information related to the R/B detection mode. Either a gpio
191 * id or a native R/B id (those supported by the NAND controller).
192 */
193 struct sunxi_nand_rb {
194 enum sunxi_nand_rb_type type;
195 union {
196 int gpio;
197 int nativeid;
198 } info;
199 };
200
201 /*
202 * Chip Select structure: stores information related to NAND Chip Select
203 *
204 * @cs: the NAND CS id used to communicate with a NAND Chip
205 * @rb: the Ready/Busy description
206 */
207 struct sunxi_nand_chip_sel {
208 u8 cs;
209 struct sunxi_nand_rb rb;
210 };
211
212 /*
213 * sunxi HW ECC infos: stores information related to HW ECC support
214 *
215 * @mode: the sunxi ECC mode field deduced from ECC requirements
216 */
217 struct sunxi_nand_hw_ecc {
218 int mode;
219 };
220
221 /*
222 * NAND chip structure: stores NAND chip device related information
223 *
224 * @node: used to store NAND chips into a list
225 * @nand: base NAND chip structure
226 * @mtd: base MTD structure
227 * @clk_rate: clk_rate required for this NAND chip
228 * @timing_cfg TIMING_CFG register value for this NAND chip
229 * @selected: current active CS
230 * @nsels: number of CS lines required by the NAND chip
231 * @sels: array of CS lines descriptions
232 */
233 struct sunxi_nand_chip {
234 struct list_head node;
235 struct nand_chip nand;
236 unsigned long clk_rate;
237 u32 timing_cfg;
238 u32 timing_ctl;
239 int selected;
240 int addr_cycles;
241 u32 addr[2];
242 int cmd_cycles;
243 u8 cmd[2];
244 int nsels;
245 struct sunxi_nand_chip_sel sels[0];
246 };
247
248 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)
249 {
250 return container_of(nand, struct sunxi_nand_chip, nand);
251 }
252
253 /*
254 * NAND Controller structure: stores sunxi NAND controller information
255 *
256 * @controller: base controller structure
257 * @dev: parent device (used to print error messages)
258 * @regs: NAND controller registers
259 * @ahb_clk: NAND Controller AHB clock
260 * @mod_clk: NAND Controller mod clock
261 * @assigned_cs: bitmask describing already assigned CS lines
262 * @clk_rate: NAND controller current clock rate
263 * @chips: a list containing all the NAND chips attached to
264 * this NAND controller
265 * @complete: a completion object used to wait for NAND
266 * controller events
267 */
268 struct sunxi_nfc {
269 struct nand_hw_control controller;
270 struct device *dev;
271 void __iomem *regs;
272 struct clk *ahb_clk;
273 struct clk *mod_clk;
274 struct reset_control *reset;
275 unsigned long assigned_cs;
276 unsigned long clk_rate;
277 struct list_head chips;
278 struct completion complete;
279 struct dma_chan *dmac;
280 };
281
282 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_hw_control *ctrl)
283 {
284 return container_of(ctrl, struct sunxi_nfc, controller);
285 }
286
287 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id)
288 {
289 struct sunxi_nfc *nfc = dev_id;
290 u32 st = readl(nfc->regs + NFC_REG_ST);
291 u32 ien = readl(nfc->regs + NFC_REG_INT);
292
293 if (!(ien & st))
294 return IRQ_NONE;
295
296 if ((ien & st) == ien)
297 complete(&nfc->complete);
298
299 writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
300 writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT);
301
302 return IRQ_HANDLED;
303 }
304
305 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events,
306 bool use_polling, unsigned int timeout_ms)
307 {
308 int ret;
309
310 if (events & ~NFC_INT_MASK)
311 return -EINVAL;
312
313 if (!timeout_ms)
314 timeout_ms = NFC_DEFAULT_TIMEOUT_MS;
315
316 if (!use_polling) {
317 init_completion(&nfc->complete);
318
319 writel(events, nfc->regs + NFC_REG_INT);
320
321 ret = wait_for_completion_timeout(&nfc->complete,
322 msecs_to_jiffies(timeout_ms));
323
324 writel(0, nfc->regs + NFC_REG_INT);
325 } else {
326 u32 status;
327
328 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
329 (status & events) == events, 1,
330 timeout_ms * 1000);
331 }
332
333 writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
334
335 if (ret)
336 dev_err(nfc->dev, "wait interrupt timedout\n");
337
338 return ret;
339 }
340
341 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc)
342 {
343 u32 status;
344 int ret;
345
346 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
347 !(status & NFC_CMD_FIFO_STATUS), 1,
348 NFC_DEFAULT_TIMEOUT_MS * 1000);
349 if (ret)
350 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n");
351
352 return ret;
353 }
354
355 static int sunxi_nfc_rst(struct sunxi_nfc *nfc)
356 {
357 u32 ctl;
358 int ret;
359
360 writel(0, nfc->regs + NFC_REG_ECC_CTL);
361 writel(NFC_RESET, nfc->regs + NFC_REG_CTL);
362
363 ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl,
364 !(ctl & NFC_RESET), 1,
365 NFC_DEFAULT_TIMEOUT_MS * 1000);
366 if (ret)
367 dev_err(nfc->dev, "wait for NAND controller reset timedout\n");
368
369 return ret;
370 }
371
372 static int sunxi_nfc_dma_op_prepare(struct mtd_info *mtd, const void *buf,
373 int chunksize, int nchunks,
374 enum dma_data_direction ddir,
375 struct scatterlist *sg)
376 {
377 struct nand_chip *nand = mtd_to_nand(mtd);
378 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
379 struct dma_async_tx_descriptor *dmad;
380 enum dma_transfer_direction tdir;
381 dma_cookie_t dmat;
382 int ret;
383
384 if (ddir == DMA_FROM_DEVICE)
385 tdir = DMA_DEV_TO_MEM;
386 else
387 tdir = DMA_MEM_TO_DEV;
388
389 sg_init_one(sg, buf, nchunks * chunksize);
390 ret = dma_map_sg(nfc->dev, sg, 1, ddir);
391 if (!ret)
392 return -ENOMEM;
393
394 dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);
395 if (!dmad) {
396 ret = -EINVAL;
397 goto err_unmap_buf;
398 }
399
400 writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,
401 nfc->regs + NFC_REG_CTL);
402 writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);
403 writel(chunksize, nfc->regs + NFC_REG_CNT);
404 dmat = dmaengine_submit(dmad);
405
406 ret = dma_submit_error(dmat);
407 if (ret)
408 goto err_clr_dma_flag;
409
410 return 0;
411
412 err_clr_dma_flag:
413 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
414 nfc->regs + NFC_REG_CTL);
415
416 err_unmap_buf:
417 dma_unmap_sg(nfc->dev, sg, 1, ddir);
418 return ret;
419 }
420
421 static void sunxi_nfc_dma_op_cleanup(struct mtd_info *mtd,
422 enum dma_data_direction ddir,
423 struct scatterlist *sg)
424 {
425 struct nand_chip *nand = mtd_to_nand(mtd);
426 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
427
428 dma_unmap_sg(nfc->dev, sg, 1, ddir);
429 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
430 nfc->regs + NFC_REG_CTL);
431 }
432
433 static int sunxi_nfc_dev_ready(struct mtd_info *mtd)
434 {
435 struct nand_chip *nand = mtd_to_nand(mtd);
436 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
437 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
438 struct sunxi_nand_rb *rb;
439 int ret;
440
441 if (sunxi_nand->selected < 0)
442 return 0;
443
444 rb = &sunxi_nand->sels[sunxi_nand->selected].rb;
445
446 switch (rb->type) {
447 case RB_NATIVE:
448 ret = !!(readl(nfc->regs + NFC_REG_ST) &
449 NFC_RB_STATE(rb->info.nativeid));
450 break;
451 case RB_GPIO:
452 ret = gpio_get_value(rb->info.gpio);
453 break;
454 case RB_NONE:
455 default:
456 ret = 0;
457 dev_err(nfc->dev, "cannot check R/B NAND status!\n");
458 break;
459 }
460
461 return ret;
462 }
463
464 static void sunxi_nfc_select_chip(struct mtd_info *mtd, int chip)
465 {
466 struct nand_chip *nand = mtd_to_nand(mtd);
467 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
468 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
469 struct sunxi_nand_chip_sel *sel;
470 u32 ctl;
471
472 if (chip > 0 && chip >= sunxi_nand->nsels)
473 return;
474
475 if (chip == sunxi_nand->selected)
476 return;
477
478 ctl = readl(nfc->regs + NFC_REG_CTL) &
479 ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN);
480
481 if (chip >= 0) {
482 sel = &sunxi_nand->sels[chip];
483
484 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN |
485 NFC_PAGE_SHIFT(nand->page_shift);
486 if (sel->rb.type == RB_NONE) {
487 nand->dev_ready = NULL;
488 } else {
489 nand->dev_ready = sunxi_nfc_dev_ready;
490 if (sel->rb.type == RB_NATIVE)
491 ctl |= NFC_RB_SEL(sel->rb.info.nativeid);
492 }
493
494 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA);
495
496 if (nfc->clk_rate != sunxi_nand->clk_rate) {
497 clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate);
498 nfc->clk_rate = sunxi_nand->clk_rate;
499 }
500 }
501
502 writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL);
503 writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG);
504 writel(ctl, nfc->regs + NFC_REG_CTL);
505
506 sunxi_nand->selected = chip;
507 }
508
509 static void sunxi_nfc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
510 {
511 struct nand_chip *nand = mtd_to_nand(mtd);
512 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
513 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
514 int ret;
515 int cnt;
516 int offs = 0;
517 u32 tmp;
518
519 while (len > offs) {
520 cnt = min(len - offs, NFC_SRAM_SIZE);
521
522 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
523 if (ret)
524 break;
525
526 writel(cnt, nfc->regs + NFC_REG_CNT);
527 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
528 writel(tmp, nfc->regs + NFC_REG_CMD);
529
530 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
531 if (ret)
532 break;
533
534 if (buf)
535 memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE,
536 cnt);
537 offs += cnt;
538 }
539 }
540
541 static void sunxi_nfc_write_buf(struct mtd_info *mtd, const uint8_t *buf,
542 int len)
543 {
544 struct nand_chip *nand = mtd_to_nand(mtd);
545 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
546 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
547 int ret;
548 int cnt;
549 int offs = 0;
550 u32 tmp;
551
552 while (len > offs) {
553 cnt = min(len - offs, NFC_SRAM_SIZE);
554
555 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
556 if (ret)
557 break;
558
559 writel(cnt, nfc->regs + NFC_REG_CNT);
560 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt);
561 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
562 NFC_ACCESS_DIR;
563 writel(tmp, nfc->regs + NFC_REG_CMD);
564
565 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
566 if (ret)
567 break;
568
569 offs += cnt;
570 }
571 }
572
573 static uint8_t sunxi_nfc_read_byte(struct mtd_info *mtd)
574 {
575 uint8_t ret;
576
577 sunxi_nfc_read_buf(mtd, &ret, 1);
578
579 return ret;
580 }
581
582 static void sunxi_nfc_cmd_ctrl(struct mtd_info *mtd, int dat,
583 unsigned int ctrl)
584 {
585 struct nand_chip *nand = mtd_to_nand(mtd);
586 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
587 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
588 int ret;
589
590 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
591 if (ret)
592 return;
593
594 if (dat == NAND_CMD_NONE && (ctrl & NAND_NCE) &&
595 !(ctrl & (NAND_CLE | NAND_ALE))) {
596 u32 cmd = 0;
597
598 if (!sunxi_nand->addr_cycles && !sunxi_nand->cmd_cycles)
599 return;
600
601 if (sunxi_nand->cmd_cycles--)
602 cmd |= NFC_SEND_CMD1 | sunxi_nand->cmd[0];
603
604 if (sunxi_nand->cmd_cycles--) {
605 cmd |= NFC_SEND_CMD2;
606 writel(sunxi_nand->cmd[1],
607 nfc->regs + NFC_REG_RCMD_SET);
608 }
609
610 sunxi_nand->cmd_cycles = 0;
611
612 if (sunxi_nand->addr_cycles) {
613 cmd |= NFC_SEND_ADR |
614 NFC_ADR_NUM(sunxi_nand->addr_cycles);
615 writel(sunxi_nand->addr[0],
616 nfc->regs + NFC_REG_ADDR_LOW);
617 }
618
619 if (sunxi_nand->addr_cycles > 4)
620 writel(sunxi_nand->addr[1],
621 nfc->regs + NFC_REG_ADDR_HIGH);
622
623 writel(cmd, nfc->regs + NFC_REG_CMD);
624 sunxi_nand->addr[0] = 0;
625 sunxi_nand->addr[1] = 0;
626 sunxi_nand->addr_cycles = 0;
627 sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
628 }
629
630 if (ctrl & NAND_CLE) {
631 sunxi_nand->cmd[sunxi_nand->cmd_cycles++] = dat;
632 } else if (ctrl & NAND_ALE) {
633 sunxi_nand->addr[sunxi_nand->addr_cycles / 4] |=
634 dat << ((sunxi_nand->addr_cycles % 4) * 8);
635 sunxi_nand->addr_cycles++;
636 }
637 }
638
639 /* These seed values have been extracted from Allwinner's BSP */
640 static const u16 sunxi_nfc_randomizer_page_seeds[] = {
641 0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72,
642 0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436,
643 0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d,
644 0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130,
645 0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56,
646 0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55,
647 0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb,
648 0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17,
649 0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62,
650 0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064,
651 0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126,
652 0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e,
653 0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3,
654 0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b,
655 0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d,
656 0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db,
657 };
658
659 /*
660 * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds
661 * have been generated using
662 * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what
663 * the randomizer engine does internally before de/scrambling OOB data.
664 *
665 * Those tables are statically defined to avoid calculating randomizer state
666 * at runtime.
667 */
668 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = {
669 0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64,
670 0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409,
671 0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617,
672 0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d,
673 0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91,
674 0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d,
675 0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab,
676 0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8,
677 0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8,
678 0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b,
679 0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5,
680 0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a,
681 0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891,
682 0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36,
683 0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd,
684 0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0,
685 };
686
687 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = {
688 0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6,
689 0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982,
690 0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9,
691 0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07,
692 0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e,
693 0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2,
694 0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c,
695 0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f,
696 0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc,
697 0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e,
698 0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8,
699 0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68,
700 0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d,
701 0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179,
702 0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601,
703 0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd,
704 };
705
706 static u16 sunxi_nfc_randomizer_step(u16 state, int count)
707 {
708 state &= 0x7fff;
709
710 /*
711 * This loop is just a simple implementation of a Fibonacci LFSR using
712 * the x16 + x15 + 1 polynomial.
713 */
714 while (count--)
715 state = ((state >> 1) |
716 (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff;
717
718 return state;
719 }
720
721 static u16 sunxi_nfc_randomizer_state(struct mtd_info *mtd, int page, bool ecc)
722 {
723 const u16 *seeds = sunxi_nfc_randomizer_page_seeds;
724 int mod = mtd_div_by_ws(mtd->erasesize, mtd);
725
726 if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds))
727 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds);
728
729 if (ecc) {
730 if (mtd->ecc_step_size == 512)
731 seeds = sunxi_nfc_randomizer_ecc512_seeds;
732 else
733 seeds = sunxi_nfc_randomizer_ecc1024_seeds;
734 }
735
736 return seeds[page % mod];
737 }
738
739 static void sunxi_nfc_randomizer_config(struct mtd_info *mtd,
740 int page, bool ecc)
741 {
742 struct nand_chip *nand = mtd_to_nand(mtd);
743 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
744 u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
745 u16 state;
746
747 if (!(nand->options & NAND_NEED_SCRAMBLING))
748 return;
749
750 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
751 state = sunxi_nfc_randomizer_state(mtd, page, ecc);
752 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK;
753 writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL);
754 }
755
756 static void sunxi_nfc_randomizer_enable(struct mtd_info *mtd)
757 {
758 struct nand_chip *nand = mtd_to_nand(mtd);
759 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
760
761 if (!(nand->options & NAND_NEED_SCRAMBLING))
762 return;
763
764 writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN,
765 nfc->regs + NFC_REG_ECC_CTL);
766 }
767
768 static void sunxi_nfc_randomizer_disable(struct mtd_info *mtd)
769 {
770 struct nand_chip *nand = mtd_to_nand(mtd);
771 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
772
773 if (!(nand->options & NAND_NEED_SCRAMBLING))
774 return;
775
776 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN,
777 nfc->regs + NFC_REG_ECC_CTL);
778 }
779
780 static void sunxi_nfc_randomize_bbm(struct mtd_info *mtd, int page, u8 *bbm)
781 {
782 u16 state = sunxi_nfc_randomizer_state(mtd, page, true);
783
784 bbm[0] ^= state;
785 bbm[1] ^= sunxi_nfc_randomizer_step(state, 8);
786 }
787
788 static void sunxi_nfc_randomizer_write_buf(struct mtd_info *mtd,
789 const uint8_t *buf, int len,
790 bool ecc, int page)
791 {
792 sunxi_nfc_randomizer_config(mtd, page, ecc);
793 sunxi_nfc_randomizer_enable(mtd);
794 sunxi_nfc_write_buf(mtd, buf, len);
795 sunxi_nfc_randomizer_disable(mtd);
796 }
797
798 static void sunxi_nfc_randomizer_read_buf(struct mtd_info *mtd, uint8_t *buf,
799 int len, bool ecc, int page)
800 {
801 sunxi_nfc_randomizer_config(mtd, page, ecc);
802 sunxi_nfc_randomizer_enable(mtd);
803 sunxi_nfc_read_buf(mtd, buf, len);
804 sunxi_nfc_randomizer_disable(mtd);
805 }
806
807 static void sunxi_nfc_hw_ecc_enable(struct mtd_info *mtd)
808 {
809 struct nand_chip *nand = mtd_to_nand(mtd);
810 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
811 struct sunxi_nand_hw_ecc *data = nand->ecc.priv;
812 u32 ecc_ctl;
813
814 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
815 ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE |
816 NFC_ECC_BLOCK_SIZE_MSK);
817 ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(data->mode) | NFC_ECC_EXCEPTION |
818 NFC_ECC_PIPELINE;
819
820 writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL);
821 }
822
823 static void sunxi_nfc_hw_ecc_disable(struct mtd_info *mtd)
824 {
825 struct nand_chip *nand = mtd_to_nand(mtd);
826 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
827
828 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN,
829 nfc->regs + NFC_REG_ECC_CTL);
830 }
831
832 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf)
833 {
834 buf[0] = user_data;
835 buf[1] = user_data >> 8;
836 buf[2] = user_data >> 16;
837 buf[3] = user_data >> 24;
838 }
839
840 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf)
841 {
842 return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24);
843 }
844
845 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct mtd_info *mtd, u8 *oob,
846 int step, bool bbm, int page)
847 {
848 struct nand_chip *nand = mtd_to_nand(mtd);
849 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
850
851 sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)),
852 oob);
853
854 /* De-randomize the Bad Block Marker. */
855 if (bbm && (nand->options & NAND_NEED_SCRAMBLING))
856 sunxi_nfc_randomize_bbm(mtd, page, oob);
857 }
858
859 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct mtd_info *mtd,
860 const u8 *oob, int step,
861 bool bbm, int page)
862 {
863 struct nand_chip *nand = mtd_to_nand(mtd);
864 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
865 u8 user_data[4];
866
867 /* Randomize the Bad Block Marker. */
868 if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
869 memcpy(user_data, oob, sizeof(user_data));
870 sunxi_nfc_randomize_bbm(mtd, page, user_data);
871 oob = user_data;
872 }
873
874 writel(sunxi_nfc_buf_to_user_data(oob),
875 nfc->regs + NFC_REG_USER_DATA(step));
876 }
877
878 static void sunxi_nfc_hw_ecc_update_stats(struct mtd_info *mtd,
879 unsigned int *max_bitflips, int ret)
880 {
881 if (ret < 0) {
882 mtd->ecc_stats.failed++;
883 } else {
884 mtd->ecc_stats.corrected += ret;
885 *max_bitflips = max_t(unsigned int, *max_bitflips, ret);
886 }
887 }
888
889 static int sunxi_nfc_hw_ecc_correct(struct mtd_info *mtd, u8 *data, u8 *oob,
890 int step, u32 status, bool *erased)
891 {
892 struct nand_chip *nand = mtd_to_nand(mtd);
893 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
894 struct nand_ecc_ctrl *ecc = &nand->ecc;
895 u32 tmp;
896
897 *erased = false;
898
899 if (status & NFC_ECC_ERR(step))
900 return -EBADMSG;
901
902 if (status & NFC_ECC_PAT_FOUND(step)) {
903 u8 pattern;
904
905 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) {
906 pattern = 0x0;
907 } else {
908 pattern = 0xff;
909 *erased = true;
910 }
911
912 if (data)
913 memset(data, pattern, ecc->size);
914
915 if (oob)
916 memset(oob, pattern, ecc->bytes + 4);
917
918 return 0;
919 }
920
921 tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step));
922
923 return NFC_ECC_ERR_CNT(step, tmp);
924 }
925
926 static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd,
927 u8 *data, int data_off,
928 u8 *oob, int oob_off,
929 int *cur_off,
930 unsigned int *max_bitflips,
931 bool bbm, bool oob_required, int page)
932 {
933 struct nand_chip *nand = mtd_to_nand(mtd);
934 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
935 struct nand_ecc_ctrl *ecc = &nand->ecc;
936 int raw_mode = 0;
937 bool erased;
938 int ret;
939
940 if (*cur_off != data_off)
941 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, data_off, -1);
942
943 sunxi_nfc_randomizer_read_buf(mtd, NULL, ecc->size, false, page);
944
945 if (data_off + ecc->size != oob_off)
946 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
947
948 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
949 if (ret)
950 return ret;
951
952 sunxi_nfc_randomizer_enable(mtd);
953 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP,
954 nfc->regs + NFC_REG_CMD);
955
956 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
957 sunxi_nfc_randomizer_disable(mtd);
958 if (ret)
959 return ret;
960
961 *cur_off = oob_off + ecc->bytes + 4;
962
963 ret = sunxi_nfc_hw_ecc_correct(mtd, data, oob_required ? oob : NULL, 0,
964 readl(nfc->regs + NFC_REG_ECC_ST),
965 &erased);
966 if (erased)
967 return 1;
968
969 if (ret < 0) {
970 /*
971 * Re-read the data with the randomizer disabled to identify
972 * bitflips in erased pages.
973 */
974 if (nand->options & NAND_NEED_SCRAMBLING) {
975 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, data_off, -1);
976 nand->read_buf(mtd, data, ecc->size);
977 } else {
978 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,
979 ecc->size);
980 }
981
982 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
983 nand->read_buf(mtd, oob, ecc->bytes + 4);
984
985 ret = nand_check_erased_ecc_chunk(data, ecc->size,
986 oob, ecc->bytes + 4,
987 NULL, 0, ecc->strength);
988 if (ret >= 0)
989 raw_mode = 1;
990 } else {
991 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);
992
993 if (oob_required) {
994 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
995 sunxi_nfc_randomizer_read_buf(mtd, oob, ecc->bytes + 4,
996 true, page);
997
998 sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, 0,
999 bbm, page);
1000 }
1001 }
1002
1003 sunxi_nfc_hw_ecc_update_stats(mtd, max_bitflips, ret);
1004
1005 return raw_mode;
1006 }
1007
1008 static void sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info *mtd,
1009 u8 *oob, int *cur_off,
1010 bool randomize, int page)
1011 {
1012 struct nand_chip *nand = mtd_to_nand(mtd);
1013 struct nand_ecc_ctrl *ecc = &nand->ecc;
1014 int offset = ((ecc->bytes + 4) * ecc->steps);
1015 int len = mtd->oobsize - offset;
1016
1017 if (len <= 0)
1018 return;
1019
1020 if (!cur_off || *cur_off != offset)
1021 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1022 offset + mtd->writesize, -1);
1023
1024 if (!randomize)
1025 sunxi_nfc_read_buf(mtd, oob + offset, len);
1026 else
1027 sunxi_nfc_randomizer_read_buf(mtd, oob + offset, len,
1028 false, page);
1029
1030 if (cur_off)
1031 *cur_off = mtd->oobsize + mtd->writesize;
1032 }
1033
1034 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf,
1035 int oob_required, int page,
1036 int nchunks)
1037 {
1038 struct nand_chip *nand = mtd_to_nand(mtd);
1039 bool randomized = nand->options & NAND_NEED_SCRAMBLING;
1040 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1041 struct nand_ecc_ctrl *ecc = &nand->ecc;
1042 unsigned int max_bitflips = 0;
1043 int ret, i, raw_mode = 0;
1044 struct scatterlist sg;
1045 u32 status;
1046
1047 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1048 if (ret)
1049 return ret;
1050
1051 ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, nchunks,
1052 DMA_FROM_DEVICE, &sg);
1053 if (ret)
1054 return ret;
1055
1056 sunxi_nfc_hw_ecc_enable(mtd);
1057 sunxi_nfc_randomizer_config(mtd, page, false);
1058 sunxi_nfc_randomizer_enable(mtd);
1059
1060 writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |
1061 NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);
1062
1063 dma_async_issue_pending(nfc->dmac);
1064
1065 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,
1066 nfc->regs + NFC_REG_CMD);
1067
1068 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
1069 if (ret)
1070 dmaengine_terminate_all(nfc->dmac);
1071
1072 sunxi_nfc_randomizer_disable(mtd);
1073 sunxi_nfc_hw_ecc_disable(mtd);
1074
1075 sunxi_nfc_dma_op_cleanup(mtd, DMA_FROM_DEVICE, &sg);
1076
1077 if (ret)
1078 return ret;
1079
1080 status = readl(nfc->regs + NFC_REG_ECC_ST);
1081
1082 for (i = 0; i < nchunks; i++) {
1083 int data_off = i * ecc->size;
1084 int oob_off = i * (ecc->bytes + 4);
1085 u8 *data = buf + data_off;
1086 u8 *oob = nand->oob_poi + oob_off;
1087 bool erased;
1088
1089 ret = sunxi_nfc_hw_ecc_correct(mtd, randomized ? data : NULL,
1090 oob_required ? oob : NULL,
1091 i, status, &erased);
1092
1093 /* ECC errors are handled in the second loop. */
1094 if (ret < 0)
1095 continue;
1096
1097 if (oob_required && !erased) {
1098 /* TODO: use DMA to retrieve OOB */
1099 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1100 mtd->writesize + oob_off, -1);
1101 nand->read_buf(mtd, oob, ecc->bytes + 4);
1102
1103 sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, i,
1104 !i, page);
1105 }
1106
1107 if (erased)
1108 raw_mode = 1;
1109
1110 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1111 }
1112
1113 if (status & NFC_ECC_ERR_MSK) {
1114 for (i = 0; i < nchunks; i++) {
1115 int data_off = i * ecc->size;
1116 int oob_off = i * (ecc->bytes + 4);
1117 u8 *data = buf + data_off;
1118 u8 *oob = nand->oob_poi + oob_off;
1119
1120 if (!(status & NFC_ECC_ERR(i)))
1121 continue;
1122
1123 /*
1124 * Re-read the data with the randomizer disabled to
1125 * identify bitflips in erased pages.
1126 */
1127 if (randomized) {
1128 /* TODO: use DMA to read page in raw mode */
1129 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1130 data_off, -1);
1131 nand->read_buf(mtd, data, ecc->size);
1132 }
1133
1134 /* TODO: use DMA to retrieve OOB */
1135 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1136 mtd->writesize + oob_off, -1);
1137 nand->read_buf(mtd, oob, ecc->bytes + 4);
1138
1139 ret = nand_check_erased_ecc_chunk(data, ecc->size,
1140 oob, ecc->bytes + 4,
1141 NULL, 0,
1142 ecc->strength);
1143 if (ret >= 0)
1144 raw_mode = 1;
1145
1146 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1147 }
1148 }
1149
1150 if (oob_required)
1151 sunxi_nfc_hw_ecc_read_extra_oob(mtd, nand->oob_poi,
1152 NULL, !raw_mode,
1153 page);
1154
1155 return max_bitflips;
1156 }
1157
1158 static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd,
1159 const u8 *data, int data_off,
1160 const u8 *oob, int oob_off,
1161 int *cur_off, bool bbm,
1162 int page)
1163 {
1164 struct nand_chip *nand = mtd_to_nand(mtd);
1165 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1166 struct nand_ecc_ctrl *ecc = &nand->ecc;
1167 int ret;
1168
1169 if (data_off != *cur_off)
1170 nand->cmdfunc(mtd, NAND_CMD_RNDIN, data_off, -1);
1171
1172 sunxi_nfc_randomizer_write_buf(mtd, data, ecc->size, false, page);
1173
1174 if (data_off + ecc->size != oob_off)
1175 nand->cmdfunc(mtd, NAND_CMD_RNDIN, oob_off, -1);
1176
1177 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1178 if (ret)
1179 return ret;
1180
1181 sunxi_nfc_randomizer_enable(mtd);
1182 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, 0, bbm, page);
1183
1184 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
1185 NFC_ACCESS_DIR | NFC_ECC_OP,
1186 nfc->regs + NFC_REG_CMD);
1187
1188 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
1189 sunxi_nfc_randomizer_disable(mtd);
1190 if (ret)
1191 return ret;
1192
1193 *cur_off = oob_off + ecc->bytes + 4;
1194
1195 return 0;
1196 }
1197
1198 static void sunxi_nfc_hw_ecc_write_extra_oob(struct mtd_info *mtd,
1199 u8 *oob, int *cur_off,
1200 int page)
1201 {
1202 struct nand_chip *nand = mtd_to_nand(mtd);
1203 struct nand_ecc_ctrl *ecc = &nand->ecc;
1204 int offset = ((ecc->bytes + 4) * ecc->steps);
1205 int len = mtd->oobsize - offset;
1206
1207 if (len <= 0)
1208 return;
1209
1210 if (!cur_off || *cur_off != offset)
1211 nand->cmdfunc(mtd, NAND_CMD_RNDIN,
1212 offset + mtd->writesize, -1);
1213
1214 sunxi_nfc_randomizer_write_buf(mtd, oob + offset, len, false, page);
1215
1216 if (cur_off)
1217 *cur_off = mtd->oobsize + mtd->writesize;
1218 }
1219
1220 static int sunxi_nfc_hw_ecc_read_page(struct mtd_info *mtd,
1221 struct nand_chip *chip, uint8_t *buf,
1222 int oob_required, int page)
1223 {
1224 struct nand_ecc_ctrl *ecc = &chip->ecc;
1225 unsigned int max_bitflips = 0;
1226 int ret, i, cur_off = 0;
1227 bool raw_mode = false;
1228
1229 sunxi_nfc_hw_ecc_enable(mtd);
1230
1231 for (i = 0; i < ecc->steps; i++) {
1232 int data_off = i * ecc->size;
1233 int oob_off = i * (ecc->bytes + 4);
1234 u8 *data = buf + data_off;
1235 u8 *oob = chip->oob_poi + oob_off;
1236
1237 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1238 oob_off + mtd->writesize,
1239 &cur_off, &max_bitflips,
1240 !i, oob_required, page);
1241 if (ret < 0)
1242 return ret;
1243 else if (ret)
1244 raw_mode = true;
1245 }
1246
1247 if (oob_required)
1248 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1249 !raw_mode, page);
1250
1251 sunxi_nfc_hw_ecc_disable(mtd);
1252
1253 return max_bitflips;
1254 }
1255
1256 static int sunxi_nfc_hw_ecc_read_page_dma(struct mtd_info *mtd,
1257 struct nand_chip *chip, u8 *buf,
1258 int oob_required, int page)
1259 {
1260 int ret;
1261
1262 ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, oob_required, page,
1263 chip->ecc.steps);
1264 if (ret >= 0)
1265 return ret;
1266
1267 /* Fallback to PIO mode */
1268 chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
1269
1270 return sunxi_nfc_hw_ecc_read_page(mtd, chip, buf, oob_required, page);
1271 }
1272
1273 static int sunxi_nfc_hw_ecc_read_subpage(struct mtd_info *mtd,
1274 struct nand_chip *chip,
1275 u32 data_offs, u32 readlen,
1276 u8 *bufpoi, int page)
1277 {
1278 struct nand_ecc_ctrl *ecc = &chip->ecc;
1279 int ret, i, cur_off = 0;
1280 unsigned int max_bitflips = 0;
1281
1282 sunxi_nfc_hw_ecc_enable(mtd);
1283
1284 chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1285 for (i = data_offs / ecc->size;
1286 i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) {
1287 int data_off = i * ecc->size;
1288 int oob_off = i * (ecc->bytes + 4);
1289 u8 *data = bufpoi + data_off;
1290 u8 *oob = chip->oob_poi + oob_off;
1291
1292 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off,
1293 oob,
1294 oob_off + mtd->writesize,
1295 &cur_off, &max_bitflips, !i,
1296 false, page);
1297 if (ret < 0)
1298 return ret;
1299 }
1300
1301 sunxi_nfc_hw_ecc_disable(mtd);
1302
1303 return max_bitflips;
1304 }
1305
1306 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct mtd_info *mtd,
1307 struct nand_chip *chip,
1308 u32 data_offs, u32 readlen,
1309 u8 *buf, int page)
1310 {
1311 int nchunks = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size);
1312 int ret;
1313
1314 ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, false, page, nchunks);
1315 if (ret >= 0)
1316 return ret;
1317
1318 /* Fallback to PIO mode */
1319 chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
1320
1321 return sunxi_nfc_hw_ecc_read_subpage(mtd, chip, data_offs, readlen,
1322 buf, page);
1323 }
1324
1325 static int sunxi_nfc_hw_ecc_write_page(struct mtd_info *mtd,
1326 struct nand_chip *chip,
1327 const uint8_t *buf, int oob_required,
1328 int page)
1329 {
1330 struct nand_ecc_ctrl *ecc = &chip->ecc;
1331 int ret, i, cur_off = 0;
1332
1333 sunxi_nfc_hw_ecc_enable(mtd);
1334
1335 for (i = 0; i < ecc->steps; i++) {
1336 int data_off = i * ecc->size;
1337 int oob_off = i * (ecc->bytes + 4);
1338 const u8 *data = buf + data_off;
1339 const u8 *oob = chip->oob_poi + oob_off;
1340
1341 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1342 oob_off + mtd->writesize,
1343 &cur_off, !i, page);
1344 if (ret)
1345 return ret;
1346 }
1347
1348 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1349 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1350 &cur_off, page);
1351
1352 sunxi_nfc_hw_ecc_disable(mtd);
1353
1354 return 0;
1355 }
1356
1357 static int sunxi_nfc_hw_ecc_write_subpage(struct mtd_info *mtd,
1358 struct nand_chip *chip,
1359 u32 data_offs, u32 data_len,
1360 const u8 *buf, int oob_required,
1361 int page)
1362 {
1363 struct nand_ecc_ctrl *ecc = &chip->ecc;
1364 int ret, i, cur_off = 0;
1365
1366 sunxi_nfc_hw_ecc_enable(mtd);
1367
1368 for (i = data_offs / ecc->size;
1369 i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) {
1370 int data_off = i * ecc->size;
1371 int oob_off = i * (ecc->bytes + 4);
1372 const u8 *data = buf + data_off;
1373 const u8 *oob = chip->oob_poi + oob_off;
1374
1375 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1376 oob_off + mtd->writesize,
1377 &cur_off, !i, page);
1378 if (ret)
1379 return ret;
1380 }
1381
1382 sunxi_nfc_hw_ecc_disable(mtd);
1383
1384 return 0;
1385 }
1386
1387 static int sunxi_nfc_hw_ecc_write_page_dma(struct mtd_info *mtd,
1388 struct nand_chip *chip,
1389 const u8 *buf,
1390 int oob_required,
1391 int page)
1392 {
1393 struct nand_chip *nand = mtd_to_nand(mtd);
1394 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1395 struct nand_ecc_ctrl *ecc = &nand->ecc;
1396 struct scatterlist sg;
1397 int ret, i;
1398
1399 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1400 if (ret)
1401 return ret;
1402
1403 ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, ecc->steps,
1404 DMA_TO_DEVICE, &sg);
1405 if (ret)
1406 goto pio_fallback;
1407
1408 for (i = 0; i < ecc->steps; i++) {
1409 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));
1410
1411 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, i, !i, page);
1412 }
1413
1414 sunxi_nfc_hw_ecc_enable(mtd);
1415 sunxi_nfc_randomizer_config(mtd, page, false);
1416 sunxi_nfc_randomizer_enable(mtd);
1417
1418 writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
1419 nfc->regs + NFC_REG_RCMD_SET);
1420
1421 dma_async_issue_pending(nfc->dmac);
1422
1423 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
1424 NFC_DATA_TRANS | NFC_ACCESS_DIR,
1425 nfc->regs + NFC_REG_CMD);
1426
1427 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
1428 if (ret)
1429 dmaengine_terminate_all(nfc->dmac);
1430
1431 sunxi_nfc_randomizer_disable(mtd);
1432 sunxi_nfc_hw_ecc_disable(mtd);
1433
1434 sunxi_nfc_dma_op_cleanup(mtd, DMA_TO_DEVICE, &sg);
1435
1436 if (ret)
1437 return ret;
1438
1439 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1440 /* TODO: use DMA to transfer extra OOB bytes ? */
1441 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1442 NULL, page);
1443
1444 return 0;
1445
1446 pio_fallback:
1447 return sunxi_nfc_hw_ecc_write_page(mtd, chip, buf, oob_required, page);
1448 }
1449
1450 static int sunxi_nfc_hw_syndrome_ecc_read_page(struct mtd_info *mtd,
1451 struct nand_chip *chip,
1452 uint8_t *buf, int oob_required,
1453 int page)
1454 {
1455 struct nand_ecc_ctrl *ecc = &chip->ecc;
1456 unsigned int max_bitflips = 0;
1457 int ret, i, cur_off = 0;
1458 bool raw_mode = false;
1459
1460 sunxi_nfc_hw_ecc_enable(mtd);
1461
1462 for (i = 0; i < ecc->steps; i++) {
1463 int data_off = i * (ecc->size + ecc->bytes + 4);
1464 int oob_off = data_off + ecc->size;
1465 u8 *data = buf + (i * ecc->size);
1466 u8 *oob = chip->oob_poi + (i * (ecc->bytes + 4));
1467
1468 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1469 oob_off, &cur_off,
1470 &max_bitflips, !i,
1471 oob_required,
1472 page);
1473 if (ret < 0)
1474 return ret;
1475 else if (ret)
1476 raw_mode = true;
1477 }
1478
1479 if (oob_required)
1480 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1481 !raw_mode, page);
1482
1483 sunxi_nfc_hw_ecc_disable(mtd);
1484
1485 return max_bitflips;
1486 }
1487
1488 static int sunxi_nfc_hw_syndrome_ecc_write_page(struct mtd_info *mtd,
1489 struct nand_chip *chip,
1490 const uint8_t *buf,
1491 int oob_required, int page)
1492 {
1493 struct nand_ecc_ctrl *ecc = &chip->ecc;
1494 int ret, i, cur_off = 0;
1495
1496 sunxi_nfc_hw_ecc_enable(mtd);
1497
1498 for (i = 0; i < ecc->steps; i++) {
1499 int data_off = i * (ecc->size + ecc->bytes + 4);
1500 int oob_off = data_off + ecc->size;
1501 const u8 *data = buf + (i * ecc->size);
1502 const u8 *oob = chip->oob_poi + (i * (ecc->bytes + 4));
1503
1504 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off,
1505 oob, oob_off, &cur_off,
1506 false, page);
1507 if (ret)
1508 return ret;
1509 }
1510
1511 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1512 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1513 &cur_off, page);
1514
1515 sunxi_nfc_hw_ecc_disable(mtd);
1516
1517 return 0;
1518 }
1519
1520 static int sunxi_nfc_hw_common_ecc_read_oob(struct mtd_info *mtd,
1521 struct nand_chip *chip,
1522 int page)
1523 {
1524 chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1525
1526 chip->pagebuf = -1;
1527
1528 return chip->ecc.read_page(mtd, chip, chip->buffers->databuf, 1, page);
1529 }
1530
1531 static int sunxi_nfc_hw_common_ecc_write_oob(struct mtd_info *mtd,
1532 struct nand_chip *chip,
1533 int page)
1534 {
1535 int ret, status;
1536
1537 chip->cmdfunc(mtd, NAND_CMD_SEQIN, 0, page);
1538
1539 chip->pagebuf = -1;
1540
1541 memset(chip->buffers->databuf, 0xff, mtd->writesize);
1542 ret = chip->ecc.write_page(mtd, chip, chip->buffers->databuf, 1, page);
1543 if (ret)
1544 return ret;
1545
1546 /* Send command to program the OOB data */
1547 chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1548
1549 status = chip->waitfunc(mtd, chip);
1550
1551 return status & NAND_STATUS_FAIL ? -EIO : 0;
1552 }
1553
1554 static const s32 tWB_lut[] = {6, 12, 16, 20};
1555 static const s32 tRHW_lut[] = {4, 8, 12, 20};
1556
1557 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
1558 u32 clk_period)
1559 {
1560 u32 clk_cycles = DIV_ROUND_UP(duration, clk_period);
1561 int i;
1562
1563 for (i = 0; i < lut_size; i++) {
1564 if (clk_cycles <= lut[i])
1565 return i;
1566 }
1567
1568 /* Doesn't fit */
1569 return -EINVAL;
1570 }
1571
1572 #define sunxi_nand_lookup_timing(l, p, c) \
1573 _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)
1574
1575 static int sunxi_nand_chip_set_timings(struct sunxi_nand_chip *chip,
1576 const struct nand_sdr_timings *timings)
1577 {
1578 struct sunxi_nfc *nfc = to_sunxi_nfc(chip->nand.controller);
1579 u32 min_clk_period = 0;
1580 s32 tWB, tADL, tWHR, tRHW, tCAD;
1581 long real_clk_rate;
1582
1583 /* T1 <=> tCLS */
1584 if (timings->tCLS_min > min_clk_period)
1585 min_clk_period = timings->tCLS_min;
1586
1587 /* T2 <=> tCLH */
1588 if (timings->tCLH_min > min_clk_period)
1589 min_clk_period = timings->tCLH_min;
1590
1591 /* T3 <=> tCS */
1592 if (timings->tCS_min > min_clk_period)
1593 min_clk_period = timings->tCS_min;
1594
1595 /* T4 <=> tCH */
1596 if (timings->tCH_min > min_clk_period)
1597 min_clk_period = timings->tCH_min;
1598
1599 /* T5 <=> tWP */
1600 if (timings->tWP_min > min_clk_period)
1601 min_clk_period = timings->tWP_min;
1602
1603 /* T6 <=> tWH */
1604 if (timings->tWH_min > min_clk_period)
1605 min_clk_period = timings->tWH_min;
1606
1607 /* T7 <=> tALS */
1608 if (timings->tALS_min > min_clk_period)
1609 min_clk_period = timings->tALS_min;
1610
1611 /* T8 <=> tDS */
1612 if (timings->tDS_min > min_clk_period)
1613 min_clk_period = timings->tDS_min;
1614
1615 /* T9 <=> tDH */
1616 if (timings->tDH_min > min_clk_period)
1617 min_clk_period = timings->tDH_min;
1618
1619 /* T10 <=> tRR */
1620 if (timings->tRR_min > (min_clk_period * 3))
1621 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);
1622
1623 /* T11 <=> tALH */
1624 if (timings->tALH_min > min_clk_period)
1625 min_clk_period = timings->tALH_min;
1626
1627 /* T12 <=> tRP */
1628 if (timings->tRP_min > min_clk_period)
1629 min_clk_period = timings->tRP_min;
1630
1631 /* T13 <=> tREH */
1632 if (timings->tREH_min > min_clk_period)
1633 min_clk_period = timings->tREH_min;
1634
1635 /* T14 <=> tRC */
1636 if (timings->tRC_min > (min_clk_period * 2))
1637 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2);
1638
1639 /* T15 <=> tWC */
1640 if (timings->tWC_min > (min_clk_period * 2))
1641 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);
1642
1643 /* T16 - T19 + tCAD */
1644 if (timings->tWB_max > (min_clk_period * 20))
1645 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);
1646
1647 if (timings->tADL_min > (min_clk_period * 32))
1648 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
1649
1650 if (timings->tWHR_min > (min_clk_period * 32))
1651 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
1652
1653 if (timings->tRHW_min > (min_clk_period * 20))
1654 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);
1655
1656 tWB = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,
1657 min_clk_period);
1658 if (tWB < 0) {
1659 dev_err(nfc->dev, "unsupported tWB\n");
1660 return tWB;
1661 }
1662
1663 tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;
1664 if (tADL > 3) {
1665 dev_err(nfc->dev, "unsupported tADL\n");
1666 return -EINVAL;
1667 }
1668
1669 tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;
1670 if (tWHR > 3) {
1671 dev_err(nfc->dev, "unsupported tWHR\n");
1672 return -EINVAL;
1673 }
1674
1675 tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,
1676 min_clk_period);
1677 if (tRHW < 0) {
1678 dev_err(nfc->dev, "unsupported tRHW\n");
1679 return tRHW;
1680 }
1681
1682 /*
1683 * TODO: according to ONFI specs this value only applies for DDR NAND,
1684 * but Allwinner seems to set this to 0x7. Mimic them for now.
1685 */
1686 tCAD = 0x7;
1687
1688 /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */
1689 chip->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD);
1690
1691 /* Convert min_clk_period from picoseconds to nanoseconds */
1692 min_clk_period = DIV_ROUND_UP(min_clk_period, 1000);
1693
1694 /*
1695 * Unlike what is stated in Allwinner datasheet, the clk_rate should
1696 * be set to (1 / min_clk_period), and not (2 / min_clk_period).
1697 * This new formula was verified with a scope and validated by
1698 * Allwinner engineers.
1699 */
1700 chip->clk_rate = NSEC_PER_SEC / min_clk_period;
1701 real_clk_rate = clk_round_rate(nfc->mod_clk, chip->clk_rate);
1702
1703 /*
1704 * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data
1705 * output cycle timings shall be used if the host drives tRC less than
1706 * 30 ns.
1707 */
1708 min_clk_period = NSEC_PER_SEC / real_clk_rate;
1709 chip->timing_ctl = ((min_clk_period * 2) < 30) ?
1710 NFC_TIMING_CTL_EDO : 0;
1711
1712 return 0;
1713 }
1714
1715 static int sunxi_nand_chip_init_timings(struct sunxi_nand_chip *chip,
1716 struct device_node *np)
1717 {
1718 struct mtd_info *mtd = nand_to_mtd(&chip->nand);
1719 const struct nand_sdr_timings *timings;
1720 int ret;
1721 int mode;
1722
1723 mode = onfi_get_async_timing_mode(&chip->nand);
1724 if (mode == ONFI_TIMING_MODE_UNKNOWN) {
1725 mode = chip->nand.onfi_timing_mode_default;
1726 } else {
1727 uint8_t feature[ONFI_SUBFEATURE_PARAM_LEN] = {};
1728 int i;
1729
1730 mode = fls(mode) - 1;
1731 if (mode < 0)
1732 mode = 0;
1733
1734 feature[0] = mode;
1735 for (i = 0; i < chip->nsels; i++) {
1736 chip->nand.select_chip(mtd, i);
1737 ret = chip->nand.onfi_set_features(mtd, &chip->nand,
1738 ONFI_FEATURE_ADDR_TIMING_MODE,
1739 feature);
1740 chip->nand.select_chip(mtd, -1);
1741 if (ret)
1742 return ret;
1743 }
1744 }
1745
1746 timings = onfi_async_timing_mode_to_sdr_timings(mode);
1747 if (IS_ERR(timings))
1748 return PTR_ERR(timings);
1749
1750 return sunxi_nand_chip_set_timings(chip, timings);
1751 }
1752
1753 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1754 struct mtd_oob_region *oobregion)
1755 {
1756 struct nand_chip *nand = mtd_to_nand(mtd);
1757 struct nand_ecc_ctrl *ecc = &nand->ecc;
1758
1759 if (section >= ecc->steps)
1760 return -ERANGE;
1761
1762 oobregion->offset = section * (ecc->bytes + 4) + 4;
1763 oobregion->length = ecc->bytes;
1764
1765 return 0;
1766 }
1767
1768 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
1769 struct mtd_oob_region *oobregion)
1770 {
1771 struct nand_chip *nand = mtd_to_nand(mtd);
1772 struct nand_ecc_ctrl *ecc = &nand->ecc;
1773
1774 if (section > ecc->steps)
1775 return -ERANGE;
1776
1777 /*
1778 * The first 2 bytes are used for BB markers, hence we
1779 * only have 2 bytes available in the first user data
1780 * section.
1781 */
1782 if (!section && ecc->mode == NAND_ECC_HW) {
1783 oobregion->offset = 2;
1784 oobregion->length = 2;
1785
1786 return 0;
1787 }
1788
1789 oobregion->offset = section * (ecc->bytes + 4);
1790
1791 if (section < ecc->steps)
1792 oobregion->length = 4;
1793 else
1794 oobregion->offset = mtd->oobsize - oobregion->offset;
1795
1796 return 0;
1797 }
1798
1799 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = {
1800 .ecc = sunxi_nand_ooblayout_ecc,
1801 .free = sunxi_nand_ooblayout_free,
1802 };
1803
1804 static int sunxi_nand_hw_common_ecc_ctrl_init(struct mtd_info *mtd,
1805 struct nand_ecc_ctrl *ecc,
1806 struct device_node *np)
1807 {
1808 static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 };
1809 struct nand_chip *nand = mtd_to_nand(mtd);
1810 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1811 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1812 struct sunxi_nand_hw_ecc *data;
1813 int nsectors;
1814 int ret;
1815 int i;
1816
1817 if (ecc->size != 512 && ecc->size != 1024)
1818 return -EINVAL;
1819
1820 data = kzalloc(sizeof(*data), GFP_KERNEL);
1821 if (!data)
1822 return -ENOMEM;
1823
1824 /* Prefer 1k ECC chunk over 512 ones */
1825 if (ecc->size == 512 && mtd->writesize > 512) {
1826 ecc->size = 1024;
1827 ecc->strength *= 2;
1828 }
1829
1830 /* Add ECC info retrieval from DT */
1831 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1832 if (ecc->strength <= strengths[i])
1833 break;
1834 }
1835
1836 if (i >= ARRAY_SIZE(strengths)) {
1837 dev_err(nfc->dev, "unsupported strength\n");
1838 ret = -ENOTSUPP;
1839 goto err;
1840 }
1841
1842 data->mode = i;
1843
1844 /* HW ECC always request ECC bytes for 1024 bytes blocks */
1845 ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8);
1846
1847 /* HW ECC always work with even numbers of ECC bytes */
1848 ecc->bytes = ALIGN(ecc->bytes, 2);
1849
1850 nsectors = mtd->writesize / ecc->size;
1851
1852 if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) {
1853 ret = -EINVAL;
1854 goto err;
1855 }
1856
1857 ecc->read_oob = sunxi_nfc_hw_common_ecc_read_oob;
1858 ecc->write_oob = sunxi_nfc_hw_common_ecc_write_oob;
1859 mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops);
1860 ecc->priv = data;
1861
1862 return 0;
1863
1864 err:
1865 kfree(data);
1866
1867 return ret;
1868 }
1869
1870 static void sunxi_nand_hw_common_ecc_ctrl_cleanup(struct nand_ecc_ctrl *ecc)
1871 {
1872 kfree(ecc->priv);
1873 }
1874
1875 static int sunxi_nand_hw_ecc_ctrl_init(struct mtd_info *mtd,
1876 struct nand_ecc_ctrl *ecc,
1877 struct device_node *np)
1878 {
1879 struct nand_chip *nand = mtd_to_nand(mtd);
1880 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1881 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1882 int ret;
1883
1884 ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
1885 if (ret)
1886 return ret;
1887
1888 if (nfc->dmac) {
1889 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
1890 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
1891 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
1892 nand->options |= NAND_USE_BOUNCE_BUFFER;
1893 } else {
1894 ecc->read_page = sunxi_nfc_hw_ecc_read_page;
1895 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1896 ecc->write_page = sunxi_nfc_hw_ecc_write_page;
1897 }
1898
1899 /* TODO: support DMA for raw accesses and subpage write */
1900 ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage;
1901 ecc->read_oob_raw = nand_read_oob_std;
1902 ecc->write_oob_raw = nand_write_oob_std;
1903 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1904
1905 return 0;
1906 }
1907
1908 static int sunxi_nand_hw_syndrome_ecc_ctrl_init(struct mtd_info *mtd,
1909 struct nand_ecc_ctrl *ecc,
1910 struct device_node *np)
1911 {
1912 int ret;
1913
1914 ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
1915 if (ret)
1916 return ret;
1917
1918 ecc->prepad = 4;
1919 ecc->read_page = sunxi_nfc_hw_syndrome_ecc_read_page;
1920 ecc->write_page = sunxi_nfc_hw_syndrome_ecc_write_page;
1921 ecc->read_oob_raw = nand_read_oob_syndrome;
1922 ecc->write_oob_raw = nand_write_oob_syndrome;
1923
1924 return 0;
1925 }
1926
1927 static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc)
1928 {
1929 switch (ecc->mode) {
1930 case NAND_ECC_HW:
1931 case NAND_ECC_HW_SYNDROME:
1932 sunxi_nand_hw_common_ecc_ctrl_cleanup(ecc);
1933 break;
1934 case NAND_ECC_NONE:
1935 default:
1936 break;
1937 }
1938 }
1939
1940 static int sunxi_nand_ecc_init(struct mtd_info *mtd, struct nand_ecc_ctrl *ecc,
1941 struct device_node *np)
1942 {
1943 struct nand_chip *nand = mtd_to_nand(mtd);
1944 int ret;
1945
1946 if (!ecc->size) {
1947 ecc->size = nand->ecc_step_ds;
1948 ecc->strength = nand->ecc_strength_ds;
1949 }
1950
1951 if (!ecc->size || !ecc->strength)
1952 return -EINVAL;
1953
1954 switch (ecc->mode) {
1955 case NAND_ECC_HW:
1956 ret = sunxi_nand_hw_ecc_ctrl_init(mtd, ecc, np);
1957 if (ret)
1958 return ret;
1959 break;
1960 case NAND_ECC_HW_SYNDROME:
1961 ret = sunxi_nand_hw_syndrome_ecc_ctrl_init(mtd, ecc, np);
1962 if (ret)
1963 return ret;
1964 break;
1965 case NAND_ECC_NONE:
1966 case NAND_ECC_SOFT:
1967 break;
1968 default:
1969 return -EINVAL;
1970 }
1971
1972 return 0;
1973 }
1974
1975 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
1976 struct device_node *np)
1977 {
1978 const struct nand_sdr_timings *timings;
1979 struct sunxi_nand_chip *chip;
1980 struct mtd_info *mtd;
1981 struct nand_chip *nand;
1982 int nsels;
1983 int ret;
1984 int i;
1985 u32 tmp;
1986
1987 if (!of_get_property(np, "reg", &nsels))
1988 return -EINVAL;
1989
1990 nsels /= sizeof(u32);
1991 if (!nsels) {
1992 dev_err(dev, "invalid reg property size\n");
1993 return -EINVAL;
1994 }
1995
1996 chip = devm_kzalloc(dev,
1997 sizeof(*chip) +
1998 (nsels * sizeof(struct sunxi_nand_chip_sel)),
1999 GFP_KERNEL);
2000 if (!chip) {
2001 dev_err(dev, "could not allocate chip\n");
2002 return -ENOMEM;
2003 }
2004
2005 chip->nsels = nsels;
2006 chip->selected = -1;
2007
2008 for (i = 0; i < nsels; i++) {
2009 ret = of_property_read_u32_index(np, "reg", i, &tmp);
2010 if (ret) {
2011 dev_err(dev, "could not retrieve reg property: %d\n",
2012 ret);
2013 return ret;
2014 }
2015
2016 if (tmp > NFC_MAX_CS) {
2017 dev_err(dev,
2018 "invalid reg value: %u (max CS = 7)\n",
2019 tmp);
2020 return -EINVAL;
2021 }
2022
2023 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
2024 dev_err(dev, "CS %d already assigned\n", tmp);
2025 return -EINVAL;
2026 }
2027
2028 chip->sels[i].cs = tmp;
2029
2030 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) &&
2031 tmp < 2) {
2032 chip->sels[i].rb.type = RB_NATIVE;
2033 chip->sels[i].rb.info.nativeid = tmp;
2034 } else {
2035 ret = of_get_named_gpio(np, "rb-gpios", i);
2036 if (ret >= 0) {
2037 tmp = ret;
2038 chip->sels[i].rb.type = RB_GPIO;
2039 chip->sels[i].rb.info.gpio = tmp;
2040 ret = devm_gpio_request(dev, tmp, "nand-rb");
2041 if (ret)
2042 return ret;
2043
2044 ret = gpio_direction_input(tmp);
2045 if (ret)
2046 return ret;
2047 } else {
2048 chip->sels[i].rb.type = RB_NONE;
2049 }
2050 }
2051 }
2052
2053 nand = &chip->nand;
2054 /* Default tR value specified in the ONFI spec (chapter 4.15.1) */
2055 nand->chip_delay = 200;
2056 nand->controller = &nfc->controller;
2057 /*
2058 * Set the ECC mode to the default value in case nothing is specified
2059 * in the DT.
2060 */
2061 nand->ecc.mode = NAND_ECC_HW;
2062 nand_set_flash_node(nand, np);
2063 nand->select_chip = sunxi_nfc_select_chip;
2064 nand->cmd_ctrl = sunxi_nfc_cmd_ctrl;
2065 nand->read_buf = sunxi_nfc_read_buf;
2066 nand->write_buf = sunxi_nfc_write_buf;
2067 nand->read_byte = sunxi_nfc_read_byte;
2068
2069 mtd = nand_to_mtd(nand);
2070 mtd->dev.parent = dev;
2071
2072 timings = onfi_async_timing_mode_to_sdr_timings(0);
2073 if (IS_ERR(timings)) {
2074 ret = PTR_ERR(timings);
2075 dev_err(dev,
2076 "could not retrieve timings for ONFI mode 0: %d\n",
2077 ret);
2078 return ret;
2079 }
2080
2081 ret = sunxi_nand_chip_set_timings(chip, timings);
2082 if (ret) {
2083 dev_err(dev, "could not configure chip timings: %d\n", ret);
2084 return ret;
2085 }
2086
2087 ret = nand_scan_ident(mtd, nsels, NULL);
2088 if (ret)
2089 return ret;
2090
2091 if (nand->bbt_options & NAND_BBT_USE_FLASH)
2092 nand->bbt_options |= NAND_BBT_NO_OOB;
2093
2094 if (nand->options & NAND_NEED_SCRAMBLING)
2095 nand->options |= NAND_NO_SUBPAGE_WRITE;
2096
2097 nand->options |= NAND_SUBPAGE_READ;
2098
2099 ret = sunxi_nand_chip_init_timings(chip, np);
2100 if (ret) {
2101 dev_err(dev, "could not configure chip timings: %d\n", ret);
2102 return ret;
2103 }
2104
2105 ret = sunxi_nand_ecc_init(mtd, &nand->ecc, np);
2106 if (ret) {
2107 dev_err(dev, "ECC init failed: %d\n", ret);
2108 return ret;
2109 }
2110
2111 ret = nand_scan_tail(mtd);
2112 if (ret) {
2113 dev_err(dev, "nand_scan_tail failed: %d\n", ret);
2114 return ret;
2115 }
2116
2117 ret = mtd_device_register(mtd, NULL, 0);
2118 if (ret) {
2119 dev_err(dev, "failed to register mtd device: %d\n", ret);
2120 nand_release(mtd);
2121 return ret;
2122 }
2123
2124 list_add_tail(&chip->node, &nfc->chips);
2125
2126 return 0;
2127 }
2128
2129 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)
2130 {
2131 struct device_node *np = dev->of_node;
2132 struct device_node *nand_np;
2133 int nchips = of_get_child_count(np);
2134 int ret;
2135
2136 if (nchips > 8) {
2137 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips);
2138 return -EINVAL;
2139 }
2140
2141 for_each_child_of_node(np, nand_np) {
2142 ret = sunxi_nand_chip_init(dev, nfc, nand_np);
2143 if (ret) {
2144 of_node_put(nand_np);
2145 return ret;
2146 }
2147 }
2148
2149 return 0;
2150 }
2151
2152 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)
2153 {
2154 struct sunxi_nand_chip *chip;
2155
2156 while (!list_empty(&nfc->chips)) {
2157 chip = list_first_entry(&nfc->chips, struct sunxi_nand_chip,
2158 node);
2159 nand_release(nand_to_mtd(&chip->nand));
2160 sunxi_nand_ecc_cleanup(&chip->nand.ecc);
2161 list_del(&chip->node);
2162 }
2163 }
2164
2165 static int sunxi_nfc_probe(struct platform_device *pdev)
2166 {
2167 struct device *dev = &pdev->dev;
2168 struct resource *r;
2169 struct sunxi_nfc *nfc;
2170 int irq;
2171 int ret;
2172
2173 nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
2174 if (!nfc)
2175 return -ENOMEM;
2176
2177 nfc->dev = dev;
2178 spin_lock_init(&nfc->controller.lock);
2179 init_waitqueue_head(&nfc->controller.wq);
2180 INIT_LIST_HEAD(&nfc->chips);
2181
2182 r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2183 nfc->regs = devm_ioremap_resource(dev, r);
2184 if (IS_ERR(nfc->regs))
2185 return PTR_ERR(nfc->regs);
2186
2187 irq = platform_get_irq(pdev, 0);
2188 if (irq < 0) {
2189 dev_err(dev, "failed to retrieve irq\n");
2190 return irq;
2191 }
2192
2193 nfc->ahb_clk = devm_clk_get(dev, "ahb");
2194 if (IS_ERR(nfc->ahb_clk)) {
2195 dev_err(dev, "failed to retrieve ahb clk\n");
2196 return PTR_ERR(nfc->ahb_clk);
2197 }
2198
2199 ret = clk_prepare_enable(nfc->ahb_clk);
2200 if (ret)
2201 return ret;
2202
2203 nfc->mod_clk = devm_clk_get(dev, "mod");
2204 if (IS_ERR(nfc->mod_clk)) {
2205 dev_err(dev, "failed to retrieve mod clk\n");
2206 ret = PTR_ERR(nfc->mod_clk);
2207 goto out_ahb_clk_unprepare;
2208 }
2209
2210 ret = clk_prepare_enable(nfc->mod_clk);
2211 if (ret)
2212 goto out_ahb_clk_unprepare;
2213
2214 nfc->reset = devm_reset_control_get_optional(dev, "ahb");
2215 if (!IS_ERR(nfc->reset)) {
2216 ret = reset_control_deassert(nfc->reset);
2217 if (ret) {
2218 dev_err(dev, "reset err %d\n", ret);
2219 goto out_mod_clk_unprepare;
2220 }
2221 } else if (PTR_ERR(nfc->reset) != -ENOENT) {
2222 ret = PTR_ERR(nfc->reset);
2223 goto out_mod_clk_unprepare;
2224 }
2225
2226 ret = sunxi_nfc_rst(nfc);
2227 if (ret)
2228 goto out_ahb_reset_reassert;
2229
2230 writel(0, nfc->regs + NFC_REG_INT);
2231 ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt,
2232 0, "sunxi-nand", nfc);
2233 if (ret)
2234 goto out_ahb_reset_reassert;
2235
2236 nfc->dmac = dma_request_slave_channel(dev, "rxtx");
2237 if (nfc->dmac) {
2238 struct dma_slave_config dmac_cfg = { };
2239
2240 dmac_cfg.src_addr = r->start + NFC_REG_IO_DATA;
2241 dmac_cfg.dst_addr = dmac_cfg.src_addr;
2242 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2243 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;
2244 dmac_cfg.src_maxburst = 4;
2245 dmac_cfg.dst_maxburst = 4;
2246 dmaengine_slave_config(nfc->dmac, &dmac_cfg);
2247 } else {
2248 dev_warn(dev, "failed to request rxtx DMA channel\n");
2249 }
2250
2251 platform_set_drvdata(pdev, nfc);
2252
2253 ret = sunxi_nand_chips_init(dev, nfc);
2254 if (ret) {
2255 dev_err(dev, "failed to init nand chips\n");
2256 goto out_release_dmac;
2257 }
2258
2259 return 0;
2260
2261 out_release_dmac:
2262 if (nfc->dmac)
2263 dma_release_channel(nfc->dmac);
2264 out_ahb_reset_reassert:
2265 if (!IS_ERR(nfc->reset))
2266 reset_control_assert(nfc->reset);
2267 out_mod_clk_unprepare:
2268 clk_disable_unprepare(nfc->mod_clk);
2269 out_ahb_clk_unprepare:
2270 clk_disable_unprepare(nfc->ahb_clk);
2271
2272 return ret;
2273 }
2274
2275 static int sunxi_nfc_remove(struct platform_device *pdev)
2276 {
2277 struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
2278
2279 sunxi_nand_chips_cleanup(nfc);
2280
2281 if (!IS_ERR(nfc->reset))
2282 reset_control_assert(nfc->reset);
2283
2284 if (nfc->dmac)
2285 dma_release_channel(nfc->dmac);
2286 clk_disable_unprepare(nfc->mod_clk);
2287 clk_disable_unprepare(nfc->ahb_clk);
2288
2289 return 0;
2290 }
2291
2292 static const struct of_device_id sunxi_nfc_ids[] = {
2293 { .compatible = "allwinner,sun4i-a10-nand" },
2294 { /* sentinel */ }
2295 };
2296 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids);
2297
2298 static struct platform_driver sunxi_nfc_driver = {
2299 .driver = {
2300 .name = "sunxi_nand",
2301 .of_match_table = sunxi_nfc_ids,
2302 },
2303 .probe = sunxi_nfc_probe,
2304 .remove = sunxi_nfc_remove,
2305 };
2306 module_platform_driver(sunxi_nfc_driver);
2307
2308 MODULE_LICENSE("GPL v2");
2309 MODULE_AUTHOR("Boris BREZILLON");
2310 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver");
2311 MODULE_ALIAS("platform:sunxi_nand");
CRIS linux kernel tree