?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost
[cris-mirror.git] / drivers / mtd / nand / sunxi_nand.c
blobf5a55c63935cbc80f8fc23c5040025096347f6cf
1 /*
2 * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com>
3 *
4 * Derived from:
5 * https://github.com/yuq/sunxi-nfc-mtd
6 * Copyright (C) 2013 Qiang Yu <yuq825@gmail.com>
7 *
8 * https://github.com/hno/Allwinner-Info
9 * Copyright (C) 2013 Henrik Nordström <Henrik Nordström>
10 *
11 * Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com>
12 * Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org>
13 *
14 * This program is free software; you can redistribute it and/or modify
15 * it under the terms of the GNU General Public License as published by
16 * the Free Software Foundation; either version 2 of the License, or
17 * (at your option) any later version.
18 *
19 * This program is distributed in the hope that it will be useful,
20 * but WITHOUT ANY WARRANTY; without even the implied warranty of
21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 * GNU General Public License for more details.
23 */
24
25 #include <linux/dma-mapping.h>
26 #include <linux/slab.h>
27 #include <linux/module.h>
28 #include <linux/moduleparam.h>
29 #include <linux/platform_device.h>
30 #include <linux/of.h>
31 #include <linux/of_device.h>
32 #include <linux/of_gpio.h>
33 #include <linux/mtd/mtd.h>
34 #include <linux/mtd/rawnand.h>
35 #include <linux/mtd/partitions.h>
36 #include <linux/clk.h>
37 #include <linux/delay.h>
38 #include <linux/dmaengine.h>
39 #include <linux/gpio.h>
40 #include <linux/interrupt.h>
41 #include <linux/iopoll.h>
42 #include <linux/reset.h>
43
44 #define NFC_REG_CTL 0x0000
45 #define NFC_REG_ST 0x0004
46 #define NFC_REG_INT 0x0008
47 #define NFC_REG_TIMING_CTL 0x000C
48 #define NFC_REG_TIMING_CFG 0x0010
49 #define NFC_REG_ADDR_LOW 0x0014
50 #define NFC_REG_ADDR_HIGH 0x0018
51 #define NFC_REG_SECTOR_NUM 0x001C
52 #define NFC_REG_CNT 0x0020
53 #define NFC_REG_CMD 0x0024
54 #define NFC_REG_RCMD_SET 0x0028
55 #define NFC_REG_WCMD_SET 0x002C
56 #define NFC_REG_IO_DATA 0x0030
57 #define NFC_REG_ECC_CTL 0x0034
58 #define NFC_REG_ECC_ST 0x0038
59 #define NFC_REG_DEBUG 0x003C
60 #define NFC_REG_ECC_ERR_CNT(x) ((0x0040 + (x)) & ~0x3)
61 #define NFC_REG_USER_DATA(x) (0x0050 + ((x) * 4))
62 #define NFC_REG_SPARE_AREA 0x00A0
63 #define NFC_REG_PAT_ID 0x00A4
64 #define NFC_RAM0_BASE 0x0400
65 #define NFC_RAM1_BASE 0x0800
66
67 /* define bit use in NFC_CTL */
68 #define NFC_EN BIT(0)
69 #define NFC_RESET BIT(1)
70 #define NFC_BUS_WIDTH_MSK BIT(2)
71 #define NFC_BUS_WIDTH_8 (0 << 2)
72 #define NFC_BUS_WIDTH_16 (1 << 2)
73 #define NFC_RB_SEL_MSK BIT(3)
74 #define NFC_RB_SEL(x) ((x) << 3)
75 #define NFC_CE_SEL_MSK GENMASK(26, 24)
76 #define NFC_CE_SEL(x) ((x) << 24)
77 #define NFC_CE_CTL BIT(6)
78 #define NFC_PAGE_SHIFT_MSK GENMASK(11, 8)
79 #define NFC_PAGE_SHIFT(x) (((x) < 10 ? 0 : (x) - 10) << 8)
80 #define NFC_SAM BIT(12)
81 #define NFC_RAM_METHOD BIT(14)
82 #define NFC_DEBUG_CTL BIT(31)
83
84 /* define bit use in NFC_ST */
85 #define NFC_RB_B2R BIT(0)
86 #define NFC_CMD_INT_FLAG BIT(1)
87 #define NFC_DMA_INT_FLAG BIT(2)
88 #define NFC_CMD_FIFO_STATUS BIT(3)
89 #define NFC_STA BIT(4)
90 #define NFC_NATCH_INT_FLAG BIT(5)
91 #define NFC_RB_STATE(x) BIT(x + 8)
92
93 /* define bit use in NFC_INT */
94 #define NFC_B2R_INT_ENABLE BIT(0)
95 #define NFC_CMD_INT_ENABLE BIT(1)
96 #define NFC_DMA_INT_ENABLE BIT(2)
97 #define NFC_INT_MASK (NFC_B2R_INT_ENABLE | \
98 NFC_CMD_INT_ENABLE | \
99 NFC_DMA_INT_ENABLE)
100
101 /* define bit use in NFC_TIMING_CTL */
102 #define NFC_TIMING_CTL_EDO BIT(8)
103
104 /* define NFC_TIMING_CFG register layout */
105 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD) \
106 (((tWB) & 0x3) | (((tADL) & 0x3) << 2) | \
107 (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) | \
108 (((tCAD) & 0x7) << 8))
109
110 /* define bit use in NFC_CMD */
111 #define NFC_CMD_LOW_BYTE_MSK GENMASK(7, 0)
112 #define NFC_CMD_HIGH_BYTE_MSK GENMASK(15, 8)
113 #define NFC_CMD(x) (x)
114 #define NFC_ADR_NUM_MSK GENMASK(18, 16)
115 #define NFC_ADR_NUM(x) (((x) - 1) << 16)
116 #define NFC_SEND_ADR BIT(19)
117 #define NFC_ACCESS_DIR BIT(20)
118 #define NFC_DATA_TRANS BIT(21)
119 #define NFC_SEND_CMD1 BIT(22)
120 #define NFC_WAIT_FLAG BIT(23)
121 #define NFC_SEND_CMD2 BIT(24)
122 #define NFC_SEQ BIT(25)
123 #define NFC_DATA_SWAP_METHOD BIT(26)
124 #define NFC_ROW_AUTO_INC BIT(27)
125 #define NFC_SEND_CMD3 BIT(28)
126 #define NFC_SEND_CMD4 BIT(29)
127 #define NFC_CMD_TYPE_MSK GENMASK(31, 30)
128 #define NFC_NORMAL_OP (0 << 30)
129 #define NFC_ECC_OP (1 << 30)
130 #define NFC_PAGE_OP (2 << 30)
131
132 /* define bit use in NFC_RCMD_SET */
133 #define NFC_READ_CMD_MSK GENMASK(7, 0)
134 #define NFC_RND_READ_CMD0_MSK GENMASK(15, 8)
135 #define NFC_RND_READ_CMD1_MSK GENMASK(23, 16)
136
137 /* define bit use in NFC_WCMD_SET */
138 #define NFC_PROGRAM_CMD_MSK GENMASK(7, 0)
139 #define NFC_RND_WRITE_CMD_MSK GENMASK(15, 8)
140 #define NFC_READ_CMD0_MSK GENMASK(23, 16)
141 #define NFC_READ_CMD1_MSK GENMASK(31, 24)
142
143 /* define bit use in NFC_ECC_CTL */
144 #define NFC_ECC_EN BIT(0)
145 #define NFC_ECC_PIPELINE BIT(3)
146 #define NFC_ECC_EXCEPTION BIT(4)
147 #define NFC_ECC_BLOCK_SIZE_MSK BIT(5)
148 #define NFC_ECC_BLOCK_512 BIT(5)
149 #define NFC_RANDOM_EN BIT(9)
150 #define NFC_RANDOM_DIRECTION BIT(10)
151 #define NFC_ECC_MODE_MSK GENMASK(15, 12)
152 #define NFC_ECC_MODE(x) ((x) << 12)
153 #define NFC_RANDOM_SEED_MSK GENMASK(30, 16)
154 #define NFC_RANDOM_SEED(x) ((x) << 16)
155
156 /* define bit use in NFC_ECC_ST */
157 #define NFC_ECC_ERR(x) BIT(x)
158 #define NFC_ECC_ERR_MSK GENMASK(15, 0)
159 #define NFC_ECC_PAT_FOUND(x) BIT(x + 16)
160 #define NFC_ECC_ERR_CNT(b, x) (((x) >> (((b) % 4) * 8)) & 0xff)
161
162 #define NFC_DEFAULT_TIMEOUT_MS 1000
163
164 #define NFC_SRAM_SIZE 1024
165
166 #define NFC_MAX_CS 7
167
168 /*
169 * Ready/Busy detection type: describes the Ready/Busy detection modes
170 *
171 * @RB_NONE: no external detection available, rely on STATUS command
172 * and software timeouts
173 * @RB_NATIVE: use sunxi NAND controller Ready/Busy support. The Ready/Busy
174 * pin of the NAND flash chip must be connected to one of the
175 * native NAND R/B pins (those which can be muxed to the NAND
176 * Controller)
177 * @RB_GPIO: use a simple GPIO to handle Ready/Busy status. The Ready/Busy
178 * pin of the NAND flash chip must be connected to a GPIO capable
179 * pin.
180 */
181 enum sunxi_nand_rb_type {
182 RB_NONE,
183 RB_NATIVE,
184 RB_GPIO,
185 };
186
187 /*
188 * Ready/Busy structure: stores information related to Ready/Busy detection
189 *
190 * @type: the Ready/Busy detection mode
191 * @info: information related to the R/B detection mode. Either a gpio
192 * id or a native R/B id (those supported by the NAND controller).
193 */
194 struct sunxi_nand_rb {
195 enum sunxi_nand_rb_type type;
196 union {
197 int gpio;
198 int nativeid;
199 } info;
200 };
201
202 /*
203 * Chip Select structure: stores information related to NAND Chip Select
204 *
205 * @cs: the NAND CS id used to communicate with a NAND Chip
206 * @rb: the Ready/Busy description
207 */
208 struct sunxi_nand_chip_sel {
209 u8 cs;
210 struct sunxi_nand_rb rb;
211 };
212
213 /*
214 * sunxi HW ECC infos: stores information related to HW ECC support
215 *
216 * @mode: the sunxi ECC mode field deduced from ECC requirements
217 */
218 struct sunxi_nand_hw_ecc {
219 int mode;
220 };
221
222 /*
223 * NAND chip structure: stores NAND chip device related information
224 *
225 * @node: used to store NAND chips into a list
226 * @nand: base NAND chip structure
227 * @mtd: base MTD structure
228 * @clk_rate: clk_rate required for this NAND chip
229 * @timing_cfg TIMING_CFG register value for this NAND chip
230 * @selected: current active CS
231 * @nsels: number of CS lines required by the NAND chip
232 * @sels: array of CS lines descriptions
233 */
234 struct sunxi_nand_chip {
235 struct list_head node;
236 struct nand_chip nand;
237 unsigned long clk_rate;
238 u32 timing_cfg;
239 u32 timing_ctl;
240 int selected;
241 int addr_cycles;
242 u32 addr[2];
243 int cmd_cycles;
244 u8 cmd[2];
245 int nsels;
246 struct sunxi_nand_chip_sel sels[0];
247 };
248
249 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)
250 {
251 return container_of(nand, struct sunxi_nand_chip, nand);
252 }
253
254 /*
255 * NAND Controller structure: stores sunxi NAND controller information
256 *
257 * @controller: base controller structure
258 * @dev: parent device (used to print error messages)
259 * @regs: NAND controller registers
260 * @ahb_clk: NAND Controller AHB clock
261 * @mod_clk: NAND Controller mod clock
262 * @assigned_cs: bitmask describing already assigned CS lines
263 * @clk_rate: NAND controller current clock rate
264 * @chips: a list containing all the NAND chips attached to
265 * this NAND controller
266 * @complete: a completion object used to wait for NAND
267 * controller events
268 */
269 struct sunxi_nfc {
270 struct nand_hw_control controller;
271 struct device *dev;
272 void __iomem *regs;
273 struct clk *ahb_clk;
274 struct clk *mod_clk;
275 struct reset_control *reset;
276 unsigned long assigned_cs;
277 unsigned long clk_rate;
278 struct list_head chips;
279 struct completion complete;
280 struct dma_chan *dmac;
281 };
282
283 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_hw_control *ctrl)
284 {
285 return container_of(ctrl, struct sunxi_nfc, controller);
286 }
287
288 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id)
289 {
290 struct sunxi_nfc *nfc = dev_id;
291 u32 st = readl(nfc->regs + NFC_REG_ST);
292 u32 ien = readl(nfc->regs + NFC_REG_INT);
293
294 if (!(ien & st))
295 return IRQ_NONE;
296
297 if ((ien & st) == ien)
298 complete(&nfc->complete);
299
300 writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
301 writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT);
302
303 return IRQ_HANDLED;
304 }
305
306 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events,
307 bool use_polling, unsigned int timeout_ms)
308 {
309 int ret;
310
311 if (events & ~NFC_INT_MASK)
312 return -EINVAL;
313
314 if (!timeout_ms)
315 timeout_ms = NFC_DEFAULT_TIMEOUT_MS;
316
317 if (!use_polling) {
318 init_completion(&nfc->complete);
319
320 writel(events, nfc->regs + NFC_REG_INT);
321
322 ret = wait_for_completion_timeout(&nfc->complete,
323 msecs_to_jiffies(timeout_ms));
324 if (!ret)
325 ret = -ETIMEDOUT;
326 else
327 ret = 0;
328
329 writel(0, nfc->regs + NFC_REG_INT);
330 } else {
331 u32 status;
332
333 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
334 (status & events) == events, 1,
335 timeout_ms * 1000);
336 }
337
338 writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
339
340 if (ret)
341 dev_err(nfc->dev, "wait interrupt timedout\n");
342
343 return ret;
344 }
345
346 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc)
347 {
348 u32 status;
349 int ret;
350
351 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
352 !(status & NFC_CMD_FIFO_STATUS), 1,
353 NFC_DEFAULT_TIMEOUT_MS * 1000);
354 if (ret)
355 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n");
356
357 return ret;
358 }
359
360 static int sunxi_nfc_rst(struct sunxi_nfc *nfc)
361 {
362 u32 ctl;
363 int ret;
364
365 writel(0, nfc->regs + NFC_REG_ECC_CTL);
366 writel(NFC_RESET, nfc->regs + NFC_REG_CTL);
367
368 ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl,
369 !(ctl & NFC_RESET), 1,
370 NFC_DEFAULT_TIMEOUT_MS * 1000);
371 if (ret)
372 dev_err(nfc->dev, "wait for NAND controller reset timedout\n");
373
374 return ret;
375 }
376
377 static int sunxi_nfc_dma_op_prepare(struct mtd_info *mtd, const void *buf,
378 int chunksize, int nchunks,
379 enum dma_data_direction ddir,
380 struct scatterlist *sg)
381 {
382 struct nand_chip *nand = mtd_to_nand(mtd);
383 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
384 struct dma_async_tx_descriptor *dmad;
385 enum dma_transfer_direction tdir;
386 dma_cookie_t dmat;
387 int ret;
388
389 if (ddir == DMA_FROM_DEVICE)
390 tdir = DMA_DEV_TO_MEM;
391 else
392 tdir = DMA_MEM_TO_DEV;
393
394 sg_init_one(sg, buf, nchunks * chunksize);
395 ret = dma_map_sg(nfc->dev, sg, 1, ddir);
396 if (!ret)
397 return -ENOMEM;
398
399 dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);
400 if (!dmad) {
401 ret = -EINVAL;
402 goto err_unmap_buf;
403 }
404
405 writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,
406 nfc->regs + NFC_REG_CTL);
407 writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);
408 writel(chunksize, nfc->regs + NFC_REG_CNT);
409 dmat = dmaengine_submit(dmad);
410
411 ret = dma_submit_error(dmat);
412 if (ret)
413 goto err_clr_dma_flag;
414
415 return 0;
416
417 err_clr_dma_flag:
418 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
419 nfc->regs + NFC_REG_CTL);
420
421 err_unmap_buf:
422 dma_unmap_sg(nfc->dev, sg, 1, ddir);
423 return ret;
424 }
425
426 static void sunxi_nfc_dma_op_cleanup(struct mtd_info *mtd,
427 enum dma_data_direction ddir,
428 struct scatterlist *sg)
429 {
430 struct nand_chip *nand = mtd_to_nand(mtd);
431 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
432
433 dma_unmap_sg(nfc->dev, sg, 1, ddir);
434 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
435 nfc->regs + NFC_REG_CTL);
436 }
437
438 static int sunxi_nfc_dev_ready(struct mtd_info *mtd)
439 {
440 struct nand_chip *nand = mtd_to_nand(mtd);
441 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
442 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
443 struct sunxi_nand_rb *rb;
444 int ret;
445
446 if (sunxi_nand->selected < 0)
447 return 0;
448
449 rb = &sunxi_nand->sels[sunxi_nand->selected].rb;
450
451 switch (rb->type) {
452 case RB_NATIVE:
453 ret = !!(readl(nfc->regs + NFC_REG_ST) &
454 NFC_RB_STATE(rb->info.nativeid));
455 break;
456 case RB_GPIO:
457 ret = gpio_get_value(rb->info.gpio);
458 break;
459 case RB_NONE:
460 default:
461 ret = 0;
462 dev_err(nfc->dev, "cannot check R/B NAND status!\n");
463 break;
464 }
465
466 return ret;
467 }
468
469 static void sunxi_nfc_select_chip(struct mtd_info *mtd, int chip)
470 {
471 struct nand_chip *nand = mtd_to_nand(mtd);
472 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
473 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
474 struct sunxi_nand_chip_sel *sel;
475 u32 ctl;
476
477 if (chip > 0 && chip >= sunxi_nand->nsels)
478 return;
479
480 if (chip == sunxi_nand->selected)
481 return;
482
483 ctl = readl(nfc->regs + NFC_REG_CTL) &
484 ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN);
485
486 if (chip >= 0) {
487 sel = &sunxi_nand->sels[chip];
488
489 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN |
490 NFC_PAGE_SHIFT(nand->page_shift);
491 if (sel->rb.type == RB_NONE) {
492 nand->dev_ready = NULL;
493 } else {
494 nand->dev_ready = sunxi_nfc_dev_ready;
495 if (sel->rb.type == RB_NATIVE)
496 ctl |= NFC_RB_SEL(sel->rb.info.nativeid);
497 }
498
499 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA);
500
501 if (nfc->clk_rate != sunxi_nand->clk_rate) {
502 clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate);
503 nfc->clk_rate = sunxi_nand->clk_rate;
504 }
505 }
506
507 writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL);
508 writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG);
509 writel(ctl, nfc->regs + NFC_REG_CTL);
510
511 sunxi_nand->selected = chip;
512 }
513
514 static void sunxi_nfc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
515 {
516 struct nand_chip *nand = mtd_to_nand(mtd);
517 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
518 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
519 int ret;
520 int cnt;
521 int offs = 0;
522 u32 tmp;
523
524 while (len > offs) {
525 bool poll = false;
526
527 cnt = min(len - offs, NFC_SRAM_SIZE);
528
529 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
530 if (ret)
531 break;
532
533 writel(cnt, nfc->regs + NFC_REG_CNT);
534 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
535 writel(tmp, nfc->regs + NFC_REG_CMD);
536
537 /* Arbitrary limit for polling mode */
538 if (cnt < 64)
539 poll = true;
540
541 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
542 if (ret)
543 break;
544
545 if (buf)
546 memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE,
547 cnt);
548 offs += cnt;
549 }
550 }
551
552 static void sunxi_nfc_write_buf(struct mtd_info *mtd, const uint8_t *buf,
553 int len)
554 {
555 struct nand_chip *nand = mtd_to_nand(mtd);
556 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
557 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
558 int ret;
559 int cnt;
560 int offs = 0;
561 u32 tmp;
562
563 while (len > offs) {
564 bool poll = false;
565
566 cnt = min(len - offs, NFC_SRAM_SIZE);
567
568 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
569 if (ret)
570 break;
571
572 writel(cnt, nfc->regs + NFC_REG_CNT);
573 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt);
574 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
575 NFC_ACCESS_DIR;
576 writel(tmp, nfc->regs + NFC_REG_CMD);
577
578 /* Arbitrary limit for polling mode */
579 if (cnt < 64)
580 poll = true;
581
582 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
583 if (ret)
584 break;
585
586 offs += cnt;
587 }
588 }
589
590 static uint8_t sunxi_nfc_read_byte(struct mtd_info *mtd)
591 {
592 uint8_t ret;
593
594 sunxi_nfc_read_buf(mtd, &ret, 1);
595
596 return ret;
597 }
598
599 static void sunxi_nfc_cmd_ctrl(struct mtd_info *mtd, int dat,
600 unsigned int ctrl)
601 {
602 struct nand_chip *nand = mtd_to_nand(mtd);
603 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
604 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
605 int ret;
606
607 if (dat == NAND_CMD_NONE && (ctrl & NAND_NCE) &&
608 !(ctrl & (NAND_CLE | NAND_ALE))) {
609 u32 cmd = 0;
610
611 if (!sunxi_nand->addr_cycles && !sunxi_nand->cmd_cycles)
612 return;
613
614 if (sunxi_nand->cmd_cycles--)
615 cmd |= NFC_SEND_CMD1 | sunxi_nand->cmd[0];
616
617 if (sunxi_nand->cmd_cycles--) {
618 cmd |= NFC_SEND_CMD2;
619 writel(sunxi_nand->cmd[1],
620 nfc->regs + NFC_REG_RCMD_SET);
621 }
622
623 sunxi_nand->cmd_cycles = 0;
624
625 if (sunxi_nand->addr_cycles) {
626 cmd |= NFC_SEND_ADR |
627 NFC_ADR_NUM(sunxi_nand->addr_cycles);
628 writel(sunxi_nand->addr[0],
629 nfc->regs + NFC_REG_ADDR_LOW);
630 }
631
632 if (sunxi_nand->addr_cycles > 4)
633 writel(sunxi_nand->addr[1],
634 nfc->regs + NFC_REG_ADDR_HIGH);
635
636 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
637 if (ret)
638 return;
639
640 writel(cmd, nfc->regs + NFC_REG_CMD);
641 sunxi_nand->addr[0] = 0;
642 sunxi_nand->addr[1] = 0;
643 sunxi_nand->addr_cycles = 0;
644 sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
645 }
646
647 if (ctrl & NAND_CLE) {
648 sunxi_nand->cmd[sunxi_nand->cmd_cycles++] = dat;
649 } else if (ctrl & NAND_ALE) {
650 sunxi_nand->addr[sunxi_nand->addr_cycles / 4] |=
651 dat << ((sunxi_nand->addr_cycles % 4) * 8);
652 sunxi_nand->addr_cycles++;
653 }
654 }
655
656 /* These seed values have been extracted from Allwinner's BSP */
657 static const u16 sunxi_nfc_randomizer_page_seeds[] = {
658 0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72,
659 0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436,
660 0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d,
661 0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130,
662 0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56,
663 0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55,
664 0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb,
665 0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17,
666 0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62,
667 0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064,
668 0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126,
669 0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e,
670 0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3,
671 0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b,
672 0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d,
673 0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db,
674 };
675
676 /*
677 * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds
678 * have been generated using
679 * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what
680 * the randomizer engine does internally before de/scrambling OOB data.
681 *
682 * Those tables are statically defined to avoid calculating randomizer state
683 * at runtime.
684 */
685 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = {
686 0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64,
687 0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409,
688 0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617,
689 0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d,
690 0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91,
691 0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d,
692 0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab,
693 0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8,
694 0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8,
695 0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b,
696 0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5,
697 0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a,
698 0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891,
699 0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36,
700 0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd,
701 0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0,
702 };
703
704 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = {
705 0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6,
706 0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982,
707 0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9,
708 0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07,
709 0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e,
710 0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2,
711 0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c,
712 0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f,
713 0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc,
714 0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e,
715 0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8,
716 0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68,
717 0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d,
718 0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179,
719 0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601,
720 0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd,
721 };
722
723 static u16 sunxi_nfc_randomizer_step(u16 state, int count)
724 {
725 state &= 0x7fff;
726
727 /*
728 * This loop is just a simple implementation of a Fibonacci LFSR using
729 * the x16 + x15 + 1 polynomial.
730 */
731 while (count--)
732 state = ((state >> 1) |
733 (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff;
734
735 return state;
736 }
737
738 static u16 sunxi_nfc_randomizer_state(struct mtd_info *mtd, int page, bool ecc)
739 {
740 const u16 *seeds = sunxi_nfc_randomizer_page_seeds;
741 int mod = mtd_div_by_ws(mtd->erasesize, mtd);
742
743 if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds))
744 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds);
745
746 if (ecc) {
747 if (mtd->ecc_step_size == 512)
748 seeds = sunxi_nfc_randomizer_ecc512_seeds;
749 else
750 seeds = sunxi_nfc_randomizer_ecc1024_seeds;
751 }
752
753 return seeds[page % mod];
754 }
755
756 static void sunxi_nfc_randomizer_config(struct mtd_info *mtd,
757 int page, bool ecc)
758 {
759 struct nand_chip *nand = mtd_to_nand(mtd);
760 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
761 u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
762 u16 state;
763
764 if (!(nand->options & NAND_NEED_SCRAMBLING))
765 return;
766
767 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
768 state = sunxi_nfc_randomizer_state(mtd, page, ecc);
769 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK;
770 writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL);
771 }
772
773 static void sunxi_nfc_randomizer_enable(struct mtd_info *mtd)
774 {
775 struct nand_chip *nand = mtd_to_nand(mtd);
776 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
777
778 if (!(nand->options & NAND_NEED_SCRAMBLING))
779 return;
780
781 writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN,
782 nfc->regs + NFC_REG_ECC_CTL);
783 }
784
785 static void sunxi_nfc_randomizer_disable(struct mtd_info *mtd)
786 {
787 struct nand_chip *nand = mtd_to_nand(mtd);
788 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
789
790 if (!(nand->options & NAND_NEED_SCRAMBLING))
791 return;
792
793 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN,
794 nfc->regs + NFC_REG_ECC_CTL);
795 }
796
797 static void sunxi_nfc_randomize_bbm(struct mtd_info *mtd, int page, u8 *bbm)
798 {
799 u16 state = sunxi_nfc_randomizer_state(mtd, page, true);
800
801 bbm[0] ^= state;
802 bbm[1] ^= sunxi_nfc_randomizer_step(state, 8);
803 }
804
805 static void sunxi_nfc_randomizer_write_buf(struct mtd_info *mtd,
806 const uint8_t *buf, int len,
807 bool ecc, int page)
808 {
809 sunxi_nfc_randomizer_config(mtd, page, ecc);
810 sunxi_nfc_randomizer_enable(mtd);
811 sunxi_nfc_write_buf(mtd, buf, len);
812 sunxi_nfc_randomizer_disable(mtd);
813 }
814
815 static void sunxi_nfc_randomizer_read_buf(struct mtd_info *mtd, uint8_t *buf,
816 int len, bool ecc, int page)
817 {
818 sunxi_nfc_randomizer_config(mtd, page, ecc);
819 sunxi_nfc_randomizer_enable(mtd);
820 sunxi_nfc_read_buf(mtd, buf, len);
821 sunxi_nfc_randomizer_disable(mtd);
822 }
823
824 static void sunxi_nfc_hw_ecc_enable(struct mtd_info *mtd)
825 {
826 struct nand_chip *nand = mtd_to_nand(mtd);
827 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
828 struct sunxi_nand_hw_ecc *data = nand->ecc.priv;
829 u32 ecc_ctl;
830
831 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
832 ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE |
833 NFC_ECC_BLOCK_SIZE_MSK);
834 ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(data->mode) | NFC_ECC_EXCEPTION |
835 NFC_ECC_PIPELINE;
836
837 if (nand->ecc.size == 512)
838 ecc_ctl |= NFC_ECC_BLOCK_512;
839
840 writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL);
841 }
842
843 static void sunxi_nfc_hw_ecc_disable(struct mtd_info *mtd)
844 {
845 struct nand_chip *nand = mtd_to_nand(mtd);
846 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
847
848 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN,
849 nfc->regs + NFC_REG_ECC_CTL);
850 }
851
852 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf)
853 {
854 buf[0] = user_data;
855 buf[1] = user_data >> 8;
856 buf[2] = user_data >> 16;
857 buf[3] = user_data >> 24;
858 }
859
860 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf)
861 {
862 return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24);
863 }
864
865 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct mtd_info *mtd, u8 *oob,
866 int step, bool bbm, int page)
867 {
868 struct nand_chip *nand = mtd_to_nand(mtd);
869 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
870
871 sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)),
872 oob);
873
874 /* De-randomize the Bad Block Marker. */
875 if (bbm && (nand->options & NAND_NEED_SCRAMBLING))
876 sunxi_nfc_randomize_bbm(mtd, page, oob);
877 }
878
879 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct mtd_info *mtd,
880 const u8 *oob, int step,
881 bool bbm, int page)
882 {
883 struct nand_chip *nand = mtd_to_nand(mtd);
884 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
885 u8 user_data[4];
886
887 /* Randomize the Bad Block Marker. */
888 if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
889 memcpy(user_data, oob, sizeof(user_data));
890 sunxi_nfc_randomize_bbm(mtd, page, user_data);
891 oob = user_data;
892 }
893
894 writel(sunxi_nfc_buf_to_user_data(oob),
895 nfc->regs + NFC_REG_USER_DATA(step));
896 }
897
898 static void sunxi_nfc_hw_ecc_update_stats(struct mtd_info *mtd,
899 unsigned int *max_bitflips, int ret)
900 {
901 if (ret < 0) {
902 mtd->ecc_stats.failed++;
903 } else {
904 mtd->ecc_stats.corrected += ret;
905 *max_bitflips = max_t(unsigned int, *max_bitflips, ret);
906 }
907 }
908
909 static int sunxi_nfc_hw_ecc_correct(struct mtd_info *mtd, u8 *data, u8 *oob,
910 int step, u32 status, bool *erased)
911 {
912 struct nand_chip *nand = mtd_to_nand(mtd);
913 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
914 struct nand_ecc_ctrl *ecc = &nand->ecc;
915 u32 tmp;
916
917 *erased = false;
918
919 if (status & NFC_ECC_ERR(step))
920 return -EBADMSG;
921
922 if (status & NFC_ECC_PAT_FOUND(step)) {
923 u8 pattern;
924
925 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) {
926 pattern = 0x0;
927 } else {
928 pattern = 0xff;
929 *erased = true;
930 }
931
932 if (data)
933 memset(data, pattern, ecc->size);
934
935 if (oob)
936 memset(oob, pattern, ecc->bytes + 4);
937
938 return 0;
939 }
940
941 tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step));
942
943 return NFC_ECC_ERR_CNT(step, tmp);
944 }
945
946 static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd,
947 u8 *data, int data_off,
948 u8 *oob, int oob_off,
949 int *cur_off,
950 unsigned int *max_bitflips,
951 bool bbm, bool oob_required, int page)
952 {
953 struct nand_chip *nand = mtd_to_nand(mtd);
954 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
955 struct nand_ecc_ctrl *ecc = &nand->ecc;
956 int raw_mode = 0;
957 bool erased;
958 int ret;
959
960 if (*cur_off != data_off)
961 nand_change_read_column_op(nand, data_off, NULL, 0, false);
962
963 sunxi_nfc_randomizer_read_buf(mtd, NULL, ecc->size, false, page);
964
965 if (data_off + ecc->size != oob_off)
966 nand_change_read_column_op(nand, oob_off, NULL, 0, false);
967
968 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
969 if (ret)
970 return ret;
971
972 sunxi_nfc_randomizer_enable(mtd);
973 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP,
974 nfc->regs + NFC_REG_CMD);
975
976 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
977 sunxi_nfc_randomizer_disable(mtd);
978 if (ret)
979 return ret;
980
981 *cur_off = oob_off + ecc->bytes + 4;
982
983 ret = sunxi_nfc_hw_ecc_correct(mtd, data, oob_required ? oob : NULL, 0,
984 readl(nfc->regs + NFC_REG_ECC_ST),
985 &erased);
986 if (erased)
987 return 1;
988
989 if (ret < 0) {
990 /*
991 * Re-read the data with the randomizer disabled to identify
992 * bitflips in erased pages.
993 */
994 if (nand->options & NAND_NEED_SCRAMBLING)
995 nand_change_read_column_op(nand, data_off, data,
996 ecc->size, false);
997 else
998 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,
999 ecc->size);
1000
1001 nand_change_read_column_op(nand, oob_off, oob, ecc->bytes + 4,
1002 false);
1003
1004 ret = nand_check_erased_ecc_chunk(data, ecc->size,
1005 oob, ecc->bytes + 4,
1006 NULL, 0, ecc->strength);
1007 if (ret >= 0)
1008 raw_mode = 1;
1009 } else {
1010 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);
1011
1012 if (oob_required) {
1013 nand_change_read_column_op(nand, oob_off, NULL, 0,
1014 false);
1015 sunxi_nfc_randomizer_read_buf(mtd, oob, ecc->bytes + 4,
1016 true, page);
1017
1018 sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, 0,
1019 bbm, page);
1020 }
1021 }
1022
1023 sunxi_nfc_hw_ecc_update_stats(mtd, max_bitflips, ret);
1024
1025 return raw_mode;
1026 }
1027
1028 static void sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info *mtd,
1029 u8 *oob, int *cur_off,
1030 bool randomize, int page)
1031 {
1032 struct nand_chip *nand = mtd_to_nand(mtd);
1033 struct nand_ecc_ctrl *ecc = &nand->ecc;
1034 int offset = ((ecc->bytes + 4) * ecc->steps);
1035 int len = mtd->oobsize - offset;
1036
1037 if (len <= 0)
1038 return;
1039
1040 if (!cur_off || *cur_off != offset)
1041 nand_change_read_column_op(nand, mtd->writesize, NULL, 0,
1042 false);
1043
1044 if (!randomize)
1045 sunxi_nfc_read_buf(mtd, oob + offset, len);
1046 else
1047 sunxi_nfc_randomizer_read_buf(mtd, oob + offset, len,
1048 false, page);
1049
1050 if (cur_off)
1051 *cur_off = mtd->oobsize + mtd->writesize;
1052 }
1053
1054 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf,
1055 int oob_required, int page,
1056 int nchunks)
1057 {
1058 struct nand_chip *nand = mtd_to_nand(mtd);
1059 bool randomized = nand->options & NAND_NEED_SCRAMBLING;
1060 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1061 struct nand_ecc_ctrl *ecc = &nand->ecc;
1062 unsigned int max_bitflips = 0;
1063 int ret, i, raw_mode = 0;
1064 struct scatterlist sg;
1065 u32 status;
1066
1067 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1068 if (ret)
1069 return ret;
1070
1071 ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, nchunks,
1072 DMA_FROM_DEVICE, &sg);
1073 if (ret)
1074 return ret;
1075
1076 sunxi_nfc_hw_ecc_enable(mtd);
1077 sunxi_nfc_randomizer_config(mtd, page, false);
1078 sunxi_nfc_randomizer_enable(mtd);
1079
1080 writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |
1081 NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);
1082
1083 dma_async_issue_pending(nfc->dmac);
1084
1085 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,
1086 nfc->regs + NFC_REG_CMD);
1087
1088 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1089 if (ret)
1090 dmaengine_terminate_all(nfc->dmac);
1091
1092 sunxi_nfc_randomizer_disable(mtd);
1093 sunxi_nfc_hw_ecc_disable(mtd);
1094
1095 sunxi_nfc_dma_op_cleanup(mtd, DMA_FROM_DEVICE, &sg);
1096
1097 if (ret)
1098 return ret;
1099
1100 status = readl(nfc->regs + NFC_REG_ECC_ST);
1101
1102 for (i = 0; i < nchunks; i++) {
1103 int data_off = i * ecc->size;
1104 int oob_off = i * (ecc->bytes + 4);
1105 u8 *data = buf + data_off;
1106 u8 *oob = nand->oob_poi + oob_off;
1107 bool erased;
1108
1109 ret = sunxi_nfc_hw_ecc_correct(mtd, randomized ? data : NULL,
1110 oob_required ? oob : NULL,
1111 i, status, &erased);
1112
1113 /* ECC errors are handled in the second loop. */
1114 if (ret < 0)
1115 continue;
1116
1117 if (oob_required && !erased) {
1118 /* TODO: use DMA to retrieve OOB */
1119 nand_change_read_column_op(nand,
1120 mtd->writesize + oob_off,
1121 oob, ecc->bytes + 4, false);
1122
1123 sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, i,
1124 !i, page);
1125 }
1126
1127 if (erased)
1128 raw_mode = 1;
1129
1130 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1131 }
1132
1133 if (status & NFC_ECC_ERR_MSK) {
1134 for (i = 0; i < nchunks; i++) {
1135 int data_off = i * ecc->size;
1136 int oob_off = i * (ecc->bytes + 4);
1137 u8 *data = buf + data_off;
1138 u8 *oob = nand->oob_poi + oob_off;
1139
1140 if (!(status & NFC_ECC_ERR(i)))
1141 continue;
1142
1143 /*
1144 * Re-read the data with the randomizer disabled to
1145 * identify bitflips in erased pages.
1146 * TODO: use DMA to read page in raw mode
1147 */
1148 if (randomized)
1149 nand_change_read_column_op(nand, data_off,
1150 data, ecc->size,
1151 false);
1152
1153 /* TODO: use DMA to retrieve OOB */
1154 nand_change_read_column_op(nand,
1155 mtd->writesize + oob_off,
1156 oob, ecc->bytes + 4, false);
1157
1158 ret = nand_check_erased_ecc_chunk(data, ecc->size,
1159 oob, ecc->bytes + 4,
1160 NULL, 0,
1161 ecc->strength);
1162 if (ret >= 0)
1163 raw_mode = 1;
1164
1165 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1166 }
1167 }
1168
1169 if (oob_required)
1170 sunxi_nfc_hw_ecc_read_extra_oob(mtd, nand->oob_poi,
1171 NULL, !raw_mode,
1172 page);
1173
1174 return max_bitflips;
1175 }
1176
1177 static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd,
1178 const u8 *data, int data_off,
1179 const u8 *oob, int oob_off,
1180 int *cur_off, bool bbm,
1181 int page)
1182 {
1183 struct nand_chip *nand = mtd_to_nand(mtd);
1184 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1185 struct nand_ecc_ctrl *ecc = &nand->ecc;
1186 int ret;
1187
1188 if (data_off != *cur_off)
1189 nand_change_write_column_op(nand, data_off, NULL, 0, false);
1190
1191 sunxi_nfc_randomizer_write_buf(mtd, data, ecc->size, false, page);
1192
1193 if (data_off + ecc->size != oob_off)
1194 nand_change_write_column_op(nand, oob_off, NULL, 0, false);
1195
1196 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1197 if (ret)
1198 return ret;
1199
1200 sunxi_nfc_randomizer_enable(mtd);
1201 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, 0, bbm, page);
1202
1203 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
1204 NFC_ACCESS_DIR | NFC_ECC_OP,
1205 nfc->regs + NFC_REG_CMD);
1206
1207 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1208 sunxi_nfc_randomizer_disable(mtd);
1209 if (ret)
1210 return ret;
1211
1212 *cur_off = oob_off + ecc->bytes + 4;
1213
1214 return 0;
1215 }
1216
1217 static void sunxi_nfc_hw_ecc_write_extra_oob(struct mtd_info *mtd,
1218 u8 *oob, int *cur_off,
1219 int page)
1220 {
1221 struct nand_chip *nand = mtd_to_nand(mtd);
1222 struct nand_ecc_ctrl *ecc = &nand->ecc;
1223 int offset = ((ecc->bytes + 4) * ecc->steps);
1224 int len = mtd->oobsize - offset;
1225
1226 if (len <= 0)
1227 return;
1228
1229 if (!cur_off || *cur_off != offset)
1230 nand_change_write_column_op(nand, offset + mtd->writesize,
1231 NULL, 0, false);
1232
1233 sunxi_nfc_randomizer_write_buf(mtd, oob + offset, len, false, page);
1234
1235 if (cur_off)
1236 *cur_off = mtd->oobsize + mtd->writesize;
1237 }
1238
1239 static int sunxi_nfc_hw_ecc_read_page(struct mtd_info *mtd,
1240 struct nand_chip *chip, uint8_t *buf,
1241 int oob_required, int page)
1242 {
1243 struct nand_ecc_ctrl *ecc = &chip->ecc;
1244 unsigned int max_bitflips = 0;
1245 int ret, i, cur_off = 0;
1246 bool raw_mode = false;
1247
1248 nand_read_page_op(chip, page, 0, NULL, 0);
1249
1250 sunxi_nfc_hw_ecc_enable(mtd);
1251
1252 for (i = 0; i < ecc->steps; i++) {
1253 int data_off = i * ecc->size;
1254 int oob_off = i * (ecc->bytes + 4);
1255 u8 *data = buf + data_off;
1256 u8 *oob = chip->oob_poi + oob_off;
1257
1258 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1259 oob_off + mtd->writesize,
1260 &cur_off, &max_bitflips,
1261 !i, oob_required, page);
1262 if (ret < 0)
1263 return ret;
1264 else if (ret)
1265 raw_mode = true;
1266 }
1267
1268 if (oob_required)
1269 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1270 !raw_mode, page);
1271
1272 sunxi_nfc_hw_ecc_disable(mtd);
1273
1274 return max_bitflips;
1275 }
1276
1277 static int sunxi_nfc_hw_ecc_read_page_dma(struct mtd_info *mtd,
1278 struct nand_chip *chip, u8 *buf,
1279 int oob_required, int page)
1280 {
1281 int ret;
1282
1283 nand_read_page_op(chip, page, 0, NULL, 0);
1284
1285 ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, oob_required, page,
1286 chip->ecc.steps);
1287 if (ret >= 0)
1288 return ret;
1289
1290 /* Fallback to PIO mode */
1291 return sunxi_nfc_hw_ecc_read_page(mtd, chip, buf, oob_required, page);
1292 }
1293
1294 static int sunxi_nfc_hw_ecc_read_subpage(struct mtd_info *mtd,
1295 struct nand_chip *chip,
1296 u32 data_offs, u32 readlen,
1297 u8 *bufpoi, int page)
1298 {
1299 struct nand_ecc_ctrl *ecc = &chip->ecc;
1300 int ret, i, cur_off = 0;
1301 unsigned int max_bitflips = 0;
1302
1303 nand_read_page_op(chip, page, 0, NULL, 0);
1304
1305 sunxi_nfc_hw_ecc_enable(mtd);
1306
1307 for (i = data_offs / ecc->size;
1308 i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) {
1309 int data_off = i * ecc->size;
1310 int oob_off = i * (ecc->bytes + 4);
1311 u8 *data = bufpoi + data_off;
1312 u8 *oob = chip->oob_poi + oob_off;
1313
1314 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off,
1315 oob,
1316 oob_off + mtd->writesize,
1317 &cur_off, &max_bitflips, !i,
1318 false, page);
1319 if (ret < 0)
1320 return ret;
1321 }
1322
1323 sunxi_nfc_hw_ecc_disable(mtd);
1324
1325 return max_bitflips;
1326 }
1327
1328 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct mtd_info *mtd,
1329 struct nand_chip *chip,
1330 u32 data_offs, u32 readlen,
1331 u8 *buf, int page)
1332 {
1333 int nchunks = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size);
1334 int ret;
1335
1336 nand_read_page_op(chip, page, 0, NULL, 0);
1337
1338 ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, false, page, nchunks);
1339 if (ret >= 0)
1340 return ret;
1341
1342 /* Fallback to PIO mode */
1343 return sunxi_nfc_hw_ecc_read_subpage(mtd, chip, data_offs, readlen,
1344 buf, page);
1345 }
1346
1347 static int sunxi_nfc_hw_ecc_write_page(struct mtd_info *mtd,
1348 struct nand_chip *chip,
1349 const uint8_t *buf, int oob_required,
1350 int page)
1351 {
1352 struct nand_ecc_ctrl *ecc = &chip->ecc;
1353 int ret, i, cur_off = 0;
1354
1355 nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1356
1357 sunxi_nfc_hw_ecc_enable(mtd);
1358
1359 for (i = 0; i < ecc->steps; i++) {
1360 int data_off = i * ecc->size;
1361 int oob_off = i * (ecc->bytes + 4);
1362 const u8 *data = buf + data_off;
1363 const u8 *oob = chip->oob_poi + oob_off;
1364
1365 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1366 oob_off + mtd->writesize,
1367 &cur_off, !i, page);
1368 if (ret)
1369 return ret;
1370 }
1371
1372 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1373 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1374 &cur_off, page);
1375
1376 sunxi_nfc_hw_ecc_disable(mtd);
1377
1378 return nand_prog_page_end_op(chip);
1379 }
1380
1381 static int sunxi_nfc_hw_ecc_write_subpage(struct mtd_info *mtd,
1382 struct nand_chip *chip,
1383 u32 data_offs, u32 data_len,
1384 const u8 *buf, int oob_required,
1385 int page)
1386 {
1387 struct nand_ecc_ctrl *ecc = &chip->ecc;
1388 int ret, i, cur_off = 0;
1389
1390 nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1391
1392 sunxi_nfc_hw_ecc_enable(mtd);
1393
1394 for (i = data_offs / ecc->size;
1395 i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) {
1396 int data_off = i * ecc->size;
1397 int oob_off = i * (ecc->bytes + 4);
1398 const u8 *data = buf + data_off;
1399 const u8 *oob = chip->oob_poi + oob_off;
1400
1401 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1402 oob_off + mtd->writesize,
1403 &cur_off, !i, page);
1404 if (ret)
1405 return ret;
1406 }
1407
1408 sunxi_nfc_hw_ecc_disable(mtd);
1409
1410 return nand_prog_page_end_op(chip);
1411 }
1412
1413 static int sunxi_nfc_hw_ecc_write_page_dma(struct mtd_info *mtd,
1414 struct nand_chip *chip,
1415 const u8 *buf,
1416 int oob_required,
1417 int page)
1418 {
1419 struct nand_chip *nand = mtd_to_nand(mtd);
1420 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1421 struct nand_ecc_ctrl *ecc = &nand->ecc;
1422 struct scatterlist sg;
1423 int ret, i;
1424
1425 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1426 if (ret)
1427 return ret;
1428
1429 ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, ecc->steps,
1430 DMA_TO_DEVICE, &sg);
1431 if (ret)
1432 goto pio_fallback;
1433
1434 for (i = 0; i < ecc->steps; i++) {
1435 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));
1436
1437 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, i, !i, page);
1438 }
1439
1440 nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1441
1442 sunxi_nfc_hw_ecc_enable(mtd);
1443 sunxi_nfc_randomizer_config(mtd, page, false);
1444 sunxi_nfc_randomizer_enable(mtd);
1445
1446 writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
1447 nfc->regs + NFC_REG_RCMD_SET);
1448
1449 dma_async_issue_pending(nfc->dmac);
1450
1451 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
1452 NFC_DATA_TRANS | NFC_ACCESS_DIR,
1453 nfc->regs + NFC_REG_CMD);
1454
1455 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1456 if (ret)
1457 dmaengine_terminate_all(nfc->dmac);
1458
1459 sunxi_nfc_randomizer_disable(mtd);
1460 sunxi_nfc_hw_ecc_disable(mtd);
1461
1462 sunxi_nfc_dma_op_cleanup(mtd, DMA_TO_DEVICE, &sg);
1463
1464 if (ret)
1465 return ret;
1466
1467 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1468 /* TODO: use DMA to transfer extra OOB bytes ? */
1469 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1470 NULL, page);
1471
1472 return nand_prog_page_end_op(chip);
1473
1474 pio_fallback:
1475 return sunxi_nfc_hw_ecc_write_page(mtd, chip, buf, oob_required, page);
1476 }
1477
1478 static int sunxi_nfc_hw_syndrome_ecc_read_page(struct mtd_info *mtd,
1479 struct nand_chip *chip,
1480 uint8_t *buf, int oob_required,
1481 int page)
1482 {
1483 struct nand_ecc_ctrl *ecc = &chip->ecc;
1484 unsigned int max_bitflips = 0;
1485 int ret, i, cur_off = 0;
1486 bool raw_mode = false;
1487
1488 nand_read_page_op(chip, page, 0, NULL, 0);
1489
1490 sunxi_nfc_hw_ecc_enable(mtd);
1491
1492 for (i = 0; i < ecc->steps; i++) {
1493 int data_off = i * (ecc->size + ecc->bytes + 4);
1494 int oob_off = data_off + ecc->size;
1495 u8 *data = buf + (i * ecc->size);
1496 u8 *oob = chip->oob_poi + (i * (ecc->bytes + 4));
1497
1498 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1499 oob_off, &cur_off,
1500 &max_bitflips, !i,
1501 oob_required,
1502 page);
1503 if (ret < 0)
1504 return ret;
1505 else if (ret)
1506 raw_mode = true;
1507 }
1508
1509 if (oob_required)
1510 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1511 !raw_mode, page);
1512
1513 sunxi_nfc_hw_ecc_disable(mtd);
1514
1515 return max_bitflips;
1516 }
1517
1518 static int sunxi_nfc_hw_syndrome_ecc_write_page(struct mtd_info *mtd,
1519 struct nand_chip *chip,
1520 const uint8_t *buf,
1521 int oob_required, int page)
1522 {
1523 struct nand_ecc_ctrl *ecc = &chip->ecc;
1524 int ret, i, cur_off = 0;
1525
1526 nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1527
1528 sunxi_nfc_hw_ecc_enable(mtd);
1529
1530 for (i = 0; i < ecc->steps; i++) {
1531 int data_off = i * (ecc->size + ecc->bytes + 4);
1532 int oob_off = data_off + ecc->size;
1533 const u8 *data = buf + (i * ecc->size);
1534 const u8 *oob = chip->oob_poi + (i * (ecc->bytes + 4));
1535
1536 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off,
1537 oob, oob_off, &cur_off,
1538 false, page);
1539 if (ret)
1540 return ret;
1541 }
1542
1543 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1544 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1545 &cur_off, page);
1546
1547 sunxi_nfc_hw_ecc_disable(mtd);
1548
1549 return nand_prog_page_end_op(chip);
1550 }
1551
1552 static int sunxi_nfc_hw_common_ecc_read_oob(struct mtd_info *mtd,
1553 struct nand_chip *chip,
1554 int page)
1555 {
1556 chip->pagebuf = -1;
1557
1558 return chip->ecc.read_page(mtd, chip, chip->data_buf, 1, page);
1559 }
1560
1561 static int sunxi_nfc_hw_common_ecc_write_oob(struct mtd_info *mtd,
1562 struct nand_chip *chip,
1563 int page)
1564 {
1565 int ret;
1566
1567 chip->pagebuf = -1;
1568
1569 memset(chip->data_buf, 0xff, mtd->writesize);
1570 ret = chip->ecc.write_page(mtd, chip, chip->data_buf, 1, page);
1571 if (ret)
1572 return ret;
1573
1574 /* Send command to program the OOB data */
1575 return nand_prog_page_end_op(chip);
1576 }
1577
1578 static const s32 tWB_lut[] = {6, 12, 16, 20};
1579 static const s32 tRHW_lut[] = {4, 8, 12, 20};
1580
1581 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
1582 u32 clk_period)
1583 {
1584 u32 clk_cycles = DIV_ROUND_UP(duration, clk_period);
1585 int i;
1586
1587 for (i = 0; i < lut_size; i++) {
1588 if (clk_cycles <= lut[i])
1589 return i;
1590 }
1591
1592 /* Doesn't fit */
1593 return -EINVAL;
1594 }
1595
1596 #define sunxi_nand_lookup_timing(l, p, c) \
1597 _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)
1598
1599 static int sunxi_nfc_setup_data_interface(struct mtd_info *mtd, int csline,
1600 const struct nand_data_interface *conf)
1601 {
1602 struct nand_chip *nand = mtd_to_nand(mtd);
1603 struct sunxi_nand_chip *chip = to_sunxi_nand(nand);
1604 struct sunxi_nfc *nfc = to_sunxi_nfc(chip->nand.controller);
1605 const struct nand_sdr_timings *timings;
1606 u32 min_clk_period = 0;
1607 s32 tWB, tADL, tWHR, tRHW, tCAD;
1608 long real_clk_rate;
1609
1610 timings = nand_get_sdr_timings(conf);
1611 if (IS_ERR(timings))
1612 return -ENOTSUPP;
1613
1614 /* T1 <=> tCLS */
1615 if (timings->tCLS_min > min_clk_period)
1616 min_clk_period = timings->tCLS_min;
1617
1618 /* T2 <=> tCLH */
1619 if (timings->tCLH_min > min_clk_period)
1620 min_clk_period = timings->tCLH_min;
1621
1622 /* T3 <=> tCS */
1623 if (timings->tCS_min > min_clk_period)
1624 min_clk_period = timings->tCS_min;
1625
1626 /* T4 <=> tCH */
1627 if (timings->tCH_min > min_clk_period)
1628 min_clk_period = timings->tCH_min;
1629
1630 /* T5 <=> tWP */
1631 if (timings->tWP_min > min_clk_period)
1632 min_clk_period = timings->tWP_min;
1633
1634 /* T6 <=> tWH */
1635 if (timings->tWH_min > min_clk_period)
1636 min_clk_period = timings->tWH_min;
1637
1638 /* T7 <=> tALS */
1639 if (timings->tALS_min > min_clk_period)
1640 min_clk_period = timings->tALS_min;
1641
1642 /* T8 <=> tDS */
1643 if (timings->tDS_min > min_clk_period)
1644 min_clk_period = timings->tDS_min;
1645
1646 /* T9 <=> tDH */
1647 if (timings->tDH_min > min_clk_period)
1648 min_clk_period = timings->tDH_min;
1649
1650 /* T10 <=> tRR */
1651 if (timings->tRR_min > (min_clk_period * 3))
1652 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);
1653
1654 /* T11 <=> tALH */
1655 if (timings->tALH_min > min_clk_period)
1656 min_clk_period = timings->tALH_min;
1657
1658 /* T12 <=> tRP */
1659 if (timings->tRP_min > min_clk_period)
1660 min_clk_period = timings->tRP_min;
1661
1662 /* T13 <=> tREH */
1663 if (timings->tREH_min > min_clk_period)
1664 min_clk_period = timings->tREH_min;
1665
1666 /* T14 <=> tRC */
1667 if (timings->tRC_min > (min_clk_period * 2))
1668 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2);
1669
1670 /* T15 <=> tWC */
1671 if (timings->tWC_min > (min_clk_period * 2))
1672 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);
1673
1674 /* T16 - T19 + tCAD */
1675 if (timings->tWB_max > (min_clk_period * 20))
1676 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);
1677
1678 if (timings->tADL_min > (min_clk_period * 32))
1679 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
1680
1681 if (timings->tWHR_min > (min_clk_period * 32))
1682 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
1683
1684 if (timings->tRHW_min > (min_clk_period * 20))
1685 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);
1686
1687 tWB = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,
1688 min_clk_period);
1689 if (tWB < 0) {
1690 dev_err(nfc->dev, "unsupported tWB\n");
1691 return tWB;
1692 }
1693
1694 tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;
1695 if (tADL > 3) {
1696 dev_err(nfc->dev, "unsupported tADL\n");
1697 return -EINVAL;
1698 }
1699
1700 tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;
1701 if (tWHR > 3) {
1702 dev_err(nfc->dev, "unsupported tWHR\n");
1703 return -EINVAL;
1704 }
1705
1706 tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,
1707 min_clk_period);
1708 if (tRHW < 0) {
1709 dev_err(nfc->dev, "unsupported tRHW\n");
1710 return tRHW;
1711 }
1712
1713 if (csline == NAND_DATA_IFACE_CHECK_ONLY)
1714 return 0;
1715
1716 /*
1717 * TODO: according to ONFI specs this value only applies for DDR NAND,
1718 * but Allwinner seems to set this to 0x7. Mimic them for now.
1719 */
1720 tCAD = 0x7;
1721
1722 /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */
1723 chip->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD);
1724
1725 /* Convert min_clk_period from picoseconds to nanoseconds */
1726 min_clk_period = DIV_ROUND_UP(min_clk_period, 1000);
1727
1728 /*
1729 * Unlike what is stated in Allwinner datasheet, the clk_rate should
1730 * be set to (1 / min_clk_period), and not (2 / min_clk_period).
1731 * This new formula was verified with a scope and validated by
1732 * Allwinner engineers.
1733 */
1734 chip->clk_rate = NSEC_PER_SEC / min_clk_period;
1735 real_clk_rate = clk_round_rate(nfc->mod_clk, chip->clk_rate);
1736 if (real_clk_rate <= 0) {
1737 dev_err(nfc->dev, "Unable to round clk %lu\n", chip->clk_rate);
1738 return -EINVAL;
1739 }
1740
1741 /*
1742 * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data
1743 * output cycle timings shall be used if the host drives tRC less than
1744 * 30 ns.
1745 */
1746 min_clk_period = NSEC_PER_SEC / real_clk_rate;
1747 chip->timing_ctl = ((min_clk_period * 2) < 30) ?
1748 NFC_TIMING_CTL_EDO : 0;
1749
1750 return 0;
1751 }
1752
1753 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1754 struct mtd_oob_region *oobregion)
1755 {
1756 struct nand_chip *nand = mtd_to_nand(mtd);
1757 struct nand_ecc_ctrl *ecc = &nand->ecc;
1758
1759 if (section >= ecc->steps)
1760 return -ERANGE;
1761
1762 oobregion->offset = section * (ecc->bytes + 4) + 4;
1763 oobregion->length = ecc->bytes;
1764
1765 return 0;
1766 }
1767
1768 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
1769 struct mtd_oob_region *oobregion)
1770 {
1771 struct nand_chip *nand = mtd_to_nand(mtd);
1772 struct nand_ecc_ctrl *ecc = &nand->ecc;
1773
1774 if (section > ecc->steps)
1775 return -ERANGE;
1776
1777 /*
1778 * The first 2 bytes are used for BB markers, hence we
1779 * only have 2 bytes available in the first user data
1780 * section.
1781 */
1782 if (!section && ecc->mode == NAND_ECC_HW) {
1783 oobregion->offset = 2;
1784 oobregion->length = 2;
1785
1786 return 0;
1787 }
1788
1789 oobregion->offset = section * (ecc->bytes + 4);
1790
1791 if (section < ecc->steps)
1792 oobregion->length = 4;
1793 else
1794 oobregion->offset = mtd->oobsize - oobregion->offset;
1795
1796 return 0;
1797 }
1798
1799 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = {
1800 .ecc = sunxi_nand_ooblayout_ecc,
1801 .free = sunxi_nand_ooblayout_free,
1802 };
1803
1804 static int sunxi_nand_hw_common_ecc_ctrl_init(struct mtd_info *mtd,
1805 struct nand_ecc_ctrl *ecc,
1806 struct device_node *np)
1807 {
1808 static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 };
1809 struct nand_chip *nand = mtd_to_nand(mtd);
1810 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1811 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1812 struct sunxi_nand_hw_ecc *data;
1813 int nsectors;
1814 int ret;
1815 int i;
1816
1817 if (ecc->options & NAND_ECC_MAXIMIZE) {
1818 int bytes;
1819
1820 ecc->size = 1024;
1821 nsectors = mtd->writesize / ecc->size;
1822
1823 /* Reserve 2 bytes for the BBM */
1824 bytes = (mtd->oobsize - 2) / nsectors;
1825
1826 /* 4 non-ECC bytes are added before each ECC bytes section */
1827 bytes -= 4;
1828
1829 /* and bytes has to be even. */
1830 if (bytes % 2)
1831 bytes--;
1832
1833 ecc->strength = bytes * 8 / fls(8 * ecc->size);
1834
1835 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1836 if (strengths[i] > ecc->strength)
1837 break;
1838 }
1839
1840 if (!i)
1841 ecc->strength = 0;
1842 else
1843 ecc->strength = strengths[i - 1];
1844 }
1845
1846 if (ecc->size != 512 && ecc->size != 1024)
1847 return -EINVAL;
1848
1849 data = kzalloc(sizeof(*data), GFP_KERNEL);
1850 if (!data)
1851 return -ENOMEM;
1852
1853 /* Prefer 1k ECC chunk over 512 ones */
1854 if (ecc->size == 512 && mtd->writesize > 512) {
1855 ecc->size = 1024;
1856 ecc->strength *= 2;
1857 }
1858
1859 /* Add ECC info retrieval from DT */
1860 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1861 if (ecc->strength <= strengths[i]) {
1862 /*
1863 * Update ecc->strength value with the actual strength
1864 * that will be used by the ECC engine.
1865 */
1866 ecc->strength = strengths[i];
1867 break;
1868 }
1869 }
1870
1871 if (i >= ARRAY_SIZE(strengths)) {
1872 dev_err(nfc->dev, "unsupported strength\n");
1873 ret = -ENOTSUPP;
1874 goto err;
1875 }
1876
1877 data->mode = i;
1878
1879 /* HW ECC always request ECC bytes for 1024 bytes blocks */
1880 ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8);
1881
1882 /* HW ECC always work with even numbers of ECC bytes */
1883 ecc->bytes = ALIGN(ecc->bytes, 2);
1884
1885 nsectors = mtd->writesize / ecc->size;
1886
1887 if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) {
1888 ret = -EINVAL;
1889 goto err;
1890 }
1891
1892 ecc->read_oob = sunxi_nfc_hw_common_ecc_read_oob;
1893 ecc->write_oob = sunxi_nfc_hw_common_ecc_write_oob;
1894 mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops);
1895 ecc->priv = data;
1896
1897 return 0;
1898
1899 err:
1900 kfree(data);
1901
1902 return ret;
1903 }
1904
1905 static void sunxi_nand_hw_common_ecc_ctrl_cleanup(struct nand_ecc_ctrl *ecc)
1906 {
1907 kfree(ecc->priv);
1908 }
1909
1910 static int sunxi_nand_hw_ecc_ctrl_init(struct mtd_info *mtd,
1911 struct nand_ecc_ctrl *ecc,
1912 struct device_node *np)
1913 {
1914 struct nand_chip *nand = mtd_to_nand(mtd);
1915 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1916 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1917 int ret;
1918
1919 ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
1920 if (ret)
1921 return ret;
1922
1923 if (nfc->dmac) {
1924 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
1925 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
1926 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
1927 nand->options |= NAND_USE_BOUNCE_BUFFER;
1928 } else {
1929 ecc->read_page = sunxi_nfc_hw_ecc_read_page;
1930 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1931 ecc->write_page = sunxi_nfc_hw_ecc_write_page;
1932 }
1933
1934 /* TODO: support DMA for raw accesses and subpage write */
1935 ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage;
1936 ecc->read_oob_raw = nand_read_oob_std;
1937 ecc->write_oob_raw = nand_write_oob_std;
1938
1939 return 0;
1940 }
1941
1942 static int sunxi_nand_hw_syndrome_ecc_ctrl_init(struct mtd_info *mtd,
1943 struct nand_ecc_ctrl *ecc,
1944 struct device_node *np)
1945 {
1946 int ret;
1947
1948 ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
1949 if (ret)
1950 return ret;
1951
1952 ecc->prepad = 4;
1953 ecc->read_page = sunxi_nfc_hw_syndrome_ecc_read_page;
1954 ecc->write_page = sunxi_nfc_hw_syndrome_ecc_write_page;
1955 ecc->read_oob_raw = nand_read_oob_syndrome;
1956 ecc->write_oob_raw = nand_write_oob_syndrome;
1957
1958 return 0;
1959 }
1960
1961 static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc)
1962 {
1963 switch (ecc->mode) {
1964 case NAND_ECC_HW:
1965 case NAND_ECC_HW_SYNDROME:
1966 sunxi_nand_hw_common_ecc_ctrl_cleanup(ecc);
1967 break;
1968 case NAND_ECC_NONE:
1969 default:
1970 break;
1971 }
1972 }
1973
1974 static int sunxi_nand_ecc_init(struct mtd_info *mtd, struct nand_ecc_ctrl *ecc,
1975 struct device_node *np)
1976 {
1977 struct nand_chip *nand = mtd_to_nand(mtd);
1978 int ret;
1979
1980 if (!ecc->size) {
1981 ecc->size = nand->ecc_step_ds;
1982 ecc->strength = nand->ecc_strength_ds;
1983 }
1984
1985 if (!ecc->size || !ecc->strength)
1986 return -EINVAL;
1987
1988 switch (ecc->mode) {
1989 case NAND_ECC_HW:
1990 ret = sunxi_nand_hw_ecc_ctrl_init(mtd, ecc, np);
1991 if (ret)
1992 return ret;
1993 break;
1994 case NAND_ECC_HW_SYNDROME:
1995 ret = sunxi_nand_hw_syndrome_ecc_ctrl_init(mtd, ecc, np);
1996 if (ret)
1997 return ret;
1998 break;
1999 case NAND_ECC_NONE:
2000 case NAND_ECC_SOFT:
2001 break;
2002 default:
2003 return -EINVAL;
2004 }
2005
2006 return 0;
2007 }
2008
2009 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
2010 struct device_node *np)
2011 {
2012 struct sunxi_nand_chip *chip;
2013 struct mtd_info *mtd;
2014 struct nand_chip *nand;
2015 int nsels;
2016 int ret;
2017 int i;
2018 u32 tmp;
2019
2020 if (!of_get_property(np, "reg", &nsels))
2021 return -EINVAL;
2022
2023 nsels /= sizeof(u32);
2024 if (!nsels) {
2025 dev_err(dev, "invalid reg property size\n");
2026 return -EINVAL;
2027 }
2028
2029 chip = devm_kzalloc(dev,
2030 sizeof(*chip) +
2031 (nsels * sizeof(struct sunxi_nand_chip_sel)),
2032 GFP_KERNEL);
2033 if (!chip) {
2034 dev_err(dev, "could not allocate chip\n");
2035 return -ENOMEM;
2036 }
2037
2038 chip->nsels = nsels;
2039 chip->selected = -1;
2040
2041 for (i = 0; i < nsels; i++) {
2042 ret = of_property_read_u32_index(np, "reg", i, &tmp);
2043 if (ret) {
2044 dev_err(dev, "could not retrieve reg property: %d\n",
2045 ret);
2046 return ret;
2047 }
2048
2049 if (tmp > NFC_MAX_CS) {
2050 dev_err(dev,
2051 "invalid reg value: %u (max CS = 7)\n",
2052 tmp);
2053 return -EINVAL;
2054 }
2055
2056 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
2057 dev_err(dev, "CS %d already assigned\n", tmp);
2058 return -EINVAL;
2059 }
2060
2061 chip->sels[i].cs = tmp;
2062
2063 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) &&
2064 tmp < 2) {
2065 chip->sels[i].rb.type = RB_NATIVE;
2066 chip->sels[i].rb.info.nativeid = tmp;
2067 } else {
2068 ret = of_get_named_gpio(np, "rb-gpios", i);
2069 if (ret >= 0) {
2070 tmp = ret;
2071 chip->sels[i].rb.type = RB_GPIO;
2072 chip->sels[i].rb.info.gpio = tmp;
2073 ret = devm_gpio_request(dev, tmp, "nand-rb");
2074 if (ret)
2075 return ret;
2076
2077 ret = gpio_direction_input(tmp);
2078 if (ret)
2079 return ret;
2080 } else {
2081 chip->sels[i].rb.type = RB_NONE;
2082 }
2083 }
2084 }
2085
2086 nand = &chip->nand;
2087 /* Default tR value specified in the ONFI spec (chapter 4.15.1) */
2088 nand->chip_delay = 200;
2089 nand->controller = &nfc->controller;
2090 /*
2091 * Set the ECC mode to the default value in case nothing is specified
2092 * in the DT.
2093 */
2094 nand->ecc.mode = NAND_ECC_HW;
2095 nand_set_flash_node(nand, np);
2096 nand->select_chip = sunxi_nfc_select_chip;
2097 nand->cmd_ctrl = sunxi_nfc_cmd_ctrl;
2098 nand->read_buf = sunxi_nfc_read_buf;
2099 nand->write_buf = sunxi_nfc_write_buf;
2100 nand->read_byte = sunxi_nfc_read_byte;
2101 nand->setup_data_interface = sunxi_nfc_setup_data_interface;
2102
2103 mtd = nand_to_mtd(nand);
2104 mtd->dev.parent = dev;
2105
2106 ret = nand_scan_ident(mtd, nsels, NULL);
2107 if (ret)
2108 return ret;
2109
2110 if (nand->bbt_options & NAND_BBT_USE_FLASH)
2111 nand->bbt_options |= NAND_BBT_NO_OOB;
2112
2113 if (nand->options & NAND_NEED_SCRAMBLING)
2114 nand->options |= NAND_NO_SUBPAGE_WRITE;
2115
2116 nand->options |= NAND_SUBPAGE_READ;
2117
2118 ret = sunxi_nand_ecc_init(mtd, &nand->ecc, np);
2119 if (ret) {
2120 dev_err(dev, "ECC init failed: %d\n", ret);
2121 return ret;
2122 }
2123
2124 ret = nand_scan_tail(mtd);
2125 if (ret) {
2126 dev_err(dev, "nand_scan_tail failed: %d\n", ret);
2127 return ret;
2128 }
2129
2130 ret = mtd_device_register(mtd, NULL, 0);
2131 if (ret) {
2132 dev_err(dev, "failed to register mtd device: %d\n", ret);
2133 nand_release(mtd);
2134 return ret;
2135 }
2136
2137 list_add_tail(&chip->node, &nfc->chips);
2138
2139 return 0;
2140 }
2141
2142 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)
2143 {
2144 struct device_node *np = dev->of_node;
2145 struct device_node *nand_np;
2146 int nchips = of_get_child_count(np);
2147 int ret;
2148
2149 if (nchips > 8) {
2150 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips);
2151 return -EINVAL;
2152 }
2153
2154 for_each_child_of_node(np, nand_np) {
2155 ret = sunxi_nand_chip_init(dev, nfc, nand_np);
2156 if (ret) {
2157 of_node_put(nand_np);
2158 return ret;
2159 }
2160 }
2161
2162 return 0;
2163 }
2164
2165 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)
2166 {
2167 struct sunxi_nand_chip *chip;
2168
2169 while (!list_empty(&nfc->chips)) {
2170 chip = list_first_entry(&nfc->chips, struct sunxi_nand_chip,
2171 node);
2172 nand_release(nand_to_mtd(&chip->nand));
2173 sunxi_nand_ecc_cleanup(&chip->nand.ecc);
2174 list_del(&chip->node);
2175 }
2176 }
2177
2178 static int sunxi_nfc_probe(struct platform_device *pdev)
2179 {
2180 struct device *dev = &pdev->dev;
2181 struct resource *r;
2182 struct sunxi_nfc *nfc;
2183 int irq;
2184 int ret;
2185
2186 nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
2187 if (!nfc)
2188 return -ENOMEM;
2189
2190 nfc->dev = dev;
2191 nand_hw_control_init(&nfc->controller);
2192 INIT_LIST_HEAD(&nfc->chips);
2193
2194 r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2195 nfc->regs = devm_ioremap_resource(dev, r);
2196 if (IS_ERR(nfc->regs))
2197 return PTR_ERR(nfc->regs);
2198
2199 irq = platform_get_irq(pdev, 0);
2200 if (irq < 0) {
2201 dev_err(dev, "failed to retrieve irq\n");
2202 return irq;
2203 }
2204
2205 nfc->ahb_clk = devm_clk_get(dev, "ahb");
2206 if (IS_ERR(nfc->ahb_clk)) {
2207 dev_err(dev, "failed to retrieve ahb clk\n");
2208 return PTR_ERR(nfc->ahb_clk);
2209 }
2210
2211 ret = clk_prepare_enable(nfc->ahb_clk);
2212 if (ret)
2213 return ret;
2214
2215 nfc->mod_clk = devm_clk_get(dev, "mod");
2216 if (IS_ERR(nfc->mod_clk)) {
2217 dev_err(dev, "failed to retrieve mod clk\n");
2218 ret = PTR_ERR(nfc->mod_clk);
2219 goto out_ahb_clk_unprepare;
2220 }
2221
2222 ret = clk_prepare_enable(nfc->mod_clk);
2223 if (ret)
2224 goto out_ahb_clk_unprepare;
2225
2226 nfc->reset = devm_reset_control_get_optional_exclusive(dev, "ahb");
2227 if (IS_ERR(nfc->reset)) {
2228 ret = PTR_ERR(nfc->reset);
2229 goto out_mod_clk_unprepare;
2230 }
2231
2232 ret = reset_control_deassert(nfc->reset);
2233 if (ret) {
2234 dev_err(dev, "reset err %d\n", ret);
2235 goto out_mod_clk_unprepare;
2236 }
2237
2238 ret = sunxi_nfc_rst(nfc);
2239 if (ret)
2240 goto out_ahb_reset_reassert;
2241
2242 writel(0, nfc->regs + NFC_REG_INT);
2243 ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt,
2244 0, "sunxi-nand", nfc);
2245 if (ret)
2246 goto out_ahb_reset_reassert;
2247
2248 nfc->dmac = dma_request_slave_channel(dev, "rxtx");
2249 if (nfc->dmac) {
2250 struct dma_slave_config dmac_cfg = { };
2251
2252 dmac_cfg.src_addr = r->start + NFC_REG_IO_DATA;
2253 dmac_cfg.dst_addr = dmac_cfg.src_addr;
2254 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2255 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;
2256 dmac_cfg.src_maxburst = 4;
2257 dmac_cfg.dst_maxburst = 4;
2258 dmaengine_slave_config(nfc->dmac, &dmac_cfg);
2259 } else {
2260 dev_warn(dev, "failed to request rxtx DMA channel\n");
2261 }
2262
2263 platform_set_drvdata(pdev, nfc);
2264
2265 ret = sunxi_nand_chips_init(dev, nfc);
2266 if (ret) {
2267 dev_err(dev, "failed to init nand chips\n");
2268 goto out_release_dmac;
2269 }
2270
2271 return 0;
2272
2273 out_release_dmac:
2274 if (nfc->dmac)
2275 dma_release_channel(nfc->dmac);
2276 out_ahb_reset_reassert:
2277 reset_control_assert(nfc->reset);
2278 out_mod_clk_unprepare:
2279 clk_disable_unprepare(nfc->mod_clk);
2280 out_ahb_clk_unprepare:
2281 clk_disable_unprepare(nfc->ahb_clk);
2282
2283 return ret;
2284 }
2285
2286 static int sunxi_nfc_remove(struct platform_device *pdev)
2287 {
2288 struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
2289
2290 sunxi_nand_chips_cleanup(nfc);
2291
2292 reset_control_assert(nfc->reset);
2293
2294 if (nfc->dmac)
2295 dma_release_channel(nfc->dmac);
2296 clk_disable_unprepare(nfc->mod_clk);
2297 clk_disable_unprepare(nfc->ahb_clk);
2298
2299 return 0;
2300 }
2301
2302 static const struct of_device_id sunxi_nfc_ids[] = {
2303 { .compatible = "allwinner,sun4i-a10-nand" },
2304 { /* sentinel */ }
2305 };
2306 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids);
2307
2308 static struct platform_driver sunxi_nfc_driver = {
2309 .driver = {
2310 .name = "sunxi_nand",
2311 .of_match_table = sunxi_nfc_ids,
2312 },
2313 .probe = sunxi_nfc_probe,
2314 .remove = sunxi_nfc_remove,
2315 };
2316 module_platform_driver(sunxi_nfc_driver);
2317
2318 MODULE_LICENSE("GPL v2");
2319 MODULE_AUTHOR("Boris BREZILLON");
2320 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver");
2321 MODULE_ALIAS("platform:sunxi_nand");
CRIS linux kernel tree