?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
x86/mm/pat: Don't report PAT on CPUs that don't support it
[linux/fpc-iii.git] / drivers / mtd / nand / sunxi_nand.c
blob0eeeb8b889ea8af83a12126ded319e84f0389618
1 /*
2 * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com>
3 *
4 * Derived from:
5 * https://github.com/yuq/sunxi-nfc-mtd
6 * Copyright (C) 2013 Qiang Yu <yuq825@gmail.com>
7 *
8 * https://github.com/hno/Allwinner-Info
9 * Copyright (C) 2013 Henrik Nordström <Henrik Nordström>
10 *
11 * Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com>
12 * Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org>
13 *
14 * This program is free software; you can redistribute it and/or modify
15 * it under the terms of the GNU General Public License as published by
16 * the Free Software Foundation; either version 2 of the License, or
17 * (at your option) any later version.
18 *
19 * This program is distributed in the hope that it will be useful,
20 * but WITHOUT ANY WARRANTY; without even the implied warranty of
21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 * GNU General Public License for more details.
23 */
24
25 #include <linux/dma-mapping.h>
26 #include <linux/slab.h>
27 #include <linux/module.h>
28 #include <linux/moduleparam.h>
29 #include <linux/platform_device.h>
30 #include <linux/of.h>
31 #include <linux/of_device.h>
32 #include <linux/of_gpio.h>
33 #include <linux/mtd/mtd.h>
34 #include <linux/mtd/nand.h>
35 #include <linux/mtd/partitions.h>
36 #include <linux/clk.h>
37 #include <linux/delay.h>
38 #include <linux/dmaengine.h>
39 #include <linux/gpio.h>
40 #include <linux/interrupt.h>
41 #include <linux/iopoll.h>
42 #include <linux/reset.h>
43
44 #define NFC_REG_CTL 0x0000
45 #define NFC_REG_ST 0x0004
46 #define NFC_REG_INT 0x0008
47 #define NFC_REG_TIMING_CTL 0x000C
48 #define NFC_REG_TIMING_CFG 0x0010
49 #define NFC_REG_ADDR_LOW 0x0014
50 #define NFC_REG_ADDR_HIGH 0x0018
51 #define NFC_REG_SECTOR_NUM 0x001C
52 #define NFC_REG_CNT 0x0020
53 #define NFC_REG_CMD 0x0024
54 #define NFC_REG_RCMD_SET 0x0028
55 #define NFC_REG_WCMD_SET 0x002C
56 #define NFC_REG_IO_DATA 0x0030
57 #define NFC_REG_ECC_CTL 0x0034
58 #define NFC_REG_ECC_ST 0x0038
59 #define NFC_REG_DEBUG 0x003C
60 #define NFC_REG_ECC_ERR_CNT(x) ((0x0040 + (x)) & ~0x3)
61 #define NFC_REG_USER_DATA(x) (0x0050 + ((x) * 4))
62 #define NFC_REG_SPARE_AREA 0x00A0
63 #define NFC_REG_PAT_ID 0x00A4
64 #define NFC_RAM0_BASE 0x0400
65 #define NFC_RAM1_BASE 0x0800
66
67 /* define bit use in NFC_CTL */
68 #define NFC_EN BIT(0)
69 #define NFC_RESET BIT(1)
70 #define NFC_BUS_WIDTH_MSK BIT(2)
71 #define NFC_BUS_WIDTH_8 (0 << 2)
72 #define NFC_BUS_WIDTH_16 (1 << 2)
73 #define NFC_RB_SEL_MSK BIT(3)
74 #define NFC_RB_SEL(x) ((x) << 3)
75 #define NFC_CE_SEL_MSK GENMASK(26, 24)
76 #define NFC_CE_SEL(x) ((x) << 24)
77 #define NFC_CE_CTL BIT(6)
78 #define NFC_PAGE_SHIFT_MSK GENMASK(11, 8)
79 #define NFC_PAGE_SHIFT(x) (((x) < 10 ? 0 : (x) - 10) << 8)
80 #define NFC_SAM BIT(12)
81 #define NFC_RAM_METHOD BIT(14)
82 #define NFC_DEBUG_CTL BIT(31)
83
84 /* define bit use in NFC_ST */
85 #define NFC_RB_B2R BIT(0)
86 #define NFC_CMD_INT_FLAG BIT(1)
87 #define NFC_DMA_INT_FLAG BIT(2)
88 #define NFC_CMD_FIFO_STATUS BIT(3)
89 #define NFC_STA BIT(4)
90 #define NFC_NATCH_INT_FLAG BIT(5)
91 #define NFC_RB_STATE(x) BIT(x + 8)
92
93 /* define bit use in NFC_INT */
94 #define NFC_B2R_INT_ENABLE BIT(0)
95 #define NFC_CMD_INT_ENABLE BIT(1)
96 #define NFC_DMA_INT_ENABLE BIT(2)
97 #define NFC_INT_MASK (NFC_B2R_INT_ENABLE | \
98 NFC_CMD_INT_ENABLE | \
99 NFC_DMA_INT_ENABLE)
100
101 /* define bit use in NFC_TIMING_CTL */
102 #define NFC_TIMING_CTL_EDO BIT(8)
103
104 /* define NFC_TIMING_CFG register layout */
105 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD) \
106 (((tWB) & 0x3) | (((tADL) & 0x3) << 2) | \
107 (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) | \
108 (((tCAD) & 0x7) << 8))
109
110 /* define bit use in NFC_CMD */
111 #define NFC_CMD_LOW_BYTE_MSK GENMASK(7, 0)
112 #define NFC_CMD_HIGH_BYTE_MSK GENMASK(15, 8)
113 #define NFC_CMD(x) (x)
114 #define NFC_ADR_NUM_MSK GENMASK(18, 16)
115 #define NFC_ADR_NUM(x) (((x) - 1) << 16)
116 #define NFC_SEND_ADR BIT(19)
117 #define NFC_ACCESS_DIR BIT(20)
118 #define NFC_DATA_TRANS BIT(21)
119 #define NFC_SEND_CMD1 BIT(22)
120 #define NFC_WAIT_FLAG BIT(23)
121 #define NFC_SEND_CMD2 BIT(24)
122 #define NFC_SEQ BIT(25)
123 #define NFC_DATA_SWAP_METHOD BIT(26)
124 #define NFC_ROW_AUTO_INC BIT(27)
125 #define NFC_SEND_CMD3 BIT(28)
126 #define NFC_SEND_CMD4 BIT(29)
127 #define NFC_CMD_TYPE_MSK GENMASK(31, 30)
128 #define NFC_NORMAL_OP (0 << 30)
129 #define NFC_ECC_OP (1 << 30)
130 #define NFC_PAGE_OP (2 << 30)
131
132 /* define bit use in NFC_RCMD_SET */
133 #define NFC_READ_CMD_MSK GENMASK(7, 0)
134 #define NFC_RND_READ_CMD0_MSK GENMASK(15, 8)
135 #define NFC_RND_READ_CMD1_MSK GENMASK(23, 16)
136
137 /* define bit use in NFC_WCMD_SET */
138 #define NFC_PROGRAM_CMD_MSK GENMASK(7, 0)
139 #define NFC_RND_WRITE_CMD_MSK GENMASK(15, 8)
140 #define NFC_READ_CMD0_MSK GENMASK(23, 16)
141 #define NFC_READ_CMD1_MSK GENMASK(31, 24)
142
143 /* define bit use in NFC_ECC_CTL */
144 #define NFC_ECC_EN BIT(0)
145 #define NFC_ECC_PIPELINE BIT(3)
146 #define NFC_ECC_EXCEPTION BIT(4)
147 #define NFC_ECC_BLOCK_SIZE_MSK BIT(5)
148 #define NFC_ECC_BLOCK_512 BIT(5)
149 #define NFC_RANDOM_EN BIT(9)
150 #define NFC_RANDOM_DIRECTION BIT(10)
151 #define NFC_ECC_MODE_MSK GENMASK(15, 12)
152 #define NFC_ECC_MODE(x) ((x) << 12)
153 #define NFC_RANDOM_SEED_MSK GENMASK(30, 16)
154 #define NFC_RANDOM_SEED(x) ((x) << 16)
155
156 /* define bit use in NFC_ECC_ST */
157 #define NFC_ECC_ERR(x) BIT(x)
158 #define NFC_ECC_ERR_MSK GENMASK(15, 0)
159 #define NFC_ECC_PAT_FOUND(x) BIT(x + 16)
160 #define NFC_ECC_ERR_CNT(b, x) (((x) >> (((b) % 4) * 8)) & 0xff)
161
162 #define NFC_DEFAULT_TIMEOUT_MS 1000
163
164 #define NFC_SRAM_SIZE 1024
165
166 #define NFC_MAX_CS 7
167
168 /*
169 * Ready/Busy detection type: describes the Ready/Busy detection modes
170 *
171 * @RB_NONE: no external detection available, rely on STATUS command
172 * and software timeouts
173 * @RB_NATIVE: use sunxi NAND controller Ready/Busy support. The Ready/Busy
174 * pin of the NAND flash chip must be connected to one of the
175 * native NAND R/B pins (those which can be muxed to the NAND
176 * Controller)
177 * @RB_GPIO: use a simple GPIO to handle Ready/Busy status. The Ready/Busy
178 * pin of the NAND flash chip must be connected to a GPIO capable
179 * pin.
180 */
181 enum sunxi_nand_rb_type {
182 RB_NONE,
183 RB_NATIVE,
184 RB_GPIO,
185 };
186
187 /*
188 * Ready/Busy structure: stores information related to Ready/Busy detection
189 *
190 * @type: the Ready/Busy detection mode
191 * @info: information related to the R/B detection mode. Either a gpio
192 * id or a native R/B id (those supported by the NAND controller).
193 */
194 struct sunxi_nand_rb {
195 enum sunxi_nand_rb_type type;
196 union {
197 int gpio;
198 int nativeid;
199 } info;
200 };
201
202 /*
203 * Chip Select structure: stores information related to NAND Chip Select
204 *
205 * @cs: the NAND CS id used to communicate with a NAND Chip
206 * @rb: the Ready/Busy description
207 */
208 struct sunxi_nand_chip_sel {
209 u8 cs;
210 struct sunxi_nand_rb rb;
211 };
212
213 /*
214 * sunxi HW ECC infos: stores information related to HW ECC support
215 *
216 * @mode: the sunxi ECC mode field deduced from ECC requirements
217 */
218 struct sunxi_nand_hw_ecc {
219 int mode;
220 };
221
222 /*
223 * NAND chip structure: stores NAND chip device related information
224 *
225 * @node: used to store NAND chips into a list
226 * @nand: base NAND chip structure
227 * @mtd: base MTD structure
228 * @clk_rate: clk_rate required for this NAND chip
229 * @timing_cfg TIMING_CFG register value for this NAND chip
230 * @selected: current active CS
231 * @nsels: number of CS lines required by the NAND chip
232 * @sels: array of CS lines descriptions
233 */
234 struct sunxi_nand_chip {
235 struct list_head node;
236 struct nand_chip nand;
237 unsigned long clk_rate;
238 u32 timing_cfg;
239 u32 timing_ctl;
240 int selected;
241 int addr_cycles;
242 u32 addr[2];
243 int cmd_cycles;
244 u8 cmd[2];
245 int nsels;
246 struct sunxi_nand_chip_sel sels[0];
247 };
248
249 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)
250 {
251 return container_of(nand, struct sunxi_nand_chip, nand);
252 }
253
254 /*
255 * NAND Controller structure: stores sunxi NAND controller information
256 *
257 * @controller: base controller structure
258 * @dev: parent device (used to print error messages)
259 * @regs: NAND controller registers
260 * @ahb_clk: NAND Controller AHB clock
261 * @mod_clk: NAND Controller mod clock
262 * @assigned_cs: bitmask describing already assigned CS lines
263 * @clk_rate: NAND controller current clock rate
264 * @chips: a list containing all the NAND chips attached to
265 * this NAND controller
266 * @complete: a completion object used to wait for NAND
267 * controller events
268 */
269 struct sunxi_nfc {
270 struct nand_hw_control controller;
271 struct device *dev;
272 void __iomem *regs;
273 struct clk *ahb_clk;
274 struct clk *mod_clk;
275 struct reset_control *reset;
276 unsigned long assigned_cs;
277 unsigned long clk_rate;
278 struct list_head chips;
279 struct completion complete;
280 struct dma_chan *dmac;
281 };
282
283 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_hw_control *ctrl)
284 {
285 return container_of(ctrl, struct sunxi_nfc, controller);
286 }
287
288 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id)
289 {
290 struct sunxi_nfc *nfc = dev_id;
291 u32 st = readl(nfc->regs + NFC_REG_ST);
292 u32 ien = readl(nfc->regs + NFC_REG_INT);
293
294 if (!(ien & st))
295 return IRQ_NONE;
296
297 if ((ien & st) == ien)
298 complete(&nfc->complete);
299
300 writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
301 writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT);
302
303 return IRQ_HANDLED;
304 }
305
306 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events,
307 bool use_polling, unsigned int timeout_ms)
308 {
309 int ret;
310
311 if (events & ~NFC_INT_MASK)
312 return -EINVAL;
313
314 if (!timeout_ms)
315 timeout_ms = NFC_DEFAULT_TIMEOUT_MS;
316
317 if (!use_polling) {
318 init_completion(&nfc->complete);
319
320 writel(events, nfc->regs + NFC_REG_INT);
321
322 ret = wait_for_completion_timeout(&nfc->complete,
323 msecs_to_jiffies(timeout_ms));
324 if (!ret)
325 ret = -ETIMEDOUT;
326 else
327 ret = 0;
328
329 writel(0, nfc->regs + NFC_REG_INT);
330 } else {
331 u32 status;
332
333 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
334 (status & events) == events, 1,
335 timeout_ms * 1000);
336 }
337
338 writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
339
340 if (ret)
341 dev_err(nfc->dev, "wait interrupt timedout\n");
342
343 return ret;
344 }
345
346 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc)
347 {
348 u32 status;
349 int ret;
350
351 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
352 !(status & NFC_CMD_FIFO_STATUS), 1,
353 NFC_DEFAULT_TIMEOUT_MS * 1000);
354 if (ret)
355 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n");
356
357 return ret;
358 }
359
360 static int sunxi_nfc_rst(struct sunxi_nfc *nfc)
361 {
362 u32 ctl;
363 int ret;
364
365 writel(0, nfc->regs + NFC_REG_ECC_CTL);
366 writel(NFC_RESET, nfc->regs + NFC_REG_CTL);
367
368 ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl,
369 !(ctl & NFC_RESET), 1,
370 NFC_DEFAULT_TIMEOUT_MS * 1000);
371 if (ret)
372 dev_err(nfc->dev, "wait for NAND controller reset timedout\n");
373
374 return ret;
375 }
376
377 static int sunxi_nfc_dma_op_prepare(struct mtd_info *mtd, const void *buf,
378 int chunksize, int nchunks,
379 enum dma_data_direction ddir,
380 struct scatterlist *sg)
381 {
382 struct nand_chip *nand = mtd_to_nand(mtd);
383 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
384 struct dma_async_tx_descriptor *dmad;
385 enum dma_transfer_direction tdir;
386 dma_cookie_t dmat;
387 int ret;
388
389 if (ddir == DMA_FROM_DEVICE)
390 tdir = DMA_DEV_TO_MEM;
391 else
392 tdir = DMA_MEM_TO_DEV;
393
394 sg_init_one(sg, buf, nchunks * chunksize);
395 ret = dma_map_sg(nfc->dev, sg, 1, ddir);
396 if (!ret)
397 return -ENOMEM;
398
399 dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);
400 if (!dmad) {
401 ret = -EINVAL;
402 goto err_unmap_buf;
403 }
404
405 writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,
406 nfc->regs + NFC_REG_CTL);
407 writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);
408 writel(chunksize, nfc->regs + NFC_REG_CNT);
409 dmat = dmaengine_submit(dmad);
410
411 ret = dma_submit_error(dmat);
412 if (ret)
413 goto err_clr_dma_flag;
414
415 return 0;
416
417 err_clr_dma_flag:
418 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
419 nfc->regs + NFC_REG_CTL);
420
421 err_unmap_buf:
422 dma_unmap_sg(nfc->dev, sg, 1, ddir);
423 return ret;
424 }
425
426 static void sunxi_nfc_dma_op_cleanup(struct mtd_info *mtd,
427 enum dma_data_direction ddir,
428 struct scatterlist *sg)
429 {
430 struct nand_chip *nand = mtd_to_nand(mtd);
431 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
432
433 dma_unmap_sg(nfc->dev, sg, 1, ddir);
434 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
435 nfc->regs + NFC_REG_CTL);
436 }
437
438 static int sunxi_nfc_dev_ready(struct mtd_info *mtd)
439 {
440 struct nand_chip *nand = mtd_to_nand(mtd);
441 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
442 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
443 struct sunxi_nand_rb *rb;
444 int ret;
445
446 if (sunxi_nand->selected < 0)
447 return 0;
448
449 rb = &sunxi_nand->sels[sunxi_nand->selected].rb;
450
451 switch (rb->type) {
452 case RB_NATIVE:
453 ret = !!(readl(nfc->regs + NFC_REG_ST) &
454 NFC_RB_STATE(rb->info.nativeid));
455 break;
456 case RB_GPIO:
457 ret = gpio_get_value(rb->info.gpio);
458 break;
459 case RB_NONE:
460 default:
461 ret = 0;
462 dev_err(nfc->dev, "cannot check R/B NAND status!\n");
463 break;
464 }
465
466 return ret;
467 }
468
469 static void sunxi_nfc_select_chip(struct mtd_info *mtd, int chip)
470 {
471 struct nand_chip *nand = mtd_to_nand(mtd);
472 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
473 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
474 struct sunxi_nand_chip_sel *sel;
475 u32 ctl;
476
477 if (chip > 0 && chip >= sunxi_nand->nsels)
478 return;
479
480 if (chip == sunxi_nand->selected)
481 return;
482
483 ctl = readl(nfc->regs + NFC_REG_CTL) &
484 ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN);
485
486 if (chip >= 0) {
487 sel = &sunxi_nand->sels[chip];
488
489 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN |
490 NFC_PAGE_SHIFT(nand->page_shift);
491 if (sel->rb.type == RB_NONE) {
492 nand->dev_ready = NULL;
493 } else {
494 nand->dev_ready = sunxi_nfc_dev_ready;
495 if (sel->rb.type == RB_NATIVE)
496 ctl |= NFC_RB_SEL(sel->rb.info.nativeid);
497 }
498
499 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA);
500
501 if (nfc->clk_rate != sunxi_nand->clk_rate) {
502 clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate);
503 nfc->clk_rate = sunxi_nand->clk_rate;
504 }
505 }
506
507 writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL);
508 writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG);
509 writel(ctl, nfc->regs + NFC_REG_CTL);
510
511 sunxi_nand->selected = chip;
512 }
513
514 static void sunxi_nfc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
515 {
516 struct nand_chip *nand = mtd_to_nand(mtd);
517 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
518 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
519 int ret;
520 int cnt;
521 int offs = 0;
522 u32 tmp;
523
524 while (len > offs) {
525 bool poll = false;
526
527 cnt = min(len - offs, NFC_SRAM_SIZE);
528
529 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
530 if (ret)
531 break;
532
533 writel(cnt, nfc->regs + NFC_REG_CNT);
534 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
535 writel(tmp, nfc->regs + NFC_REG_CMD);
536
537 /* Arbitrary limit for polling mode */
538 if (cnt < 64)
539 poll = true;
540
541 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
542 if (ret)
543 break;
544
545 if (buf)
546 memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE,
547 cnt);
548 offs += cnt;
549 }
550 }
551
552 static void sunxi_nfc_write_buf(struct mtd_info *mtd, const uint8_t *buf,
553 int len)
554 {
555 struct nand_chip *nand = mtd_to_nand(mtd);
556 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
557 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
558 int ret;
559 int cnt;
560 int offs = 0;
561 u32 tmp;
562
563 while (len > offs) {
564 bool poll = false;
565
566 cnt = min(len - offs, NFC_SRAM_SIZE);
567
568 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
569 if (ret)
570 break;
571
572 writel(cnt, nfc->regs + NFC_REG_CNT);
573 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt);
574 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
575 NFC_ACCESS_DIR;
576 writel(tmp, nfc->regs + NFC_REG_CMD);
577
578 /* Arbitrary limit for polling mode */
579 if (cnt < 64)
580 poll = true;
581
582 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
583 if (ret)
584 break;
585
586 offs += cnt;
587 }
588 }
589
590 static uint8_t sunxi_nfc_read_byte(struct mtd_info *mtd)
591 {
592 uint8_t ret;
593
594 sunxi_nfc_read_buf(mtd, &ret, 1);
595
596 return ret;
597 }
598
599 static void sunxi_nfc_cmd_ctrl(struct mtd_info *mtd, int dat,
600 unsigned int ctrl)
601 {
602 struct nand_chip *nand = mtd_to_nand(mtd);
603 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
604 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
605 int ret;
606
607 if (dat == NAND_CMD_NONE && (ctrl & NAND_NCE) &&
608 !(ctrl & (NAND_CLE | NAND_ALE))) {
609 u32 cmd = 0;
610
611 if (!sunxi_nand->addr_cycles && !sunxi_nand->cmd_cycles)
612 return;
613
614 if (sunxi_nand->cmd_cycles--)
615 cmd |= NFC_SEND_CMD1 | sunxi_nand->cmd[0];
616
617 if (sunxi_nand->cmd_cycles--) {
618 cmd |= NFC_SEND_CMD2;
619 writel(sunxi_nand->cmd[1],
620 nfc->regs + NFC_REG_RCMD_SET);
621 }
622
623 sunxi_nand->cmd_cycles = 0;
624
625 if (sunxi_nand->addr_cycles) {
626 cmd |= NFC_SEND_ADR |
627 NFC_ADR_NUM(sunxi_nand->addr_cycles);
628 writel(sunxi_nand->addr[0],
629 nfc->regs + NFC_REG_ADDR_LOW);
630 }
631
632 if (sunxi_nand->addr_cycles > 4)
633 writel(sunxi_nand->addr[1],
634 nfc->regs + NFC_REG_ADDR_HIGH);
635
636 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
637 if (ret)
638 return;
639
640 writel(cmd, nfc->regs + NFC_REG_CMD);
641 sunxi_nand->addr[0] = 0;
642 sunxi_nand->addr[1] = 0;
643 sunxi_nand->addr_cycles = 0;
644 sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, true, 0);
645 }
646
647 if (ctrl & NAND_CLE) {
648 sunxi_nand->cmd[sunxi_nand->cmd_cycles++] = dat;
649 } else if (ctrl & NAND_ALE) {
650 sunxi_nand->addr[sunxi_nand->addr_cycles / 4] |=
651 dat << ((sunxi_nand->addr_cycles % 4) * 8);
652 sunxi_nand->addr_cycles++;
653 }
654 }
655
656 /* These seed values have been extracted from Allwinner's BSP */
657 static const u16 sunxi_nfc_randomizer_page_seeds[] = {
658 0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72,
659 0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436,
660 0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d,
661 0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130,
662 0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56,
663 0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55,
664 0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb,
665 0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17,
666 0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62,
667 0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064,
668 0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126,
669 0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e,
670 0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3,
671 0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b,
672 0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d,
673 0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db,
674 };
675
676 /*
677 * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds
678 * have been generated using
679 * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what
680 * the randomizer engine does internally before de/scrambling OOB data.
681 *
682 * Those tables are statically defined to avoid calculating randomizer state
683 * at runtime.
684 */
685 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = {
686 0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64,
687 0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409,
688 0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617,
689 0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d,
690 0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91,
691 0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d,
692 0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab,
693 0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8,
694 0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8,
695 0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b,
696 0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5,
697 0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a,
698 0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891,
699 0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36,
700 0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd,
701 0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0,
702 };
703
704 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = {
705 0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6,
706 0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982,
707 0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9,
708 0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07,
709 0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e,
710 0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2,
711 0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c,
712 0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f,
713 0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc,
714 0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e,
715 0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8,
716 0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68,
717 0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d,
718 0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179,
719 0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601,
720 0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd,
721 };
722
723 static u16 sunxi_nfc_randomizer_step(u16 state, int count)
724 {
725 state &= 0x7fff;
726
727 /*
728 * This loop is just a simple implementation of a Fibonacci LFSR using
729 * the x16 + x15 + 1 polynomial.
730 */
731 while (count--)
732 state = ((state >> 1) |
733 (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff;
734
735 return state;
736 }
737
738 static u16 sunxi_nfc_randomizer_state(struct mtd_info *mtd, int page, bool ecc)
739 {
740 const u16 *seeds = sunxi_nfc_randomizer_page_seeds;
741 int mod = mtd_div_by_ws(mtd->erasesize, mtd);
742
743 if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds))
744 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds);
745
746 if (ecc) {
747 if (mtd->ecc_step_size == 512)
748 seeds = sunxi_nfc_randomizer_ecc512_seeds;
749 else
750 seeds = sunxi_nfc_randomizer_ecc1024_seeds;
751 }
752
753 return seeds[page % mod];
754 }
755
756 static void sunxi_nfc_randomizer_config(struct mtd_info *mtd,
757 int page, bool ecc)
758 {
759 struct nand_chip *nand = mtd_to_nand(mtd);
760 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
761 u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
762 u16 state;
763
764 if (!(nand->options & NAND_NEED_SCRAMBLING))
765 return;
766
767 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
768 state = sunxi_nfc_randomizer_state(mtd, page, ecc);
769 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK;
770 writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL);
771 }
772
773 static void sunxi_nfc_randomizer_enable(struct mtd_info *mtd)
774 {
775 struct nand_chip *nand = mtd_to_nand(mtd);
776 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
777
778 if (!(nand->options & NAND_NEED_SCRAMBLING))
779 return;
780
781 writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN,
782 nfc->regs + NFC_REG_ECC_CTL);
783 }
784
785 static void sunxi_nfc_randomizer_disable(struct mtd_info *mtd)
786 {
787 struct nand_chip *nand = mtd_to_nand(mtd);
788 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
789
790 if (!(nand->options & NAND_NEED_SCRAMBLING))
791 return;
792
793 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN,
794 nfc->regs + NFC_REG_ECC_CTL);
795 }
796
797 static void sunxi_nfc_randomize_bbm(struct mtd_info *mtd, int page, u8 *bbm)
798 {
799 u16 state = sunxi_nfc_randomizer_state(mtd, page, true);
800
801 bbm[0] ^= state;
802 bbm[1] ^= sunxi_nfc_randomizer_step(state, 8);
803 }
804
805 static void sunxi_nfc_randomizer_write_buf(struct mtd_info *mtd,
806 const uint8_t *buf, int len,
807 bool ecc, int page)
808 {
809 sunxi_nfc_randomizer_config(mtd, page, ecc);
810 sunxi_nfc_randomizer_enable(mtd);
811 sunxi_nfc_write_buf(mtd, buf, len);
812 sunxi_nfc_randomizer_disable(mtd);
813 }
814
815 static void sunxi_nfc_randomizer_read_buf(struct mtd_info *mtd, uint8_t *buf,
816 int len, bool ecc, int page)
817 {
818 sunxi_nfc_randomizer_config(mtd, page, ecc);
819 sunxi_nfc_randomizer_enable(mtd);
820 sunxi_nfc_read_buf(mtd, buf, len);
821 sunxi_nfc_randomizer_disable(mtd);
822 }
823
824 static void sunxi_nfc_hw_ecc_enable(struct mtd_info *mtd)
825 {
826 struct nand_chip *nand = mtd_to_nand(mtd);
827 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
828 struct sunxi_nand_hw_ecc *data = nand->ecc.priv;
829 u32 ecc_ctl;
830
831 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
832 ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE |
833 NFC_ECC_BLOCK_SIZE_MSK);
834 ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(data->mode) | NFC_ECC_EXCEPTION |
835 NFC_ECC_PIPELINE;
836
837 if (nand->ecc.size == 512)
838 ecc_ctl |= NFC_ECC_BLOCK_512;
839
840 writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL);
841 }
842
843 static void sunxi_nfc_hw_ecc_disable(struct mtd_info *mtd)
844 {
845 struct nand_chip *nand = mtd_to_nand(mtd);
846 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
847
848 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN,
849 nfc->regs + NFC_REG_ECC_CTL);
850 }
851
852 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf)
853 {
854 buf[0] = user_data;
855 buf[1] = user_data >> 8;
856 buf[2] = user_data >> 16;
857 buf[3] = user_data >> 24;
858 }
859
860 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf)
861 {
862 return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24);
863 }
864
865 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct mtd_info *mtd, u8 *oob,
866 int step, bool bbm, int page)
867 {
868 struct nand_chip *nand = mtd_to_nand(mtd);
869 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
870
871 sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)),
872 oob);
873
874 /* De-randomize the Bad Block Marker. */
875 if (bbm && (nand->options & NAND_NEED_SCRAMBLING))
876 sunxi_nfc_randomize_bbm(mtd, page, oob);
877 }
878
879 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct mtd_info *mtd,
880 const u8 *oob, int step,
881 bool bbm, int page)
882 {
883 struct nand_chip *nand = mtd_to_nand(mtd);
884 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
885 u8 user_data[4];
886
887 /* Randomize the Bad Block Marker. */
888 if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
889 memcpy(user_data, oob, sizeof(user_data));
890 sunxi_nfc_randomize_bbm(mtd, page, user_data);
891 oob = user_data;
892 }
893
894 writel(sunxi_nfc_buf_to_user_data(oob),
895 nfc->regs + NFC_REG_USER_DATA(step));
896 }
897
898 static void sunxi_nfc_hw_ecc_update_stats(struct mtd_info *mtd,
899 unsigned int *max_bitflips, int ret)
900 {
901 if (ret < 0) {
902 mtd->ecc_stats.failed++;
903 } else {
904 mtd->ecc_stats.corrected += ret;
905 *max_bitflips = max_t(unsigned int, *max_bitflips, ret);
906 }
907 }
908
909 static int sunxi_nfc_hw_ecc_correct(struct mtd_info *mtd, u8 *data, u8 *oob,
910 int step, u32 status, bool *erased)
911 {
912 struct nand_chip *nand = mtd_to_nand(mtd);
913 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
914 struct nand_ecc_ctrl *ecc = &nand->ecc;
915 u32 tmp;
916
917 *erased = false;
918
919 if (status & NFC_ECC_ERR(step))
920 return -EBADMSG;
921
922 if (status & NFC_ECC_PAT_FOUND(step)) {
923 u8 pattern;
924
925 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) {
926 pattern = 0x0;
927 } else {
928 pattern = 0xff;
929 *erased = true;
930 }
931
932 if (data)
933 memset(data, pattern, ecc->size);
934
935 if (oob)
936 memset(oob, pattern, ecc->bytes + 4);
937
938 return 0;
939 }
940
941 tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step));
942
943 return NFC_ECC_ERR_CNT(step, tmp);
944 }
945
946 static int sunxi_nfc_hw_ecc_read_chunk(struct mtd_info *mtd,
947 u8 *data, int data_off,
948 u8 *oob, int oob_off,
949 int *cur_off,
950 unsigned int *max_bitflips,
951 bool bbm, bool oob_required, int page)
952 {
953 struct nand_chip *nand = mtd_to_nand(mtd);
954 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
955 struct nand_ecc_ctrl *ecc = &nand->ecc;
956 int raw_mode = 0;
957 bool erased;
958 int ret;
959
960 if (*cur_off != data_off)
961 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, data_off, -1);
962
963 sunxi_nfc_randomizer_read_buf(mtd, NULL, ecc->size, false, page);
964
965 if (data_off + ecc->size != oob_off)
966 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
967
968 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
969 if (ret)
970 return ret;
971
972 sunxi_nfc_randomizer_enable(mtd);
973 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP,
974 nfc->regs + NFC_REG_CMD);
975
976 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
977 sunxi_nfc_randomizer_disable(mtd);
978 if (ret)
979 return ret;
980
981 *cur_off = oob_off + ecc->bytes + 4;
982
983 ret = sunxi_nfc_hw_ecc_correct(mtd, data, oob_required ? oob : NULL, 0,
984 readl(nfc->regs + NFC_REG_ECC_ST),
985 &erased);
986 if (erased)
987 return 1;
988
989 if (ret < 0) {
990 /*
991 * Re-read the data with the randomizer disabled to identify
992 * bitflips in erased pages.
993 */
994 if (nand->options & NAND_NEED_SCRAMBLING) {
995 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, data_off, -1);
996 nand->read_buf(mtd, data, ecc->size);
997 } else {
998 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,
999 ecc->size);
1000 }
1001
1002 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
1003 nand->read_buf(mtd, oob, ecc->bytes + 4);
1004
1005 ret = nand_check_erased_ecc_chunk(data, ecc->size,
1006 oob, ecc->bytes + 4,
1007 NULL, 0, ecc->strength);
1008 if (ret >= 0)
1009 raw_mode = 1;
1010 } else {
1011 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);
1012
1013 if (oob_required) {
1014 nand->cmdfunc(mtd, NAND_CMD_RNDOUT, oob_off, -1);
1015 sunxi_nfc_randomizer_read_buf(mtd, oob, ecc->bytes + 4,
1016 true, page);
1017
1018 sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, 0,
1019 bbm, page);
1020 }
1021 }
1022
1023 sunxi_nfc_hw_ecc_update_stats(mtd, max_bitflips, ret);
1024
1025 return raw_mode;
1026 }
1027
1028 static void sunxi_nfc_hw_ecc_read_extra_oob(struct mtd_info *mtd,
1029 u8 *oob, int *cur_off,
1030 bool randomize, int page)
1031 {
1032 struct nand_chip *nand = mtd_to_nand(mtd);
1033 struct nand_ecc_ctrl *ecc = &nand->ecc;
1034 int offset = ((ecc->bytes + 4) * ecc->steps);
1035 int len = mtd->oobsize - offset;
1036
1037 if (len <= 0)
1038 return;
1039
1040 if (!cur_off || *cur_off != offset)
1041 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1042 offset + mtd->writesize, -1);
1043
1044 if (!randomize)
1045 sunxi_nfc_read_buf(mtd, oob + offset, len);
1046 else
1047 sunxi_nfc_randomizer_read_buf(mtd, oob + offset, len,
1048 false, page);
1049
1050 if (cur_off)
1051 *cur_off = mtd->oobsize + mtd->writesize;
1052 }
1053
1054 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct mtd_info *mtd, uint8_t *buf,
1055 int oob_required, int page,
1056 int nchunks)
1057 {
1058 struct nand_chip *nand = mtd_to_nand(mtd);
1059 bool randomized = nand->options & NAND_NEED_SCRAMBLING;
1060 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1061 struct nand_ecc_ctrl *ecc = &nand->ecc;
1062 unsigned int max_bitflips = 0;
1063 int ret, i, raw_mode = 0;
1064 struct scatterlist sg;
1065 u32 status;
1066
1067 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1068 if (ret)
1069 return ret;
1070
1071 ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, nchunks,
1072 DMA_FROM_DEVICE, &sg);
1073 if (ret)
1074 return ret;
1075
1076 sunxi_nfc_hw_ecc_enable(mtd);
1077 sunxi_nfc_randomizer_config(mtd, page, false);
1078 sunxi_nfc_randomizer_enable(mtd);
1079
1080 writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |
1081 NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);
1082
1083 dma_async_issue_pending(nfc->dmac);
1084
1085 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,
1086 nfc->regs + NFC_REG_CMD);
1087
1088 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1089 if (ret)
1090 dmaengine_terminate_all(nfc->dmac);
1091
1092 sunxi_nfc_randomizer_disable(mtd);
1093 sunxi_nfc_hw_ecc_disable(mtd);
1094
1095 sunxi_nfc_dma_op_cleanup(mtd, DMA_FROM_DEVICE, &sg);
1096
1097 if (ret)
1098 return ret;
1099
1100 status = readl(nfc->regs + NFC_REG_ECC_ST);
1101
1102 for (i = 0; i < nchunks; i++) {
1103 int data_off = i * ecc->size;
1104 int oob_off = i * (ecc->bytes + 4);
1105 u8 *data = buf + data_off;
1106 u8 *oob = nand->oob_poi + oob_off;
1107 bool erased;
1108
1109 ret = sunxi_nfc_hw_ecc_correct(mtd, randomized ? data : NULL,
1110 oob_required ? oob : NULL,
1111 i, status, &erased);
1112
1113 /* ECC errors are handled in the second loop. */
1114 if (ret < 0)
1115 continue;
1116
1117 if (oob_required && !erased) {
1118 /* TODO: use DMA to retrieve OOB */
1119 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1120 mtd->writesize + oob_off, -1);
1121 nand->read_buf(mtd, oob, ecc->bytes + 4);
1122
1123 sunxi_nfc_hw_ecc_get_prot_oob_bytes(mtd, oob, i,
1124 !i, page);
1125 }
1126
1127 if (erased)
1128 raw_mode = 1;
1129
1130 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1131 }
1132
1133 if (status & NFC_ECC_ERR_MSK) {
1134 for (i = 0; i < nchunks; i++) {
1135 int data_off = i * ecc->size;
1136 int oob_off = i * (ecc->bytes + 4);
1137 u8 *data = buf + data_off;
1138 u8 *oob = nand->oob_poi + oob_off;
1139
1140 if (!(status & NFC_ECC_ERR(i)))
1141 continue;
1142
1143 /*
1144 * Re-read the data with the randomizer disabled to
1145 * identify bitflips in erased pages.
1146 */
1147 if (randomized) {
1148 /* TODO: use DMA to read page in raw mode */
1149 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1150 data_off, -1);
1151 nand->read_buf(mtd, data, ecc->size);
1152 }
1153
1154 /* TODO: use DMA to retrieve OOB */
1155 nand->cmdfunc(mtd, NAND_CMD_RNDOUT,
1156 mtd->writesize + oob_off, -1);
1157 nand->read_buf(mtd, oob, ecc->bytes + 4);
1158
1159 ret = nand_check_erased_ecc_chunk(data, ecc->size,
1160 oob, ecc->bytes + 4,
1161 NULL, 0,
1162 ecc->strength);
1163 if (ret >= 0)
1164 raw_mode = 1;
1165
1166 sunxi_nfc_hw_ecc_update_stats(mtd, &max_bitflips, ret);
1167 }
1168 }
1169
1170 if (oob_required)
1171 sunxi_nfc_hw_ecc_read_extra_oob(mtd, nand->oob_poi,
1172 NULL, !raw_mode,
1173 page);
1174
1175 return max_bitflips;
1176 }
1177
1178 static int sunxi_nfc_hw_ecc_write_chunk(struct mtd_info *mtd,
1179 const u8 *data, int data_off,
1180 const u8 *oob, int oob_off,
1181 int *cur_off, bool bbm,
1182 int page)
1183 {
1184 struct nand_chip *nand = mtd_to_nand(mtd);
1185 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1186 struct nand_ecc_ctrl *ecc = &nand->ecc;
1187 int ret;
1188
1189 if (data_off != *cur_off)
1190 nand->cmdfunc(mtd, NAND_CMD_RNDIN, data_off, -1);
1191
1192 sunxi_nfc_randomizer_write_buf(mtd, data, ecc->size, false, page);
1193
1194 if (data_off + ecc->size != oob_off)
1195 nand->cmdfunc(mtd, NAND_CMD_RNDIN, oob_off, -1);
1196
1197 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1198 if (ret)
1199 return ret;
1200
1201 sunxi_nfc_randomizer_enable(mtd);
1202 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, 0, bbm, page);
1203
1204 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
1205 NFC_ACCESS_DIR | NFC_ECC_OP,
1206 nfc->regs + NFC_REG_CMD);
1207
1208 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1209 sunxi_nfc_randomizer_disable(mtd);
1210 if (ret)
1211 return ret;
1212
1213 *cur_off = oob_off + ecc->bytes + 4;
1214
1215 return 0;
1216 }
1217
1218 static void sunxi_nfc_hw_ecc_write_extra_oob(struct mtd_info *mtd,
1219 u8 *oob, int *cur_off,
1220 int page)
1221 {
1222 struct nand_chip *nand = mtd_to_nand(mtd);
1223 struct nand_ecc_ctrl *ecc = &nand->ecc;
1224 int offset = ((ecc->bytes + 4) * ecc->steps);
1225 int len = mtd->oobsize - offset;
1226
1227 if (len <= 0)
1228 return;
1229
1230 if (!cur_off || *cur_off != offset)
1231 nand->cmdfunc(mtd, NAND_CMD_RNDIN,
1232 offset + mtd->writesize, -1);
1233
1234 sunxi_nfc_randomizer_write_buf(mtd, oob + offset, len, false, page);
1235
1236 if (cur_off)
1237 *cur_off = mtd->oobsize + mtd->writesize;
1238 }
1239
1240 static int sunxi_nfc_hw_ecc_read_page(struct mtd_info *mtd,
1241 struct nand_chip *chip, uint8_t *buf,
1242 int oob_required, int page)
1243 {
1244 struct nand_ecc_ctrl *ecc = &chip->ecc;
1245 unsigned int max_bitflips = 0;
1246 int ret, i, cur_off = 0;
1247 bool raw_mode = false;
1248
1249 sunxi_nfc_hw_ecc_enable(mtd);
1250
1251 for (i = 0; i < ecc->steps; i++) {
1252 int data_off = i * ecc->size;
1253 int oob_off = i * (ecc->bytes + 4);
1254 u8 *data = buf + data_off;
1255 u8 *oob = chip->oob_poi + oob_off;
1256
1257 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1258 oob_off + mtd->writesize,
1259 &cur_off, &max_bitflips,
1260 !i, oob_required, page);
1261 if (ret < 0)
1262 return ret;
1263 else if (ret)
1264 raw_mode = true;
1265 }
1266
1267 if (oob_required)
1268 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1269 !raw_mode, page);
1270
1271 sunxi_nfc_hw_ecc_disable(mtd);
1272
1273 return max_bitflips;
1274 }
1275
1276 static int sunxi_nfc_hw_ecc_read_page_dma(struct mtd_info *mtd,
1277 struct nand_chip *chip, u8 *buf,
1278 int oob_required, int page)
1279 {
1280 int ret;
1281
1282 ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, oob_required, page,
1283 chip->ecc.steps);
1284 if (ret >= 0)
1285 return ret;
1286
1287 /* Fallback to PIO mode */
1288 chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
1289
1290 return sunxi_nfc_hw_ecc_read_page(mtd, chip, buf, oob_required, page);
1291 }
1292
1293 static int sunxi_nfc_hw_ecc_read_subpage(struct mtd_info *mtd,
1294 struct nand_chip *chip,
1295 u32 data_offs, u32 readlen,
1296 u8 *bufpoi, int page)
1297 {
1298 struct nand_ecc_ctrl *ecc = &chip->ecc;
1299 int ret, i, cur_off = 0;
1300 unsigned int max_bitflips = 0;
1301
1302 sunxi_nfc_hw_ecc_enable(mtd);
1303
1304 chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1305 for (i = data_offs / ecc->size;
1306 i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) {
1307 int data_off = i * ecc->size;
1308 int oob_off = i * (ecc->bytes + 4);
1309 u8 *data = bufpoi + data_off;
1310 u8 *oob = chip->oob_poi + oob_off;
1311
1312 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off,
1313 oob,
1314 oob_off + mtd->writesize,
1315 &cur_off, &max_bitflips, !i,
1316 false, page);
1317 if (ret < 0)
1318 return ret;
1319 }
1320
1321 sunxi_nfc_hw_ecc_disable(mtd);
1322
1323 return max_bitflips;
1324 }
1325
1326 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct mtd_info *mtd,
1327 struct nand_chip *chip,
1328 u32 data_offs, u32 readlen,
1329 u8 *buf, int page)
1330 {
1331 int nchunks = DIV_ROUND_UP(data_offs + readlen, chip->ecc.size);
1332 int ret;
1333
1334 ret = sunxi_nfc_hw_ecc_read_chunks_dma(mtd, buf, false, page, nchunks);
1335 if (ret >= 0)
1336 return ret;
1337
1338 /* Fallback to PIO mode */
1339 chip->cmdfunc(mtd, NAND_CMD_RNDOUT, 0, -1);
1340
1341 return sunxi_nfc_hw_ecc_read_subpage(mtd, chip, data_offs, readlen,
1342 buf, page);
1343 }
1344
1345 static int sunxi_nfc_hw_ecc_write_page(struct mtd_info *mtd,
1346 struct nand_chip *chip,
1347 const uint8_t *buf, int oob_required,
1348 int page)
1349 {
1350 struct nand_ecc_ctrl *ecc = &chip->ecc;
1351 int ret, i, cur_off = 0;
1352
1353 sunxi_nfc_hw_ecc_enable(mtd);
1354
1355 for (i = 0; i < ecc->steps; i++) {
1356 int data_off = i * ecc->size;
1357 int oob_off = i * (ecc->bytes + 4);
1358 const u8 *data = buf + data_off;
1359 const u8 *oob = chip->oob_poi + oob_off;
1360
1361 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1362 oob_off + mtd->writesize,
1363 &cur_off, !i, page);
1364 if (ret)
1365 return ret;
1366 }
1367
1368 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1369 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1370 &cur_off, page);
1371
1372 sunxi_nfc_hw_ecc_disable(mtd);
1373
1374 return 0;
1375 }
1376
1377 static int sunxi_nfc_hw_ecc_write_subpage(struct mtd_info *mtd,
1378 struct nand_chip *chip,
1379 u32 data_offs, u32 data_len,
1380 const u8 *buf, int oob_required,
1381 int page)
1382 {
1383 struct nand_ecc_ctrl *ecc = &chip->ecc;
1384 int ret, i, cur_off = 0;
1385
1386 sunxi_nfc_hw_ecc_enable(mtd);
1387
1388 for (i = data_offs / ecc->size;
1389 i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) {
1390 int data_off = i * ecc->size;
1391 int oob_off = i * (ecc->bytes + 4);
1392 const u8 *data = buf + data_off;
1393 const u8 *oob = chip->oob_poi + oob_off;
1394
1395 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off, oob,
1396 oob_off + mtd->writesize,
1397 &cur_off, !i, page);
1398 if (ret)
1399 return ret;
1400 }
1401
1402 sunxi_nfc_hw_ecc_disable(mtd);
1403
1404 return 0;
1405 }
1406
1407 static int sunxi_nfc_hw_ecc_write_page_dma(struct mtd_info *mtd,
1408 struct nand_chip *chip,
1409 const u8 *buf,
1410 int oob_required,
1411 int page)
1412 {
1413 struct nand_chip *nand = mtd_to_nand(mtd);
1414 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1415 struct nand_ecc_ctrl *ecc = &nand->ecc;
1416 struct scatterlist sg;
1417 int ret, i;
1418
1419 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1420 if (ret)
1421 return ret;
1422
1423 ret = sunxi_nfc_dma_op_prepare(mtd, buf, ecc->size, ecc->steps,
1424 DMA_TO_DEVICE, &sg);
1425 if (ret)
1426 goto pio_fallback;
1427
1428 for (i = 0; i < ecc->steps; i++) {
1429 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));
1430
1431 sunxi_nfc_hw_ecc_set_prot_oob_bytes(mtd, oob, i, !i, page);
1432 }
1433
1434 sunxi_nfc_hw_ecc_enable(mtd);
1435 sunxi_nfc_randomizer_config(mtd, page, false);
1436 sunxi_nfc_randomizer_enable(mtd);
1437
1438 writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
1439 nfc->regs + NFC_REG_RCMD_SET);
1440
1441 dma_async_issue_pending(nfc->dmac);
1442
1443 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
1444 NFC_DATA_TRANS | NFC_ACCESS_DIR,
1445 nfc->regs + NFC_REG_CMD);
1446
1447 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1448 if (ret)
1449 dmaengine_terminate_all(nfc->dmac);
1450
1451 sunxi_nfc_randomizer_disable(mtd);
1452 sunxi_nfc_hw_ecc_disable(mtd);
1453
1454 sunxi_nfc_dma_op_cleanup(mtd, DMA_TO_DEVICE, &sg);
1455
1456 if (ret)
1457 return ret;
1458
1459 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1460 /* TODO: use DMA to transfer extra OOB bytes ? */
1461 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1462 NULL, page);
1463
1464 return 0;
1465
1466 pio_fallback:
1467 return sunxi_nfc_hw_ecc_write_page(mtd, chip, buf, oob_required, page);
1468 }
1469
1470 static int sunxi_nfc_hw_syndrome_ecc_read_page(struct mtd_info *mtd,
1471 struct nand_chip *chip,
1472 uint8_t *buf, int oob_required,
1473 int page)
1474 {
1475 struct nand_ecc_ctrl *ecc = &chip->ecc;
1476 unsigned int max_bitflips = 0;
1477 int ret, i, cur_off = 0;
1478 bool raw_mode = false;
1479
1480 sunxi_nfc_hw_ecc_enable(mtd);
1481
1482 for (i = 0; i < ecc->steps; i++) {
1483 int data_off = i * (ecc->size + ecc->bytes + 4);
1484 int oob_off = data_off + ecc->size;
1485 u8 *data = buf + (i * ecc->size);
1486 u8 *oob = chip->oob_poi + (i * (ecc->bytes + 4));
1487
1488 ret = sunxi_nfc_hw_ecc_read_chunk(mtd, data, data_off, oob,
1489 oob_off, &cur_off,
1490 &max_bitflips, !i,
1491 oob_required,
1492 page);
1493 if (ret < 0)
1494 return ret;
1495 else if (ret)
1496 raw_mode = true;
1497 }
1498
1499 if (oob_required)
1500 sunxi_nfc_hw_ecc_read_extra_oob(mtd, chip->oob_poi, &cur_off,
1501 !raw_mode, page);
1502
1503 sunxi_nfc_hw_ecc_disable(mtd);
1504
1505 return max_bitflips;
1506 }
1507
1508 static int sunxi_nfc_hw_syndrome_ecc_write_page(struct mtd_info *mtd,
1509 struct nand_chip *chip,
1510 const uint8_t *buf,
1511 int oob_required, int page)
1512 {
1513 struct nand_ecc_ctrl *ecc = &chip->ecc;
1514 int ret, i, cur_off = 0;
1515
1516 sunxi_nfc_hw_ecc_enable(mtd);
1517
1518 for (i = 0; i < ecc->steps; i++) {
1519 int data_off = i * (ecc->size + ecc->bytes + 4);
1520 int oob_off = data_off + ecc->size;
1521 const u8 *data = buf + (i * ecc->size);
1522 const u8 *oob = chip->oob_poi + (i * (ecc->bytes + 4));
1523
1524 ret = sunxi_nfc_hw_ecc_write_chunk(mtd, data, data_off,
1525 oob, oob_off, &cur_off,
1526 false, page);
1527 if (ret)
1528 return ret;
1529 }
1530
1531 if (oob_required || (chip->options & NAND_NEED_SCRAMBLING))
1532 sunxi_nfc_hw_ecc_write_extra_oob(mtd, chip->oob_poi,
1533 &cur_off, page);
1534
1535 sunxi_nfc_hw_ecc_disable(mtd);
1536
1537 return 0;
1538 }
1539
1540 static int sunxi_nfc_hw_common_ecc_read_oob(struct mtd_info *mtd,
1541 struct nand_chip *chip,
1542 int page)
1543 {
1544 chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1545
1546 chip->pagebuf = -1;
1547
1548 return chip->ecc.read_page(mtd, chip, chip->buffers->databuf, 1, page);
1549 }
1550
1551 static int sunxi_nfc_hw_common_ecc_write_oob(struct mtd_info *mtd,
1552 struct nand_chip *chip,
1553 int page)
1554 {
1555 int ret, status;
1556
1557 chip->cmdfunc(mtd, NAND_CMD_SEQIN, 0, page);
1558
1559 chip->pagebuf = -1;
1560
1561 memset(chip->buffers->databuf, 0xff, mtd->writesize);
1562 ret = chip->ecc.write_page(mtd, chip, chip->buffers->databuf, 1, page);
1563 if (ret)
1564 return ret;
1565
1566 /* Send command to program the OOB data */
1567 chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1568
1569 status = chip->waitfunc(mtd, chip);
1570
1571 return status & NAND_STATUS_FAIL ? -EIO : 0;
1572 }
1573
1574 static const s32 tWB_lut[] = {6, 12, 16, 20};
1575 static const s32 tRHW_lut[] = {4, 8, 12, 20};
1576
1577 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
1578 u32 clk_period)
1579 {
1580 u32 clk_cycles = DIV_ROUND_UP(duration, clk_period);
1581 int i;
1582
1583 for (i = 0; i < lut_size; i++) {
1584 if (clk_cycles <= lut[i])
1585 return i;
1586 }
1587
1588 /* Doesn't fit */
1589 return -EINVAL;
1590 }
1591
1592 #define sunxi_nand_lookup_timing(l, p, c) \
1593 _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)
1594
1595 static int sunxi_nfc_setup_data_interface(struct mtd_info *mtd,
1596 const struct nand_data_interface *conf,
1597 bool check_only)
1598 {
1599 struct nand_chip *nand = mtd_to_nand(mtd);
1600 struct sunxi_nand_chip *chip = to_sunxi_nand(nand);
1601 struct sunxi_nfc *nfc = to_sunxi_nfc(chip->nand.controller);
1602 const struct nand_sdr_timings *timings;
1603 u32 min_clk_period = 0;
1604 s32 tWB, tADL, tWHR, tRHW, tCAD;
1605 long real_clk_rate;
1606
1607 timings = nand_get_sdr_timings(conf);
1608 if (IS_ERR(timings))
1609 return -ENOTSUPP;
1610
1611 /* T1 <=> tCLS */
1612 if (timings->tCLS_min > min_clk_period)
1613 min_clk_period = timings->tCLS_min;
1614
1615 /* T2 <=> tCLH */
1616 if (timings->tCLH_min > min_clk_period)
1617 min_clk_period = timings->tCLH_min;
1618
1619 /* T3 <=> tCS */
1620 if (timings->tCS_min > min_clk_period)
1621 min_clk_period = timings->tCS_min;
1622
1623 /* T4 <=> tCH */
1624 if (timings->tCH_min > min_clk_period)
1625 min_clk_period = timings->tCH_min;
1626
1627 /* T5 <=> tWP */
1628 if (timings->tWP_min > min_clk_period)
1629 min_clk_period = timings->tWP_min;
1630
1631 /* T6 <=> tWH */
1632 if (timings->tWH_min > min_clk_period)
1633 min_clk_period = timings->tWH_min;
1634
1635 /* T7 <=> tALS */
1636 if (timings->tALS_min > min_clk_period)
1637 min_clk_period = timings->tALS_min;
1638
1639 /* T8 <=> tDS */
1640 if (timings->tDS_min > min_clk_period)
1641 min_clk_period = timings->tDS_min;
1642
1643 /* T9 <=> tDH */
1644 if (timings->tDH_min > min_clk_period)
1645 min_clk_period = timings->tDH_min;
1646
1647 /* T10 <=> tRR */
1648 if (timings->tRR_min > (min_clk_period * 3))
1649 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);
1650
1651 /* T11 <=> tALH */
1652 if (timings->tALH_min > min_clk_period)
1653 min_clk_period = timings->tALH_min;
1654
1655 /* T12 <=> tRP */
1656 if (timings->tRP_min > min_clk_period)
1657 min_clk_period = timings->tRP_min;
1658
1659 /* T13 <=> tREH */
1660 if (timings->tREH_min > min_clk_period)
1661 min_clk_period = timings->tREH_min;
1662
1663 /* T14 <=> tRC */
1664 if (timings->tRC_min > (min_clk_period * 2))
1665 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2);
1666
1667 /* T15 <=> tWC */
1668 if (timings->tWC_min > (min_clk_period * 2))
1669 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);
1670
1671 /* T16 - T19 + tCAD */
1672 if (timings->tWB_max > (min_clk_period * 20))
1673 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);
1674
1675 if (timings->tADL_min > (min_clk_period * 32))
1676 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
1677
1678 if (timings->tWHR_min > (min_clk_period * 32))
1679 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
1680
1681 if (timings->tRHW_min > (min_clk_period * 20))
1682 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);
1683
1684 tWB = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,
1685 min_clk_period);
1686 if (tWB < 0) {
1687 dev_err(nfc->dev, "unsupported tWB\n");
1688 return tWB;
1689 }
1690
1691 tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;
1692 if (tADL > 3) {
1693 dev_err(nfc->dev, "unsupported tADL\n");
1694 return -EINVAL;
1695 }
1696
1697 tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;
1698 if (tWHR > 3) {
1699 dev_err(nfc->dev, "unsupported tWHR\n");
1700 return -EINVAL;
1701 }
1702
1703 tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,
1704 min_clk_period);
1705 if (tRHW < 0) {
1706 dev_err(nfc->dev, "unsupported tRHW\n");
1707 return tRHW;
1708 }
1709
1710 if (check_only)
1711 return 0;
1712
1713 /*
1714 * TODO: according to ONFI specs this value only applies for DDR NAND,
1715 * but Allwinner seems to set this to 0x7. Mimic them for now.
1716 */
1717 tCAD = 0x7;
1718
1719 /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */
1720 chip->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD);
1721
1722 /* Convert min_clk_period from picoseconds to nanoseconds */
1723 min_clk_period = DIV_ROUND_UP(min_clk_period, 1000);
1724
1725 /*
1726 * Unlike what is stated in Allwinner datasheet, the clk_rate should
1727 * be set to (1 / min_clk_period), and not (2 / min_clk_period).
1728 * This new formula was verified with a scope and validated by
1729 * Allwinner engineers.
1730 */
1731 chip->clk_rate = NSEC_PER_SEC / min_clk_period;
1732 real_clk_rate = clk_round_rate(nfc->mod_clk, chip->clk_rate);
1733
1734 /*
1735 * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data
1736 * output cycle timings shall be used if the host drives tRC less than
1737 * 30 ns.
1738 */
1739 min_clk_period = NSEC_PER_SEC / real_clk_rate;
1740 chip->timing_ctl = ((min_clk_period * 2) < 30) ?
1741 NFC_TIMING_CTL_EDO : 0;
1742
1743 return 0;
1744 }
1745
1746 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1747 struct mtd_oob_region *oobregion)
1748 {
1749 struct nand_chip *nand = mtd_to_nand(mtd);
1750 struct nand_ecc_ctrl *ecc = &nand->ecc;
1751
1752 if (section >= ecc->steps)
1753 return -ERANGE;
1754
1755 oobregion->offset = section * (ecc->bytes + 4) + 4;
1756 oobregion->length = ecc->bytes;
1757
1758 return 0;
1759 }
1760
1761 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
1762 struct mtd_oob_region *oobregion)
1763 {
1764 struct nand_chip *nand = mtd_to_nand(mtd);
1765 struct nand_ecc_ctrl *ecc = &nand->ecc;
1766
1767 if (section > ecc->steps)
1768 return -ERANGE;
1769
1770 /*
1771 * The first 2 bytes are used for BB markers, hence we
1772 * only have 2 bytes available in the first user data
1773 * section.
1774 */
1775 if (!section && ecc->mode == NAND_ECC_HW) {
1776 oobregion->offset = 2;
1777 oobregion->length = 2;
1778
1779 return 0;
1780 }
1781
1782 oobregion->offset = section * (ecc->bytes + 4);
1783
1784 if (section < ecc->steps)
1785 oobregion->length = 4;
1786 else
1787 oobregion->offset = mtd->oobsize - oobregion->offset;
1788
1789 return 0;
1790 }
1791
1792 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = {
1793 .ecc = sunxi_nand_ooblayout_ecc,
1794 .free = sunxi_nand_ooblayout_free,
1795 };
1796
1797 static int sunxi_nand_hw_common_ecc_ctrl_init(struct mtd_info *mtd,
1798 struct nand_ecc_ctrl *ecc,
1799 struct device_node *np)
1800 {
1801 static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 };
1802 struct nand_chip *nand = mtd_to_nand(mtd);
1803 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1804 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1805 struct sunxi_nand_hw_ecc *data;
1806 int nsectors;
1807 int ret;
1808 int i;
1809
1810 if (ecc->options & NAND_ECC_MAXIMIZE) {
1811 int bytes;
1812
1813 ecc->size = 1024;
1814 nsectors = mtd->writesize / ecc->size;
1815
1816 /* Reserve 2 bytes for the BBM */
1817 bytes = (mtd->oobsize - 2) / nsectors;
1818
1819 /* 4 non-ECC bytes are added before each ECC bytes section */
1820 bytes -= 4;
1821
1822 /* and bytes has to be even. */
1823 if (bytes % 2)
1824 bytes--;
1825
1826 ecc->strength = bytes * 8 / fls(8 * ecc->size);
1827
1828 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1829 if (strengths[i] > ecc->strength)
1830 break;
1831 }
1832
1833 if (!i)
1834 ecc->strength = 0;
1835 else
1836 ecc->strength = strengths[i - 1];
1837 }
1838
1839 if (ecc->size != 512 && ecc->size != 1024)
1840 return -EINVAL;
1841
1842 data = kzalloc(sizeof(*data), GFP_KERNEL);
1843 if (!data)
1844 return -ENOMEM;
1845
1846 /* Prefer 1k ECC chunk over 512 ones */
1847 if (ecc->size == 512 && mtd->writesize > 512) {
1848 ecc->size = 1024;
1849 ecc->strength *= 2;
1850 }
1851
1852 /* Add ECC info retrieval from DT */
1853 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1854 if (ecc->strength <= strengths[i])
1855 break;
1856 }
1857
1858 if (i >= ARRAY_SIZE(strengths)) {
1859 dev_err(nfc->dev, "unsupported strength\n");
1860 ret = -ENOTSUPP;
1861 goto err;
1862 }
1863
1864 data->mode = i;
1865
1866 /* HW ECC always request ECC bytes for 1024 bytes blocks */
1867 ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8);
1868
1869 /* HW ECC always work with even numbers of ECC bytes */
1870 ecc->bytes = ALIGN(ecc->bytes, 2);
1871
1872 nsectors = mtd->writesize / ecc->size;
1873
1874 if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) {
1875 ret = -EINVAL;
1876 goto err;
1877 }
1878
1879 ecc->read_oob = sunxi_nfc_hw_common_ecc_read_oob;
1880 ecc->write_oob = sunxi_nfc_hw_common_ecc_write_oob;
1881 mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops);
1882 ecc->priv = data;
1883
1884 return 0;
1885
1886 err:
1887 kfree(data);
1888
1889 return ret;
1890 }
1891
1892 static void sunxi_nand_hw_common_ecc_ctrl_cleanup(struct nand_ecc_ctrl *ecc)
1893 {
1894 kfree(ecc->priv);
1895 }
1896
1897 static int sunxi_nand_hw_ecc_ctrl_init(struct mtd_info *mtd,
1898 struct nand_ecc_ctrl *ecc,
1899 struct device_node *np)
1900 {
1901 struct nand_chip *nand = mtd_to_nand(mtd);
1902 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1903 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1904 int ret;
1905
1906 ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
1907 if (ret)
1908 return ret;
1909
1910 if (nfc->dmac) {
1911 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
1912 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
1913 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
1914 nand->options |= NAND_USE_BOUNCE_BUFFER;
1915 } else {
1916 ecc->read_page = sunxi_nfc_hw_ecc_read_page;
1917 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1918 ecc->write_page = sunxi_nfc_hw_ecc_write_page;
1919 }
1920
1921 /* TODO: support DMA for raw accesses and subpage write */
1922 ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage;
1923 ecc->read_oob_raw = nand_read_oob_std;
1924 ecc->write_oob_raw = nand_write_oob_std;
1925 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1926
1927 return 0;
1928 }
1929
1930 static int sunxi_nand_hw_syndrome_ecc_ctrl_init(struct mtd_info *mtd,
1931 struct nand_ecc_ctrl *ecc,
1932 struct device_node *np)
1933 {
1934 int ret;
1935
1936 ret = sunxi_nand_hw_common_ecc_ctrl_init(mtd, ecc, np);
1937 if (ret)
1938 return ret;
1939
1940 ecc->prepad = 4;
1941 ecc->read_page = sunxi_nfc_hw_syndrome_ecc_read_page;
1942 ecc->write_page = sunxi_nfc_hw_syndrome_ecc_write_page;
1943 ecc->read_oob_raw = nand_read_oob_syndrome;
1944 ecc->write_oob_raw = nand_write_oob_syndrome;
1945
1946 return 0;
1947 }
1948
1949 static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc)
1950 {
1951 switch (ecc->mode) {
1952 case NAND_ECC_HW:
1953 case NAND_ECC_HW_SYNDROME:
1954 sunxi_nand_hw_common_ecc_ctrl_cleanup(ecc);
1955 break;
1956 case NAND_ECC_NONE:
1957 default:
1958 break;
1959 }
1960 }
1961
1962 static int sunxi_nand_ecc_init(struct mtd_info *mtd, struct nand_ecc_ctrl *ecc,
1963 struct device_node *np)
1964 {
1965 struct nand_chip *nand = mtd_to_nand(mtd);
1966 int ret;
1967
1968 if (!ecc->size) {
1969 ecc->size = nand->ecc_step_ds;
1970 ecc->strength = nand->ecc_strength_ds;
1971 }
1972
1973 if (!ecc->size || !ecc->strength)
1974 return -EINVAL;
1975
1976 switch (ecc->mode) {
1977 case NAND_ECC_HW:
1978 ret = sunxi_nand_hw_ecc_ctrl_init(mtd, ecc, np);
1979 if (ret)
1980 return ret;
1981 break;
1982 case NAND_ECC_HW_SYNDROME:
1983 ret = sunxi_nand_hw_syndrome_ecc_ctrl_init(mtd, ecc, np);
1984 if (ret)
1985 return ret;
1986 break;
1987 case NAND_ECC_NONE:
1988 case NAND_ECC_SOFT:
1989 break;
1990 default:
1991 return -EINVAL;
1992 }
1993
1994 return 0;
1995 }
1996
1997 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
1998 struct device_node *np)
1999 {
2000 struct sunxi_nand_chip *chip;
2001 struct mtd_info *mtd;
2002 struct nand_chip *nand;
2003 int nsels;
2004 int ret;
2005 int i;
2006 u32 tmp;
2007
2008 if (!of_get_property(np, "reg", &nsels))
2009 return -EINVAL;
2010
2011 nsels /= sizeof(u32);
2012 if (!nsels) {
2013 dev_err(dev, "invalid reg property size\n");
2014 return -EINVAL;
2015 }
2016
2017 chip = devm_kzalloc(dev,
2018 sizeof(*chip) +
2019 (nsels * sizeof(struct sunxi_nand_chip_sel)),
2020 GFP_KERNEL);
2021 if (!chip) {
2022 dev_err(dev, "could not allocate chip\n");
2023 return -ENOMEM;
2024 }
2025
2026 chip->nsels = nsels;
2027 chip->selected = -1;
2028
2029 for (i = 0; i < nsels; i++) {
2030 ret = of_property_read_u32_index(np, "reg", i, &tmp);
2031 if (ret) {
2032 dev_err(dev, "could not retrieve reg property: %d\n",
2033 ret);
2034 return ret;
2035 }
2036
2037 if (tmp > NFC_MAX_CS) {
2038 dev_err(dev,
2039 "invalid reg value: %u (max CS = 7)\n",
2040 tmp);
2041 return -EINVAL;
2042 }
2043
2044 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
2045 dev_err(dev, "CS %d already assigned\n", tmp);
2046 return -EINVAL;
2047 }
2048
2049 chip->sels[i].cs = tmp;
2050
2051 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) &&
2052 tmp < 2) {
2053 chip->sels[i].rb.type = RB_NATIVE;
2054 chip->sels[i].rb.info.nativeid = tmp;
2055 } else {
2056 ret = of_get_named_gpio(np, "rb-gpios", i);
2057 if (ret >= 0) {
2058 tmp = ret;
2059 chip->sels[i].rb.type = RB_GPIO;
2060 chip->sels[i].rb.info.gpio = tmp;
2061 ret = devm_gpio_request(dev, tmp, "nand-rb");
2062 if (ret)
2063 return ret;
2064
2065 ret = gpio_direction_input(tmp);
2066 if (ret)
2067 return ret;
2068 } else {
2069 chip->sels[i].rb.type = RB_NONE;
2070 }
2071 }
2072 }
2073
2074 nand = &chip->nand;
2075 /* Default tR value specified in the ONFI spec (chapter 4.15.1) */
2076 nand->chip_delay = 200;
2077 nand->controller = &nfc->controller;
2078 /*
2079 * Set the ECC mode to the default value in case nothing is specified
2080 * in the DT.
2081 */
2082 nand->ecc.mode = NAND_ECC_HW;
2083 nand_set_flash_node(nand, np);
2084 nand->select_chip = sunxi_nfc_select_chip;
2085 nand->cmd_ctrl = sunxi_nfc_cmd_ctrl;
2086 nand->read_buf = sunxi_nfc_read_buf;
2087 nand->write_buf = sunxi_nfc_write_buf;
2088 nand->read_byte = sunxi_nfc_read_byte;
2089 nand->setup_data_interface = sunxi_nfc_setup_data_interface;
2090
2091 mtd = nand_to_mtd(nand);
2092 mtd->dev.parent = dev;
2093
2094 ret = nand_scan_ident(mtd, nsels, NULL);
2095 if (ret)
2096 return ret;
2097
2098 if (nand->bbt_options & NAND_BBT_USE_FLASH)
2099 nand->bbt_options |= NAND_BBT_NO_OOB;
2100
2101 if (nand->options & NAND_NEED_SCRAMBLING)
2102 nand->options |= NAND_NO_SUBPAGE_WRITE;
2103
2104 nand->options |= NAND_SUBPAGE_READ;
2105
2106 ret = sunxi_nand_ecc_init(mtd, &nand->ecc, np);
2107 if (ret) {
2108 dev_err(dev, "ECC init failed: %d\n", ret);
2109 return ret;
2110 }
2111
2112 ret = nand_scan_tail(mtd);
2113 if (ret) {
2114 dev_err(dev, "nand_scan_tail failed: %d\n", ret);
2115 return ret;
2116 }
2117
2118 ret = mtd_device_register(mtd, NULL, 0);
2119 if (ret) {
2120 dev_err(dev, "failed to register mtd device: %d\n", ret);
2121 nand_release(mtd);
2122 return ret;
2123 }
2124
2125 list_add_tail(&chip->node, &nfc->chips);
2126
2127 return 0;
2128 }
2129
2130 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)
2131 {
2132 struct device_node *np = dev->of_node;
2133 struct device_node *nand_np;
2134 int nchips = of_get_child_count(np);
2135 int ret;
2136
2137 if (nchips > 8) {
2138 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips);
2139 return -EINVAL;
2140 }
2141
2142 for_each_child_of_node(np, nand_np) {
2143 ret = sunxi_nand_chip_init(dev, nfc, nand_np);
2144 if (ret) {
2145 of_node_put(nand_np);
2146 return ret;
2147 }
2148 }
2149
2150 return 0;
2151 }
2152
2153 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)
2154 {
2155 struct sunxi_nand_chip *chip;
2156
2157 while (!list_empty(&nfc->chips)) {
2158 chip = list_first_entry(&nfc->chips, struct sunxi_nand_chip,
2159 node);
2160 nand_release(nand_to_mtd(&chip->nand));
2161 sunxi_nand_ecc_cleanup(&chip->nand.ecc);
2162 list_del(&chip->node);
2163 }
2164 }
2165
2166 static int sunxi_nfc_probe(struct platform_device *pdev)
2167 {
2168 struct device *dev = &pdev->dev;
2169 struct resource *r;
2170 struct sunxi_nfc *nfc;
2171 int irq;
2172 int ret;
2173
2174 nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
2175 if (!nfc)
2176 return -ENOMEM;
2177
2178 nfc->dev = dev;
2179 nand_hw_control_init(&nfc->controller);
2180 INIT_LIST_HEAD(&nfc->chips);
2181
2182 r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2183 nfc->regs = devm_ioremap_resource(dev, r);
2184 if (IS_ERR(nfc->regs))
2185 return PTR_ERR(nfc->regs);
2186
2187 irq = platform_get_irq(pdev, 0);
2188 if (irq < 0) {
2189 dev_err(dev, "failed to retrieve irq\n");
2190 return irq;
2191 }
2192
2193 nfc->ahb_clk = devm_clk_get(dev, "ahb");
2194 if (IS_ERR(nfc->ahb_clk)) {
2195 dev_err(dev, "failed to retrieve ahb clk\n");
2196 return PTR_ERR(nfc->ahb_clk);
2197 }
2198
2199 ret = clk_prepare_enable(nfc->ahb_clk);
2200 if (ret)
2201 return ret;
2202
2203 nfc->mod_clk = devm_clk_get(dev, "mod");
2204 if (IS_ERR(nfc->mod_clk)) {
2205 dev_err(dev, "failed to retrieve mod clk\n");
2206 ret = PTR_ERR(nfc->mod_clk);
2207 goto out_ahb_clk_unprepare;
2208 }
2209
2210 ret = clk_prepare_enable(nfc->mod_clk);
2211 if (ret)
2212 goto out_ahb_clk_unprepare;
2213
2214 nfc->reset = devm_reset_control_get_optional(dev, "ahb");
2215 if (!IS_ERR(nfc->reset)) {
2216 ret = reset_control_deassert(nfc->reset);
2217 if (ret) {
2218 dev_err(dev, "reset err %d\n", ret);
2219 goto out_mod_clk_unprepare;
2220 }
2221 } else if (PTR_ERR(nfc->reset) != -ENOENT) {
2222 ret = PTR_ERR(nfc->reset);
2223 goto out_mod_clk_unprepare;
2224 }
2225
2226 ret = sunxi_nfc_rst(nfc);
2227 if (ret)
2228 goto out_ahb_reset_reassert;
2229
2230 writel(0, nfc->regs + NFC_REG_INT);
2231 ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt,
2232 0, "sunxi-nand", nfc);
2233 if (ret)
2234 goto out_ahb_reset_reassert;
2235
2236 nfc->dmac = dma_request_slave_channel(dev, "rxtx");
2237 if (nfc->dmac) {
2238 struct dma_slave_config dmac_cfg = { };
2239
2240 dmac_cfg.src_addr = r->start + NFC_REG_IO_DATA;
2241 dmac_cfg.dst_addr = dmac_cfg.src_addr;
2242 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2243 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;
2244 dmac_cfg.src_maxburst = 4;
2245 dmac_cfg.dst_maxburst = 4;
2246 dmaengine_slave_config(nfc->dmac, &dmac_cfg);
2247 } else {
2248 dev_warn(dev, "failed to request rxtx DMA channel\n");
2249 }
2250
2251 platform_set_drvdata(pdev, nfc);
2252
2253 ret = sunxi_nand_chips_init(dev, nfc);
2254 if (ret) {
2255 dev_err(dev, "failed to init nand chips\n");
2256 goto out_release_dmac;
2257 }
2258
2259 return 0;
2260
2261 out_release_dmac:
2262 if (nfc->dmac)
2263 dma_release_channel(nfc->dmac);
2264 out_ahb_reset_reassert:
2265 if (!IS_ERR(nfc->reset))
2266 reset_control_assert(nfc->reset);
2267 out_mod_clk_unprepare:
2268 clk_disable_unprepare(nfc->mod_clk);
2269 out_ahb_clk_unprepare:
2270 clk_disable_unprepare(nfc->ahb_clk);
2271
2272 return ret;
2273 }
2274
2275 static int sunxi_nfc_remove(struct platform_device *pdev)
2276 {
2277 struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
2278
2279 sunxi_nand_chips_cleanup(nfc);
2280
2281 if (!IS_ERR(nfc->reset))
2282 reset_control_assert(nfc->reset);
2283
2284 if (nfc->dmac)
2285 dma_release_channel(nfc->dmac);
2286 clk_disable_unprepare(nfc->mod_clk);
2287 clk_disable_unprepare(nfc->ahb_clk);
2288
2289 return 0;
2290 }
2291
2292 static const struct of_device_id sunxi_nfc_ids[] = {
2293 { .compatible = "allwinner,sun4i-a10-nand" },
2294 { /* sentinel */ }
2295 };
2296 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids);
2297
2298 static struct platform_driver sunxi_nfc_driver = {
2299 .driver = {
2300 .name = "sunxi_nand",
2301 .of_match_table = sunxi_nfc_ids,
2302 },
2303 .probe = sunxi_nfc_probe,
2304 .remove = sunxi_nfc_remove,
2305 };
2306 module_platform_driver(sunxi_nfc_driver);
2307
2308 MODULE_LICENSE("GPL v2");
2309 MODULE_AUTHOR("Boris BREZILLON");
2310 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver");
2311 MODULE_ALIAS("platform:sunxi_nand");
Linux with added FPC-III support