?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Revert "mtd: rawnand: sunxi: Add A23/A33 DMA support"
[linux-2.6/linux-2.6-arm.git] / drivers / mtd / nand / raw / sunxi_nand.c
blobe93f39bc2bc55dd215fc12882965b1a22d332917
1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3 * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com>
4 *
5 * Derived from:
6 * https://github.com/yuq/sunxi-nfc-mtd
7 * Copyright (C) 2013 Qiang Yu <yuq825@gmail.com>
8 *
9 * https://github.com/hno/Allwinner-Info
10 * Copyright (C) 2013 Henrik Nordström <Henrik Nordström>
11 *
12 * Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com>
13 * Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org>
14 */
15
16 #include <linux/dma-mapping.h>
17 #include <linux/slab.h>
18 #include <linux/module.h>
19 #include <linux/moduleparam.h>
20 #include <linux/platform_device.h>
21 #include <linux/of.h>
22 #include <linux/of_device.h>
23 #include <linux/mtd/mtd.h>
24 #include <linux/mtd/rawnand.h>
25 #include <linux/mtd/partitions.h>
26 #include <linux/clk.h>
27 #include <linux/delay.h>
28 #include <linux/dmaengine.h>
29 #include <linux/interrupt.h>
30 #include <linux/iopoll.h>
31 #include <linux/reset.h>
32
33 #define NFC_REG_CTL 0x0000
34 #define NFC_REG_ST 0x0004
35 #define NFC_REG_INT 0x0008
36 #define NFC_REG_TIMING_CTL 0x000C
37 #define NFC_REG_TIMING_CFG 0x0010
38 #define NFC_REG_ADDR_LOW 0x0014
39 #define NFC_REG_ADDR_HIGH 0x0018
40 #define NFC_REG_SECTOR_NUM 0x001C
41 #define NFC_REG_CNT 0x0020
42 #define NFC_REG_CMD 0x0024
43 #define NFC_REG_RCMD_SET 0x0028
44 #define NFC_REG_WCMD_SET 0x002C
45 #define NFC_REG_A10_IO_DATA 0x0030
46 #define NFC_REG_ECC_CTL 0x0034
47 #define NFC_REG_ECC_ST 0x0038
48 #define NFC_REG_DEBUG 0x003C
49 #define NFC_REG_ECC_ERR_CNT(x) ((0x0040 + (x)) & ~0x3)
50 #define NFC_REG_USER_DATA(x) (0x0050 + ((x) * 4))
51 #define NFC_REG_SPARE_AREA 0x00A0
52 #define NFC_REG_PAT_ID 0x00A4
53 #define NFC_RAM0_BASE 0x0400
54 #define NFC_RAM1_BASE 0x0800
55
56 /* define bit use in NFC_CTL */
57 #define NFC_EN BIT(0)
58 #define NFC_RESET BIT(1)
59 #define NFC_BUS_WIDTH_MSK BIT(2)
60 #define NFC_BUS_WIDTH_8 (0 << 2)
61 #define NFC_BUS_WIDTH_16 (1 << 2)
62 #define NFC_RB_SEL_MSK BIT(3)
63 #define NFC_RB_SEL(x) ((x) << 3)
64 #define NFC_CE_SEL_MSK GENMASK(26, 24)
65 #define NFC_CE_SEL(x) ((x) << 24)
66 #define NFC_CE_CTL BIT(6)
67 #define NFC_PAGE_SHIFT_MSK GENMASK(11, 8)
68 #define NFC_PAGE_SHIFT(x) (((x) < 10 ? 0 : (x) - 10) << 8)
69 #define NFC_SAM BIT(12)
70 #define NFC_RAM_METHOD BIT(14)
71 #define NFC_DEBUG_CTL BIT(31)
72
73 /* define bit use in NFC_ST */
74 #define NFC_RB_B2R BIT(0)
75 #define NFC_CMD_INT_FLAG BIT(1)
76 #define NFC_DMA_INT_FLAG BIT(2)
77 #define NFC_CMD_FIFO_STATUS BIT(3)
78 #define NFC_STA BIT(4)
79 #define NFC_NATCH_INT_FLAG BIT(5)
80 #define NFC_RB_STATE(x) BIT(x + 8)
81
82 /* define bit use in NFC_INT */
83 #define NFC_B2R_INT_ENABLE BIT(0)
84 #define NFC_CMD_INT_ENABLE BIT(1)
85 #define NFC_DMA_INT_ENABLE BIT(2)
86 #define NFC_INT_MASK (NFC_B2R_INT_ENABLE | \
87 NFC_CMD_INT_ENABLE | \
88 NFC_DMA_INT_ENABLE)
89
90 /* define bit use in NFC_TIMING_CTL */
91 #define NFC_TIMING_CTL_EDO BIT(8)
92
93 /* define NFC_TIMING_CFG register layout */
94 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD) \
95 (((tWB) & 0x3) | (((tADL) & 0x3) << 2) | \
96 (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) | \
97 (((tCAD) & 0x7) << 8))
98
99 /* define bit use in NFC_CMD */
100 #define NFC_CMD_LOW_BYTE_MSK GENMASK(7, 0)
101 #define NFC_CMD_HIGH_BYTE_MSK GENMASK(15, 8)
102 #define NFC_CMD(x) (x)
103 #define NFC_ADR_NUM_MSK GENMASK(18, 16)
104 #define NFC_ADR_NUM(x) (((x) - 1) << 16)
105 #define NFC_SEND_ADR BIT(19)
106 #define NFC_ACCESS_DIR BIT(20)
107 #define NFC_DATA_TRANS BIT(21)
108 #define NFC_SEND_CMD1 BIT(22)
109 #define NFC_WAIT_FLAG BIT(23)
110 #define NFC_SEND_CMD2 BIT(24)
111 #define NFC_SEQ BIT(25)
112 #define NFC_DATA_SWAP_METHOD BIT(26)
113 #define NFC_ROW_AUTO_INC BIT(27)
114 #define NFC_SEND_CMD3 BIT(28)
115 #define NFC_SEND_CMD4 BIT(29)
116 #define NFC_CMD_TYPE_MSK GENMASK(31, 30)
117 #define NFC_NORMAL_OP (0 << 30)
118 #define NFC_ECC_OP (1 << 30)
119 #define NFC_PAGE_OP (2U << 30)
120
121 /* define bit use in NFC_RCMD_SET */
122 #define NFC_READ_CMD_MSK GENMASK(7, 0)
123 #define NFC_RND_READ_CMD0_MSK GENMASK(15, 8)
124 #define NFC_RND_READ_CMD1_MSK GENMASK(23, 16)
125
126 /* define bit use in NFC_WCMD_SET */
127 #define NFC_PROGRAM_CMD_MSK GENMASK(7, 0)
128 #define NFC_RND_WRITE_CMD_MSK GENMASK(15, 8)
129 #define NFC_READ_CMD0_MSK GENMASK(23, 16)
130 #define NFC_READ_CMD1_MSK GENMASK(31, 24)
131
132 /* define bit use in NFC_ECC_CTL */
133 #define NFC_ECC_EN BIT(0)
134 #define NFC_ECC_PIPELINE BIT(3)
135 #define NFC_ECC_EXCEPTION BIT(4)
136 #define NFC_ECC_BLOCK_SIZE_MSK BIT(5)
137 #define NFC_ECC_BLOCK_512 BIT(5)
138 #define NFC_RANDOM_EN BIT(9)
139 #define NFC_RANDOM_DIRECTION BIT(10)
140 #define NFC_ECC_MODE_MSK GENMASK(15, 12)
141 #define NFC_ECC_MODE(x) ((x) << 12)
142 #define NFC_RANDOM_SEED_MSK GENMASK(30, 16)
143 #define NFC_RANDOM_SEED(x) ((x) << 16)
144
145 /* define bit use in NFC_ECC_ST */
146 #define NFC_ECC_ERR(x) BIT(x)
147 #define NFC_ECC_ERR_MSK GENMASK(15, 0)
148 #define NFC_ECC_PAT_FOUND(x) BIT(x + 16)
149 #define NFC_ECC_ERR_CNT(b, x) (((x) >> (((b) % 4) * 8)) & 0xff)
150
151 #define NFC_DEFAULT_TIMEOUT_MS 1000
152
153 #define NFC_SRAM_SIZE 1024
154
155 #define NFC_MAX_CS 7
156
157 /**
158 * struct sunxi_nand_chip_sel - stores information related to NAND Chip Select
159 *
160 * @cs: the NAND CS id used to communicate with a NAND Chip
161 * @rb: the Ready/Busy pin ID. -1 means no R/B pin connected to the NFC
162 */
163 struct sunxi_nand_chip_sel {
164 u8 cs;
165 s8 rb;
166 };
167
168 /**
169 * struct sunxi_nand_hw_ecc - stores information related to HW ECC support
170 *
171 * @mode: the sunxi ECC mode field deduced from ECC requirements
172 */
173 struct sunxi_nand_hw_ecc {
174 int mode;
175 };
176
177 /**
178 * struct sunxi_nand_chip - stores NAND chip device related information
179 *
180 * @node: used to store NAND chips into a list
181 * @nand: base NAND chip structure
182 * @clk_rate: clk_rate required for this NAND chip
183 * @timing_cfg: TIMING_CFG register value for this NAND chip
184 * @timing_ctl: TIMING_CTL register value for this NAND chip
185 * @nsels: number of CS lines required by the NAND chip
186 * @sels: array of CS lines descriptions
187 */
188 struct sunxi_nand_chip {
189 struct list_head node;
190 struct nand_chip nand;
191 unsigned long clk_rate;
192 u32 timing_cfg;
193 u32 timing_ctl;
194 int nsels;
195 struct sunxi_nand_chip_sel sels[0];
196 };
197
198 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)
199 {
200 return container_of(nand, struct sunxi_nand_chip, nand);
201 }
202
203 /*
204 * NAND Controller capabilities structure: stores NAND controller capabilities
205 * for distinction between compatible strings.
206 *
207 * @reg_io_data: I/O data register
208 * @dma_maxburst: DMA maxburst
209 */
210 struct sunxi_nfc_caps {
211 unsigned int reg_io_data;
212 unsigned int dma_maxburst;
213 };
214
215 /**
216 * struct sunxi_nfc - stores sunxi NAND controller information
217 *
218 * @controller: base controller structure
219 * @dev: parent device (used to print error messages)
220 * @regs: NAND controller registers
221 * @ahb_clk: NAND controller AHB clock
222 * @mod_clk: NAND controller mod clock
223 * @reset: NAND controller reset line
224 * @assigned_cs: bitmask describing already assigned CS lines
225 * @clk_rate: NAND controller current clock rate
226 * @chips: a list containing all the NAND chips attached to this NAND
227 * controller
228 * @complete: a completion object used to wait for NAND controller events
229 * @dmac: the DMA channel attached to the NAND controller
230 */
231 struct sunxi_nfc {
232 struct nand_controller controller;
233 struct device *dev;
234 void __iomem *regs;
235 struct clk *ahb_clk;
236 struct clk *mod_clk;
237 struct reset_control *reset;
238 unsigned long assigned_cs;
239 unsigned long clk_rate;
240 struct list_head chips;
241 struct completion complete;
242 struct dma_chan *dmac;
243 const struct sunxi_nfc_caps *caps;
244 };
245
246 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_controller *ctrl)
247 {
248 return container_of(ctrl, struct sunxi_nfc, controller);
249 }
250
251 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id)
252 {
253 struct sunxi_nfc *nfc = dev_id;
254 u32 st = readl(nfc->regs + NFC_REG_ST);
255 u32 ien = readl(nfc->regs + NFC_REG_INT);
256
257 if (!(ien & st))
258 return IRQ_NONE;
259
260 if ((ien & st) == ien)
261 complete(&nfc->complete);
262
263 writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
264 writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT);
265
266 return IRQ_HANDLED;
267 }
268
269 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events,
270 bool use_polling, unsigned int timeout_ms)
271 {
272 int ret;
273
274 if (events & ~NFC_INT_MASK)
275 return -EINVAL;
276
277 if (!timeout_ms)
278 timeout_ms = NFC_DEFAULT_TIMEOUT_MS;
279
280 if (!use_polling) {
281 init_completion(&nfc->complete);
282
283 writel(events, nfc->regs + NFC_REG_INT);
284
285 ret = wait_for_completion_timeout(&nfc->complete,
286 msecs_to_jiffies(timeout_ms));
287 if (!ret)
288 ret = -ETIMEDOUT;
289 else
290 ret = 0;
291
292 writel(0, nfc->regs + NFC_REG_INT);
293 } else {
294 u32 status;
295
296 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
297 (status & events) == events, 1,
298 timeout_ms * 1000);
299 }
300
301 writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
302
303 if (ret)
304 dev_err(nfc->dev, "wait interrupt timedout\n");
305
306 return ret;
307 }
308
309 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc)
310 {
311 u32 status;
312 int ret;
313
314 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
315 !(status & NFC_CMD_FIFO_STATUS), 1,
316 NFC_DEFAULT_TIMEOUT_MS * 1000);
317 if (ret)
318 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n");
319
320 return ret;
321 }
322
323 static int sunxi_nfc_rst(struct sunxi_nfc *nfc)
324 {
325 u32 ctl;
326 int ret;
327
328 writel(0, nfc->regs + NFC_REG_ECC_CTL);
329 writel(NFC_RESET, nfc->regs + NFC_REG_CTL);
330
331 ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl,
332 !(ctl & NFC_RESET), 1,
333 NFC_DEFAULT_TIMEOUT_MS * 1000);
334 if (ret)
335 dev_err(nfc->dev, "wait for NAND controller reset timedout\n");
336
337 return ret;
338 }
339
340 static int sunxi_nfc_dma_op_prepare(struct sunxi_nfc *nfc, const void *buf,
341 int chunksize, int nchunks,
342 enum dma_data_direction ddir,
343 struct scatterlist *sg)
344 {
345 struct dma_async_tx_descriptor *dmad;
346 enum dma_transfer_direction tdir;
347 dma_cookie_t dmat;
348 int ret;
349
350 if (ddir == DMA_FROM_DEVICE)
351 tdir = DMA_DEV_TO_MEM;
352 else
353 tdir = DMA_MEM_TO_DEV;
354
355 sg_init_one(sg, buf, nchunks * chunksize);
356 ret = dma_map_sg(nfc->dev, sg, 1, ddir);
357 if (!ret)
358 return -ENOMEM;
359
360 dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);
361 if (!dmad) {
362 ret = -EINVAL;
363 goto err_unmap_buf;
364 }
365
366 writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,
367 nfc->regs + NFC_REG_CTL);
368 writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);
369 writel(chunksize, nfc->regs + NFC_REG_CNT);
370 dmat = dmaengine_submit(dmad);
371
372 ret = dma_submit_error(dmat);
373 if (ret)
374 goto err_clr_dma_flag;
375
376 return 0;
377
378 err_clr_dma_flag:
379 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
380 nfc->regs + NFC_REG_CTL);
381
382 err_unmap_buf:
383 dma_unmap_sg(nfc->dev, sg, 1, ddir);
384 return ret;
385 }
386
387 static void sunxi_nfc_dma_op_cleanup(struct sunxi_nfc *nfc,
388 enum dma_data_direction ddir,
389 struct scatterlist *sg)
390 {
391 dma_unmap_sg(nfc->dev, sg, 1, ddir);
392 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
393 nfc->regs + NFC_REG_CTL);
394 }
395
396 static void sunxi_nfc_select_chip(struct nand_chip *nand, unsigned int cs)
397 {
398 struct mtd_info *mtd = nand_to_mtd(nand);
399 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
400 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
401 struct sunxi_nand_chip_sel *sel;
402 u32 ctl;
403
404 if (cs > 0 && cs >= sunxi_nand->nsels)
405 return;
406
407 ctl = readl(nfc->regs + NFC_REG_CTL) &
408 ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN);
409
410 sel = &sunxi_nand->sels[cs];
411 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN | NFC_PAGE_SHIFT(nand->page_shift);
412 if (sel->rb >= 0)
413 ctl |= NFC_RB_SEL(sel->rb);
414
415 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA);
416
417 if (nfc->clk_rate != sunxi_nand->clk_rate) {
418 clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate);
419 nfc->clk_rate = sunxi_nand->clk_rate;
420 }
421
422 writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL);
423 writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG);
424 writel(ctl, nfc->regs + NFC_REG_CTL);
425 }
426
427 static void sunxi_nfc_read_buf(struct nand_chip *nand, uint8_t *buf, int len)
428 {
429 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
430 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
431 int ret;
432 int cnt;
433 int offs = 0;
434 u32 tmp;
435
436 while (len > offs) {
437 bool poll = false;
438
439 cnt = min(len - offs, NFC_SRAM_SIZE);
440
441 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
442 if (ret)
443 break;
444
445 writel(cnt, nfc->regs + NFC_REG_CNT);
446 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
447 writel(tmp, nfc->regs + NFC_REG_CMD);
448
449 /* Arbitrary limit for polling mode */
450 if (cnt < 64)
451 poll = true;
452
453 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
454 if (ret)
455 break;
456
457 if (buf)
458 memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE,
459 cnt);
460 offs += cnt;
461 }
462 }
463
464 static void sunxi_nfc_write_buf(struct nand_chip *nand, const uint8_t *buf,
465 int len)
466 {
467 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
468 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
469 int ret;
470 int cnt;
471 int offs = 0;
472 u32 tmp;
473
474 while (len > offs) {
475 bool poll = false;
476
477 cnt = min(len - offs, NFC_SRAM_SIZE);
478
479 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
480 if (ret)
481 break;
482
483 writel(cnt, nfc->regs + NFC_REG_CNT);
484 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt);
485 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
486 NFC_ACCESS_DIR;
487 writel(tmp, nfc->regs + NFC_REG_CMD);
488
489 /* Arbitrary limit for polling mode */
490 if (cnt < 64)
491 poll = true;
492
493 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
494 if (ret)
495 break;
496
497 offs += cnt;
498 }
499 }
500
501 /* These seed values have been extracted from Allwinner's BSP */
502 static const u16 sunxi_nfc_randomizer_page_seeds[] = {
503 0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72,
504 0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436,
505 0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d,
506 0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130,
507 0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56,
508 0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55,
509 0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb,
510 0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17,
511 0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62,
512 0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064,
513 0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126,
514 0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e,
515 0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3,
516 0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b,
517 0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d,
518 0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db,
519 };
520
521 /*
522 * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds
523 * have been generated using
524 * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what
525 * the randomizer engine does internally before de/scrambling OOB data.
526 *
527 * Those tables are statically defined to avoid calculating randomizer state
528 * at runtime.
529 */
530 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = {
531 0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64,
532 0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409,
533 0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617,
534 0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d,
535 0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91,
536 0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d,
537 0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab,
538 0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8,
539 0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8,
540 0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b,
541 0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5,
542 0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a,
543 0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891,
544 0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36,
545 0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd,
546 0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0,
547 };
548
549 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = {
550 0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6,
551 0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982,
552 0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9,
553 0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07,
554 0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e,
555 0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2,
556 0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c,
557 0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f,
558 0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc,
559 0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e,
560 0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8,
561 0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68,
562 0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d,
563 0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179,
564 0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601,
565 0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd,
566 };
567
568 static u16 sunxi_nfc_randomizer_step(u16 state, int count)
569 {
570 state &= 0x7fff;
571
572 /*
573 * This loop is just a simple implementation of a Fibonacci LFSR using
574 * the x16 + x15 + 1 polynomial.
575 */
576 while (count--)
577 state = ((state >> 1) |
578 (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff;
579
580 return state;
581 }
582
583 static u16 sunxi_nfc_randomizer_state(struct nand_chip *nand, int page,
584 bool ecc)
585 {
586 struct mtd_info *mtd = nand_to_mtd(nand);
587 const u16 *seeds = sunxi_nfc_randomizer_page_seeds;
588 int mod = mtd_div_by_ws(mtd->erasesize, mtd);
589
590 if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds))
591 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds);
592
593 if (ecc) {
594 if (mtd->ecc_step_size == 512)
595 seeds = sunxi_nfc_randomizer_ecc512_seeds;
596 else
597 seeds = sunxi_nfc_randomizer_ecc1024_seeds;
598 }
599
600 return seeds[page % mod];
601 }
602
603 static void sunxi_nfc_randomizer_config(struct nand_chip *nand, int page,
604 bool ecc)
605 {
606 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
607 u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
608 u16 state;
609
610 if (!(nand->options & NAND_NEED_SCRAMBLING))
611 return;
612
613 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
614 state = sunxi_nfc_randomizer_state(nand, page, ecc);
615 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK;
616 writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL);
617 }
618
619 static void sunxi_nfc_randomizer_enable(struct nand_chip *nand)
620 {
621 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
622
623 if (!(nand->options & NAND_NEED_SCRAMBLING))
624 return;
625
626 writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN,
627 nfc->regs + NFC_REG_ECC_CTL);
628 }
629
630 static void sunxi_nfc_randomizer_disable(struct nand_chip *nand)
631 {
632 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
633
634 if (!(nand->options & NAND_NEED_SCRAMBLING))
635 return;
636
637 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN,
638 nfc->regs + NFC_REG_ECC_CTL);
639 }
640
641 static void sunxi_nfc_randomize_bbm(struct nand_chip *nand, int page, u8 *bbm)
642 {
643 u16 state = sunxi_nfc_randomizer_state(nand, page, true);
644
645 bbm[0] ^= state;
646 bbm[1] ^= sunxi_nfc_randomizer_step(state, 8);
647 }
648
649 static void sunxi_nfc_randomizer_write_buf(struct nand_chip *nand,
650 const uint8_t *buf, int len,
651 bool ecc, int page)
652 {
653 sunxi_nfc_randomizer_config(nand, page, ecc);
654 sunxi_nfc_randomizer_enable(nand);
655 sunxi_nfc_write_buf(nand, buf, len);
656 sunxi_nfc_randomizer_disable(nand);
657 }
658
659 static void sunxi_nfc_randomizer_read_buf(struct nand_chip *nand, uint8_t *buf,
660 int len, bool ecc, int page)
661 {
662 sunxi_nfc_randomizer_config(nand, page, ecc);
663 sunxi_nfc_randomizer_enable(nand);
664 sunxi_nfc_read_buf(nand, buf, len);
665 sunxi_nfc_randomizer_disable(nand);
666 }
667
668 static void sunxi_nfc_hw_ecc_enable(struct nand_chip *nand)
669 {
670 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
671 struct sunxi_nand_hw_ecc *data = nand->ecc.priv;
672 u32 ecc_ctl;
673
674 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
675 ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE |
676 NFC_ECC_BLOCK_SIZE_MSK);
677 ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(data->mode) | NFC_ECC_EXCEPTION |
678 NFC_ECC_PIPELINE;
679
680 if (nand->ecc.size == 512)
681 ecc_ctl |= NFC_ECC_BLOCK_512;
682
683 writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL);
684 }
685
686 static void sunxi_nfc_hw_ecc_disable(struct nand_chip *nand)
687 {
688 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
689
690 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN,
691 nfc->regs + NFC_REG_ECC_CTL);
692 }
693
694 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf)
695 {
696 buf[0] = user_data;
697 buf[1] = user_data >> 8;
698 buf[2] = user_data >> 16;
699 buf[3] = user_data >> 24;
700 }
701
702 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf)
703 {
704 return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24);
705 }
706
707 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct nand_chip *nand, u8 *oob,
708 int step, bool bbm, int page)
709 {
710 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
711
712 sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)),
713 oob);
714
715 /* De-randomize the Bad Block Marker. */
716 if (bbm && (nand->options & NAND_NEED_SCRAMBLING))
717 sunxi_nfc_randomize_bbm(nand, page, oob);
718 }
719
720 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct nand_chip *nand,
721 const u8 *oob, int step,
722 bool bbm, int page)
723 {
724 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
725 u8 user_data[4];
726
727 /* Randomize the Bad Block Marker. */
728 if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
729 memcpy(user_data, oob, sizeof(user_data));
730 sunxi_nfc_randomize_bbm(nand, page, user_data);
731 oob = user_data;
732 }
733
734 writel(sunxi_nfc_buf_to_user_data(oob),
735 nfc->regs + NFC_REG_USER_DATA(step));
736 }
737
738 static void sunxi_nfc_hw_ecc_update_stats(struct nand_chip *nand,
739 unsigned int *max_bitflips, int ret)
740 {
741 struct mtd_info *mtd = nand_to_mtd(nand);
742
743 if (ret < 0) {
744 mtd->ecc_stats.failed++;
745 } else {
746 mtd->ecc_stats.corrected += ret;
747 *max_bitflips = max_t(unsigned int, *max_bitflips, ret);
748 }
749 }
750
751 static int sunxi_nfc_hw_ecc_correct(struct nand_chip *nand, u8 *data, u8 *oob,
752 int step, u32 status, bool *erased)
753 {
754 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
755 struct nand_ecc_ctrl *ecc = &nand->ecc;
756 u32 tmp;
757
758 *erased = false;
759
760 if (status & NFC_ECC_ERR(step))
761 return -EBADMSG;
762
763 if (status & NFC_ECC_PAT_FOUND(step)) {
764 u8 pattern;
765
766 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) {
767 pattern = 0x0;
768 } else {
769 pattern = 0xff;
770 *erased = true;
771 }
772
773 if (data)
774 memset(data, pattern, ecc->size);
775
776 if (oob)
777 memset(oob, pattern, ecc->bytes + 4);
778
779 return 0;
780 }
781
782 tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step));
783
784 return NFC_ECC_ERR_CNT(step, tmp);
785 }
786
787 static int sunxi_nfc_hw_ecc_read_chunk(struct nand_chip *nand,
788 u8 *data, int data_off,
789 u8 *oob, int oob_off,
790 int *cur_off,
791 unsigned int *max_bitflips,
792 bool bbm, bool oob_required, int page)
793 {
794 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
795 struct nand_ecc_ctrl *ecc = &nand->ecc;
796 int raw_mode = 0;
797 bool erased;
798 int ret;
799
800 if (*cur_off != data_off)
801 nand_change_read_column_op(nand, data_off, NULL, 0, false);
802
803 sunxi_nfc_randomizer_read_buf(nand, NULL, ecc->size, false, page);
804
805 if (data_off + ecc->size != oob_off)
806 nand_change_read_column_op(nand, oob_off, NULL, 0, false);
807
808 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
809 if (ret)
810 return ret;
811
812 sunxi_nfc_randomizer_enable(nand);
813 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP,
814 nfc->regs + NFC_REG_CMD);
815
816 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
817 sunxi_nfc_randomizer_disable(nand);
818 if (ret)
819 return ret;
820
821 *cur_off = oob_off + ecc->bytes + 4;
822
823 ret = sunxi_nfc_hw_ecc_correct(nand, data, oob_required ? oob : NULL, 0,
824 readl(nfc->regs + NFC_REG_ECC_ST),
825 &erased);
826 if (erased)
827 return 1;
828
829 if (ret < 0) {
830 /*
831 * Re-read the data with the randomizer disabled to identify
832 * bitflips in erased pages.
833 */
834 if (nand->options & NAND_NEED_SCRAMBLING)
835 nand_change_read_column_op(nand, data_off, data,
836 ecc->size, false);
837 else
838 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,
839 ecc->size);
840
841 nand_change_read_column_op(nand, oob_off, oob, ecc->bytes + 4,
842 false);
843
844 ret = nand_check_erased_ecc_chunk(data, ecc->size,
845 oob, ecc->bytes + 4,
846 NULL, 0, ecc->strength);
847 if (ret >= 0)
848 raw_mode = 1;
849 } else {
850 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);
851
852 if (oob_required) {
853 nand_change_read_column_op(nand, oob_off, NULL, 0,
854 false);
855 sunxi_nfc_randomizer_read_buf(nand, oob, ecc->bytes + 4,
856 true, page);
857
858 sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, 0,
859 bbm, page);
860 }
861 }
862
863 sunxi_nfc_hw_ecc_update_stats(nand, max_bitflips, ret);
864
865 return raw_mode;
866 }
867
868 static void sunxi_nfc_hw_ecc_read_extra_oob(struct nand_chip *nand,
869 u8 *oob, int *cur_off,
870 bool randomize, int page)
871 {
872 struct mtd_info *mtd = nand_to_mtd(nand);
873 struct nand_ecc_ctrl *ecc = &nand->ecc;
874 int offset = ((ecc->bytes + 4) * ecc->steps);
875 int len = mtd->oobsize - offset;
876
877 if (len <= 0)
878 return;
879
880 if (!cur_off || *cur_off != offset)
881 nand_change_read_column_op(nand, mtd->writesize, NULL, 0,
882 false);
883
884 if (!randomize)
885 sunxi_nfc_read_buf(nand, oob + offset, len);
886 else
887 sunxi_nfc_randomizer_read_buf(nand, oob + offset, len,
888 false, page);
889
890 if (cur_off)
891 *cur_off = mtd->oobsize + mtd->writesize;
892 }
893
894 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, uint8_t *buf,
895 int oob_required, int page,
896 int nchunks)
897 {
898 bool randomized = nand->options & NAND_NEED_SCRAMBLING;
899 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
900 struct mtd_info *mtd = nand_to_mtd(nand);
901 struct nand_ecc_ctrl *ecc = &nand->ecc;
902 unsigned int max_bitflips = 0;
903 int ret, i, raw_mode = 0;
904 struct scatterlist sg;
905 u32 status;
906
907 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
908 if (ret)
909 return ret;
910
911 ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, nchunks,
912 DMA_FROM_DEVICE, &sg);
913 if (ret)
914 return ret;
915
916 sunxi_nfc_hw_ecc_enable(nand);
917 sunxi_nfc_randomizer_config(nand, page, false);
918 sunxi_nfc_randomizer_enable(nand);
919
920 writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |
921 NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);
922
923 dma_async_issue_pending(nfc->dmac);
924
925 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,
926 nfc->regs + NFC_REG_CMD);
927
928 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
929 if (ret)
930 dmaengine_terminate_all(nfc->dmac);
931
932 sunxi_nfc_randomizer_disable(nand);
933 sunxi_nfc_hw_ecc_disable(nand);
934
935 sunxi_nfc_dma_op_cleanup(nfc, DMA_FROM_DEVICE, &sg);
936
937 if (ret)
938 return ret;
939
940 status = readl(nfc->regs + NFC_REG_ECC_ST);
941
942 for (i = 0; i < nchunks; i++) {
943 int data_off = i * ecc->size;
944 int oob_off = i * (ecc->bytes + 4);
945 u8 *data = buf + data_off;
946 u8 *oob = nand->oob_poi + oob_off;
947 bool erased;
948
949 ret = sunxi_nfc_hw_ecc_correct(nand, randomized ? data : NULL,
950 oob_required ? oob : NULL,
951 i, status, &erased);
952
953 /* ECC errors are handled in the second loop. */
954 if (ret < 0)
955 continue;
956
957 if (oob_required && !erased) {
958 /* TODO: use DMA to retrieve OOB */
959 nand_change_read_column_op(nand,
960 mtd->writesize + oob_off,
961 oob, ecc->bytes + 4, false);
962
963 sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, i,
964 !i, page);
965 }
966
967 if (erased)
968 raw_mode = 1;
969
970 sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, ret);
971 }
972
973 if (status & NFC_ECC_ERR_MSK) {
974 for (i = 0; i < nchunks; i++) {
975 int data_off = i * ecc->size;
976 int oob_off = i * (ecc->bytes + 4);
977 u8 *data = buf + data_off;
978 u8 *oob = nand->oob_poi + oob_off;
979
980 if (!(status & NFC_ECC_ERR(i)))
981 continue;
982
983 /*
984 * Re-read the data with the randomizer disabled to
985 * identify bitflips in erased pages.
986 * TODO: use DMA to read page in raw mode
987 */
988 if (randomized)
989 nand_change_read_column_op(nand, data_off,
990 data, ecc->size,
991 false);
992
993 /* TODO: use DMA to retrieve OOB */
994 nand_change_read_column_op(nand,
995 mtd->writesize + oob_off,
996 oob, ecc->bytes + 4, false);
997
998 ret = nand_check_erased_ecc_chunk(data, ecc->size,
999 oob, ecc->bytes + 4,
1000 NULL, 0,
1001 ecc->strength);
1002 if (ret >= 0)
1003 raw_mode = 1;
1004
1005 sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, ret);
1006 }
1007 }
1008
1009 if (oob_required)
1010 sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi,
1011 NULL, !raw_mode,
1012 page);
1013
1014 return max_bitflips;
1015 }
1016
1017 static int sunxi_nfc_hw_ecc_write_chunk(struct nand_chip *nand,
1018 const u8 *data, int data_off,
1019 const u8 *oob, int oob_off,
1020 int *cur_off, bool bbm,
1021 int page)
1022 {
1023 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1024 struct nand_ecc_ctrl *ecc = &nand->ecc;
1025 int ret;
1026
1027 if (data_off != *cur_off)
1028 nand_change_write_column_op(nand, data_off, NULL, 0, false);
1029
1030 sunxi_nfc_randomizer_write_buf(nand, data, ecc->size, false, page);
1031
1032 if (data_off + ecc->size != oob_off)
1033 nand_change_write_column_op(nand, oob_off, NULL, 0, false);
1034
1035 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1036 if (ret)
1037 return ret;
1038
1039 sunxi_nfc_randomizer_enable(nand);
1040 sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, 0, bbm, page);
1041
1042 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
1043 NFC_ACCESS_DIR | NFC_ECC_OP,
1044 nfc->regs + NFC_REG_CMD);
1045
1046 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1047 sunxi_nfc_randomizer_disable(nand);
1048 if (ret)
1049 return ret;
1050
1051 *cur_off = oob_off + ecc->bytes + 4;
1052
1053 return 0;
1054 }
1055
1056 static void sunxi_nfc_hw_ecc_write_extra_oob(struct nand_chip *nand,
1057 u8 *oob, int *cur_off,
1058 int page)
1059 {
1060 struct mtd_info *mtd = nand_to_mtd(nand);
1061 struct nand_ecc_ctrl *ecc = &nand->ecc;
1062 int offset = ((ecc->bytes + 4) * ecc->steps);
1063 int len = mtd->oobsize - offset;
1064
1065 if (len <= 0)
1066 return;
1067
1068 if (!cur_off || *cur_off != offset)
1069 nand_change_write_column_op(nand, offset + mtd->writesize,
1070 NULL, 0, false);
1071
1072 sunxi_nfc_randomizer_write_buf(nand, oob + offset, len, false, page);
1073
1074 if (cur_off)
1075 *cur_off = mtd->oobsize + mtd->writesize;
1076 }
1077
1078 static int sunxi_nfc_hw_ecc_read_page(struct nand_chip *nand, uint8_t *buf,
1079 int oob_required, int page)
1080 {
1081 struct mtd_info *mtd = nand_to_mtd(nand);
1082 struct nand_ecc_ctrl *ecc = &nand->ecc;
1083 unsigned int max_bitflips = 0;
1084 int ret, i, cur_off = 0;
1085 bool raw_mode = false;
1086
1087 sunxi_nfc_select_chip(nand, nand->cur_cs);
1088
1089 nand_read_page_op(nand, page, 0, NULL, 0);
1090
1091 sunxi_nfc_hw_ecc_enable(nand);
1092
1093 for (i = 0; i < ecc->steps; i++) {
1094 int data_off = i * ecc->size;
1095 int oob_off = i * (ecc->bytes + 4);
1096 u8 *data = buf + data_off;
1097 u8 *oob = nand->oob_poi + oob_off;
1098
1099 ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off, oob,
1100 oob_off + mtd->writesize,
1101 &cur_off, &max_bitflips,
1102 !i, oob_required, page);
1103 if (ret < 0)
1104 return ret;
1105 else if (ret)
1106 raw_mode = true;
1107 }
1108
1109 if (oob_required)
1110 sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi, &cur_off,
1111 !raw_mode, page);
1112
1113 sunxi_nfc_hw_ecc_disable(nand);
1114
1115 return max_bitflips;
1116 }
1117
1118 static int sunxi_nfc_hw_ecc_read_page_dma(struct nand_chip *nand, u8 *buf,
1119 int oob_required, int page)
1120 {
1121 int ret;
1122
1123 sunxi_nfc_select_chip(nand, nand->cur_cs);
1124
1125 nand_read_page_op(nand, page, 0, NULL, 0);
1126
1127 ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, oob_required, page,
1128 nand->ecc.steps);
1129 if (ret >= 0)
1130 return ret;
1131
1132 /* Fallback to PIO mode */
1133 return sunxi_nfc_hw_ecc_read_page(nand, buf, oob_required, page);
1134 }
1135
1136 static int sunxi_nfc_hw_ecc_read_subpage(struct nand_chip *nand,
1137 u32 data_offs, u32 readlen,
1138 u8 *bufpoi, int page)
1139 {
1140 struct mtd_info *mtd = nand_to_mtd(nand);
1141 struct nand_ecc_ctrl *ecc = &nand->ecc;
1142 int ret, i, cur_off = 0;
1143 unsigned int max_bitflips = 0;
1144
1145 sunxi_nfc_select_chip(nand, nand->cur_cs);
1146
1147 nand_read_page_op(nand, page, 0, NULL, 0);
1148
1149 sunxi_nfc_hw_ecc_enable(nand);
1150
1151 for (i = data_offs / ecc->size;
1152 i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) {
1153 int data_off = i * ecc->size;
1154 int oob_off = i * (ecc->bytes + 4);
1155 u8 *data = bufpoi + data_off;
1156 u8 *oob = nand->oob_poi + oob_off;
1157
1158 ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off,
1159 oob,
1160 oob_off + mtd->writesize,
1161 &cur_off, &max_bitflips, !i,
1162 false, page);
1163 if (ret < 0)
1164 return ret;
1165 }
1166
1167 sunxi_nfc_hw_ecc_disable(nand);
1168
1169 return max_bitflips;
1170 }
1171
1172 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct nand_chip *nand,
1173 u32 data_offs, u32 readlen,
1174 u8 *buf, int page)
1175 {
1176 int nchunks = DIV_ROUND_UP(data_offs + readlen, nand->ecc.size);
1177 int ret;
1178
1179 sunxi_nfc_select_chip(nand, nand->cur_cs);
1180
1181 nand_read_page_op(nand, page, 0, NULL, 0);
1182
1183 ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, false, page, nchunks);
1184 if (ret >= 0)
1185 return ret;
1186
1187 /* Fallback to PIO mode */
1188 return sunxi_nfc_hw_ecc_read_subpage(nand, data_offs, readlen,
1189 buf, page);
1190 }
1191
1192 static int sunxi_nfc_hw_ecc_write_page(struct nand_chip *nand,
1193 const uint8_t *buf, int oob_required,
1194 int page)
1195 {
1196 struct mtd_info *mtd = nand_to_mtd(nand);
1197 struct nand_ecc_ctrl *ecc = &nand->ecc;
1198 int ret, i, cur_off = 0;
1199
1200 sunxi_nfc_select_chip(nand, nand->cur_cs);
1201
1202 nand_prog_page_begin_op(nand, page, 0, NULL, 0);
1203
1204 sunxi_nfc_hw_ecc_enable(nand);
1205
1206 for (i = 0; i < ecc->steps; i++) {
1207 int data_off = i * ecc->size;
1208 int oob_off = i * (ecc->bytes + 4);
1209 const u8 *data = buf + data_off;
1210 const u8 *oob = nand->oob_poi + oob_off;
1211
1212 ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob,
1213 oob_off + mtd->writesize,
1214 &cur_off, !i, page);
1215 if (ret)
1216 return ret;
1217 }
1218
1219 if (oob_required || (nand->options & NAND_NEED_SCRAMBLING))
1220 sunxi_nfc_hw_ecc_write_extra_oob(nand, nand->oob_poi,
1221 &cur_off, page);
1222
1223 sunxi_nfc_hw_ecc_disable(nand);
1224
1225 return nand_prog_page_end_op(nand);
1226 }
1227
1228 static int sunxi_nfc_hw_ecc_write_subpage(struct nand_chip *nand,
1229 u32 data_offs, u32 data_len,
1230 const u8 *buf, int oob_required,
1231 int page)
1232 {
1233 struct mtd_info *mtd = nand_to_mtd(nand);
1234 struct nand_ecc_ctrl *ecc = &nand->ecc;
1235 int ret, i, cur_off = 0;
1236
1237 sunxi_nfc_select_chip(nand, nand->cur_cs);
1238
1239 nand_prog_page_begin_op(nand, page, 0, NULL, 0);
1240
1241 sunxi_nfc_hw_ecc_enable(nand);
1242
1243 for (i = data_offs / ecc->size;
1244 i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) {
1245 int data_off = i * ecc->size;
1246 int oob_off = i * (ecc->bytes + 4);
1247 const u8 *data = buf + data_off;
1248 const u8 *oob = nand->oob_poi + oob_off;
1249
1250 ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob,
1251 oob_off + mtd->writesize,
1252 &cur_off, !i, page);
1253 if (ret)
1254 return ret;
1255 }
1256
1257 sunxi_nfc_hw_ecc_disable(nand);
1258
1259 return nand_prog_page_end_op(nand);
1260 }
1261
1262 static int sunxi_nfc_hw_ecc_write_page_dma(struct nand_chip *nand,
1263 const u8 *buf,
1264 int oob_required,
1265 int page)
1266 {
1267 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1268 struct nand_ecc_ctrl *ecc = &nand->ecc;
1269 struct scatterlist sg;
1270 int ret, i;
1271
1272 sunxi_nfc_select_chip(nand, nand->cur_cs);
1273
1274 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1275 if (ret)
1276 return ret;
1277
1278 ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, ecc->steps,
1279 DMA_TO_DEVICE, &sg);
1280 if (ret)
1281 goto pio_fallback;
1282
1283 for (i = 0; i < ecc->steps; i++) {
1284 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));
1285
1286 sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, i, !i, page);
1287 }
1288
1289 nand_prog_page_begin_op(nand, page, 0, NULL, 0);
1290
1291 sunxi_nfc_hw_ecc_enable(nand);
1292 sunxi_nfc_randomizer_config(nand, page, false);
1293 sunxi_nfc_randomizer_enable(nand);
1294
1295 writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
1296 nfc->regs + NFC_REG_WCMD_SET);
1297
1298 dma_async_issue_pending(nfc->dmac);
1299
1300 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
1301 NFC_DATA_TRANS | NFC_ACCESS_DIR,
1302 nfc->regs + NFC_REG_CMD);
1303
1304 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1305 if (ret)
1306 dmaengine_terminate_all(nfc->dmac);
1307
1308 sunxi_nfc_randomizer_disable(nand);
1309 sunxi_nfc_hw_ecc_disable(nand);
1310
1311 sunxi_nfc_dma_op_cleanup(nfc, DMA_TO_DEVICE, &sg);
1312
1313 if (ret)
1314 return ret;
1315
1316 if (oob_required || (nand->options & NAND_NEED_SCRAMBLING))
1317 /* TODO: use DMA to transfer extra OOB bytes ? */
1318 sunxi_nfc_hw_ecc_write_extra_oob(nand, nand->oob_poi,
1319 NULL, page);
1320
1321 return nand_prog_page_end_op(nand);
1322
1323 pio_fallback:
1324 return sunxi_nfc_hw_ecc_write_page(nand, buf, oob_required, page);
1325 }
1326
1327 static int sunxi_nfc_hw_ecc_read_oob(struct nand_chip *nand, int page)
1328 {
1329 u8 *buf = nand_get_data_buf(nand);
1330
1331 return nand->ecc.read_page(nand, buf, 1, page);
1332 }
1333
1334 static int sunxi_nfc_hw_ecc_write_oob(struct nand_chip *nand, int page)
1335 {
1336 struct mtd_info *mtd = nand_to_mtd(nand);
1337 u8 *buf = nand_get_data_buf(nand);
1338 int ret;
1339
1340 memset(buf, 0xff, mtd->writesize);
1341 ret = nand->ecc.write_page(nand, buf, 1, page);
1342 if (ret)
1343 return ret;
1344
1345 /* Send command to program the OOB data */
1346 return nand_prog_page_end_op(nand);
1347 }
1348
1349 static const s32 tWB_lut[] = {6, 12, 16, 20};
1350 static const s32 tRHW_lut[] = {4, 8, 12, 20};
1351
1352 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
1353 u32 clk_period)
1354 {
1355 u32 clk_cycles = DIV_ROUND_UP(duration, clk_period);
1356 int i;
1357
1358 for (i = 0; i < lut_size; i++) {
1359 if (clk_cycles <= lut[i])
1360 return i;
1361 }
1362
1363 /* Doesn't fit */
1364 return -EINVAL;
1365 }
1366
1367 #define sunxi_nand_lookup_timing(l, p, c) \
1368 _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)
1369
1370 static int sunxi_nfc_setup_data_interface(struct nand_chip *nand, int csline,
1371 const struct nand_data_interface *conf)
1372 {
1373 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1374 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1375 const struct nand_sdr_timings *timings;
1376 u32 min_clk_period = 0;
1377 s32 tWB, tADL, tWHR, tRHW, tCAD;
1378 long real_clk_rate;
1379
1380 timings = nand_get_sdr_timings(conf);
1381 if (IS_ERR(timings))
1382 return -ENOTSUPP;
1383
1384 /* T1 <=> tCLS */
1385 if (timings->tCLS_min > min_clk_period)
1386 min_clk_period = timings->tCLS_min;
1387
1388 /* T2 <=> tCLH */
1389 if (timings->tCLH_min > min_clk_period)
1390 min_clk_period = timings->tCLH_min;
1391
1392 /* T3 <=> tCS */
1393 if (timings->tCS_min > min_clk_period)
1394 min_clk_period = timings->tCS_min;
1395
1396 /* T4 <=> tCH */
1397 if (timings->tCH_min > min_clk_period)
1398 min_clk_period = timings->tCH_min;
1399
1400 /* T5 <=> tWP */
1401 if (timings->tWP_min > min_clk_period)
1402 min_clk_period = timings->tWP_min;
1403
1404 /* T6 <=> tWH */
1405 if (timings->tWH_min > min_clk_period)
1406 min_clk_period = timings->tWH_min;
1407
1408 /* T7 <=> tALS */
1409 if (timings->tALS_min > min_clk_period)
1410 min_clk_period = timings->tALS_min;
1411
1412 /* T8 <=> tDS */
1413 if (timings->tDS_min > min_clk_period)
1414 min_clk_period = timings->tDS_min;
1415
1416 /* T9 <=> tDH */
1417 if (timings->tDH_min > min_clk_period)
1418 min_clk_period = timings->tDH_min;
1419
1420 /* T10 <=> tRR */
1421 if (timings->tRR_min > (min_clk_period * 3))
1422 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);
1423
1424 /* T11 <=> tALH */
1425 if (timings->tALH_min > min_clk_period)
1426 min_clk_period = timings->tALH_min;
1427
1428 /* T12 <=> tRP */
1429 if (timings->tRP_min > min_clk_period)
1430 min_clk_period = timings->tRP_min;
1431
1432 /* T13 <=> tREH */
1433 if (timings->tREH_min > min_clk_period)
1434 min_clk_period = timings->tREH_min;
1435
1436 /* T14 <=> tRC */
1437 if (timings->tRC_min > (min_clk_period * 2))
1438 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2);
1439
1440 /* T15 <=> tWC */
1441 if (timings->tWC_min > (min_clk_period * 2))
1442 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);
1443
1444 /* T16 - T19 + tCAD */
1445 if (timings->tWB_max > (min_clk_period * 20))
1446 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);
1447
1448 if (timings->tADL_min > (min_clk_period * 32))
1449 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
1450
1451 if (timings->tWHR_min > (min_clk_period * 32))
1452 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
1453
1454 if (timings->tRHW_min > (min_clk_period * 20))
1455 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);
1456
1457 /*
1458 * In non-EDO, tREA should be less than tRP to guarantee that the
1459 * controller does not sample the IO lines too early. Unfortunately,
1460 * the sunxi NAND controller does not allow us to have different
1461 * values for tRP and tREH (tRP = tREH = tRW / 2).
1462 *
1463 * We have 2 options to overcome this limitation:
1464 *
1465 * 1/ Extend tRC to fulfil the tREA <= tRC / 2 constraint
1466 * 2/ Use EDO mode (only works if timings->tRLOH > 0)
1467 */
1468 if (timings->tREA_max > min_clk_period && !timings->tRLOH_min)
1469 min_clk_period = timings->tREA_max;
1470
1471 tWB = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,
1472 min_clk_period);
1473 if (tWB < 0) {
1474 dev_err(nfc->dev, "unsupported tWB\n");
1475 return tWB;
1476 }
1477
1478 tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;
1479 if (tADL > 3) {
1480 dev_err(nfc->dev, "unsupported tADL\n");
1481 return -EINVAL;
1482 }
1483
1484 tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;
1485 if (tWHR > 3) {
1486 dev_err(nfc->dev, "unsupported tWHR\n");
1487 return -EINVAL;
1488 }
1489
1490 tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,
1491 min_clk_period);
1492 if (tRHW < 0) {
1493 dev_err(nfc->dev, "unsupported tRHW\n");
1494 return tRHW;
1495 }
1496
1497 if (csline == NAND_DATA_IFACE_CHECK_ONLY)
1498 return 0;
1499
1500 /*
1501 * TODO: according to ONFI specs this value only applies for DDR NAND,
1502 * but Allwinner seems to set this to 0x7. Mimic them for now.
1503 */
1504 tCAD = 0x7;
1505
1506 /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */
1507 sunxi_nand->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD);
1508
1509 /* Convert min_clk_period from picoseconds to nanoseconds */
1510 min_clk_period = DIV_ROUND_UP(min_clk_period, 1000);
1511
1512 /*
1513 * Unlike what is stated in Allwinner datasheet, the clk_rate should
1514 * be set to (1 / min_clk_period), and not (2 / min_clk_period).
1515 * This new formula was verified with a scope and validated by
1516 * Allwinner engineers.
1517 */
1518 sunxi_nand->clk_rate = NSEC_PER_SEC / min_clk_period;
1519 real_clk_rate = clk_round_rate(nfc->mod_clk, sunxi_nand->clk_rate);
1520 if (real_clk_rate <= 0) {
1521 dev_err(nfc->dev, "Unable to round clk %lu\n",
1522 sunxi_nand->clk_rate);
1523 return -EINVAL;
1524 }
1525
1526 sunxi_nand->timing_ctl = 0;
1527
1528 /*
1529 * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data
1530 * output cycle timings shall be used if the host drives tRC less than
1531 * 30 ns. We should also use EDO mode if tREA is bigger than tRP.
1532 */
1533 min_clk_period = NSEC_PER_SEC / real_clk_rate;
1534 if (min_clk_period * 2 < 30 || min_clk_period * 1000 < timings->tREA_max)
1535 sunxi_nand->timing_ctl = NFC_TIMING_CTL_EDO;
1536
1537 return 0;
1538 }
1539
1540 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1541 struct mtd_oob_region *oobregion)
1542 {
1543 struct nand_chip *nand = mtd_to_nand(mtd);
1544 struct nand_ecc_ctrl *ecc = &nand->ecc;
1545
1546 if (section >= ecc->steps)
1547 return -ERANGE;
1548
1549 oobregion->offset = section * (ecc->bytes + 4) + 4;
1550 oobregion->length = ecc->bytes;
1551
1552 return 0;
1553 }
1554
1555 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
1556 struct mtd_oob_region *oobregion)
1557 {
1558 struct nand_chip *nand = mtd_to_nand(mtd);
1559 struct nand_ecc_ctrl *ecc = &nand->ecc;
1560
1561 if (section > ecc->steps)
1562 return -ERANGE;
1563
1564 /*
1565 * The first 2 bytes are used for BB markers, hence we
1566 * only have 2 bytes available in the first user data
1567 * section.
1568 */
1569 if (!section && ecc->mode == NAND_ECC_HW) {
1570 oobregion->offset = 2;
1571 oobregion->length = 2;
1572
1573 return 0;
1574 }
1575
1576 oobregion->offset = section * (ecc->bytes + 4);
1577
1578 if (section < ecc->steps)
1579 oobregion->length = 4;
1580 else
1581 oobregion->offset = mtd->oobsize - oobregion->offset;
1582
1583 return 0;
1584 }
1585
1586 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = {
1587 .ecc = sunxi_nand_ooblayout_ecc,
1588 .free = sunxi_nand_ooblayout_free,
1589 };
1590
1591 static void sunxi_nand_hw_ecc_ctrl_cleanup(struct nand_ecc_ctrl *ecc)
1592 {
1593 kfree(ecc->priv);
1594 }
1595
1596 static int sunxi_nand_hw_ecc_ctrl_init(struct nand_chip *nand,
1597 struct nand_ecc_ctrl *ecc,
1598 struct device_node *np)
1599 {
1600 static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 };
1601 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1602 struct mtd_info *mtd = nand_to_mtd(nand);
1603 struct sunxi_nand_hw_ecc *data;
1604 int nsectors;
1605 int ret;
1606 int i;
1607
1608 if (ecc->options & NAND_ECC_MAXIMIZE) {
1609 int bytes;
1610
1611 ecc->size = 1024;
1612 nsectors = mtd->writesize / ecc->size;
1613
1614 /* Reserve 2 bytes for the BBM */
1615 bytes = (mtd->oobsize - 2) / nsectors;
1616
1617 /* 4 non-ECC bytes are added before each ECC bytes section */
1618 bytes -= 4;
1619
1620 /* and bytes has to be even. */
1621 if (bytes % 2)
1622 bytes--;
1623
1624 ecc->strength = bytes * 8 / fls(8 * ecc->size);
1625
1626 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1627 if (strengths[i] > ecc->strength)
1628 break;
1629 }
1630
1631 if (!i)
1632 ecc->strength = 0;
1633 else
1634 ecc->strength = strengths[i - 1];
1635 }
1636
1637 if (ecc->size != 512 && ecc->size != 1024)
1638 return -EINVAL;
1639
1640 data = kzalloc(sizeof(*data), GFP_KERNEL);
1641 if (!data)
1642 return -ENOMEM;
1643
1644 /* Prefer 1k ECC chunk over 512 ones */
1645 if (ecc->size == 512 && mtd->writesize > 512) {
1646 ecc->size = 1024;
1647 ecc->strength *= 2;
1648 }
1649
1650 /* Add ECC info retrieval from DT */
1651 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1652 if (ecc->strength <= strengths[i]) {
1653 /*
1654 * Update ecc->strength value with the actual strength
1655 * that will be used by the ECC engine.
1656 */
1657 ecc->strength = strengths[i];
1658 break;
1659 }
1660 }
1661
1662 if (i >= ARRAY_SIZE(strengths)) {
1663 dev_err(nfc->dev, "unsupported strength\n");
1664 ret = -ENOTSUPP;
1665 goto err;
1666 }
1667
1668 data->mode = i;
1669
1670 /* HW ECC always request ECC bytes for 1024 bytes blocks */
1671 ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8);
1672
1673 /* HW ECC always work with even numbers of ECC bytes */
1674 ecc->bytes = ALIGN(ecc->bytes, 2);
1675
1676 nsectors = mtd->writesize / ecc->size;
1677
1678 if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) {
1679 ret = -EINVAL;
1680 goto err;
1681 }
1682
1683 ecc->read_oob = sunxi_nfc_hw_ecc_read_oob;
1684 ecc->write_oob = sunxi_nfc_hw_ecc_write_oob;
1685 mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops);
1686 ecc->priv = data;
1687
1688 if (nfc->dmac) {
1689 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
1690 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
1691 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
1692 nand->options |= NAND_USE_BOUNCE_BUFFER;
1693 } else {
1694 ecc->read_page = sunxi_nfc_hw_ecc_read_page;
1695 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1696 ecc->write_page = sunxi_nfc_hw_ecc_write_page;
1697 }
1698
1699 /* TODO: support DMA for raw accesses and subpage write */
1700 ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage;
1701 ecc->read_oob_raw = nand_read_oob_std;
1702 ecc->write_oob_raw = nand_write_oob_std;
1703
1704 return 0;
1705
1706 err:
1707 kfree(data);
1708
1709 return ret;
1710 }
1711
1712 static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc)
1713 {
1714 switch (ecc->mode) {
1715 case NAND_ECC_HW:
1716 sunxi_nand_hw_ecc_ctrl_cleanup(ecc);
1717 break;
1718 case NAND_ECC_NONE:
1719 default:
1720 break;
1721 }
1722 }
1723
1724 static int sunxi_nand_attach_chip(struct nand_chip *nand)
1725 {
1726 struct nand_ecc_ctrl *ecc = &nand->ecc;
1727 struct device_node *np = nand_get_flash_node(nand);
1728 int ret;
1729
1730 if (nand->bbt_options & NAND_BBT_USE_FLASH)
1731 nand->bbt_options |= NAND_BBT_NO_OOB;
1732
1733 if (nand->options & NAND_NEED_SCRAMBLING)
1734 nand->options |= NAND_NO_SUBPAGE_WRITE;
1735
1736 nand->options |= NAND_SUBPAGE_READ;
1737
1738 if (!ecc->size) {
1739 ecc->size = nand->base.eccreq.step_size;
1740 ecc->strength = nand->base.eccreq.strength;
1741 }
1742
1743 if (!ecc->size || !ecc->strength)
1744 return -EINVAL;
1745
1746 switch (ecc->mode) {
1747 case NAND_ECC_HW:
1748 ret = sunxi_nand_hw_ecc_ctrl_init(nand, ecc, np);
1749 if (ret)
1750 return ret;
1751 break;
1752 case NAND_ECC_NONE:
1753 case NAND_ECC_SOFT:
1754 break;
1755 default:
1756 return -EINVAL;
1757 }
1758
1759 return 0;
1760 }
1761
1762 static int sunxi_nfc_exec_subop(struct nand_chip *nand,
1763 const struct nand_subop *subop)
1764 {
1765 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1766 u32 cmd = 0, extcmd = 0, cnt = 0, addrs[2] = { };
1767 unsigned int i, j, remaining, start;
1768 void *inbuf = NULL;
1769 int ret;
1770
1771 for (i = 0; i < subop->ninstrs; i++) {
1772 const struct nand_op_instr *instr = &subop->instrs[i];
1773
1774 switch (instr->type) {
1775 case NAND_OP_CMD_INSTR:
1776 if (cmd & NFC_SEND_CMD1) {
1777 if (WARN_ON(cmd & NFC_SEND_CMD2))
1778 return -EINVAL;
1779
1780 cmd |= NFC_SEND_CMD2;
1781 extcmd |= instr->ctx.cmd.opcode;
1782 } else {
1783 cmd |= NFC_SEND_CMD1 |
1784 NFC_CMD(instr->ctx.cmd.opcode);
1785 }
1786 break;
1787
1788 case NAND_OP_ADDR_INSTR:
1789 remaining = nand_subop_get_num_addr_cyc(subop, i);
1790 start = nand_subop_get_addr_start_off(subop, i);
1791 for (j = 0; j < 8 && j + start < remaining; j++) {
1792 u32 addr = instr->ctx.addr.addrs[j + start];
1793
1794 addrs[j / 4] |= addr << (j % 4) * 8;
1795 }
1796
1797 if (j)
1798 cmd |= NFC_SEND_ADR | NFC_ADR_NUM(j);
1799
1800 break;
1801
1802 case NAND_OP_DATA_IN_INSTR:
1803 case NAND_OP_DATA_OUT_INSTR:
1804 start = nand_subop_get_data_start_off(subop, i);
1805 remaining = nand_subop_get_data_len(subop, i);
1806 cnt = min_t(u32, remaining, NFC_SRAM_SIZE);
1807 cmd |= NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
1808
1809 if (instr->type == NAND_OP_DATA_OUT_INSTR) {
1810 cmd |= NFC_ACCESS_DIR;
1811 memcpy_toio(nfc->regs + NFC_RAM0_BASE,
1812 instr->ctx.data.buf.out + start,
1813 cnt);
1814 } else {
1815 inbuf = instr->ctx.data.buf.in + start;
1816 }
1817
1818 break;
1819
1820 case NAND_OP_WAITRDY_INSTR:
1821 cmd |= NFC_WAIT_FLAG;
1822 break;
1823 }
1824 }
1825
1826 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1827 if (ret)
1828 return ret;
1829
1830 if (cmd & NFC_SEND_ADR) {
1831 writel(addrs[0], nfc->regs + NFC_REG_ADDR_LOW);
1832 writel(addrs[1], nfc->regs + NFC_REG_ADDR_HIGH);
1833 }
1834
1835 if (cmd & NFC_SEND_CMD2)
1836 writel(extcmd,
1837 nfc->regs +
1838 (cmd & NFC_ACCESS_DIR ?
1839 NFC_REG_WCMD_SET : NFC_REG_RCMD_SET));
1840
1841 if (cmd & NFC_DATA_TRANS)
1842 writel(cnt, nfc->regs + NFC_REG_CNT);
1843
1844 writel(cmd, nfc->regs + NFC_REG_CMD);
1845
1846 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG,
1847 !(cmd & NFC_WAIT_FLAG) && cnt < 64,
1848 0);
1849 if (ret)
1850 return ret;
1851
1852 if (inbuf)
1853 memcpy_fromio(inbuf, nfc->regs + NFC_RAM0_BASE, cnt);
1854
1855 return 0;
1856 }
1857
1858 static int sunxi_nfc_soft_waitrdy(struct nand_chip *nand,
1859 const struct nand_subop *subop)
1860 {
1861 return nand_soft_waitrdy(nand,
1862 subop->instrs[0].ctx.waitrdy.timeout_ms);
1863 }
1864
1865 static const struct nand_op_parser sunxi_nfc_op_parser = NAND_OP_PARSER(
1866 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1867 NAND_OP_PARSER_PAT_CMD_ELEM(true),
1868 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1869 NAND_OP_PARSER_PAT_CMD_ELEM(true),
1870 NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
1871 NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)),
1872 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1873 NAND_OP_PARSER_PAT_CMD_ELEM(true),
1874 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1875 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024),
1876 NAND_OP_PARSER_PAT_CMD_ELEM(true),
1877 NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)),
1878 );
1879
1880 static const struct nand_op_parser sunxi_nfc_norb_op_parser = NAND_OP_PARSER(
1881 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1882 NAND_OP_PARSER_PAT_CMD_ELEM(true),
1883 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1884 NAND_OP_PARSER_PAT_CMD_ELEM(true),
1885 NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)),
1886 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1887 NAND_OP_PARSER_PAT_CMD_ELEM(true),
1888 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1889 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024),
1890 NAND_OP_PARSER_PAT_CMD_ELEM(true)),
1891 NAND_OP_PARSER_PATTERN(sunxi_nfc_soft_waitrdy,
1892 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
1893 );
1894
1895 static int sunxi_nfc_exec_op(struct nand_chip *nand,
1896 const struct nand_operation *op, bool check_only)
1897 {
1898 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1899 const struct nand_op_parser *parser;
1900
1901 sunxi_nfc_select_chip(nand, op->cs);
1902
1903 if (sunxi_nand->sels[op->cs].rb >= 0)
1904 parser = &sunxi_nfc_op_parser;
1905 else
1906 parser = &sunxi_nfc_norb_op_parser;
1907
1908 return nand_op_parser_exec_op(nand, parser, op, check_only);
1909 }
1910
1911 static const struct nand_controller_ops sunxi_nand_controller_ops = {
1912 .attach_chip = sunxi_nand_attach_chip,
1913 .setup_data_interface = sunxi_nfc_setup_data_interface,
1914 .exec_op = sunxi_nfc_exec_op,
1915 };
1916
1917 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
1918 struct device_node *np)
1919 {
1920 struct sunxi_nand_chip *sunxi_nand;
1921 struct mtd_info *mtd;
1922 struct nand_chip *nand;
1923 int nsels;
1924 int ret;
1925 int i;
1926 u32 tmp;
1927
1928 if (!of_get_property(np, "reg", &nsels))
1929 return -EINVAL;
1930
1931 nsels /= sizeof(u32);
1932 if (!nsels) {
1933 dev_err(dev, "invalid reg property size\n");
1934 return -EINVAL;
1935 }
1936
1937 sunxi_nand = devm_kzalloc(dev, struct_size(sunxi_nand, sels, nsels),
1938 GFP_KERNEL);
1939 if (!sunxi_nand) {
1940 dev_err(dev, "could not allocate chip\n");
1941 return -ENOMEM;
1942 }
1943
1944 sunxi_nand->nsels = nsels;
1945
1946 for (i = 0; i < nsels; i++) {
1947 ret = of_property_read_u32_index(np, "reg", i, &tmp);
1948 if (ret) {
1949 dev_err(dev, "could not retrieve reg property: %d\n",
1950 ret);
1951 return ret;
1952 }
1953
1954 if (tmp > NFC_MAX_CS) {
1955 dev_err(dev,
1956 "invalid reg value: %u (max CS = 7)\n",
1957 tmp);
1958 return -EINVAL;
1959 }
1960
1961 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
1962 dev_err(dev, "CS %d already assigned\n", tmp);
1963 return -EINVAL;
1964 }
1965
1966 sunxi_nand->sels[i].cs = tmp;
1967
1968 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) &&
1969 tmp < 2)
1970 sunxi_nand->sels[i].rb = tmp;
1971 else
1972 sunxi_nand->sels[i].rb = -1;
1973 }
1974
1975 nand = &sunxi_nand->nand;
1976 /* Default tR value specified in the ONFI spec (chapter 4.15.1) */
1977 nand->controller = &nfc->controller;
1978 nand->controller->ops = &sunxi_nand_controller_ops;
1979
1980 /*
1981 * Set the ECC mode to the default value in case nothing is specified
1982 * in the DT.
1983 */
1984 nand->ecc.mode = NAND_ECC_HW;
1985 nand_set_flash_node(nand, np);
1986
1987 mtd = nand_to_mtd(nand);
1988 mtd->dev.parent = dev;
1989
1990 ret = nand_scan(nand, nsels);
1991 if (ret)
1992 return ret;
1993
1994 ret = mtd_device_register(mtd, NULL, 0);
1995 if (ret) {
1996 dev_err(dev, "failed to register mtd device: %d\n", ret);
1997 nand_release(nand);
1998 return ret;
1999 }
2000
2001 list_add_tail(&sunxi_nand->node, &nfc->chips);
2002
2003 return 0;
2004 }
2005
2006 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)
2007 {
2008 struct device_node *np = dev->of_node;
2009 struct device_node *nand_np;
2010 int nchips = of_get_child_count(np);
2011 int ret;
2012
2013 if (nchips > 8) {
2014 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips);
2015 return -EINVAL;
2016 }
2017
2018 for_each_child_of_node(np, nand_np) {
2019 ret = sunxi_nand_chip_init(dev, nfc, nand_np);
2020 if (ret) {
2021 of_node_put(nand_np);
2022 return ret;
2023 }
2024 }
2025
2026 return 0;
2027 }
2028
2029 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)
2030 {
2031 struct sunxi_nand_chip *sunxi_nand;
2032
2033 while (!list_empty(&nfc->chips)) {
2034 sunxi_nand = list_first_entry(&nfc->chips,
2035 struct sunxi_nand_chip,
2036 node);
2037 nand_release(&sunxi_nand->nand);
2038 sunxi_nand_ecc_cleanup(&sunxi_nand->nand.ecc);
2039 list_del(&sunxi_nand->node);
2040 }
2041 }
2042
2043 static int sunxi_nfc_probe(struct platform_device *pdev)
2044 {
2045 struct device *dev = &pdev->dev;
2046 struct resource *r;
2047 struct sunxi_nfc *nfc;
2048 int irq;
2049 int ret;
2050
2051 nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
2052 if (!nfc)
2053 return -ENOMEM;
2054
2055 nfc->dev = dev;
2056 nand_controller_init(&nfc->controller);
2057 INIT_LIST_HEAD(&nfc->chips);
2058
2059 r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2060 nfc->regs = devm_ioremap_resource(dev, r);
2061 if (IS_ERR(nfc->regs))
2062 return PTR_ERR(nfc->regs);
2063
2064 irq = platform_get_irq(pdev, 0);
2065 if (irq < 0) {
2066 dev_err(dev, "failed to retrieve irq\n");
2067 return irq;
2068 }
2069
2070 nfc->ahb_clk = devm_clk_get(dev, "ahb");
2071 if (IS_ERR(nfc->ahb_clk)) {
2072 dev_err(dev, "failed to retrieve ahb clk\n");
2073 return PTR_ERR(nfc->ahb_clk);
2074 }
2075
2076 ret = clk_prepare_enable(nfc->ahb_clk);
2077 if (ret)
2078 return ret;
2079
2080 nfc->mod_clk = devm_clk_get(dev, "mod");
2081 if (IS_ERR(nfc->mod_clk)) {
2082 dev_err(dev, "failed to retrieve mod clk\n");
2083 ret = PTR_ERR(nfc->mod_clk);
2084 goto out_ahb_clk_unprepare;
2085 }
2086
2087 ret = clk_prepare_enable(nfc->mod_clk);
2088 if (ret)
2089 goto out_ahb_clk_unprepare;
2090
2091 nfc->reset = devm_reset_control_get_optional_exclusive(dev, "ahb");
2092 if (IS_ERR(nfc->reset)) {
2093 ret = PTR_ERR(nfc->reset);
2094 goto out_mod_clk_unprepare;
2095 }
2096
2097 ret = reset_control_deassert(nfc->reset);
2098 if (ret) {
2099 dev_err(dev, "reset err %d\n", ret);
2100 goto out_mod_clk_unprepare;
2101 }
2102
2103 nfc->caps = of_device_get_match_data(&pdev->dev);
2104 if (!nfc->caps) {
2105 ret = -EINVAL;
2106 goto out_ahb_reset_reassert;
2107 }
2108
2109 ret = sunxi_nfc_rst(nfc);
2110 if (ret)
2111 goto out_ahb_reset_reassert;
2112
2113 writel(0, nfc->regs + NFC_REG_INT);
2114 ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt,
2115 0, "sunxi-nand", nfc);
2116 if (ret)
2117 goto out_ahb_reset_reassert;
2118
2119 nfc->dmac = dma_request_slave_channel(dev, "rxtx");
2120 if (nfc->dmac) {
2121 struct dma_slave_config dmac_cfg = { };
2122
2123 dmac_cfg.src_addr = r->start + nfc->caps->reg_io_data;
2124 dmac_cfg.dst_addr = dmac_cfg.src_addr;
2125 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2126 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;
2127 dmac_cfg.src_maxburst = nfc->caps->dma_maxburst;
2128 dmac_cfg.dst_maxburst = nfc->caps->dma_maxburst;
2129 dmaengine_slave_config(nfc->dmac, &dmac_cfg);
2130 } else {
2131 dev_warn(dev, "failed to request rxtx DMA channel\n");
2132 }
2133
2134 platform_set_drvdata(pdev, nfc);
2135
2136 ret = sunxi_nand_chips_init(dev, nfc);
2137 if (ret) {
2138 dev_err(dev, "failed to init nand chips\n");
2139 goto out_release_dmac;
2140 }
2141
2142 return 0;
2143
2144 out_release_dmac:
2145 if (nfc->dmac)
2146 dma_release_channel(nfc->dmac);
2147 out_ahb_reset_reassert:
2148 reset_control_assert(nfc->reset);
2149 out_mod_clk_unprepare:
2150 clk_disable_unprepare(nfc->mod_clk);
2151 out_ahb_clk_unprepare:
2152 clk_disable_unprepare(nfc->ahb_clk);
2153
2154 return ret;
2155 }
2156
2157 static int sunxi_nfc_remove(struct platform_device *pdev)
2158 {
2159 struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
2160
2161 sunxi_nand_chips_cleanup(nfc);
2162
2163 reset_control_assert(nfc->reset);
2164
2165 if (nfc->dmac)
2166 dma_release_channel(nfc->dmac);
2167 clk_disable_unprepare(nfc->mod_clk);
2168 clk_disable_unprepare(nfc->ahb_clk);
2169
2170 return 0;
2171 }
2172
2173 static const struct sunxi_nfc_caps sunxi_nfc_a10_caps = {
2174 .reg_io_data = NFC_REG_A10_IO_DATA,
2175 .dma_maxburst = 4,
2176 };
2177
2178 static const struct of_device_id sunxi_nfc_ids[] = {
2179 {
2180 .compatible = "allwinner,sun4i-a10-nand",
2181 .data = &sunxi_nfc_a10_caps,
2182 },
2183 { /* sentinel */ }
2184 };
2185 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids);
2186
2187 static struct platform_driver sunxi_nfc_driver = {
2188 .driver = {
2189 .name = "sunxi_nand",
2190 .of_match_table = sunxi_nfc_ids,
2191 },
2192 .probe = sunxi_nfc_probe,
2193 .remove = sunxi_nfc_remove,
2194 };
2195 module_platform_driver(sunxi_nfc_driver);
2196
2197 MODULE_LICENSE("GPL");
2198 MODULE_AUTHOR("Boris BREZILLON");
2199 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver");
2200 MODULE_ALIAS("platform:sunxi_nand");
Mirror of linux-2.6-arm at arm.linux.org.uk