?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
mtd: rawnand: Get rid of chip->ecc_{strength,step}_ds
[linux/fpc-iii.git] / drivers / mtd / nand / raw / sunxi_nand.c
blob13f03324d36a48177be7c7720a9c101da19fdc18
1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3 * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com>
4 *
5 * Derived from:
6 * https://github.com/yuq/sunxi-nfc-mtd
7 * Copyright (C) 2013 Qiang Yu <yuq825@gmail.com>
8 *
9 * https://github.com/hno/Allwinner-Info
10 * Copyright (C) 2013 Henrik Nordström <Henrik Nordström>
11 *
12 * Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com>
13 * Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org>
14 */
15
16 #include <linux/dma-mapping.h>
17 #include <linux/slab.h>
18 #include <linux/module.h>
19 #include <linux/moduleparam.h>
20 #include <linux/platform_device.h>
21 #include <linux/of.h>
22 #include <linux/of_device.h>
23 #include <linux/mtd/mtd.h>
24 #include <linux/mtd/rawnand.h>
25 #include <linux/mtd/partitions.h>
26 #include <linux/clk.h>
27 #include <linux/delay.h>
28 #include <linux/dmaengine.h>
29 #include <linux/interrupt.h>
30 #include <linux/iopoll.h>
31 #include <linux/reset.h>
32
33 #define NFC_REG_CTL 0x0000
34 #define NFC_REG_ST 0x0004
35 #define NFC_REG_INT 0x0008
36 #define NFC_REG_TIMING_CTL 0x000C
37 #define NFC_REG_TIMING_CFG 0x0010
38 #define NFC_REG_ADDR_LOW 0x0014
39 #define NFC_REG_ADDR_HIGH 0x0018
40 #define NFC_REG_SECTOR_NUM 0x001C
41 #define NFC_REG_CNT 0x0020
42 #define NFC_REG_CMD 0x0024
43 #define NFC_REG_RCMD_SET 0x0028
44 #define NFC_REG_WCMD_SET 0x002C
45 #define NFC_REG_IO_DATA 0x0030
46 #define NFC_REG_ECC_CTL 0x0034
47 #define NFC_REG_ECC_ST 0x0038
48 #define NFC_REG_DEBUG 0x003C
49 #define NFC_REG_ECC_ERR_CNT(x) ((0x0040 + (x)) & ~0x3)
50 #define NFC_REG_USER_DATA(x) (0x0050 + ((x) * 4))
51 #define NFC_REG_SPARE_AREA 0x00A0
52 #define NFC_REG_PAT_ID 0x00A4
53 #define NFC_RAM0_BASE 0x0400
54 #define NFC_RAM1_BASE 0x0800
55
56 /* define bit use in NFC_CTL */
57 #define NFC_EN BIT(0)
58 #define NFC_RESET BIT(1)
59 #define NFC_BUS_WIDTH_MSK BIT(2)
60 #define NFC_BUS_WIDTH_8 (0 << 2)
61 #define NFC_BUS_WIDTH_16 (1 << 2)
62 #define NFC_RB_SEL_MSK BIT(3)
63 #define NFC_RB_SEL(x) ((x) << 3)
64 #define NFC_CE_SEL_MSK GENMASK(26, 24)
65 #define NFC_CE_SEL(x) ((x) << 24)
66 #define NFC_CE_CTL BIT(6)
67 #define NFC_PAGE_SHIFT_MSK GENMASK(11, 8)
68 #define NFC_PAGE_SHIFT(x) (((x) < 10 ? 0 : (x) - 10) << 8)
69 #define NFC_SAM BIT(12)
70 #define NFC_RAM_METHOD BIT(14)
71 #define NFC_DEBUG_CTL BIT(31)
72
73 /* define bit use in NFC_ST */
74 #define NFC_RB_B2R BIT(0)
75 #define NFC_CMD_INT_FLAG BIT(1)
76 #define NFC_DMA_INT_FLAG BIT(2)
77 #define NFC_CMD_FIFO_STATUS BIT(3)
78 #define NFC_STA BIT(4)
79 #define NFC_NATCH_INT_FLAG BIT(5)
80 #define NFC_RB_STATE(x) BIT(x + 8)
81
82 /* define bit use in NFC_INT */
83 #define NFC_B2R_INT_ENABLE BIT(0)
84 #define NFC_CMD_INT_ENABLE BIT(1)
85 #define NFC_DMA_INT_ENABLE BIT(2)
86 #define NFC_INT_MASK (NFC_B2R_INT_ENABLE | \
87 NFC_CMD_INT_ENABLE | \
88 NFC_DMA_INT_ENABLE)
89
90 /* define bit use in NFC_TIMING_CTL */
91 #define NFC_TIMING_CTL_EDO BIT(8)
92
93 /* define NFC_TIMING_CFG register layout */
94 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD) \
95 (((tWB) & 0x3) | (((tADL) & 0x3) << 2) | \
96 (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) | \
97 (((tCAD) & 0x7) << 8))
98
99 /* define bit use in NFC_CMD */
100 #define NFC_CMD_LOW_BYTE_MSK GENMASK(7, 0)
101 #define NFC_CMD_HIGH_BYTE_MSK GENMASK(15, 8)
102 #define NFC_CMD(x) (x)
103 #define NFC_ADR_NUM_MSK GENMASK(18, 16)
104 #define NFC_ADR_NUM(x) (((x) - 1) << 16)
105 #define NFC_SEND_ADR BIT(19)
106 #define NFC_ACCESS_DIR BIT(20)
107 #define NFC_DATA_TRANS BIT(21)
108 #define NFC_SEND_CMD1 BIT(22)
109 #define NFC_WAIT_FLAG BIT(23)
110 #define NFC_SEND_CMD2 BIT(24)
111 #define NFC_SEQ BIT(25)
112 #define NFC_DATA_SWAP_METHOD BIT(26)
113 #define NFC_ROW_AUTO_INC BIT(27)
114 #define NFC_SEND_CMD3 BIT(28)
115 #define NFC_SEND_CMD4 BIT(29)
116 #define NFC_CMD_TYPE_MSK GENMASK(31, 30)
117 #define NFC_NORMAL_OP (0 << 30)
118 #define NFC_ECC_OP (1 << 30)
119 #define NFC_PAGE_OP (2U << 30)
120
121 /* define bit use in NFC_RCMD_SET */
122 #define NFC_READ_CMD_MSK GENMASK(7, 0)
123 #define NFC_RND_READ_CMD0_MSK GENMASK(15, 8)
124 #define NFC_RND_READ_CMD1_MSK GENMASK(23, 16)
125
126 /* define bit use in NFC_WCMD_SET */
127 #define NFC_PROGRAM_CMD_MSK GENMASK(7, 0)
128 #define NFC_RND_WRITE_CMD_MSK GENMASK(15, 8)
129 #define NFC_READ_CMD0_MSK GENMASK(23, 16)
130 #define NFC_READ_CMD1_MSK GENMASK(31, 24)
131
132 /* define bit use in NFC_ECC_CTL */
133 #define NFC_ECC_EN BIT(0)
134 #define NFC_ECC_PIPELINE BIT(3)
135 #define NFC_ECC_EXCEPTION BIT(4)
136 #define NFC_ECC_BLOCK_SIZE_MSK BIT(5)
137 #define NFC_ECC_BLOCK_512 BIT(5)
138 #define NFC_RANDOM_EN BIT(9)
139 #define NFC_RANDOM_DIRECTION BIT(10)
140 #define NFC_ECC_MODE_MSK GENMASK(15, 12)
141 #define NFC_ECC_MODE(x) ((x) << 12)
142 #define NFC_RANDOM_SEED_MSK GENMASK(30, 16)
143 #define NFC_RANDOM_SEED(x) ((x) << 16)
144
145 /* define bit use in NFC_ECC_ST */
146 #define NFC_ECC_ERR(x) BIT(x)
147 #define NFC_ECC_ERR_MSK GENMASK(15, 0)
148 #define NFC_ECC_PAT_FOUND(x) BIT(x + 16)
149 #define NFC_ECC_ERR_CNT(b, x) (((x) >> (((b) % 4) * 8)) & 0xff)
150
151 #define NFC_DEFAULT_TIMEOUT_MS 1000
152
153 #define NFC_SRAM_SIZE 1024
154
155 #define NFC_MAX_CS 7
156
157 /**
158 * struct sunxi_nand_chip_sel - stores information related to NAND Chip Select
159 *
160 * @cs: the NAND CS id used to communicate with a NAND Chip
161 * @rb: the Ready/Busy pin ID. -1 means no R/B pin connected to the NFC
162 */
163 struct sunxi_nand_chip_sel {
164 u8 cs;
165 s8 rb;
166 };
167
168 /**
169 * struct sunxi_nand_hw_ecc - stores information related to HW ECC support
170 *
171 * @mode: the sunxi ECC mode field deduced from ECC requirements
172 */
173 struct sunxi_nand_hw_ecc {
174 int mode;
175 };
176
177 /**
178 * struct sunxi_nand_chip - stores NAND chip device related information
179 *
180 * @node: used to store NAND chips into a list
181 * @nand: base NAND chip structure
182 * @clk_rate: clk_rate required for this NAND chip
183 * @timing_cfg: TIMING_CFG register value for this NAND chip
184 * @timing_ctl: TIMING_CTL register value for this NAND chip
185 * @nsels: number of CS lines required by the NAND chip
186 * @sels: array of CS lines descriptions
187 */
188 struct sunxi_nand_chip {
189 struct list_head node;
190 struct nand_chip nand;
191 unsigned long clk_rate;
192 u32 timing_cfg;
193 u32 timing_ctl;
194 int nsels;
195 struct sunxi_nand_chip_sel sels[0];
196 };
197
198 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)
199 {
200 return container_of(nand, struct sunxi_nand_chip, nand);
201 }
202
203 /**
204 * struct sunxi_nfc - stores sunxi NAND controller information
205 *
206 * @controller: base controller structure
207 * @dev: parent device (used to print error messages)
208 * @regs: NAND controller registers
209 * @ahb_clk: NAND controller AHB clock
210 * @mod_clk: NAND controller mod clock
211 * @reset: NAND controller reset line
212 * @assigned_cs: bitmask describing already assigned CS lines
213 * @clk_rate: NAND controller current clock rate
214 * @chips: a list containing all the NAND chips attached to this NAND
215 * controller
216 * @complete: a completion object used to wait for NAND controller events
217 * @dmac: the DMA channel attached to the NAND controller
218 */
219 struct sunxi_nfc {
220 struct nand_controller controller;
221 struct device *dev;
222 void __iomem *regs;
223 struct clk *ahb_clk;
224 struct clk *mod_clk;
225 struct reset_control *reset;
226 unsigned long assigned_cs;
227 unsigned long clk_rate;
228 struct list_head chips;
229 struct completion complete;
230 struct dma_chan *dmac;
231 };
232
233 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_controller *ctrl)
234 {
235 return container_of(ctrl, struct sunxi_nfc, controller);
236 }
237
238 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id)
239 {
240 struct sunxi_nfc *nfc = dev_id;
241 u32 st = readl(nfc->regs + NFC_REG_ST);
242 u32 ien = readl(nfc->regs + NFC_REG_INT);
243
244 if (!(ien & st))
245 return IRQ_NONE;
246
247 if ((ien & st) == ien)
248 complete(&nfc->complete);
249
250 writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
251 writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT);
252
253 return IRQ_HANDLED;
254 }
255
256 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events,
257 bool use_polling, unsigned int timeout_ms)
258 {
259 int ret;
260
261 if (events & ~NFC_INT_MASK)
262 return -EINVAL;
263
264 if (!timeout_ms)
265 timeout_ms = NFC_DEFAULT_TIMEOUT_MS;
266
267 if (!use_polling) {
268 init_completion(&nfc->complete);
269
270 writel(events, nfc->regs + NFC_REG_INT);
271
272 ret = wait_for_completion_timeout(&nfc->complete,
273 msecs_to_jiffies(timeout_ms));
274 if (!ret)
275 ret = -ETIMEDOUT;
276 else
277 ret = 0;
278
279 writel(0, nfc->regs + NFC_REG_INT);
280 } else {
281 u32 status;
282
283 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
284 (status & events) == events, 1,
285 timeout_ms * 1000);
286 }
287
288 writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
289
290 if (ret)
291 dev_err(nfc->dev, "wait interrupt timedout\n");
292
293 return ret;
294 }
295
296 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc)
297 {
298 u32 status;
299 int ret;
300
301 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
302 !(status & NFC_CMD_FIFO_STATUS), 1,
303 NFC_DEFAULT_TIMEOUT_MS * 1000);
304 if (ret)
305 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n");
306
307 return ret;
308 }
309
310 static int sunxi_nfc_rst(struct sunxi_nfc *nfc)
311 {
312 u32 ctl;
313 int ret;
314
315 writel(0, nfc->regs + NFC_REG_ECC_CTL);
316 writel(NFC_RESET, nfc->regs + NFC_REG_CTL);
317
318 ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl,
319 !(ctl & NFC_RESET), 1,
320 NFC_DEFAULT_TIMEOUT_MS * 1000);
321 if (ret)
322 dev_err(nfc->dev, "wait for NAND controller reset timedout\n");
323
324 return ret;
325 }
326
327 static int sunxi_nfc_dma_op_prepare(struct sunxi_nfc *nfc, const void *buf,
328 int chunksize, int nchunks,
329 enum dma_data_direction ddir,
330 struct scatterlist *sg)
331 {
332 struct dma_async_tx_descriptor *dmad;
333 enum dma_transfer_direction tdir;
334 dma_cookie_t dmat;
335 int ret;
336
337 if (ddir == DMA_FROM_DEVICE)
338 tdir = DMA_DEV_TO_MEM;
339 else
340 tdir = DMA_MEM_TO_DEV;
341
342 sg_init_one(sg, buf, nchunks * chunksize);
343 ret = dma_map_sg(nfc->dev, sg, 1, ddir);
344 if (!ret)
345 return -ENOMEM;
346
347 dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);
348 if (!dmad) {
349 ret = -EINVAL;
350 goto err_unmap_buf;
351 }
352
353 writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,
354 nfc->regs + NFC_REG_CTL);
355 writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);
356 writel(chunksize, nfc->regs + NFC_REG_CNT);
357 dmat = dmaengine_submit(dmad);
358
359 ret = dma_submit_error(dmat);
360 if (ret)
361 goto err_clr_dma_flag;
362
363 return 0;
364
365 err_clr_dma_flag:
366 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
367 nfc->regs + NFC_REG_CTL);
368
369 err_unmap_buf:
370 dma_unmap_sg(nfc->dev, sg, 1, ddir);
371 return ret;
372 }
373
374 static void sunxi_nfc_dma_op_cleanup(struct sunxi_nfc *nfc,
375 enum dma_data_direction ddir,
376 struct scatterlist *sg)
377 {
378 dma_unmap_sg(nfc->dev, sg, 1, ddir);
379 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
380 nfc->regs + NFC_REG_CTL);
381 }
382
383 static void sunxi_nfc_select_chip(struct nand_chip *nand, unsigned int cs)
384 {
385 struct mtd_info *mtd = nand_to_mtd(nand);
386 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
387 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
388 struct sunxi_nand_chip_sel *sel;
389 u32 ctl;
390
391 if (cs > 0 && cs >= sunxi_nand->nsels)
392 return;
393
394 ctl = readl(nfc->regs + NFC_REG_CTL) &
395 ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN);
396
397 sel = &sunxi_nand->sels[cs];
398 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN | NFC_PAGE_SHIFT(nand->page_shift);
399 if (sel->rb >= 0)
400 ctl |= NFC_RB_SEL(sel->rb);
401
402 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA);
403
404 if (nfc->clk_rate != sunxi_nand->clk_rate) {
405 clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate);
406 nfc->clk_rate = sunxi_nand->clk_rate;
407 }
408
409 writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL);
410 writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG);
411 writel(ctl, nfc->regs + NFC_REG_CTL);
412 }
413
414 static void sunxi_nfc_read_buf(struct nand_chip *nand, uint8_t *buf, int len)
415 {
416 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
417 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
418 int ret;
419 int cnt;
420 int offs = 0;
421 u32 tmp;
422
423 while (len > offs) {
424 bool poll = false;
425
426 cnt = min(len - offs, NFC_SRAM_SIZE);
427
428 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
429 if (ret)
430 break;
431
432 writel(cnt, nfc->regs + NFC_REG_CNT);
433 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
434 writel(tmp, nfc->regs + NFC_REG_CMD);
435
436 /* Arbitrary limit for polling mode */
437 if (cnt < 64)
438 poll = true;
439
440 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
441 if (ret)
442 break;
443
444 if (buf)
445 memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE,
446 cnt);
447 offs += cnt;
448 }
449 }
450
451 static void sunxi_nfc_write_buf(struct nand_chip *nand, const uint8_t *buf,
452 int len)
453 {
454 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
455 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
456 int ret;
457 int cnt;
458 int offs = 0;
459 u32 tmp;
460
461 while (len > offs) {
462 bool poll = false;
463
464 cnt = min(len - offs, NFC_SRAM_SIZE);
465
466 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
467 if (ret)
468 break;
469
470 writel(cnt, nfc->regs + NFC_REG_CNT);
471 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt);
472 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
473 NFC_ACCESS_DIR;
474 writel(tmp, nfc->regs + NFC_REG_CMD);
475
476 /* Arbitrary limit for polling mode */
477 if (cnt < 64)
478 poll = true;
479
480 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
481 if (ret)
482 break;
483
484 offs += cnt;
485 }
486 }
487
488 /* These seed values have been extracted from Allwinner's BSP */
489 static const u16 sunxi_nfc_randomizer_page_seeds[] = {
490 0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72,
491 0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436,
492 0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d,
493 0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130,
494 0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56,
495 0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55,
496 0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb,
497 0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17,
498 0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62,
499 0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064,
500 0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126,
501 0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e,
502 0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3,
503 0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b,
504 0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d,
505 0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db,
506 };
507
508 /*
509 * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds
510 * have been generated using
511 * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what
512 * the randomizer engine does internally before de/scrambling OOB data.
513 *
514 * Those tables are statically defined to avoid calculating randomizer state
515 * at runtime.
516 */
517 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = {
518 0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64,
519 0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409,
520 0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617,
521 0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d,
522 0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91,
523 0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d,
524 0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab,
525 0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8,
526 0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8,
527 0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b,
528 0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5,
529 0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a,
530 0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891,
531 0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36,
532 0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd,
533 0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0,
534 };
535
536 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = {
537 0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6,
538 0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982,
539 0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9,
540 0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07,
541 0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e,
542 0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2,
543 0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c,
544 0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f,
545 0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc,
546 0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e,
547 0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8,
548 0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68,
549 0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d,
550 0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179,
551 0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601,
552 0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd,
553 };
554
555 static u16 sunxi_nfc_randomizer_step(u16 state, int count)
556 {
557 state &= 0x7fff;
558
559 /*
560 * This loop is just a simple implementation of a Fibonacci LFSR using
561 * the x16 + x15 + 1 polynomial.
562 */
563 while (count--)
564 state = ((state >> 1) |
565 (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff;
566
567 return state;
568 }
569
570 static u16 sunxi_nfc_randomizer_state(struct nand_chip *nand, int page,
571 bool ecc)
572 {
573 struct mtd_info *mtd = nand_to_mtd(nand);
574 const u16 *seeds = sunxi_nfc_randomizer_page_seeds;
575 int mod = mtd_div_by_ws(mtd->erasesize, mtd);
576
577 if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds))
578 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds);
579
580 if (ecc) {
581 if (mtd->ecc_step_size == 512)
582 seeds = sunxi_nfc_randomizer_ecc512_seeds;
583 else
584 seeds = sunxi_nfc_randomizer_ecc1024_seeds;
585 }
586
587 return seeds[page % mod];
588 }
589
590 static void sunxi_nfc_randomizer_config(struct nand_chip *nand, int page,
591 bool ecc)
592 {
593 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
594 u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
595 u16 state;
596
597 if (!(nand->options & NAND_NEED_SCRAMBLING))
598 return;
599
600 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
601 state = sunxi_nfc_randomizer_state(nand, page, ecc);
602 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK;
603 writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL);
604 }
605
606 static void sunxi_nfc_randomizer_enable(struct nand_chip *nand)
607 {
608 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
609
610 if (!(nand->options & NAND_NEED_SCRAMBLING))
611 return;
612
613 writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN,
614 nfc->regs + NFC_REG_ECC_CTL);
615 }
616
617 static void sunxi_nfc_randomizer_disable(struct nand_chip *nand)
618 {
619 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
620
621 if (!(nand->options & NAND_NEED_SCRAMBLING))
622 return;
623
624 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN,
625 nfc->regs + NFC_REG_ECC_CTL);
626 }
627
628 static void sunxi_nfc_randomize_bbm(struct nand_chip *nand, int page, u8 *bbm)
629 {
630 u16 state = sunxi_nfc_randomizer_state(nand, page, true);
631
632 bbm[0] ^= state;
633 bbm[1] ^= sunxi_nfc_randomizer_step(state, 8);
634 }
635
636 static void sunxi_nfc_randomizer_write_buf(struct nand_chip *nand,
637 const uint8_t *buf, int len,
638 bool ecc, int page)
639 {
640 sunxi_nfc_randomizer_config(nand, page, ecc);
641 sunxi_nfc_randomizer_enable(nand);
642 sunxi_nfc_write_buf(nand, buf, len);
643 sunxi_nfc_randomizer_disable(nand);
644 }
645
646 static void sunxi_nfc_randomizer_read_buf(struct nand_chip *nand, uint8_t *buf,
647 int len, bool ecc, int page)
648 {
649 sunxi_nfc_randomizer_config(nand, page, ecc);
650 sunxi_nfc_randomizer_enable(nand);
651 sunxi_nfc_read_buf(nand, buf, len);
652 sunxi_nfc_randomizer_disable(nand);
653 }
654
655 static void sunxi_nfc_hw_ecc_enable(struct nand_chip *nand)
656 {
657 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
658 struct sunxi_nand_hw_ecc *data = nand->ecc.priv;
659 u32 ecc_ctl;
660
661 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
662 ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE |
663 NFC_ECC_BLOCK_SIZE_MSK);
664 ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(data->mode) | NFC_ECC_EXCEPTION |
665 NFC_ECC_PIPELINE;
666
667 if (nand->ecc.size == 512)
668 ecc_ctl |= NFC_ECC_BLOCK_512;
669
670 writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL);
671 }
672
673 static void sunxi_nfc_hw_ecc_disable(struct nand_chip *nand)
674 {
675 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
676
677 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN,
678 nfc->regs + NFC_REG_ECC_CTL);
679 }
680
681 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf)
682 {
683 buf[0] = user_data;
684 buf[1] = user_data >> 8;
685 buf[2] = user_data >> 16;
686 buf[3] = user_data >> 24;
687 }
688
689 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf)
690 {
691 return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24);
692 }
693
694 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct nand_chip *nand, u8 *oob,
695 int step, bool bbm, int page)
696 {
697 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
698
699 sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)),
700 oob);
701
702 /* De-randomize the Bad Block Marker. */
703 if (bbm && (nand->options & NAND_NEED_SCRAMBLING))
704 sunxi_nfc_randomize_bbm(nand, page, oob);
705 }
706
707 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct nand_chip *nand,
708 const u8 *oob, int step,
709 bool bbm, int page)
710 {
711 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
712 u8 user_data[4];
713
714 /* Randomize the Bad Block Marker. */
715 if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
716 memcpy(user_data, oob, sizeof(user_data));
717 sunxi_nfc_randomize_bbm(nand, page, user_data);
718 oob = user_data;
719 }
720
721 writel(sunxi_nfc_buf_to_user_data(oob),
722 nfc->regs + NFC_REG_USER_DATA(step));
723 }
724
725 static void sunxi_nfc_hw_ecc_update_stats(struct nand_chip *nand,
726 unsigned int *max_bitflips, int ret)
727 {
728 struct mtd_info *mtd = nand_to_mtd(nand);
729
730 if (ret < 0) {
731 mtd->ecc_stats.failed++;
732 } else {
733 mtd->ecc_stats.corrected += ret;
734 *max_bitflips = max_t(unsigned int, *max_bitflips, ret);
735 }
736 }
737
738 static int sunxi_nfc_hw_ecc_correct(struct nand_chip *nand, u8 *data, u8 *oob,
739 int step, u32 status, bool *erased)
740 {
741 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
742 struct nand_ecc_ctrl *ecc = &nand->ecc;
743 u32 tmp;
744
745 *erased = false;
746
747 if (status & NFC_ECC_ERR(step))
748 return -EBADMSG;
749
750 if (status & NFC_ECC_PAT_FOUND(step)) {
751 u8 pattern;
752
753 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) {
754 pattern = 0x0;
755 } else {
756 pattern = 0xff;
757 *erased = true;
758 }
759
760 if (data)
761 memset(data, pattern, ecc->size);
762
763 if (oob)
764 memset(oob, pattern, ecc->bytes + 4);
765
766 return 0;
767 }
768
769 tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step));
770
771 return NFC_ECC_ERR_CNT(step, tmp);
772 }
773
774 static int sunxi_nfc_hw_ecc_read_chunk(struct nand_chip *nand,
775 u8 *data, int data_off,
776 u8 *oob, int oob_off,
777 int *cur_off,
778 unsigned int *max_bitflips,
779 bool bbm, bool oob_required, int page)
780 {
781 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
782 struct nand_ecc_ctrl *ecc = &nand->ecc;
783 int raw_mode = 0;
784 bool erased;
785 int ret;
786
787 if (*cur_off != data_off)
788 nand_change_read_column_op(nand, data_off, NULL, 0, false);
789
790 sunxi_nfc_randomizer_read_buf(nand, NULL, ecc->size, false, page);
791
792 if (data_off + ecc->size != oob_off)
793 nand_change_read_column_op(nand, oob_off, NULL, 0, false);
794
795 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
796 if (ret)
797 return ret;
798
799 sunxi_nfc_randomizer_enable(nand);
800 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP,
801 nfc->regs + NFC_REG_CMD);
802
803 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
804 sunxi_nfc_randomizer_disable(nand);
805 if (ret)
806 return ret;
807
808 *cur_off = oob_off + ecc->bytes + 4;
809
810 ret = sunxi_nfc_hw_ecc_correct(nand, data, oob_required ? oob : NULL, 0,
811 readl(nfc->regs + NFC_REG_ECC_ST),
812 &erased);
813 if (erased)
814 return 1;
815
816 if (ret < 0) {
817 /*
818 * Re-read the data with the randomizer disabled to identify
819 * bitflips in erased pages.
820 */
821 if (nand->options & NAND_NEED_SCRAMBLING)
822 nand_change_read_column_op(nand, data_off, data,
823 ecc->size, false);
824 else
825 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,
826 ecc->size);
827
828 nand_change_read_column_op(nand, oob_off, oob, ecc->bytes + 4,
829 false);
830
831 ret = nand_check_erased_ecc_chunk(data, ecc->size,
832 oob, ecc->bytes + 4,
833 NULL, 0, ecc->strength);
834 if (ret >= 0)
835 raw_mode = 1;
836 } else {
837 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);
838
839 if (oob_required) {
840 nand_change_read_column_op(nand, oob_off, NULL, 0,
841 false);
842 sunxi_nfc_randomizer_read_buf(nand, oob, ecc->bytes + 4,
843 true, page);
844
845 sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, 0,
846 bbm, page);
847 }
848 }
849
850 sunxi_nfc_hw_ecc_update_stats(nand, max_bitflips, ret);
851
852 return raw_mode;
853 }
854
855 static void sunxi_nfc_hw_ecc_read_extra_oob(struct nand_chip *nand,
856 u8 *oob, int *cur_off,
857 bool randomize, int page)
858 {
859 struct mtd_info *mtd = nand_to_mtd(nand);
860 struct nand_ecc_ctrl *ecc = &nand->ecc;
861 int offset = ((ecc->bytes + 4) * ecc->steps);
862 int len = mtd->oobsize - offset;
863
864 if (len <= 0)
865 return;
866
867 if (!cur_off || *cur_off != offset)
868 nand_change_read_column_op(nand, mtd->writesize, NULL, 0,
869 false);
870
871 if (!randomize)
872 sunxi_nfc_read_buf(nand, oob + offset, len);
873 else
874 sunxi_nfc_randomizer_read_buf(nand, oob + offset, len,
875 false, page);
876
877 if (cur_off)
878 *cur_off = mtd->oobsize + mtd->writesize;
879 }
880
881 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, uint8_t *buf,
882 int oob_required, int page,
883 int nchunks)
884 {
885 bool randomized = nand->options & NAND_NEED_SCRAMBLING;
886 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
887 struct mtd_info *mtd = nand_to_mtd(nand);
888 struct nand_ecc_ctrl *ecc = &nand->ecc;
889 unsigned int max_bitflips = 0;
890 int ret, i, raw_mode = 0;
891 struct scatterlist sg;
892 u32 status;
893
894 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
895 if (ret)
896 return ret;
897
898 ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, nchunks,
899 DMA_FROM_DEVICE, &sg);
900 if (ret)
901 return ret;
902
903 sunxi_nfc_hw_ecc_enable(nand);
904 sunxi_nfc_randomizer_config(nand, page, false);
905 sunxi_nfc_randomizer_enable(nand);
906
907 writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |
908 NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);
909
910 dma_async_issue_pending(nfc->dmac);
911
912 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,
913 nfc->regs + NFC_REG_CMD);
914
915 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
916 if (ret)
917 dmaengine_terminate_all(nfc->dmac);
918
919 sunxi_nfc_randomizer_disable(nand);
920 sunxi_nfc_hw_ecc_disable(nand);
921
922 sunxi_nfc_dma_op_cleanup(nfc, DMA_FROM_DEVICE, &sg);
923
924 if (ret)
925 return ret;
926
927 status = readl(nfc->regs + NFC_REG_ECC_ST);
928
929 for (i = 0; i < nchunks; i++) {
930 int data_off = i * ecc->size;
931 int oob_off = i * (ecc->bytes + 4);
932 u8 *data = buf + data_off;
933 u8 *oob = nand->oob_poi + oob_off;
934 bool erased;
935
936 ret = sunxi_nfc_hw_ecc_correct(nand, randomized ? data : NULL,
937 oob_required ? oob : NULL,
938 i, status, &erased);
939
940 /* ECC errors are handled in the second loop. */
941 if (ret < 0)
942 continue;
943
944 if (oob_required && !erased) {
945 /* TODO: use DMA to retrieve OOB */
946 nand_change_read_column_op(nand,
947 mtd->writesize + oob_off,
948 oob, ecc->bytes + 4, false);
949
950 sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, i,
951 !i, page);
952 }
953
954 if (erased)
955 raw_mode = 1;
956
957 sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, ret);
958 }
959
960 if (status & NFC_ECC_ERR_MSK) {
961 for (i = 0; i < nchunks; i++) {
962 int data_off = i * ecc->size;
963 int oob_off = i * (ecc->bytes + 4);
964 u8 *data = buf + data_off;
965 u8 *oob = nand->oob_poi + oob_off;
966
967 if (!(status & NFC_ECC_ERR(i)))
968 continue;
969
970 /*
971 * Re-read the data with the randomizer disabled to
972 * identify bitflips in erased pages.
973 * TODO: use DMA to read page in raw mode
974 */
975 if (randomized)
976 nand_change_read_column_op(nand, data_off,
977 data, ecc->size,
978 false);
979
980 /* TODO: use DMA to retrieve OOB */
981 nand_change_read_column_op(nand,
982 mtd->writesize + oob_off,
983 oob, ecc->bytes + 4, false);
984
985 ret = nand_check_erased_ecc_chunk(data, ecc->size,
986 oob, ecc->bytes + 4,
987 NULL, 0,
988 ecc->strength);
989 if (ret >= 0)
990 raw_mode = 1;
991
992 sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, ret);
993 }
994 }
995
996 if (oob_required)
997 sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi,
998 NULL, !raw_mode,
999 page);
1000
1001 return max_bitflips;
1002 }
1003
1004 static int sunxi_nfc_hw_ecc_write_chunk(struct nand_chip *nand,
1005 const u8 *data, int data_off,
1006 const u8 *oob, int oob_off,
1007 int *cur_off, bool bbm,
1008 int page)
1009 {
1010 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1011 struct nand_ecc_ctrl *ecc = &nand->ecc;
1012 int ret;
1013
1014 if (data_off != *cur_off)
1015 nand_change_write_column_op(nand, data_off, NULL, 0, false);
1016
1017 sunxi_nfc_randomizer_write_buf(nand, data, ecc->size, false, page);
1018
1019 if (data_off + ecc->size != oob_off)
1020 nand_change_write_column_op(nand, oob_off, NULL, 0, false);
1021
1022 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1023 if (ret)
1024 return ret;
1025
1026 sunxi_nfc_randomizer_enable(nand);
1027 sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, 0, bbm, page);
1028
1029 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
1030 NFC_ACCESS_DIR | NFC_ECC_OP,
1031 nfc->regs + NFC_REG_CMD);
1032
1033 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1034 sunxi_nfc_randomizer_disable(nand);
1035 if (ret)
1036 return ret;
1037
1038 *cur_off = oob_off + ecc->bytes + 4;
1039
1040 return 0;
1041 }
1042
1043 static void sunxi_nfc_hw_ecc_write_extra_oob(struct nand_chip *nand,
1044 u8 *oob, int *cur_off,
1045 int page)
1046 {
1047 struct mtd_info *mtd = nand_to_mtd(nand);
1048 struct nand_ecc_ctrl *ecc = &nand->ecc;
1049 int offset = ((ecc->bytes + 4) * ecc->steps);
1050 int len = mtd->oobsize - offset;
1051
1052 if (len <= 0)
1053 return;
1054
1055 if (!cur_off || *cur_off != offset)
1056 nand_change_write_column_op(nand, offset + mtd->writesize,
1057 NULL, 0, false);
1058
1059 sunxi_nfc_randomizer_write_buf(nand, oob + offset, len, false, page);
1060
1061 if (cur_off)
1062 *cur_off = mtd->oobsize + mtd->writesize;
1063 }
1064
1065 static int sunxi_nfc_hw_ecc_read_page(struct nand_chip *nand, uint8_t *buf,
1066 int oob_required, int page)
1067 {
1068 struct mtd_info *mtd = nand_to_mtd(nand);
1069 struct nand_ecc_ctrl *ecc = &nand->ecc;
1070 unsigned int max_bitflips = 0;
1071 int ret, i, cur_off = 0;
1072 bool raw_mode = false;
1073
1074 sunxi_nfc_select_chip(nand, nand->cur_cs);
1075
1076 nand_read_page_op(nand, page, 0, NULL, 0);
1077
1078 sunxi_nfc_hw_ecc_enable(nand);
1079
1080 for (i = 0; i < ecc->steps; i++) {
1081 int data_off = i * ecc->size;
1082 int oob_off = i * (ecc->bytes + 4);
1083 u8 *data = buf + data_off;
1084 u8 *oob = nand->oob_poi + oob_off;
1085
1086 ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off, oob,
1087 oob_off + mtd->writesize,
1088 &cur_off, &max_bitflips,
1089 !i, oob_required, page);
1090 if (ret < 0)
1091 return ret;
1092 else if (ret)
1093 raw_mode = true;
1094 }
1095
1096 if (oob_required)
1097 sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi, &cur_off,
1098 !raw_mode, page);
1099
1100 sunxi_nfc_hw_ecc_disable(nand);
1101
1102 return max_bitflips;
1103 }
1104
1105 static int sunxi_nfc_hw_ecc_read_page_dma(struct nand_chip *nand, u8 *buf,
1106 int oob_required, int page)
1107 {
1108 int ret;
1109
1110 sunxi_nfc_select_chip(nand, nand->cur_cs);
1111
1112 nand_read_page_op(nand, page, 0, NULL, 0);
1113
1114 ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, oob_required, page,
1115 nand->ecc.steps);
1116 if (ret >= 0)
1117 return ret;
1118
1119 /* Fallback to PIO mode */
1120 return sunxi_nfc_hw_ecc_read_page(nand, buf, oob_required, page);
1121 }
1122
1123 static int sunxi_nfc_hw_ecc_read_subpage(struct nand_chip *nand,
1124 u32 data_offs, u32 readlen,
1125 u8 *bufpoi, int page)
1126 {
1127 struct mtd_info *mtd = nand_to_mtd(nand);
1128 struct nand_ecc_ctrl *ecc = &nand->ecc;
1129 int ret, i, cur_off = 0;
1130 unsigned int max_bitflips = 0;
1131
1132 sunxi_nfc_select_chip(nand, nand->cur_cs);
1133
1134 nand_read_page_op(nand, page, 0, NULL, 0);
1135
1136 sunxi_nfc_hw_ecc_enable(nand);
1137
1138 for (i = data_offs / ecc->size;
1139 i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) {
1140 int data_off = i * ecc->size;
1141 int oob_off = i * (ecc->bytes + 4);
1142 u8 *data = bufpoi + data_off;
1143 u8 *oob = nand->oob_poi + oob_off;
1144
1145 ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off,
1146 oob,
1147 oob_off + mtd->writesize,
1148 &cur_off, &max_bitflips, !i,
1149 false, page);
1150 if (ret < 0)
1151 return ret;
1152 }
1153
1154 sunxi_nfc_hw_ecc_disable(nand);
1155
1156 return max_bitflips;
1157 }
1158
1159 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct nand_chip *nand,
1160 u32 data_offs, u32 readlen,
1161 u8 *buf, int page)
1162 {
1163 int nchunks = DIV_ROUND_UP(data_offs + readlen, nand->ecc.size);
1164 int ret;
1165
1166 sunxi_nfc_select_chip(nand, nand->cur_cs);
1167
1168 nand_read_page_op(nand, page, 0, NULL, 0);
1169
1170 ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, false, page, nchunks);
1171 if (ret >= 0)
1172 return ret;
1173
1174 /* Fallback to PIO mode */
1175 return sunxi_nfc_hw_ecc_read_subpage(nand, data_offs, readlen,
1176 buf, page);
1177 }
1178
1179 static int sunxi_nfc_hw_ecc_write_page(struct nand_chip *nand,
1180 const uint8_t *buf, int oob_required,
1181 int page)
1182 {
1183 struct mtd_info *mtd = nand_to_mtd(nand);
1184 struct nand_ecc_ctrl *ecc = &nand->ecc;
1185 int ret, i, cur_off = 0;
1186
1187 sunxi_nfc_select_chip(nand, nand->cur_cs);
1188
1189 nand_prog_page_begin_op(nand, page, 0, NULL, 0);
1190
1191 sunxi_nfc_hw_ecc_enable(nand);
1192
1193 for (i = 0; i < ecc->steps; i++) {
1194 int data_off = i * ecc->size;
1195 int oob_off = i * (ecc->bytes + 4);
1196 const u8 *data = buf + data_off;
1197 const u8 *oob = nand->oob_poi + oob_off;
1198
1199 ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob,
1200 oob_off + mtd->writesize,
1201 &cur_off, !i, page);
1202 if (ret)
1203 return ret;
1204 }
1205
1206 if (oob_required || (nand->options & NAND_NEED_SCRAMBLING))
1207 sunxi_nfc_hw_ecc_write_extra_oob(nand, nand->oob_poi,
1208 &cur_off, page);
1209
1210 sunxi_nfc_hw_ecc_disable(nand);
1211
1212 return nand_prog_page_end_op(nand);
1213 }
1214
1215 static int sunxi_nfc_hw_ecc_write_subpage(struct nand_chip *nand,
1216 u32 data_offs, u32 data_len,
1217 const u8 *buf, int oob_required,
1218 int page)
1219 {
1220 struct mtd_info *mtd = nand_to_mtd(nand);
1221 struct nand_ecc_ctrl *ecc = &nand->ecc;
1222 int ret, i, cur_off = 0;
1223
1224 sunxi_nfc_select_chip(nand, nand->cur_cs);
1225
1226 nand_prog_page_begin_op(nand, page, 0, NULL, 0);
1227
1228 sunxi_nfc_hw_ecc_enable(nand);
1229
1230 for (i = data_offs / ecc->size;
1231 i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) {
1232 int data_off = i * ecc->size;
1233 int oob_off = i * (ecc->bytes + 4);
1234 const u8 *data = buf + data_off;
1235 const u8 *oob = nand->oob_poi + oob_off;
1236
1237 ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob,
1238 oob_off + mtd->writesize,
1239 &cur_off, !i, page);
1240 if (ret)
1241 return ret;
1242 }
1243
1244 sunxi_nfc_hw_ecc_disable(nand);
1245
1246 return nand_prog_page_end_op(nand);
1247 }
1248
1249 static int sunxi_nfc_hw_ecc_write_page_dma(struct nand_chip *nand,
1250 const u8 *buf,
1251 int oob_required,
1252 int page)
1253 {
1254 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1255 struct nand_ecc_ctrl *ecc = &nand->ecc;
1256 struct scatterlist sg;
1257 int ret, i;
1258
1259 sunxi_nfc_select_chip(nand, nand->cur_cs);
1260
1261 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1262 if (ret)
1263 return ret;
1264
1265 ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, ecc->steps,
1266 DMA_TO_DEVICE, &sg);
1267 if (ret)
1268 goto pio_fallback;
1269
1270 for (i = 0; i < ecc->steps; i++) {
1271 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));
1272
1273 sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, i, !i, page);
1274 }
1275
1276 nand_prog_page_begin_op(nand, page, 0, NULL, 0);
1277
1278 sunxi_nfc_hw_ecc_enable(nand);
1279 sunxi_nfc_randomizer_config(nand, page, false);
1280 sunxi_nfc_randomizer_enable(nand);
1281
1282 writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
1283 nfc->regs + NFC_REG_WCMD_SET);
1284
1285 dma_async_issue_pending(nfc->dmac);
1286
1287 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
1288 NFC_DATA_TRANS | NFC_ACCESS_DIR,
1289 nfc->regs + NFC_REG_CMD);
1290
1291 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1292 if (ret)
1293 dmaengine_terminate_all(nfc->dmac);
1294
1295 sunxi_nfc_randomizer_disable(nand);
1296 sunxi_nfc_hw_ecc_disable(nand);
1297
1298 sunxi_nfc_dma_op_cleanup(nfc, DMA_TO_DEVICE, &sg);
1299
1300 if (ret)
1301 return ret;
1302
1303 if (oob_required || (nand->options & NAND_NEED_SCRAMBLING))
1304 /* TODO: use DMA to transfer extra OOB bytes ? */
1305 sunxi_nfc_hw_ecc_write_extra_oob(nand, nand->oob_poi,
1306 NULL, page);
1307
1308 return nand_prog_page_end_op(nand);
1309
1310 pio_fallback:
1311 return sunxi_nfc_hw_ecc_write_page(nand, buf, oob_required, page);
1312 }
1313
1314 static int sunxi_nfc_hw_ecc_read_oob(struct nand_chip *nand, int page)
1315 {
1316 u8 *buf = nand_get_data_buf(nand);
1317
1318 return nand->ecc.read_page(nand, buf, 1, page);
1319 }
1320
1321 static int sunxi_nfc_hw_ecc_write_oob(struct nand_chip *nand, int page)
1322 {
1323 struct mtd_info *mtd = nand_to_mtd(nand);
1324 u8 *buf = nand_get_data_buf(nand);
1325 int ret;
1326
1327 memset(buf, 0xff, mtd->writesize);
1328 ret = nand->ecc.write_page(nand, buf, 1, page);
1329 if (ret)
1330 return ret;
1331
1332 /* Send command to program the OOB data */
1333 return nand_prog_page_end_op(nand);
1334 }
1335
1336 static const s32 tWB_lut[] = {6, 12, 16, 20};
1337 static const s32 tRHW_lut[] = {4, 8, 12, 20};
1338
1339 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
1340 u32 clk_period)
1341 {
1342 u32 clk_cycles = DIV_ROUND_UP(duration, clk_period);
1343 int i;
1344
1345 for (i = 0; i < lut_size; i++) {
1346 if (clk_cycles <= lut[i])
1347 return i;
1348 }
1349
1350 /* Doesn't fit */
1351 return -EINVAL;
1352 }
1353
1354 #define sunxi_nand_lookup_timing(l, p, c) \
1355 _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)
1356
1357 static int sunxi_nfc_setup_data_interface(struct nand_chip *nand, int csline,
1358 const struct nand_data_interface *conf)
1359 {
1360 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1361 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1362 const struct nand_sdr_timings *timings;
1363 u32 min_clk_period = 0;
1364 s32 tWB, tADL, tWHR, tRHW, tCAD;
1365 long real_clk_rate;
1366
1367 timings = nand_get_sdr_timings(conf);
1368 if (IS_ERR(timings))
1369 return -ENOTSUPP;
1370
1371 /* T1 <=> tCLS */
1372 if (timings->tCLS_min > min_clk_period)
1373 min_clk_period = timings->tCLS_min;
1374
1375 /* T2 <=> tCLH */
1376 if (timings->tCLH_min > min_clk_period)
1377 min_clk_period = timings->tCLH_min;
1378
1379 /* T3 <=> tCS */
1380 if (timings->tCS_min > min_clk_period)
1381 min_clk_period = timings->tCS_min;
1382
1383 /* T4 <=> tCH */
1384 if (timings->tCH_min > min_clk_period)
1385 min_clk_period = timings->tCH_min;
1386
1387 /* T5 <=> tWP */
1388 if (timings->tWP_min > min_clk_period)
1389 min_clk_period = timings->tWP_min;
1390
1391 /* T6 <=> tWH */
1392 if (timings->tWH_min > min_clk_period)
1393 min_clk_period = timings->tWH_min;
1394
1395 /* T7 <=> tALS */
1396 if (timings->tALS_min > min_clk_period)
1397 min_clk_period = timings->tALS_min;
1398
1399 /* T8 <=> tDS */
1400 if (timings->tDS_min > min_clk_period)
1401 min_clk_period = timings->tDS_min;
1402
1403 /* T9 <=> tDH */
1404 if (timings->tDH_min > min_clk_period)
1405 min_clk_period = timings->tDH_min;
1406
1407 /* T10 <=> tRR */
1408 if (timings->tRR_min > (min_clk_period * 3))
1409 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);
1410
1411 /* T11 <=> tALH */
1412 if (timings->tALH_min > min_clk_period)
1413 min_clk_period = timings->tALH_min;
1414
1415 /* T12 <=> tRP */
1416 if (timings->tRP_min > min_clk_period)
1417 min_clk_period = timings->tRP_min;
1418
1419 /* T13 <=> tREH */
1420 if (timings->tREH_min > min_clk_period)
1421 min_clk_period = timings->tREH_min;
1422
1423 /* T14 <=> tRC */
1424 if (timings->tRC_min > (min_clk_period * 2))
1425 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2);
1426
1427 /* T15 <=> tWC */
1428 if (timings->tWC_min > (min_clk_period * 2))
1429 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);
1430
1431 /* T16 - T19 + tCAD */
1432 if (timings->tWB_max > (min_clk_period * 20))
1433 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);
1434
1435 if (timings->tADL_min > (min_clk_period * 32))
1436 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
1437
1438 if (timings->tWHR_min > (min_clk_period * 32))
1439 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
1440
1441 if (timings->tRHW_min > (min_clk_period * 20))
1442 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);
1443
1444 /*
1445 * In non-EDO, tREA should be less than tRP to guarantee that the
1446 * controller does not sample the IO lines too early. Unfortunately,
1447 * the sunxi NAND controller does not allow us to have different
1448 * values for tRP and tREH (tRP = tREH = tRW / 2).
1449 *
1450 * We have 2 options to overcome this limitation:
1451 *
1452 * 1/ Extend tRC to fulfil the tREA <= tRC / 2 constraint
1453 * 2/ Use EDO mode (only works if timings->tRLOH > 0)
1454 */
1455 if (timings->tREA_max > min_clk_period && !timings->tRLOH_min)
1456 min_clk_period = timings->tREA_max;
1457
1458 tWB = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,
1459 min_clk_period);
1460 if (tWB < 0) {
1461 dev_err(nfc->dev, "unsupported tWB\n");
1462 return tWB;
1463 }
1464
1465 tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;
1466 if (tADL > 3) {
1467 dev_err(nfc->dev, "unsupported tADL\n");
1468 return -EINVAL;
1469 }
1470
1471 tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;
1472 if (tWHR > 3) {
1473 dev_err(nfc->dev, "unsupported tWHR\n");
1474 return -EINVAL;
1475 }
1476
1477 tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,
1478 min_clk_period);
1479 if (tRHW < 0) {
1480 dev_err(nfc->dev, "unsupported tRHW\n");
1481 return tRHW;
1482 }
1483
1484 if (csline == NAND_DATA_IFACE_CHECK_ONLY)
1485 return 0;
1486
1487 /*
1488 * TODO: according to ONFI specs this value only applies for DDR NAND,
1489 * but Allwinner seems to set this to 0x7. Mimic them for now.
1490 */
1491 tCAD = 0x7;
1492
1493 /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */
1494 sunxi_nand->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD);
1495
1496 /* Convert min_clk_period from picoseconds to nanoseconds */
1497 min_clk_period = DIV_ROUND_UP(min_clk_period, 1000);
1498
1499 /*
1500 * Unlike what is stated in Allwinner datasheet, the clk_rate should
1501 * be set to (1 / min_clk_period), and not (2 / min_clk_period).
1502 * This new formula was verified with a scope and validated by
1503 * Allwinner engineers.
1504 */
1505 sunxi_nand->clk_rate = NSEC_PER_SEC / min_clk_period;
1506 real_clk_rate = clk_round_rate(nfc->mod_clk, sunxi_nand->clk_rate);
1507 if (real_clk_rate <= 0) {
1508 dev_err(nfc->dev, "Unable to round clk %lu\n",
1509 sunxi_nand->clk_rate);
1510 return -EINVAL;
1511 }
1512
1513 sunxi_nand->timing_ctl = 0;
1514
1515 /*
1516 * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data
1517 * output cycle timings shall be used if the host drives tRC less than
1518 * 30 ns. We should also use EDO mode if tREA is bigger than tRP.
1519 */
1520 min_clk_period = NSEC_PER_SEC / real_clk_rate;
1521 if (min_clk_period * 2 < 30 || min_clk_period * 1000 < timings->tREA_max)
1522 sunxi_nand->timing_ctl = NFC_TIMING_CTL_EDO;
1523
1524 return 0;
1525 }
1526
1527 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1528 struct mtd_oob_region *oobregion)
1529 {
1530 struct nand_chip *nand = mtd_to_nand(mtd);
1531 struct nand_ecc_ctrl *ecc = &nand->ecc;
1532
1533 if (section >= ecc->steps)
1534 return -ERANGE;
1535
1536 oobregion->offset = section * (ecc->bytes + 4) + 4;
1537 oobregion->length = ecc->bytes;
1538
1539 return 0;
1540 }
1541
1542 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
1543 struct mtd_oob_region *oobregion)
1544 {
1545 struct nand_chip *nand = mtd_to_nand(mtd);
1546 struct nand_ecc_ctrl *ecc = &nand->ecc;
1547
1548 if (section > ecc->steps)
1549 return -ERANGE;
1550
1551 /*
1552 * The first 2 bytes are used for BB markers, hence we
1553 * only have 2 bytes available in the first user data
1554 * section.
1555 */
1556 if (!section && ecc->mode == NAND_ECC_HW) {
1557 oobregion->offset = 2;
1558 oobregion->length = 2;
1559
1560 return 0;
1561 }
1562
1563 oobregion->offset = section * (ecc->bytes + 4);
1564
1565 if (section < ecc->steps)
1566 oobregion->length = 4;
1567 else
1568 oobregion->offset = mtd->oobsize - oobregion->offset;
1569
1570 return 0;
1571 }
1572
1573 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = {
1574 .ecc = sunxi_nand_ooblayout_ecc,
1575 .free = sunxi_nand_ooblayout_free,
1576 };
1577
1578 static void sunxi_nand_hw_ecc_ctrl_cleanup(struct nand_ecc_ctrl *ecc)
1579 {
1580 kfree(ecc->priv);
1581 }
1582
1583 static int sunxi_nand_hw_ecc_ctrl_init(struct nand_chip *nand,
1584 struct nand_ecc_ctrl *ecc,
1585 struct device_node *np)
1586 {
1587 static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 };
1588 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1589 struct mtd_info *mtd = nand_to_mtd(nand);
1590 struct sunxi_nand_hw_ecc *data;
1591 int nsectors;
1592 int ret;
1593 int i;
1594
1595 if (ecc->options & NAND_ECC_MAXIMIZE) {
1596 int bytes;
1597
1598 ecc->size = 1024;
1599 nsectors = mtd->writesize / ecc->size;
1600
1601 /* Reserve 2 bytes for the BBM */
1602 bytes = (mtd->oobsize - 2) / nsectors;
1603
1604 /* 4 non-ECC bytes are added before each ECC bytes section */
1605 bytes -= 4;
1606
1607 /* and bytes has to be even. */
1608 if (bytes % 2)
1609 bytes--;
1610
1611 ecc->strength = bytes * 8 / fls(8 * ecc->size);
1612
1613 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1614 if (strengths[i] > ecc->strength)
1615 break;
1616 }
1617
1618 if (!i)
1619 ecc->strength = 0;
1620 else
1621 ecc->strength = strengths[i - 1];
1622 }
1623
1624 if (ecc->size != 512 && ecc->size != 1024)
1625 return -EINVAL;
1626
1627 data = kzalloc(sizeof(*data), GFP_KERNEL);
1628 if (!data)
1629 return -ENOMEM;
1630
1631 /* Prefer 1k ECC chunk over 512 ones */
1632 if (ecc->size == 512 && mtd->writesize > 512) {
1633 ecc->size = 1024;
1634 ecc->strength *= 2;
1635 }
1636
1637 /* Add ECC info retrieval from DT */
1638 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1639 if (ecc->strength <= strengths[i]) {
1640 /*
1641 * Update ecc->strength value with the actual strength
1642 * that will be used by the ECC engine.
1643 */
1644 ecc->strength = strengths[i];
1645 break;
1646 }
1647 }
1648
1649 if (i >= ARRAY_SIZE(strengths)) {
1650 dev_err(nfc->dev, "unsupported strength\n");
1651 ret = -ENOTSUPP;
1652 goto err;
1653 }
1654
1655 data->mode = i;
1656
1657 /* HW ECC always request ECC bytes for 1024 bytes blocks */
1658 ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8);
1659
1660 /* HW ECC always work with even numbers of ECC bytes */
1661 ecc->bytes = ALIGN(ecc->bytes, 2);
1662
1663 nsectors = mtd->writesize / ecc->size;
1664
1665 if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) {
1666 ret = -EINVAL;
1667 goto err;
1668 }
1669
1670 ecc->read_oob = sunxi_nfc_hw_ecc_read_oob;
1671 ecc->write_oob = sunxi_nfc_hw_ecc_write_oob;
1672 mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops);
1673 ecc->priv = data;
1674
1675 if (nfc->dmac) {
1676 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
1677 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
1678 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
1679 nand->options |= NAND_USE_BOUNCE_BUFFER;
1680 } else {
1681 ecc->read_page = sunxi_nfc_hw_ecc_read_page;
1682 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1683 ecc->write_page = sunxi_nfc_hw_ecc_write_page;
1684 }
1685
1686 /* TODO: support DMA for raw accesses and subpage write */
1687 ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage;
1688 ecc->read_oob_raw = nand_read_oob_std;
1689 ecc->write_oob_raw = nand_write_oob_std;
1690
1691 return 0;
1692
1693 err:
1694 kfree(data);
1695
1696 return ret;
1697 }
1698
1699 static void sunxi_nand_ecc_cleanup(struct nand_ecc_ctrl *ecc)
1700 {
1701 switch (ecc->mode) {
1702 case NAND_ECC_HW:
1703 sunxi_nand_hw_ecc_ctrl_cleanup(ecc);
1704 break;
1705 case NAND_ECC_NONE:
1706 default:
1707 break;
1708 }
1709 }
1710
1711 static int sunxi_nand_attach_chip(struct nand_chip *nand)
1712 {
1713 struct nand_ecc_ctrl *ecc = &nand->ecc;
1714 struct device_node *np = nand_get_flash_node(nand);
1715 int ret;
1716
1717 if (nand->bbt_options & NAND_BBT_USE_FLASH)
1718 nand->bbt_options |= NAND_BBT_NO_OOB;
1719
1720 if (nand->options & NAND_NEED_SCRAMBLING)
1721 nand->options |= NAND_NO_SUBPAGE_WRITE;
1722
1723 nand->options |= NAND_SUBPAGE_READ;
1724
1725 if (!ecc->size) {
1726 ecc->size = nand->base.eccreq.step_size;
1727 ecc->strength = nand->base.eccreq.strength;
1728 }
1729
1730 if (!ecc->size || !ecc->strength)
1731 return -EINVAL;
1732
1733 switch (ecc->mode) {
1734 case NAND_ECC_HW:
1735 ret = sunxi_nand_hw_ecc_ctrl_init(nand, ecc, np);
1736 if (ret)
1737 return ret;
1738 break;
1739 case NAND_ECC_NONE:
1740 case NAND_ECC_SOFT:
1741 break;
1742 default:
1743 return -EINVAL;
1744 }
1745
1746 return 0;
1747 }
1748
1749 static int sunxi_nfc_exec_subop(struct nand_chip *nand,
1750 const struct nand_subop *subop)
1751 {
1752 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1753 u32 cmd = 0, extcmd = 0, cnt = 0, addrs[2] = { };
1754 unsigned int i, j, remaining, start;
1755 void *inbuf = NULL;
1756 int ret;
1757
1758 for (i = 0; i < subop->ninstrs; i++) {
1759 const struct nand_op_instr *instr = &subop->instrs[i];
1760
1761 switch (instr->type) {
1762 case NAND_OP_CMD_INSTR:
1763 if (cmd & NFC_SEND_CMD1) {
1764 if (WARN_ON(cmd & NFC_SEND_CMD2))
1765 return -EINVAL;
1766
1767 cmd |= NFC_SEND_CMD2;
1768 extcmd |= instr->ctx.cmd.opcode;
1769 } else {
1770 cmd |= NFC_SEND_CMD1 |
1771 NFC_CMD(instr->ctx.cmd.opcode);
1772 }
1773 break;
1774
1775 case NAND_OP_ADDR_INSTR:
1776 remaining = nand_subop_get_num_addr_cyc(subop, i);
1777 start = nand_subop_get_addr_start_off(subop, i);
1778 for (j = 0; j < 8 && j + start < remaining; j++) {
1779 u32 addr = instr->ctx.addr.addrs[j + start];
1780
1781 addrs[j / 4] |= addr << (j % 4) * 8;
1782 }
1783
1784 if (j)
1785 cmd |= NFC_SEND_ADR | NFC_ADR_NUM(j);
1786
1787 break;
1788
1789 case NAND_OP_DATA_IN_INSTR:
1790 case NAND_OP_DATA_OUT_INSTR:
1791 start = nand_subop_get_data_start_off(subop, i);
1792 remaining = nand_subop_get_data_len(subop, i);
1793 cnt = min_t(u32, remaining, NFC_SRAM_SIZE);
1794 cmd |= NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
1795
1796 if (instr->type == NAND_OP_DATA_OUT_INSTR) {
1797 cmd |= NFC_ACCESS_DIR;
1798 memcpy_toio(nfc->regs + NFC_RAM0_BASE,
1799 instr->ctx.data.buf.out + start,
1800 cnt);
1801 } else {
1802 inbuf = instr->ctx.data.buf.in + start;
1803 }
1804
1805 break;
1806
1807 case NAND_OP_WAITRDY_INSTR:
1808 cmd |= NFC_WAIT_FLAG;
1809 break;
1810 }
1811 }
1812
1813 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1814 if (ret)
1815 return ret;
1816
1817 if (cmd & NFC_SEND_ADR) {
1818 writel(addrs[0], nfc->regs + NFC_REG_ADDR_LOW);
1819 writel(addrs[1], nfc->regs + NFC_REG_ADDR_HIGH);
1820 }
1821
1822 if (cmd & NFC_SEND_CMD2)
1823 writel(extcmd,
1824 nfc->regs +
1825 (cmd & NFC_ACCESS_DIR ?
1826 NFC_REG_WCMD_SET : NFC_REG_RCMD_SET));
1827
1828 if (cmd & NFC_DATA_TRANS)
1829 writel(cnt, nfc->regs + NFC_REG_CNT);
1830
1831 writel(cmd, nfc->regs + NFC_REG_CMD);
1832
1833 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG,
1834 !(cmd & NFC_WAIT_FLAG) && cnt < 64,
1835 0);
1836 if (ret)
1837 return ret;
1838
1839 if (inbuf)
1840 memcpy_fromio(inbuf, nfc->regs + NFC_RAM0_BASE, cnt);
1841
1842 return 0;
1843 }
1844
1845 static int sunxi_nfc_soft_waitrdy(struct nand_chip *nand,
1846 const struct nand_subop *subop)
1847 {
1848 return nand_soft_waitrdy(nand,
1849 subop->instrs[0].ctx.waitrdy.timeout_ms);
1850 }
1851
1852 static const struct nand_op_parser sunxi_nfc_op_parser = NAND_OP_PARSER(
1853 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1854 NAND_OP_PARSER_PAT_CMD_ELEM(true),
1855 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1856 NAND_OP_PARSER_PAT_CMD_ELEM(true),
1857 NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
1858 NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)),
1859 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1860 NAND_OP_PARSER_PAT_CMD_ELEM(true),
1861 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1862 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024),
1863 NAND_OP_PARSER_PAT_CMD_ELEM(true),
1864 NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)),
1865 );
1866
1867 static const struct nand_op_parser sunxi_nfc_norb_op_parser = NAND_OP_PARSER(
1868 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1869 NAND_OP_PARSER_PAT_CMD_ELEM(true),
1870 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1871 NAND_OP_PARSER_PAT_CMD_ELEM(true),
1872 NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)),
1873 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1874 NAND_OP_PARSER_PAT_CMD_ELEM(true),
1875 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1876 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024),
1877 NAND_OP_PARSER_PAT_CMD_ELEM(true)),
1878 NAND_OP_PARSER_PATTERN(sunxi_nfc_soft_waitrdy,
1879 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
1880 );
1881
1882 static int sunxi_nfc_exec_op(struct nand_chip *nand,
1883 const struct nand_operation *op, bool check_only)
1884 {
1885 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1886 const struct nand_op_parser *parser;
1887
1888 sunxi_nfc_select_chip(nand, op->cs);
1889
1890 if (sunxi_nand->sels[op->cs].rb >= 0)
1891 parser = &sunxi_nfc_op_parser;
1892 else
1893 parser = &sunxi_nfc_norb_op_parser;
1894
1895 return nand_op_parser_exec_op(nand, parser, op, check_only);
1896 }
1897
1898 static const struct nand_controller_ops sunxi_nand_controller_ops = {
1899 .attach_chip = sunxi_nand_attach_chip,
1900 .setup_data_interface = sunxi_nfc_setup_data_interface,
1901 .exec_op = sunxi_nfc_exec_op,
1902 };
1903
1904 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
1905 struct device_node *np)
1906 {
1907 struct sunxi_nand_chip *sunxi_nand;
1908 struct mtd_info *mtd;
1909 struct nand_chip *nand;
1910 int nsels;
1911 int ret;
1912 int i;
1913 u32 tmp;
1914
1915 if (!of_get_property(np, "reg", &nsels))
1916 return -EINVAL;
1917
1918 nsels /= sizeof(u32);
1919 if (!nsels) {
1920 dev_err(dev, "invalid reg property size\n");
1921 return -EINVAL;
1922 }
1923
1924 sunxi_nand = devm_kzalloc(dev, struct_size(sunxi_nand, sels, nsels),
1925 GFP_KERNEL);
1926 if (!sunxi_nand) {
1927 dev_err(dev, "could not allocate chip\n");
1928 return -ENOMEM;
1929 }
1930
1931 sunxi_nand->nsels = nsels;
1932
1933 for (i = 0; i < nsels; i++) {
1934 ret = of_property_read_u32_index(np, "reg", i, &tmp);
1935 if (ret) {
1936 dev_err(dev, "could not retrieve reg property: %d\n",
1937 ret);
1938 return ret;
1939 }
1940
1941 if (tmp > NFC_MAX_CS) {
1942 dev_err(dev,
1943 "invalid reg value: %u (max CS = 7)\n",
1944 tmp);
1945 return -EINVAL;
1946 }
1947
1948 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
1949 dev_err(dev, "CS %d already assigned\n", tmp);
1950 return -EINVAL;
1951 }
1952
1953 sunxi_nand->sels[i].cs = tmp;
1954
1955 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) &&
1956 tmp < 2)
1957 sunxi_nand->sels[i].rb = tmp;
1958 else
1959 sunxi_nand->sels[i].rb = -1;
1960 }
1961
1962 nand = &sunxi_nand->nand;
1963 /* Default tR value specified in the ONFI spec (chapter 4.15.1) */
1964 nand->controller = &nfc->controller;
1965 nand->controller->ops = &sunxi_nand_controller_ops;
1966
1967 /*
1968 * Set the ECC mode to the default value in case nothing is specified
1969 * in the DT.
1970 */
1971 nand->ecc.mode = NAND_ECC_HW;
1972 nand_set_flash_node(nand, np);
1973
1974 mtd = nand_to_mtd(nand);
1975 mtd->dev.parent = dev;
1976
1977 ret = nand_scan(nand, nsels);
1978 if (ret)
1979 return ret;
1980
1981 ret = mtd_device_register(mtd, NULL, 0);
1982 if (ret) {
1983 dev_err(dev, "failed to register mtd device: %d\n", ret);
1984 nand_release(nand);
1985 return ret;
1986 }
1987
1988 list_add_tail(&sunxi_nand->node, &nfc->chips);
1989
1990 return 0;
1991 }
1992
1993 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)
1994 {
1995 struct device_node *np = dev->of_node;
1996 struct device_node *nand_np;
1997 int nchips = of_get_child_count(np);
1998 int ret;
1999
2000 if (nchips > 8) {
2001 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips);
2002 return -EINVAL;
2003 }
2004
2005 for_each_child_of_node(np, nand_np) {
2006 ret = sunxi_nand_chip_init(dev, nfc, nand_np);
2007 if (ret) {
2008 of_node_put(nand_np);
2009 return ret;
2010 }
2011 }
2012
2013 return 0;
2014 }
2015
2016 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)
2017 {
2018 struct sunxi_nand_chip *sunxi_nand;
2019
2020 while (!list_empty(&nfc->chips)) {
2021 sunxi_nand = list_first_entry(&nfc->chips,
2022 struct sunxi_nand_chip,
2023 node);
2024 nand_release(&sunxi_nand->nand);
2025 sunxi_nand_ecc_cleanup(&sunxi_nand->nand.ecc);
2026 list_del(&sunxi_nand->node);
2027 }
2028 }
2029
2030 static int sunxi_nfc_probe(struct platform_device *pdev)
2031 {
2032 struct device *dev = &pdev->dev;
2033 struct resource *r;
2034 struct sunxi_nfc *nfc;
2035 int irq;
2036 int ret;
2037
2038 nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
2039 if (!nfc)
2040 return -ENOMEM;
2041
2042 nfc->dev = dev;
2043 nand_controller_init(&nfc->controller);
2044 INIT_LIST_HEAD(&nfc->chips);
2045
2046 r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2047 nfc->regs = devm_ioremap_resource(dev, r);
2048 if (IS_ERR(nfc->regs))
2049 return PTR_ERR(nfc->regs);
2050
2051 irq = platform_get_irq(pdev, 0);
2052 if (irq < 0) {
2053 dev_err(dev, "failed to retrieve irq\n");
2054 return irq;
2055 }
2056
2057 nfc->ahb_clk = devm_clk_get(dev, "ahb");
2058 if (IS_ERR(nfc->ahb_clk)) {
2059 dev_err(dev, "failed to retrieve ahb clk\n");
2060 return PTR_ERR(nfc->ahb_clk);
2061 }
2062
2063 ret = clk_prepare_enable(nfc->ahb_clk);
2064 if (ret)
2065 return ret;
2066
2067 nfc->mod_clk = devm_clk_get(dev, "mod");
2068 if (IS_ERR(nfc->mod_clk)) {
2069 dev_err(dev, "failed to retrieve mod clk\n");
2070 ret = PTR_ERR(nfc->mod_clk);
2071 goto out_ahb_clk_unprepare;
2072 }
2073
2074 ret = clk_prepare_enable(nfc->mod_clk);
2075 if (ret)
2076 goto out_ahb_clk_unprepare;
2077
2078 nfc->reset = devm_reset_control_get_optional_exclusive(dev, "ahb");
2079 if (IS_ERR(nfc->reset)) {
2080 ret = PTR_ERR(nfc->reset);
2081 goto out_mod_clk_unprepare;
2082 }
2083
2084 ret = reset_control_deassert(nfc->reset);
2085 if (ret) {
2086 dev_err(dev, "reset err %d\n", ret);
2087 goto out_mod_clk_unprepare;
2088 }
2089
2090 ret = sunxi_nfc_rst(nfc);
2091 if (ret)
2092 goto out_ahb_reset_reassert;
2093
2094 writel(0, nfc->regs + NFC_REG_INT);
2095 ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt,
2096 0, "sunxi-nand", nfc);
2097 if (ret)
2098 goto out_ahb_reset_reassert;
2099
2100 nfc->dmac = dma_request_slave_channel(dev, "rxtx");
2101 if (nfc->dmac) {
2102 struct dma_slave_config dmac_cfg = { };
2103
2104 dmac_cfg.src_addr = r->start + NFC_REG_IO_DATA;
2105 dmac_cfg.dst_addr = dmac_cfg.src_addr;
2106 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2107 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;
2108 dmac_cfg.src_maxburst = 4;
2109 dmac_cfg.dst_maxburst = 4;
2110 dmaengine_slave_config(nfc->dmac, &dmac_cfg);
2111 } else {
2112 dev_warn(dev, "failed to request rxtx DMA channel\n");
2113 }
2114
2115 platform_set_drvdata(pdev, nfc);
2116
2117 ret = sunxi_nand_chips_init(dev, nfc);
2118 if (ret) {
2119 dev_err(dev, "failed to init nand chips\n");
2120 goto out_release_dmac;
2121 }
2122
2123 return 0;
2124
2125 out_release_dmac:
2126 if (nfc->dmac)
2127 dma_release_channel(nfc->dmac);
2128 out_ahb_reset_reassert:
2129 reset_control_assert(nfc->reset);
2130 out_mod_clk_unprepare:
2131 clk_disable_unprepare(nfc->mod_clk);
2132 out_ahb_clk_unprepare:
2133 clk_disable_unprepare(nfc->ahb_clk);
2134
2135 return ret;
2136 }
2137
2138 static int sunxi_nfc_remove(struct platform_device *pdev)
2139 {
2140 struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
2141
2142 sunxi_nand_chips_cleanup(nfc);
2143
2144 reset_control_assert(nfc->reset);
2145
2146 if (nfc->dmac)
2147 dma_release_channel(nfc->dmac);
2148 clk_disable_unprepare(nfc->mod_clk);
2149 clk_disable_unprepare(nfc->ahb_clk);
2150
2151 return 0;
2152 }
2153
2154 static const struct of_device_id sunxi_nfc_ids[] = {
2155 { .compatible = "allwinner,sun4i-a10-nand" },
2156 { /* sentinel */ }
2157 };
2158 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids);
2159
2160 static struct platform_driver sunxi_nfc_driver = {
2161 .driver = {
2162 .name = "sunxi_nand",
2163 .of_match_table = sunxi_nfc_ids,
2164 },
2165 .probe = sunxi_nfc_probe,
2166 .remove = sunxi_nfc_remove,
2167 };
2168 module_platform_driver(sunxi_nfc_driver);
2169
2170 MODULE_LICENSE("GPL");
2171 MODULE_AUTHOR("Boris BREZILLON");
2172 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver");
2173 MODULE_ALIAS("platform:sunxi_nand");
Linux with added FPC-III support