?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'trace-v5.11-rc2' of git://git.kernel.org/pub/scm/linux/kernel/git/rostedt...
[linux/fpc-iii.git] / drivers / mtd / nand / raw / sunxi_nand.c
blob923a9e236fcf62c92b466f6a95a8b952ff168899
1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3 * Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com>
4 *
5 * Derived from:
6 * https://github.com/yuq/sunxi-nfc-mtd
7 * Copyright (C) 2013 Qiang Yu <yuq825@gmail.com>
8 *
9 * https://github.com/hno/Allwinner-Info
10 * Copyright (C) 2013 Henrik Nordström <Henrik Nordström>
11 *
12 * Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com>
13 * Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org>
14 */
15
16 #include <linux/dma-mapping.h>
17 #include <linux/slab.h>
18 #include <linux/module.h>
19 #include <linux/moduleparam.h>
20 #include <linux/platform_device.h>
21 #include <linux/of.h>
22 #include <linux/of_device.h>
23 #include <linux/mtd/mtd.h>
24 #include <linux/mtd/rawnand.h>
25 #include <linux/mtd/partitions.h>
26 #include <linux/clk.h>
27 #include <linux/delay.h>
28 #include <linux/dmaengine.h>
29 #include <linux/interrupt.h>
30 #include <linux/iopoll.h>
31 #include <linux/reset.h>
32
33 #define NFC_REG_CTL 0x0000
34 #define NFC_REG_ST 0x0004
35 #define NFC_REG_INT 0x0008
36 #define NFC_REG_TIMING_CTL 0x000C
37 #define NFC_REG_TIMING_CFG 0x0010
38 #define NFC_REG_ADDR_LOW 0x0014
39 #define NFC_REG_ADDR_HIGH 0x0018
40 #define NFC_REG_SECTOR_NUM 0x001C
41 #define NFC_REG_CNT 0x0020
42 #define NFC_REG_CMD 0x0024
43 #define NFC_REG_RCMD_SET 0x0028
44 #define NFC_REG_WCMD_SET 0x002C
45 #define NFC_REG_A10_IO_DATA 0x0030
46 #define NFC_REG_A23_IO_DATA 0x0300
47 #define NFC_REG_ECC_CTL 0x0034
48 #define NFC_REG_ECC_ST 0x0038
49 #define NFC_REG_DEBUG 0x003C
50 #define NFC_REG_ECC_ERR_CNT(x) ((0x0040 + (x)) & ~0x3)
51 #define NFC_REG_USER_DATA(x) (0x0050 + ((x) * 4))
52 #define NFC_REG_SPARE_AREA 0x00A0
53 #define NFC_REG_PAT_ID 0x00A4
54 #define NFC_REG_MDMA_ADDR 0x00C0
55 #define NFC_REG_MDMA_CNT 0x00C4
56 #define NFC_RAM0_BASE 0x0400
57 #define NFC_RAM1_BASE 0x0800
58
59 /* define bit use in NFC_CTL */
60 #define NFC_EN BIT(0)
61 #define NFC_RESET BIT(1)
62 #define NFC_BUS_WIDTH_MSK BIT(2)
63 #define NFC_BUS_WIDTH_8 (0 << 2)
64 #define NFC_BUS_WIDTH_16 (1 << 2)
65 #define NFC_RB_SEL_MSK BIT(3)
66 #define NFC_RB_SEL(x) ((x) << 3)
67 #define NFC_CE_SEL_MSK GENMASK(26, 24)
68 #define NFC_CE_SEL(x) ((x) << 24)
69 #define NFC_CE_CTL BIT(6)
70 #define NFC_PAGE_SHIFT_MSK GENMASK(11, 8)
71 #define NFC_PAGE_SHIFT(x) (((x) < 10 ? 0 : (x) - 10) << 8)
72 #define NFC_SAM BIT(12)
73 #define NFC_RAM_METHOD BIT(14)
74 #define NFC_DMA_TYPE_NORMAL BIT(15)
75 #define NFC_DEBUG_CTL BIT(31)
76
77 /* define bit use in NFC_ST */
78 #define NFC_RB_B2R BIT(0)
79 #define NFC_CMD_INT_FLAG BIT(1)
80 #define NFC_DMA_INT_FLAG BIT(2)
81 #define NFC_CMD_FIFO_STATUS BIT(3)
82 #define NFC_STA BIT(4)
83 #define NFC_NATCH_INT_FLAG BIT(5)
84 #define NFC_RB_STATE(x) BIT(x + 8)
85
86 /* define bit use in NFC_INT */
87 #define NFC_B2R_INT_ENABLE BIT(0)
88 #define NFC_CMD_INT_ENABLE BIT(1)
89 #define NFC_DMA_INT_ENABLE BIT(2)
90 #define NFC_INT_MASK (NFC_B2R_INT_ENABLE | \
91 NFC_CMD_INT_ENABLE | \
92 NFC_DMA_INT_ENABLE)
93
94 /* define bit use in NFC_TIMING_CTL */
95 #define NFC_TIMING_CTL_EDO BIT(8)
96
97 /* define NFC_TIMING_CFG register layout */
98 #define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD) \
99 (((tWB) & 0x3) | (((tADL) & 0x3) << 2) | \
100 (((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) | \
101 (((tCAD) & 0x7) << 8))
102
103 /* define bit use in NFC_CMD */
104 #define NFC_CMD_LOW_BYTE_MSK GENMASK(7, 0)
105 #define NFC_CMD_HIGH_BYTE_MSK GENMASK(15, 8)
106 #define NFC_CMD(x) (x)
107 #define NFC_ADR_NUM_MSK GENMASK(18, 16)
108 #define NFC_ADR_NUM(x) (((x) - 1) << 16)
109 #define NFC_SEND_ADR BIT(19)
110 #define NFC_ACCESS_DIR BIT(20)
111 #define NFC_DATA_TRANS BIT(21)
112 #define NFC_SEND_CMD1 BIT(22)
113 #define NFC_WAIT_FLAG BIT(23)
114 #define NFC_SEND_CMD2 BIT(24)
115 #define NFC_SEQ BIT(25)
116 #define NFC_DATA_SWAP_METHOD BIT(26)
117 #define NFC_ROW_AUTO_INC BIT(27)
118 #define NFC_SEND_CMD3 BIT(28)
119 #define NFC_SEND_CMD4 BIT(29)
120 #define NFC_CMD_TYPE_MSK GENMASK(31, 30)
121 #define NFC_NORMAL_OP (0 << 30)
122 #define NFC_ECC_OP (1 << 30)
123 #define NFC_PAGE_OP (2U << 30)
124
125 /* define bit use in NFC_RCMD_SET */
126 #define NFC_READ_CMD_MSK GENMASK(7, 0)
127 #define NFC_RND_READ_CMD0_MSK GENMASK(15, 8)
128 #define NFC_RND_READ_CMD1_MSK GENMASK(23, 16)
129
130 /* define bit use in NFC_WCMD_SET */
131 #define NFC_PROGRAM_CMD_MSK GENMASK(7, 0)
132 #define NFC_RND_WRITE_CMD_MSK GENMASK(15, 8)
133 #define NFC_READ_CMD0_MSK GENMASK(23, 16)
134 #define NFC_READ_CMD1_MSK GENMASK(31, 24)
135
136 /* define bit use in NFC_ECC_CTL */
137 #define NFC_ECC_EN BIT(0)
138 #define NFC_ECC_PIPELINE BIT(3)
139 #define NFC_ECC_EXCEPTION BIT(4)
140 #define NFC_ECC_BLOCK_SIZE_MSK BIT(5)
141 #define NFC_ECC_BLOCK_512 BIT(5)
142 #define NFC_RANDOM_EN BIT(9)
143 #define NFC_RANDOM_DIRECTION BIT(10)
144 #define NFC_ECC_MODE_MSK GENMASK(15, 12)
145 #define NFC_ECC_MODE(x) ((x) << 12)
146 #define NFC_RANDOM_SEED_MSK GENMASK(30, 16)
147 #define NFC_RANDOM_SEED(x) ((x) << 16)
148
149 /* define bit use in NFC_ECC_ST */
150 #define NFC_ECC_ERR(x) BIT(x)
151 #define NFC_ECC_ERR_MSK GENMASK(15, 0)
152 #define NFC_ECC_PAT_FOUND(x) BIT(x + 16)
153 #define NFC_ECC_ERR_CNT(b, x) (((x) >> (((b) % 4) * 8)) & 0xff)
154
155 #define NFC_DEFAULT_TIMEOUT_MS 1000
156
157 #define NFC_SRAM_SIZE 1024
158
159 #define NFC_MAX_CS 7
160
161 /**
162 * struct sunxi_nand_chip_sel - stores information related to NAND Chip Select
163 *
164 * @cs: the NAND CS id used to communicate with a NAND Chip
165 * @rb: the Ready/Busy pin ID. -1 means no R/B pin connected to the NFC
166 */
167 struct sunxi_nand_chip_sel {
168 u8 cs;
169 s8 rb;
170 };
171
172 /**
173 * struct sunxi_nand_hw_ecc - stores information related to HW ECC support
174 *
175 * @mode: the sunxi ECC mode field deduced from ECC requirements
176 */
177 struct sunxi_nand_hw_ecc {
178 int mode;
179 };
180
181 /**
182 * struct sunxi_nand_chip - stores NAND chip device related information
183 *
184 * @node: used to store NAND chips into a list
185 * @nand: base NAND chip structure
186 * @ecc: ECC controller structure
187 * @clk_rate: clk_rate required for this NAND chip
188 * @timing_cfg: TIMING_CFG register value for this NAND chip
189 * @timing_ctl: TIMING_CTL register value for this NAND chip
190 * @nsels: number of CS lines required by the NAND chip
191 * @sels: array of CS lines descriptions
192 */
193 struct sunxi_nand_chip {
194 struct list_head node;
195 struct nand_chip nand;
196 struct sunxi_nand_hw_ecc *ecc;
197 unsigned long clk_rate;
198 u32 timing_cfg;
199 u32 timing_ctl;
200 int nsels;
201 struct sunxi_nand_chip_sel sels[];
202 };
203
204 static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)
205 {
206 return container_of(nand, struct sunxi_nand_chip, nand);
207 }
208
209 /*
210 * NAND Controller capabilities structure: stores NAND controller capabilities
211 * for distinction between compatible strings.
212 *
213 * @has_mdma: Use mbus dma mode, otherwise general dma
214 * through MBUS on A23/A33 needs extra configuration.
215 * @reg_io_data: I/O data register
216 * @dma_maxburst: DMA maxburst
217 */
218 struct sunxi_nfc_caps {
219 bool has_mdma;
220 unsigned int reg_io_data;
221 unsigned int dma_maxburst;
222 };
223
224 /**
225 * struct sunxi_nfc - stores sunxi NAND controller information
226 *
227 * @controller: base controller structure
228 * @dev: parent device (used to print error messages)
229 * @regs: NAND controller registers
230 * @ahb_clk: NAND controller AHB clock
231 * @mod_clk: NAND controller mod clock
232 * @reset: NAND controller reset line
233 * @assigned_cs: bitmask describing already assigned CS lines
234 * @clk_rate: NAND controller current clock rate
235 * @chips: a list containing all the NAND chips attached to this NAND
236 * controller
237 * @complete: a completion object used to wait for NAND controller events
238 * @dmac: the DMA channel attached to the NAND controller
239 * @caps: NAND Controller capabilities
240 */
241 struct sunxi_nfc {
242 struct nand_controller controller;
243 struct device *dev;
244 void __iomem *regs;
245 struct clk *ahb_clk;
246 struct clk *mod_clk;
247 struct reset_control *reset;
248 unsigned long assigned_cs;
249 unsigned long clk_rate;
250 struct list_head chips;
251 struct completion complete;
252 struct dma_chan *dmac;
253 const struct sunxi_nfc_caps *caps;
254 };
255
256 static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_controller *ctrl)
257 {
258 return container_of(ctrl, struct sunxi_nfc, controller);
259 }
260
261 static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id)
262 {
263 struct sunxi_nfc *nfc = dev_id;
264 u32 st = readl(nfc->regs + NFC_REG_ST);
265 u32 ien = readl(nfc->regs + NFC_REG_INT);
266
267 if (!(ien & st))
268 return IRQ_NONE;
269
270 if ((ien & st) == ien)
271 complete(&nfc->complete);
272
273 writel(st & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
274 writel(~st & ien & NFC_INT_MASK, nfc->regs + NFC_REG_INT);
275
276 return IRQ_HANDLED;
277 }
278
279 static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events,
280 bool use_polling, unsigned int timeout_ms)
281 {
282 int ret;
283
284 if (events & ~NFC_INT_MASK)
285 return -EINVAL;
286
287 if (!timeout_ms)
288 timeout_ms = NFC_DEFAULT_TIMEOUT_MS;
289
290 if (!use_polling) {
291 init_completion(&nfc->complete);
292
293 writel(events, nfc->regs + NFC_REG_INT);
294
295 ret = wait_for_completion_timeout(&nfc->complete,
296 msecs_to_jiffies(timeout_ms));
297 if (!ret)
298 ret = -ETIMEDOUT;
299 else
300 ret = 0;
301
302 writel(0, nfc->regs + NFC_REG_INT);
303 } else {
304 u32 status;
305
306 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
307 (status & events) == events, 1,
308 timeout_ms * 1000);
309 }
310
311 writel(events & NFC_INT_MASK, nfc->regs + NFC_REG_ST);
312
313 if (ret)
314 dev_err(nfc->dev, "wait interrupt timedout\n");
315
316 return ret;
317 }
318
319 static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc)
320 {
321 u32 status;
322 int ret;
323
324 ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
325 !(status & NFC_CMD_FIFO_STATUS), 1,
326 NFC_DEFAULT_TIMEOUT_MS * 1000);
327 if (ret)
328 dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n");
329
330 return ret;
331 }
332
333 static int sunxi_nfc_rst(struct sunxi_nfc *nfc)
334 {
335 u32 ctl;
336 int ret;
337
338 writel(0, nfc->regs + NFC_REG_ECC_CTL);
339 writel(NFC_RESET, nfc->regs + NFC_REG_CTL);
340
341 ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl,
342 !(ctl & NFC_RESET), 1,
343 NFC_DEFAULT_TIMEOUT_MS * 1000);
344 if (ret)
345 dev_err(nfc->dev, "wait for NAND controller reset timedout\n");
346
347 return ret;
348 }
349
350 static int sunxi_nfc_dma_op_prepare(struct sunxi_nfc *nfc, const void *buf,
351 int chunksize, int nchunks,
352 enum dma_data_direction ddir,
353 struct scatterlist *sg)
354 {
355 struct dma_async_tx_descriptor *dmad;
356 enum dma_transfer_direction tdir;
357 dma_cookie_t dmat;
358 int ret;
359
360 if (ddir == DMA_FROM_DEVICE)
361 tdir = DMA_DEV_TO_MEM;
362 else
363 tdir = DMA_MEM_TO_DEV;
364
365 sg_init_one(sg, buf, nchunks * chunksize);
366 ret = dma_map_sg(nfc->dev, sg, 1, ddir);
367 if (!ret)
368 return -ENOMEM;
369
370 if (!nfc->caps->has_mdma) {
371 dmad = dmaengine_prep_slave_sg(nfc->dmac, sg, 1, tdir, DMA_CTRL_ACK);
372 if (!dmad) {
373 ret = -EINVAL;
374 goto err_unmap_buf;
375 }
376 }
377
378 writel(readl(nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,
379 nfc->regs + NFC_REG_CTL);
380 writel(nchunks, nfc->regs + NFC_REG_SECTOR_NUM);
381 writel(chunksize, nfc->regs + NFC_REG_CNT);
382
383 if (nfc->caps->has_mdma) {
384 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_DMA_TYPE_NORMAL,
385 nfc->regs + NFC_REG_CTL);
386 writel(chunksize * nchunks, nfc->regs + NFC_REG_MDMA_CNT);
387 writel(sg_dma_address(sg), nfc->regs + NFC_REG_MDMA_ADDR);
388 } else {
389 dmat = dmaengine_submit(dmad);
390
391 ret = dma_submit_error(dmat);
392 if (ret)
393 goto err_clr_dma_flag;
394 }
395
396 return 0;
397
398 err_clr_dma_flag:
399 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
400 nfc->regs + NFC_REG_CTL);
401
402 err_unmap_buf:
403 dma_unmap_sg(nfc->dev, sg, 1, ddir);
404 return ret;
405 }
406
407 static void sunxi_nfc_dma_op_cleanup(struct sunxi_nfc *nfc,
408 enum dma_data_direction ddir,
409 struct scatterlist *sg)
410 {
411 dma_unmap_sg(nfc->dev, sg, 1, ddir);
412 writel(readl(nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
413 nfc->regs + NFC_REG_CTL);
414 }
415
416 static void sunxi_nfc_select_chip(struct nand_chip *nand, unsigned int cs)
417 {
418 struct mtd_info *mtd = nand_to_mtd(nand);
419 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
420 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
421 struct sunxi_nand_chip_sel *sel;
422 u32 ctl;
423
424 if (cs > 0 && cs >= sunxi_nand->nsels)
425 return;
426
427 ctl = readl(nfc->regs + NFC_REG_CTL) &
428 ~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN);
429
430 sel = &sunxi_nand->sels[cs];
431 ctl |= NFC_CE_SEL(sel->cs) | NFC_EN | NFC_PAGE_SHIFT(nand->page_shift);
432 if (sel->rb >= 0)
433 ctl |= NFC_RB_SEL(sel->rb);
434
435 writel(mtd->writesize, nfc->regs + NFC_REG_SPARE_AREA);
436
437 if (nfc->clk_rate != sunxi_nand->clk_rate) {
438 clk_set_rate(nfc->mod_clk, sunxi_nand->clk_rate);
439 nfc->clk_rate = sunxi_nand->clk_rate;
440 }
441
442 writel(sunxi_nand->timing_ctl, nfc->regs + NFC_REG_TIMING_CTL);
443 writel(sunxi_nand->timing_cfg, nfc->regs + NFC_REG_TIMING_CFG);
444 writel(ctl, nfc->regs + NFC_REG_CTL);
445 }
446
447 static void sunxi_nfc_read_buf(struct nand_chip *nand, uint8_t *buf, int len)
448 {
449 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
450 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
451 int ret;
452 int cnt;
453 int offs = 0;
454 u32 tmp;
455
456 while (len > offs) {
457 bool poll = false;
458
459 cnt = min(len - offs, NFC_SRAM_SIZE);
460
461 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
462 if (ret)
463 break;
464
465 writel(cnt, nfc->regs + NFC_REG_CNT);
466 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
467 writel(tmp, nfc->regs + NFC_REG_CMD);
468
469 /* Arbitrary limit for polling mode */
470 if (cnt < 64)
471 poll = true;
472
473 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
474 if (ret)
475 break;
476
477 if (buf)
478 memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE,
479 cnt);
480 offs += cnt;
481 }
482 }
483
484 static void sunxi_nfc_write_buf(struct nand_chip *nand, const uint8_t *buf,
485 int len)
486 {
487 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
488 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
489 int ret;
490 int cnt;
491 int offs = 0;
492 u32 tmp;
493
494 while (len > offs) {
495 bool poll = false;
496
497 cnt = min(len - offs, NFC_SRAM_SIZE);
498
499 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
500 if (ret)
501 break;
502
503 writel(cnt, nfc->regs + NFC_REG_CNT);
504 memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt);
505 tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
506 NFC_ACCESS_DIR;
507 writel(tmp, nfc->regs + NFC_REG_CMD);
508
509 /* Arbitrary limit for polling mode */
510 if (cnt < 64)
511 poll = true;
512
513 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, poll, 0);
514 if (ret)
515 break;
516
517 offs += cnt;
518 }
519 }
520
521 /* These seed values have been extracted from Allwinner's BSP */
522 static const u16 sunxi_nfc_randomizer_page_seeds[] = {
523 0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72,
524 0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436,
525 0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d,
526 0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130,
527 0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56,
528 0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55,
529 0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb,
530 0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17,
531 0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62,
532 0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064,
533 0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126,
534 0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e,
535 0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3,
536 0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b,
537 0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d,
538 0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db,
539 };
540
541 /*
542 * sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds
543 * have been generated using
544 * sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what
545 * the randomizer engine does internally before de/scrambling OOB data.
546 *
547 * Those tables are statically defined to avoid calculating randomizer state
548 * at runtime.
549 */
550 static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = {
551 0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64,
552 0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409,
553 0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617,
554 0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d,
555 0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91,
556 0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d,
557 0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab,
558 0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8,
559 0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8,
560 0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b,
561 0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5,
562 0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a,
563 0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891,
564 0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36,
565 0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd,
566 0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0,
567 };
568
569 static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = {
570 0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6,
571 0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982,
572 0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9,
573 0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07,
574 0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e,
575 0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2,
576 0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c,
577 0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f,
578 0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc,
579 0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e,
580 0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8,
581 0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68,
582 0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d,
583 0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179,
584 0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601,
585 0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd,
586 };
587
588 static u16 sunxi_nfc_randomizer_step(u16 state, int count)
589 {
590 state &= 0x7fff;
591
592 /*
593 * This loop is just a simple implementation of a Fibonacci LFSR using
594 * the x16 + x15 + 1 polynomial.
595 */
596 while (count--)
597 state = ((state >> 1) |
598 (((state ^ (state >> 1)) & 1) << 14)) & 0x7fff;
599
600 return state;
601 }
602
603 static u16 sunxi_nfc_randomizer_state(struct nand_chip *nand, int page,
604 bool ecc)
605 {
606 struct mtd_info *mtd = nand_to_mtd(nand);
607 const u16 *seeds = sunxi_nfc_randomizer_page_seeds;
608 int mod = mtd_div_by_ws(mtd->erasesize, mtd);
609
610 if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds))
611 mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds);
612
613 if (ecc) {
614 if (mtd->ecc_step_size == 512)
615 seeds = sunxi_nfc_randomizer_ecc512_seeds;
616 else
617 seeds = sunxi_nfc_randomizer_ecc1024_seeds;
618 }
619
620 return seeds[page % mod];
621 }
622
623 static void sunxi_nfc_randomizer_config(struct nand_chip *nand, int page,
624 bool ecc)
625 {
626 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
627 u32 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
628 u16 state;
629
630 if (!(nand->options & NAND_NEED_SCRAMBLING))
631 return;
632
633 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
634 state = sunxi_nfc_randomizer_state(nand, page, ecc);
635 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK;
636 writel(ecc_ctl | NFC_RANDOM_SEED(state), nfc->regs + NFC_REG_ECC_CTL);
637 }
638
639 static void sunxi_nfc_randomizer_enable(struct nand_chip *nand)
640 {
641 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
642
643 if (!(nand->options & NAND_NEED_SCRAMBLING))
644 return;
645
646 writel(readl(nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN,
647 nfc->regs + NFC_REG_ECC_CTL);
648 }
649
650 static void sunxi_nfc_randomizer_disable(struct nand_chip *nand)
651 {
652 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
653
654 if (!(nand->options & NAND_NEED_SCRAMBLING))
655 return;
656
657 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN,
658 nfc->regs + NFC_REG_ECC_CTL);
659 }
660
661 static void sunxi_nfc_randomize_bbm(struct nand_chip *nand, int page, u8 *bbm)
662 {
663 u16 state = sunxi_nfc_randomizer_state(nand, page, true);
664
665 bbm[0] ^= state;
666 bbm[1] ^= sunxi_nfc_randomizer_step(state, 8);
667 }
668
669 static void sunxi_nfc_randomizer_write_buf(struct nand_chip *nand,
670 const uint8_t *buf, int len,
671 bool ecc, int page)
672 {
673 sunxi_nfc_randomizer_config(nand, page, ecc);
674 sunxi_nfc_randomizer_enable(nand);
675 sunxi_nfc_write_buf(nand, buf, len);
676 sunxi_nfc_randomizer_disable(nand);
677 }
678
679 static void sunxi_nfc_randomizer_read_buf(struct nand_chip *nand, uint8_t *buf,
680 int len, bool ecc, int page)
681 {
682 sunxi_nfc_randomizer_config(nand, page, ecc);
683 sunxi_nfc_randomizer_enable(nand);
684 sunxi_nfc_read_buf(nand, buf, len);
685 sunxi_nfc_randomizer_disable(nand);
686 }
687
688 static void sunxi_nfc_hw_ecc_enable(struct nand_chip *nand)
689 {
690 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
691 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
692 u32 ecc_ctl;
693
694 ecc_ctl = readl(nfc->regs + NFC_REG_ECC_CTL);
695 ecc_ctl &= ~(NFC_ECC_MODE_MSK | NFC_ECC_PIPELINE |
696 NFC_ECC_BLOCK_SIZE_MSK);
697 ecc_ctl |= NFC_ECC_EN | NFC_ECC_MODE(sunxi_nand->ecc->mode) |
698 NFC_ECC_EXCEPTION | NFC_ECC_PIPELINE;
699
700 if (nand->ecc.size == 512)
701 ecc_ctl |= NFC_ECC_BLOCK_512;
702
703 writel(ecc_ctl, nfc->regs + NFC_REG_ECC_CTL);
704 }
705
706 static void sunxi_nfc_hw_ecc_disable(struct nand_chip *nand)
707 {
708 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
709
710 writel(readl(nfc->regs + NFC_REG_ECC_CTL) & ~NFC_ECC_EN,
711 nfc->regs + NFC_REG_ECC_CTL);
712 }
713
714 static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf)
715 {
716 buf[0] = user_data;
717 buf[1] = user_data >> 8;
718 buf[2] = user_data >> 16;
719 buf[3] = user_data >> 24;
720 }
721
722 static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf)
723 {
724 return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24);
725 }
726
727 static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct nand_chip *nand, u8 *oob,
728 int step, bool bbm, int page)
729 {
730 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
731
732 sunxi_nfc_user_data_to_buf(readl(nfc->regs + NFC_REG_USER_DATA(step)),
733 oob);
734
735 /* De-randomize the Bad Block Marker. */
736 if (bbm && (nand->options & NAND_NEED_SCRAMBLING))
737 sunxi_nfc_randomize_bbm(nand, page, oob);
738 }
739
740 static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct nand_chip *nand,
741 const u8 *oob, int step,
742 bool bbm, int page)
743 {
744 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
745 u8 user_data[4];
746
747 /* Randomize the Bad Block Marker. */
748 if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
749 memcpy(user_data, oob, sizeof(user_data));
750 sunxi_nfc_randomize_bbm(nand, page, user_data);
751 oob = user_data;
752 }
753
754 writel(sunxi_nfc_buf_to_user_data(oob),
755 nfc->regs + NFC_REG_USER_DATA(step));
756 }
757
758 static void sunxi_nfc_hw_ecc_update_stats(struct nand_chip *nand,
759 unsigned int *max_bitflips, int ret)
760 {
761 struct mtd_info *mtd = nand_to_mtd(nand);
762
763 if (ret < 0) {
764 mtd->ecc_stats.failed++;
765 } else {
766 mtd->ecc_stats.corrected += ret;
767 *max_bitflips = max_t(unsigned int, *max_bitflips, ret);
768 }
769 }
770
771 static int sunxi_nfc_hw_ecc_correct(struct nand_chip *nand, u8 *data, u8 *oob,
772 int step, u32 status, bool *erased)
773 {
774 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
775 struct nand_ecc_ctrl *ecc = &nand->ecc;
776 u32 tmp;
777
778 *erased = false;
779
780 if (status & NFC_ECC_ERR(step))
781 return -EBADMSG;
782
783 if (status & NFC_ECC_PAT_FOUND(step)) {
784 u8 pattern;
785
786 if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) {
787 pattern = 0x0;
788 } else {
789 pattern = 0xff;
790 *erased = true;
791 }
792
793 if (data)
794 memset(data, pattern, ecc->size);
795
796 if (oob)
797 memset(oob, pattern, ecc->bytes + 4);
798
799 return 0;
800 }
801
802 tmp = readl(nfc->regs + NFC_REG_ECC_ERR_CNT(step));
803
804 return NFC_ECC_ERR_CNT(step, tmp);
805 }
806
807 static int sunxi_nfc_hw_ecc_read_chunk(struct nand_chip *nand,
808 u8 *data, int data_off,
809 u8 *oob, int oob_off,
810 int *cur_off,
811 unsigned int *max_bitflips,
812 bool bbm, bool oob_required, int page)
813 {
814 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
815 struct nand_ecc_ctrl *ecc = &nand->ecc;
816 int raw_mode = 0;
817 bool erased;
818 int ret;
819
820 if (*cur_off != data_off)
821 nand_change_read_column_op(nand, data_off, NULL, 0, false);
822
823 sunxi_nfc_randomizer_read_buf(nand, NULL, ecc->size, false, page);
824
825 if (data_off + ecc->size != oob_off)
826 nand_change_read_column_op(nand, oob_off, NULL, 0, false);
827
828 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
829 if (ret)
830 return ret;
831
832 sunxi_nfc_randomizer_enable(nand);
833 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP,
834 nfc->regs + NFC_REG_CMD);
835
836 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
837 sunxi_nfc_randomizer_disable(nand);
838 if (ret)
839 return ret;
840
841 *cur_off = oob_off + ecc->bytes + 4;
842
843 ret = sunxi_nfc_hw_ecc_correct(nand, data, oob_required ? oob : NULL, 0,
844 readl(nfc->regs + NFC_REG_ECC_ST),
845 &erased);
846 if (erased)
847 return 1;
848
849 if (ret < 0) {
850 /*
851 * Re-read the data with the randomizer disabled to identify
852 * bitflips in erased pages.
853 */
854 if (nand->options & NAND_NEED_SCRAMBLING)
855 nand_change_read_column_op(nand, data_off, data,
856 ecc->size, false);
857 else
858 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,
859 ecc->size);
860
861 nand_change_read_column_op(nand, oob_off, oob, ecc->bytes + 4,
862 false);
863
864 ret = nand_check_erased_ecc_chunk(data, ecc->size,
865 oob, ecc->bytes + 4,
866 NULL, 0, ecc->strength);
867 if (ret >= 0)
868 raw_mode = 1;
869 } else {
870 memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);
871
872 if (oob_required) {
873 nand_change_read_column_op(nand, oob_off, NULL, 0,
874 false);
875 sunxi_nfc_randomizer_read_buf(nand, oob, ecc->bytes + 4,
876 true, page);
877
878 sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, 0,
879 bbm, page);
880 }
881 }
882
883 sunxi_nfc_hw_ecc_update_stats(nand, max_bitflips, ret);
884
885 return raw_mode;
886 }
887
888 static void sunxi_nfc_hw_ecc_read_extra_oob(struct nand_chip *nand,
889 u8 *oob, int *cur_off,
890 bool randomize, int page)
891 {
892 struct mtd_info *mtd = nand_to_mtd(nand);
893 struct nand_ecc_ctrl *ecc = &nand->ecc;
894 int offset = ((ecc->bytes + 4) * ecc->steps);
895 int len = mtd->oobsize - offset;
896
897 if (len <= 0)
898 return;
899
900 if (!cur_off || *cur_off != offset)
901 nand_change_read_column_op(nand, mtd->writesize, NULL, 0,
902 false);
903
904 if (!randomize)
905 sunxi_nfc_read_buf(nand, oob + offset, len);
906 else
907 sunxi_nfc_randomizer_read_buf(nand, oob + offset, len,
908 false, page);
909
910 if (cur_off)
911 *cur_off = mtd->oobsize + mtd->writesize;
912 }
913
914 static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, uint8_t *buf,
915 int oob_required, int page,
916 int nchunks)
917 {
918 bool randomized = nand->options & NAND_NEED_SCRAMBLING;
919 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
920 struct mtd_info *mtd = nand_to_mtd(nand);
921 struct nand_ecc_ctrl *ecc = &nand->ecc;
922 unsigned int max_bitflips = 0;
923 int ret, i, raw_mode = 0;
924 struct scatterlist sg;
925 u32 status, wait;
926
927 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
928 if (ret)
929 return ret;
930
931 ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, nchunks,
932 DMA_FROM_DEVICE, &sg);
933 if (ret)
934 return ret;
935
936 sunxi_nfc_hw_ecc_enable(nand);
937 sunxi_nfc_randomizer_config(nand, page, false);
938 sunxi_nfc_randomizer_enable(nand);
939
940 writel((NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |
941 NAND_CMD_READSTART, nfc->regs + NFC_REG_RCMD_SET);
942
943 wait = NFC_CMD_INT_FLAG;
944
945 if (nfc->caps->has_mdma)
946 wait |= NFC_DMA_INT_FLAG;
947 else
948 dma_async_issue_pending(nfc->dmac);
949
950 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,
951 nfc->regs + NFC_REG_CMD);
952
953 ret = sunxi_nfc_wait_events(nfc, wait, false, 0);
954 if (ret && !nfc->caps->has_mdma)
955 dmaengine_terminate_all(nfc->dmac);
956
957 sunxi_nfc_randomizer_disable(nand);
958 sunxi_nfc_hw_ecc_disable(nand);
959
960 sunxi_nfc_dma_op_cleanup(nfc, DMA_FROM_DEVICE, &sg);
961
962 if (ret)
963 return ret;
964
965 status = readl(nfc->regs + NFC_REG_ECC_ST);
966
967 for (i = 0; i < nchunks; i++) {
968 int data_off = i * ecc->size;
969 int oob_off = i * (ecc->bytes + 4);
970 u8 *data = buf + data_off;
971 u8 *oob = nand->oob_poi + oob_off;
972 bool erased;
973
974 ret = sunxi_nfc_hw_ecc_correct(nand, randomized ? data : NULL,
975 oob_required ? oob : NULL,
976 i, status, &erased);
977
978 /* ECC errors are handled in the second loop. */
979 if (ret < 0)
980 continue;
981
982 if (oob_required && !erased) {
983 /* TODO: use DMA to retrieve OOB */
984 nand_change_read_column_op(nand,
985 mtd->writesize + oob_off,
986 oob, ecc->bytes + 4, false);
987
988 sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, i,
989 !i, page);
990 }
991
992 if (erased)
993 raw_mode = 1;
994
995 sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, ret);
996 }
997
998 if (status & NFC_ECC_ERR_MSK) {
999 for (i = 0; i < nchunks; i++) {
1000 int data_off = i * ecc->size;
1001 int oob_off = i * (ecc->bytes + 4);
1002 u8 *data = buf + data_off;
1003 u8 *oob = nand->oob_poi + oob_off;
1004
1005 if (!(status & NFC_ECC_ERR(i)))
1006 continue;
1007
1008 /*
1009 * Re-read the data with the randomizer disabled to
1010 * identify bitflips in erased pages.
1011 * TODO: use DMA to read page in raw mode
1012 */
1013 if (randomized)
1014 nand_change_read_column_op(nand, data_off,
1015 data, ecc->size,
1016 false);
1017
1018 /* TODO: use DMA to retrieve OOB */
1019 nand_change_read_column_op(nand,
1020 mtd->writesize + oob_off,
1021 oob, ecc->bytes + 4, false);
1022
1023 ret = nand_check_erased_ecc_chunk(data, ecc->size,
1024 oob, ecc->bytes + 4,
1025 NULL, 0,
1026 ecc->strength);
1027 if (ret >= 0)
1028 raw_mode = 1;
1029
1030 sunxi_nfc_hw_ecc_update_stats(nand, &max_bitflips, ret);
1031 }
1032 }
1033
1034 if (oob_required)
1035 sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi,
1036 NULL, !raw_mode,
1037 page);
1038
1039 return max_bitflips;
1040 }
1041
1042 static int sunxi_nfc_hw_ecc_write_chunk(struct nand_chip *nand,
1043 const u8 *data, int data_off,
1044 const u8 *oob, int oob_off,
1045 int *cur_off, bool bbm,
1046 int page)
1047 {
1048 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1049 struct nand_ecc_ctrl *ecc = &nand->ecc;
1050 int ret;
1051
1052 if (data_off != *cur_off)
1053 nand_change_write_column_op(nand, data_off, NULL, 0, false);
1054
1055 sunxi_nfc_randomizer_write_buf(nand, data, ecc->size, false, page);
1056
1057 if (data_off + ecc->size != oob_off)
1058 nand_change_write_column_op(nand, oob_off, NULL, 0, false);
1059
1060 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1061 if (ret)
1062 return ret;
1063
1064 sunxi_nfc_randomizer_enable(nand);
1065 sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, 0, bbm, page);
1066
1067 writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
1068 NFC_ACCESS_DIR | NFC_ECC_OP,
1069 nfc->regs + NFC_REG_CMD);
1070
1071 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, false, 0);
1072 sunxi_nfc_randomizer_disable(nand);
1073 if (ret)
1074 return ret;
1075
1076 *cur_off = oob_off + ecc->bytes + 4;
1077
1078 return 0;
1079 }
1080
1081 static void sunxi_nfc_hw_ecc_write_extra_oob(struct nand_chip *nand,
1082 u8 *oob, int *cur_off,
1083 int page)
1084 {
1085 struct mtd_info *mtd = nand_to_mtd(nand);
1086 struct nand_ecc_ctrl *ecc = &nand->ecc;
1087 int offset = ((ecc->bytes + 4) * ecc->steps);
1088 int len = mtd->oobsize - offset;
1089
1090 if (len <= 0)
1091 return;
1092
1093 if (!cur_off || *cur_off != offset)
1094 nand_change_write_column_op(nand, offset + mtd->writesize,
1095 NULL, 0, false);
1096
1097 sunxi_nfc_randomizer_write_buf(nand, oob + offset, len, false, page);
1098
1099 if (cur_off)
1100 *cur_off = mtd->oobsize + mtd->writesize;
1101 }
1102
1103 static int sunxi_nfc_hw_ecc_read_page(struct nand_chip *nand, uint8_t *buf,
1104 int oob_required, int page)
1105 {
1106 struct mtd_info *mtd = nand_to_mtd(nand);
1107 struct nand_ecc_ctrl *ecc = &nand->ecc;
1108 unsigned int max_bitflips = 0;
1109 int ret, i, cur_off = 0;
1110 bool raw_mode = false;
1111
1112 sunxi_nfc_select_chip(nand, nand->cur_cs);
1113
1114 nand_read_page_op(nand, page, 0, NULL, 0);
1115
1116 sunxi_nfc_hw_ecc_enable(nand);
1117
1118 for (i = 0; i < ecc->steps; i++) {
1119 int data_off = i * ecc->size;
1120 int oob_off = i * (ecc->bytes + 4);
1121 u8 *data = buf + data_off;
1122 u8 *oob = nand->oob_poi + oob_off;
1123
1124 ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off, oob,
1125 oob_off + mtd->writesize,
1126 &cur_off, &max_bitflips,
1127 !i, oob_required, page);
1128 if (ret < 0)
1129 return ret;
1130 else if (ret)
1131 raw_mode = true;
1132 }
1133
1134 if (oob_required)
1135 sunxi_nfc_hw_ecc_read_extra_oob(nand, nand->oob_poi, &cur_off,
1136 !raw_mode, page);
1137
1138 sunxi_nfc_hw_ecc_disable(nand);
1139
1140 return max_bitflips;
1141 }
1142
1143 static int sunxi_nfc_hw_ecc_read_page_dma(struct nand_chip *nand, u8 *buf,
1144 int oob_required, int page)
1145 {
1146 int ret;
1147
1148 sunxi_nfc_select_chip(nand, nand->cur_cs);
1149
1150 nand_read_page_op(nand, page, 0, NULL, 0);
1151
1152 ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, oob_required, page,
1153 nand->ecc.steps);
1154 if (ret >= 0)
1155 return ret;
1156
1157 /* Fallback to PIO mode */
1158 return sunxi_nfc_hw_ecc_read_page(nand, buf, oob_required, page);
1159 }
1160
1161 static int sunxi_nfc_hw_ecc_read_subpage(struct nand_chip *nand,
1162 u32 data_offs, u32 readlen,
1163 u8 *bufpoi, int page)
1164 {
1165 struct mtd_info *mtd = nand_to_mtd(nand);
1166 struct nand_ecc_ctrl *ecc = &nand->ecc;
1167 int ret, i, cur_off = 0;
1168 unsigned int max_bitflips = 0;
1169
1170 sunxi_nfc_select_chip(nand, nand->cur_cs);
1171
1172 nand_read_page_op(nand, page, 0, NULL, 0);
1173
1174 sunxi_nfc_hw_ecc_enable(nand);
1175
1176 for (i = data_offs / ecc->size;
1177 i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) {
1178 int data_off = i * ecc->size;
1179 int oob_off = i * (ecc->bytes + 4);
1180 u8 *data = bufpoi + data_off;
1181 u8 *oob = nand->oob_poi + oob_off;
1182
1183 ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off,
1184 oob,
1185 oob_off + mtd->writesize,
1186 &cur_off, &max_bitflips, !i,
1187 false, page);
1188 if (ret < 0)
1189 return ret;
1190 }
1191
1192 sunxi_nfc_hw_ecc_disable(nand);
1193
1194 return max_bitflips;
1195 }
1196
1197 static int sunxi_nfc_hw_ecc_read_subpage_dma(struct nand_chip *nand,
1198 u32 data_offs, u32 readlen,
1199 u8 *buf, int page)
1200 {
1201 int nchunks = DIV_ROUND_UP(data_offs + readlen, nand->ecc.size);
1202 int ret;
1203
1204 sunxi_nfc_select_chip(nand, nand->cur_cs);
1205
1206 nand_read_page_op(nand, page, 0, NULL, 0);
1207
1208 ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, false, page, nchunks);
1209 if (ret >= 0)
1210 return ret;
1211
1212 /* Fallback to PIO mode */
1213 return sunxi_nfc_hw_ecc_read_subpage(nand, data_offs, readlen,
1214 buf, page);
1215 }
1216
1217 static int sunxi_nfc_hw_ecc_write_page(struct nand_chip *nand,
1218 const uint8_t *buf, int oob_required,
1219 int page)
1220 {
1221 struct mtd_info *mtd = nand_to_mtd(nand);
1222 struct nand_ecc_ctrl *ecc = &nand->ecc;
1223 int ret, i, cur_off = 0;
1224
1225 sunxi_nfc_select_chip(nand, nand->cur_cs);
1226
1227 nand_prog_page_begin_op(nand, page, 0, NULL, 0);
1228
1229 sunxi_nfc_hw_ecc_enable(nand);
1230
1231 for (i = 0; i < ecc->steps; i++) {
1232 int data_off = i * ecc->size;
1233 int oob_off = i * (ecc->bytes + 4);
1234 const u8 *data = buf + data_off;
1235 const u8 *oob = nand->oob_poi + oob_off;
1236
1237 ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob,
1238 oob_off + mtd->writesize,
1239 &cur_off, !i, page);
1240 if (ret)
1241 return ret;
1242 }
1243
1244 if (oob_required || (nand->options & NAND_NEED_SCRAMBLING))
1245 sunxi_nfc_hw_ecc_write_extra_oob(nand, nand->oob_poi,
1246 &cur_off, page);
1247
1248 sunxi_nfc_hw_ecc_disable(nand);
1249
1250 return nand_prog_page_end_op(nand);
1251 }
1252
1253 static int sunxi_nfc_hw_ecc_write_subpage(struct nand_chip *nand,
1254 u32 data_offs, u32 data_len,
1255 const u8 *buf, int oob_required,
1256 int page)
1257 {
1258 struct mtd_info *mtd = nand_to_mtd(nand);
1259 struct nand_ecc_ctrl *ecc = &nand->ecc;
1260 int ret, i, cur_off = 0;
1261
1262 sunxi_nfc_select_chip(nand, nand->cur_cs);
1263
1264 nand_prog_page_begin_op(nand, page, 0, NULL, 0);
1265
1266 sunxi_nfc_hw_ecc_enable(nand);
1267
1268 for (i = data_offs / ecc->size;
1269 i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) {
1270 int data_off = i * ecc->size;
1271 int oob_off = i * (ecc->bytes + 4);
1272 const u8 *data = buf + data_off;
1273 const u8 *oob = nand->oob_poi + oob_off;
1274
1275 ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob,
1276 oob_off + mtd->writesize,
1277 &cur_off, !i, page);
1278 if (ret)
1279 return ret;
1280 }
1281
1282 sunxi_nfc_hw_ecc_disable(nand);
1283
1284 return nand_prog_page_end_op(nand);
1285 }
1286
1287 static int sunxi_nfc_hw_ecc_write_page_dma(struct nand_chip *nand,
1288 const u8 *buf,
1289 int oob_required,
1290 int page)
1291 {
1292 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1293 struct nand_ecc_ctrl *ecc = &nand->ecc;
1294 struct scatterlist sg;
1295 u32 wait;
1296 int ret, i;
1297
1298 sunxi_nfc_select_chip(nand, nand->cur_cs);
1299
1300 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1301 if (ret)
1302 return ret;
1303
1304 ret = sunxi_nfc_dma_op_prepare(nfc, buf, ecc->size, ecc->steps,
1305 DMA_TO_DEVICE, &sg);
1306 if (ret)
1307 goto pio_fallback;
1308
1309 for (i = 0; i < ecc->steps; i++) {
1310 const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));
1311
1312 sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, i, !i, page);
1313 }
1314
1315 nand_prog_page_begin_op(nand, page, 0, NULL, 0);
1316
1317 sunxi_nfc_hw_ecc_enable(nand);
1318 sunxi_nfc_randomizer_config(nand, page, false);
1319 sunxi_nfc_randomizer_enable(nand);
1320
1321 writel((NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
1322 nfc->regs + NFC_REG_WCMD_SET);
1323
1324 wait = NFC_CMD_INT_FLAG;
1325
1326 if (nfc->caps->has_mdma)
1327 wait |= NFC_DMA_INT_FLAG;
1328 else
1329 dma_async_issue_pending(nfc->dmac);
1330
1331 writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
1332 NFC_DATA_TRANS | NFC_ACCESS_DIR,
1333 nfc->regs + NFC_REG_CMD);
1334
1335 ret = sunxi_nfc_wait_events(nfc, wait, false, 0);
1336 if (ret && !nfc->caps->has_mdma)
1337 dmaengine_terminate_all(nfc->dmac);
1338
1339 sunxi_nfc_randomizer_disable(nand);
1340 sunxi_nfc_hw_ecc_disable(nand);
1341
1342 sunxi_nfc_dma_op_cleanup(nfc, DMA_TO_DEVICE, &sg);
1343
1344 if (ret)
1345 return ret;
1346
1347 if (oob_required || (nand->options & NAND_NEED_SCRAMBLING))
1348 /* TODO: use DMA to transfer extra OOB bytes ? */
1349 sunxi_nfc_hw_ecc_write_extra_oob(nand, nand->oob_poi,
1350 NULL, page);
1351
1352 return nand_prog_page_end_op(nand);
1353
1354 pio_fallback:
1355 return sunxi_nfc_hw_ecc_write_page(nand, buf, oob_required, page);
1356 }
1357
1358 static int sunxi_nfc_hw_ecc_read_oob(struct nand_chip *nand, int page)
1359 {
1360 u8 *buf = nand_get_data_buf(nand);
1361
1362 return nand->ecc.read_page(nand, buf, 1, page);
1363 }
1364
1365 static int sunxi_nfc_hw_ecc_write_oob(struct nand_chip *nand, int page)
1366 {
1367 struct mtd_info *mtd = nand_to_mtd(nand);
1368 u8 *buf = nand_get_data_buf(nand);
1369 int ret;
1370
1371 memset(buf, 0xff, mtd->writesize);
1372 ret = nand->ecc.write_page(nand, buf, 1, page);
1373 if (ret)
1374 return ret;
1375
1376 /* Send command to program the OOB data */
1377 return nand_prog_page_end_op(nand);
1378 }
1379
1380 static const s32 tWB_lut[] = {6, 12, 16, 20};
1381 static const s32 tRHW_lut[] = {4, 8, 12, 20};
1382
1383 static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
1384 u32 clk_period)
1385 {
1386 u32 clk_cycles = DIV_ROUND_UP(duration, clk_period);
1387 int i;
1388
1389 for (i = 0; i < lut_size; i++) {
1390 if (clk_cycles <= lut[i])
1391 return i;
1392 }
1393
1394 /* Doesn't fit */
1395 return -EINVAL;
1396 }
1397
1398 #define sunxi_nand_lookup_timing(l, p, c) \
1399 _sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)
1400
1401 static int sunxi_nfc_setup_interface(struct nand_chip *nand, int csline,
1402 const struct nand_interface_config *conf)
1403 {
1404 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1405 struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
1406 const struct nand_sdr_timings *timings;
1407 u32 min_clk_period = 0;
1408 s32 tWB, tADL, tWHR, tRHW, tCAD;
1409 long real_clk_rate;
1410
1411 timings = nand_get_sdr_timings(conf);
1412 if (IS_ERR(timings))
1413 return -ENOTSUPP;
1414
1415 /* T1 <=> tCLS */
1416 if (timings->tCLS_min > min_clk_period)
1417 min_clk_period = timings->tCLS_min;
1418
1419 /* T2 <=> tCLH */
1420 if (timings->tCLH_min > min_clk_period)
1421 min_clk_period = timings->tCLH_min;
1422
1423 /* T3 <=> tCS */
1424 if (timings->tCS_min > min_clk_period)
1425 min_clk_period = timings->tCS_min;
1426
1427 /* T4 <=> tCH */
1428 if (timings->tCH_min > min_clk_period)
1429 min_clk_period = timings->tCH_min;
1430
1431 /* T5 <=> tWP */
1432 if (timings->tWP_min > min_clk_period)
1433 min_clk_period = timings->tWP_min;
1434
1435 /* T6 <=> tWH */
1436 if (timings->tWH_min > min_clk_period)
1437 min_clk_period = timings->tWH_min;
1438
1439 /* T7 <=> tALS */
1440 if (timings->tALS_min > min_clk_period)
1441 min_clk_period = timings->tALS_min;
1442
1443 /* T8 <=> tDS */
1444 if (timings->tDS_min > min_clk_period)
1445 min_clk_period = timings->tDS_min;
1446
1447 /* T9 <=> tDH */
1448 if (timings->tDH_min > min_clk_period)
1449 min_clk_period = timings->tDH_min;
1450
1451 /* T10 <=> tRR */
1452 if (timings->tRR_min > (min_clk_period * 3))
1453 min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);
1454
1455 /* T11 <=> tALH */
1456 if (timings->tALH_min > min_clk_period)
1457 min_clk_period = timings->tALH_min;
1458
1459 /* T12 <=> tRP */
1460 if (timings->tRP_min > min_clk_period)
1461 min_clk_period = timings->tRP_min;
1462
1463 /* T13 <=> tREH */
1464 if (timings->tREH_min > min_clk_period)
1465 min_clk_period = timings->tREH_min;
1466
1467 /* T14 <=> tRC */
1468 if (timings->tRC_min > (min_clk_period * 2))
1469 min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2);
1470
1471 /* T15 <=> tWC */
1472 if (timings->tWC_min > (min_clk_period * 2))
1473 min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);
1474
1475 /* T16 - T19 + tCAD */
1476 if (timings->tWB_max > (min_clk_period * 20))
1477 min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);
1478
1479 if (timings->tADL_min > (min_clk_period * 32))
1480 min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
1481
1482 if (timings->tWHR_min > (min_clk_period * 32))
1483 min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
1484
1485 if (timings->tRHW_min > (min_clk_period * 20))
1486 min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);
1487
1488 /*
1489 * In non-EDO, tREA should be less than tRP to guarantee that the
1490 * controller does not sample the IO lines too early. Unfortunately,
1491 * the sunxi NAND controller does not allow us to have different
1492 * values for tRP and tREH (tRP = tREH = tRW / 2).
1493 *
1494 * We have 2 options to overcome this limitation:
1495 *
1496 * 1/ Extend tRC to fulfil the tREA <= tRC / 2 constraint
1497 * 2/ Use EDO mode (only works if timings->tRLOH > 0)
1498 */
1499 if (timings->tREA_max > min_clk_period && !timings->tRLOH_min)
1500 min_clk_period = timings->tREA_max;
1501
1502 tWB = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,
1503 min_clk_period);
1504 if (tWB < 0) {
1505 dev_err(nfc->dev, "unsupported tWB\n");
1506 return tWB;
1507 }
1508
1509 tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;
1510 if (tADL > 3) {
1511 dev_err(nfc->dev, "unsupported tADL\n");
1512 return -EINVAL;
1513 }
1514
1515 tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;
1516 if (tWHR > 3) {
1517 dev_err(nfc->dev, "unsupported tWHR\n");
1518 return -EINVAL;
1519 }
1520
1521 tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,
1522 min_clk_period);
1523 if (tRHW < 0) {
1524 dev_err(nfc->dev, "unsupported tRHW\n");
1525 return tRHW;
1526 }
1527
1528 if (csline == NAND_DATA_IFACE_CHECK_ONLY)
1529 return 0;
1530
1531 /*
1532 * TODO: according to ONFI specs this value only applies for DDR NAND,
1533 * but Allwinner seems to set this to 0x7. Mimic them for now.
1534 */
1535 tCAD = 0x7;
1536
1537 /* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */
1538 sunxi_nand->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD);
1539
1540 /* Convert min_clk_period from picoseconds to nanoseconds */
1541 min_clk_period = DIV_ROUND_UP(min_clk_period, 1000);
1542
1543 /*
1544 * Unlike what is stated in Allwinner datasheet, the clk_rate should
1545 * be set to (1 / min_clk_period), and not (2 / min_clk_period).
1546 * This new formula was verified with a scope and validated by
1547 * Allwinner engineers.
1548 */
1549 sunxi_nand->clk_rate = NSEC_PER_SEC / min_clk_period;
1550 real_clk_rate = clk_round_rate(nfc->mod_clk, sunxi_nand->clk_rate);
1551 if (real_clk_rate <= 0) {
1552 dev_err(nfc->dev, "Unable to round clk %lu\n",
1553 sunxi_nand->clk_rate);
1554 return -EINVAL;
1555 }
1556
1557 sunxi_nand->timing_ctl = 0;
1558
1559 /*
1560 * ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data
1561 * output cycle timings shall be used if the host drives tRC less than
1562 * 30 ns. We should also use EDO mode if tREA is bigger than tRP.
1563 */
1564 min_clk_period = NSEC_PER_SEC / real_clk_rate;
1565 if (min_clk_period * 2 < 30 || min_clk_period * 1000 < timings->tREA_max)
1566 sunxi_nand->timing_ctl = NFC_TIMING_CTL_EDO;
1567
1568 return 0;
1569 }
1570
1571 static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1572 struct mtd_oob_region *oobregion)
1573 {
1574 struct nand_chip *nand = mtd_to_nand(mtd);
1575 struct nand_ecc_ctrl *ecc = &nand->ecc;
1576
1577 if (section >= ecc->steps)
1578 return -ERANGE;
1579
1580 oobregion->offset = section * (ecc->bytes + 4) + 4;
1581 oobregion->length = ecc->bytes;
1582
1583 return 0;
1584 }
1585
1586 static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
1587 struct mtd_oob_region *oobregion)
1588 {
1589 struct nand_chip *nand = mtd_to_nand(mtd);
1590 struct nand_ecc_ctrl *ecc = &nand->ecc;
1591
1592 if (section > ecc->steps)
1593 return -ERANGE;
1594
1595 /*
1596 * The first 2 bytes are used for BB markers, hence we
1597 * only have 2 bytes available in the first user data
1598 * section.
1599 */
1600 if (!section && ecc->engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST) {
1601 oobregion->offset = 2;
1602 oobregion->length = 2;
1603
1604 return 0;
1605 }
1606
1607 oobregion->offset = section * (ecc->bytes + 4);
1608
1609 if (section < ecc->steps)
1610 oobregion->length = 4;
1611 else
1612 oobregion->offset = mtd->oobsize - oobregion->offset;
1613
1614 return 0;
1615 }
1616
1617 static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = {
1618 .ecc = sunxi_nand_ooblayout_ecc,
1619 .free = sunxi_nand_ooblayout_free,
1620 };
1621
1622 static void sunxi_nand_hw_ecc_ctrl_cleanup(struct sunxi_nand_chip *sunxi_nand)
1623 {
1624 kfree(sunxi_nand->ecc);
1625 }
1626
1627 static int sunxi_nand_hw_ecc_ctrl_init(struct nand_chip *nand,
1628 struct nand_ecc_ctrl *ecc,
1629 struct device_node *np)
1630 {
1631 static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 };
1632 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1633 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1634 struct mtd_info *mtd = nand_to_mtd(nand);
1635 struct nand_device *nanddev = mtd_to_nanddev(mtd);
1636 int nsectors;
1637 int ret;
1638 int i;
1639
1640 if (nanddev->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH) {
1641 int bytes;
1642
1643 ecc->size = 1024;
1644 nsectors = mtd->writesize / ecc->size;
1645
1646 /* Reserve 2 bytes for the BBM */
1647 bytes = (mtd->oobsize - 2) / nsectors;
1648
1649 /* 4 non-ECC bytes are added before each ECC bytes section */
1650 bytes -= 4;
1651
1652 /* and bytes has to be even. */
1653 if (bytes % 2)
1654 bytes--;
1655
1656 ecc->strength = bytes * 8 / fls(8 * ecc->size);
1657
1658 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1659 if (strengths[i] > ecc->strength)
1660 break;
1661 }
1662
1663 if (!i)
1664 ecc->strength = 0;
1665 else
1666 ecc->strength = strengths[i - 1];
1667 }
1668
1669 if (ecc->size != 512 && ecc->size != 1024)
1670 return -EINVAL;
1671
1672 sunxi_nand->ecc = kzalloc(sizeof(*sunxi_nand->ecc), GFP_KERNEL);
1673 if (!sunxi_nand->ecc)
1674 return -ENOMEM;
1675
1676 /* Prefer 1k ECC chunk over 512 ones */
1677 if (ecc->size == 512 && mtd->writesize > 512) {
1678 ecc->size = 1024;
1679 ecc->strength *= 2;
1680 }
1681
1682 /* Add ECC info retrieval from DT */
1683 for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1684 if (ecc->strength <= strengths[i]) {
1685 /*
1686 * Update ecc->strength value with the actual strength
1687 * that will be used by the ECC engine.
1688 */
1689 ecc->strength = strengths[i];
1690 break;
1691 }
1692 }
1693
1694 if (i >= ARRAY_SIZE(strengths)) {
1695 dev_err(nfc->dev, "unsupported strength\n");
1696 ret = -ENOTSUPP;
1697 goto err;
1698 }
1699
1700 sunxi_nand->ecc->mode = i;
1701
1702 /* HW ECC always request ECC bytes for 1024 bytes blocks */
1703 ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8);
1704
1705 /* HW ECC always work with even numbers of ECC bytes */
1706 ecc->bytes = ALIGN(ecc->bytes, 2);
1707
1708 nsectors = mtd->writesize / ecc->size;
1709
1710 if (mtd->oobsize < ((ecc->bytes + 4) * nsectors)) {
1711 ret = -EINVAL;
1712 goto err;
1713 }
1714
1715 ecc->read_oob = sunxi_nfc_hw_ecc_read_oob;
1716 ecc->write_oob = sunxi_nfc_hw_ecc_write_oob;
1717 mtd_set_ooblayout(mtd, &sunxi_nand_ooblayout_ops);
1718
1719 if (nfc->dmac || nfc->caps->has_mdma) {
1720 ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
1721 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
1722 ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
1723 nand->options |= NAND_USES_DMA;
1724 } else {
1725 ecc->read_page = sunxi_nfc_hw_ecc_read_page;
1726 ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1727 ecc->write_page = sunxi_nfc_hw_ecc_write_page;
1728 }
1729
1730 /* TODO: support DMA for raw accesses and subpage write */
1731 ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage;
1732 ecc->read_oob_raw = nand_read_oob_std;
1733 ecc->write_oob_raw = nand_write_oob_std;
1734
1735 return 0;
1736
1737 err:
1738 kfree(sunxi_nand->ecc);
1739
1740 return ret;
1741 }
1742
1743 static void sunxi_nand_ecc_cleanup(struct sunxi_nand_chip *sunxi_nand)
1744 {
1745 struct nand_ecc_ctrl *ecc = &sunxi_nand->nand.ecc;
1746
1747 switch (ecc->engine_type) {
1748 case NAND_ECC_ENGINE_TYPE_ON_HOST:
1749 sunxi_nand_hw_ecc_ctrl_cleanup(sunxi_nand);
1750 break;
1751 case NAND_ECC_ENGINE_TYPE_NONE:
1752 default:
1753 break;
1754 }
1755 }
1756
1757 static int sunxi_nand_attach_chip(struct nand_chip *nand)
1758 {
1759 const struct nand_ecc_props *requirements =
1760 nanddev_get_ecc_requirements(&nand->base);
1761 struct nand_ecc_ctrl *ecc = &nand->ecc;
1762 struct device_node *np = nand_get_flash_node(nand);
1763 int ret;
1764
1765 if (nand->bbt_options & NAND_BBT_USE_FLASH)
1766 nand->bbt_options |= NAND_BBT_NO_OOB;
1767
1768 if (nand->options & NAND_NEED_SCRAMBLING)
1769 nand->options |= NAND_NO_SUBPAGE_WRITE;
1770
1771 nand->options |= NAND_SUBPAGE_READ;
1772
1773 if (!ecc->size) {
1774 ecc->size = requirements->step_size;
1775 ecc->strength = requirements->strength;
1776 }
1777
1778 if (!ecc->size || !ecc->strength)
1779 return -EINVAL;
1780
1781 switch (ecc->engine_type) {
1782 case NAND_ECC_ENGINE_TYPE_ON_HOST:
1783 ret = sunxi_nand_hw_ecc_ctrl_init(nand, ecc, np);
1784 if (ret)
1785 return ret;
1786 break;
1787 case NAND_ECC_ENGINE_TYPE_NONE:
1788 case NAND_ECC_ENGINE_TYPE_SOFT:
1789 break;
1790 default:
1791 return -EINVAL;
1792 }
1793
1794 return 0;
1795 }
1796
1797 static int sunxi_nfc_exec_subop(struct nand_chip *nand,
1798 const struct nand_subop *subop)
1799 {
1800 struct sunxi_nfc *nfc = to_sunxi_nfc(nand->controller);
1801 u32 cmd = 0, extcmd = 0, cnt = 0, addrs[2] = { };
1802 unsigned int i, j, remaining, start;
1803 void *inbuf = NULL;
1804 int ret;
1805
1806 for (i = 0; i < subop->ninstrs; i++) {
1807 const struct nand_op_instr *instr = &subop->instrs[i];
1808
1809 switch (instr->type) {
1810 case NAND_OP_CMD_INSTR:
1811 if (cmd & NFC_SEND_CMD1) {
1812 if (WARN_ON(cmd & NFC_SEND_CMD2))
1813 return -EINVAL;
1814
1815 cmd |= NFC_SEND_CMD2;
1816 extcmd |= instr->ctx.cmd.opcode;
1817 } else {
1818 cmd |= NFC_SEND_CMD1 |
1819 NFC_CMD(instr->ctx.cmd.opcode);
1820 }
1821 break;
1822
1823 case NAND_OP_ADDR_INSTR:
1824 remaining = nand_subop_get_num_addr_cyc(subop, i);
1825 start = nand_subop_get_addr_start_off(subop, i);
1826 for (j = 0; j < 8 && j + start < remaining; j++) {
1827 u32 addr = instr->ctx.addr.addrs[j + start];
1828
1829 addrs[j / 4] |= addr << (j % 4) * 8;
1830 }
1831
1832 if (j)
1833 cmd |= NFC_SEND_ADR | NFC_ADR_NUM(j);
1834
1835 break;
1836
1837 case NAND_OP_DATA_IN_INSTR:
1838 case NAND_OP_DATA_OUT_INSTR:
1839 start = nand_subop_get_data_start_off(subop, i);
1840 remaining = nand_subop_get_data_len(subop, i);
1841 cnt = min_t(u32, remaining, NFC_SRAM_SIZE);
1842 cmd |= NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
1843
1844 if (instr->type == NAND_OP_DATA_OUT_INSTR) {
1845 cmd |= NFC_ACCESS_DIR;
1846 memcpy_toio(nfc->regs + NFC_RAM0_BASE,
1847 instr->ctx.data.buf.out + start,
1848 cnt);
1849 } else {
1850 inbuf = instr->ctx.data.buf.in + start;
1851 }
1852
1853 break;
1854
1855 case NAND_OP_WAITRDY_INSTR:
1856 cmd |= NFC_WAIT_FLAG;
1857 break;
1858 }
1859 }
1860
1861 ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1862 if (ret)
1863 return ret;
1864
1865 if (cmd & NFC_SEND_ADR) {
1866 writel(addrs[0], nfc->regs + NFC_REG_ADDR_LOW);
1867 writel(addrs[1], nfc->regs + NFC_REG_ADDR_HIGH);
1868 }
1869
1870 if (cmd & NFC_SEND_CMD2)
1871 writel(extcmd,
1872 nfc->regs +
1873 (cmd & NFC_ACCESS_DIR ?
1874 NFC_REG_WCMD_SET : NFC_REG_RCMD_SET));
1875
1876 if (cmd & NFC_DATA_TRANS)
1877 writel(cnt, nfc->regs + NFC_REG_CNT);
1878
1879 writel(cmd, nfc->regs + NFC_REG_CMD);
1880
1881 ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG,
1882 !(cmd & NFC_WAIT_FLAG) && cnt < 64,
1883 0);
1884 if (ret)
1885 return ret;
1886
1887 if (inbuf)
1888 memcpy_fromio(inbuf, nfc->regs + NFC_RAM0_BASE, cnt);
1889
1890 return 0;
1891 }
1892
1893 static int sunxi_nfc_soft_waitrdy(struct nand_chip *nand,
1894 const struct nand_subop *subop)
1895 {
1896 return nand_soft_waitrdy(nand,
1897 subop->instrs[0].ctx.waitrdy.timeout_ms);
1898 }
1899
1900 static const struct nand_op_parser sunxi_nfc_op_parser = NAND_OP_PARSER(
1901 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1902 NAND_OP_PARSER_PAT_CMD_ELEM(true),
1903 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1904 NAND_OP_PARSER_PAT_CMD_ELEM(true),
1905 NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
1906 NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)),
1907 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1908 NAND_OP_PARSER_PAT_CMD_ELEM(true),
1909 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1910 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024),
1911 NAND_OP_PARSER_PAT_CMD_ELEM(true),
1912 NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)),
1913 );
1914
1915 static const struct nand_op_parser sunxi_nfc_norb_op_parser = NAND_OP_PARSER(
1916 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1917 NAND_OP_PARSER_PAT_CMD_ELEM(true),
1918 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1919 NAND_OP_PARSER_PAT_CMD_ELEM(true),
1920 NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)),
1921 NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1922 NAND_OP_PARSER_PAT_CMD_ELEM(true),
1923 NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1924 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024),
1925 NAND_OP_PARSER_PAT_CMD_ELEM(true)),
1926 NAND_OP_PARSER_PATTERN(sunxi_nfc_soft_waitrdy,
1927 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
1928 );
1929
1930 static int sunxi_nfc_exec_op(struct nand_chip *nand,
1931 const struct nand_operation *op, bool check_only)
1932 {
1933 struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1934 const struct nand_op_parser *parser;
1935
1936 if (!check_only)
1937 sunxi_nfc_select_chip(nand, op->cs);
1938
1939 if (sunxi_nand->sels[op->cs].rb >= 0)
1940 parser = &sunxi_nfc_op_parser;
1941 else
1942 parser = &sunxi_nfc_norb_op_parser;
1943
1944 return nand_op_parser_exec_op(nand, parser, op, check_only);
1945 }
1946
1947 static const struct nand_controller_ops sunxi_nand_controller_ops = {
1948 .attach_chip = sunxi_nand_attach_chip,
1949 .setup_interface = sunxi_nfc_setup_interface,
1950 .exec_op = sunxi_nfc_exec_op,
1951 };
1952
1953 static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
1954 struct device_node *np)
1955 {
1956 struct sunxi_nand_chip *sunxi_nand;
1957 struct mtd_info *mtd;
1958 struct nand_chip *nand;
1959 int nsels;
1960 int ret;
1961 int i;
1962 u32 tmp;
1963
1964 if (!of_get_property(np, "reg", &nsels))
1965 return -EINVAL;
1966
1967 nsels /= sizeof(u32);
1968 if (!nsels) {
1969 dev_err(dev, "invalid reg property size\n");
1970 return -EINVAL;
1971 }
1972
1973 sunxi_nand = devm_kzalloc(dev, struct_size(sunxi_nand, sels, nsels),
1974 GFP_KERNEL);
1975 if (!sunxi_nand) {
1976 dev_err(dev, "could not allocate chip\n");
1977 return -ENOMEM;
1978 }
1979
1980 sunxi_nand->nsels = nsels;
1981
1982 for (i = 0; i < nsels; i++) {
1983 ret = of_property_read_u32_index(np, "reg", i, &tmp);
1984 if (ret) {
1985 dev_err(dev, "could not retrieve reg property: %d\n",
1986 ret);
1987 return ret;
1988 }
1989
1990 if (tmp > NFC_MAX_CS) {
1991 dev_err(dev,
1992 "invalid reg value: %u (max CS = 7)\n",
1993 tmp);
1994 return -EINVAL;
1995 }
1996
1997 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
1998 dev_err(dev, "CS %d already assigned\n", tmp);
1999 return -EINVAL;
2000 }
2001
2002 sunxi_nand->sels[i].cs = tmp;
2003
2004 if (!of_property_read_u32_index(np, "allwinner,rb", i, &tmp) &&
2005 tmp < 2)
2006 sunxi_nand->sels[i].rb = tmp;
2007 else
2008 sunxi_nand->sels[i].rb = -1;
2009 }
2010
2011 nand = &sunxi_nand->nand;
2012 /* Default tR value specified in the ONFI spec (chapter 4.15.1) */
2013 nand->controller = &nfc->controller;
2014 nand->controller->ops = &sunxi_nand_controller_ops;
2015
2016 /*
2017 * Set the ECC mode to the default value in case nothing is specified
2018 * in the DT.
2019 */
2020 nand->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
2021 nand_set_flash_node(nand, np);
2022
2023 mtd = nand_to_mtd(nand);
2024 mtd->dev.parent = dev;
2025
2026 ret = nand_scan(nand, nsels);
2027 if (ret)
2028 return ret;
2029
2030 ret = mtd_device_register(mtd, NULL, 0);
2031 if (ret) {
2032 dev_err(dev, "failed to register mtd device: %d\n", ret);
2033 nand_cleanup(nand);
2034 return ret;
2035 }
2036
2037 list_add_tail(&sunxi_nand->node, &nfc->chips);
2038
2039 return 0;
2040 }
2041
2042 static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)
2043 {
2044 struct device_node *np = dev->of_node;
2045 struct device_node *nand_np;
2046 int nchips = of_get_child_count(np);
2047 int ret;
2048
2049 if (nchips > 8) {
2050 dev_err(dev, "too many NAND chips: %d (max = 8)\n", nchips);
2051 return -EINVAL;
2052 }
2053
2054 for_each_child_of_node(np, nand_np) {
2055 ret = sunxi_nand_chip_init(dev, nfc, nand_np);
2056 if (ret) {
2057 of_node_put(nand_np);
2058 return ret;
2059 }
2060 }
2061
2062 return 0;
2063 }
2064
2065 static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)
2066 {
2067 struct sunxi_nand_chip *sunxi_nand;
2068 struct nand_chip *chip;
2069 int ret;
2070
2071 while (!list_empty(&nfc->chips)) {
2072 sunxi_nand = list_first_entry(&nfc->chips,
2073 struct sunxi_nand_chip,
2074 node);
2075 chip = &sunxi_nand->nand;
2076 ret = mtd_device_unregister(nand_to_mtd(chip));
2077 WARN_ON(ret);
2078 nand_cleanup(chip);
2079 sunxi_nand_ecc_cleanup(sunxi_nand);
2080 list_del(&sunxi_nand->node);
2081 }
2082 }
2083
2084 static int sunxi_nfc_dma_init(struct sunxi_nfc *nfc, struct resource *r)
2085 {
2086 int ret;
2087
2088 if (nfc->caps->has_mdma)
2089 return 0;
2090
2091 nfc->dmac = dma_request_chan(nfc->dev, "rxtx");
2092 if (IS_ERR(nfc->dmac)) {
2093 ret = PTR_ERR(nfc->dmac);
2094 if (ret == -EPROBE_DEFER)
2095 return ret;
2096
2097 /* Ignore errors to fall back to PIO mode */
2098 dev_warn(nfc->dev, "failed to request rxtx DMA channel: %d\n", ret);
2099 nfc->dmac = NULL;
2100 } else {
2101 struct dma_slave_config dmac_cfg = { };
2102
2103 dmac_cfg.src_addr = r->start + nfc->caps->reg_io_data;
2104 dmac_cfg.dst_addr = dmac_cfg.src_addr;
2105 dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2106 dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;
2107 dmac_cfg.src_maxburst = nfc->caps->dma_maxburst;
2108 dmac_cfg.dst_maxburst = nfc->caps->dma_maxburst;
2109 dmaengine_slave_config(nfc->dmac, &dmac_cfg);
2110 }
2111 return 0;
2112 }
2113
2114 static int sunxi_nfc_probe(struct platform_device *pdev)
2115 {
2116 struct device *dev = &pdev->dev;
2117 struct resource *r;
2118 struct sunxi_nfc *nfc;
2119 int irq;
2120 int ret;
2121
2122 nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
2123 if (!nfc)
2124 return -ENOMEM;
2125
2126 nfc->dev = dev;
2127 nand_controller_init(&nfc->controller);
2128 INIT_LIST_HEAD(&nfc->chips);
2129
2130 r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2131 nfc->regs = devm_ioremap_resource(dev, r);
2132 if (IS_ERR(nfc->regs))
2133 return PTR_ERR(nfc->regs);
2134
2135 irq = platform_get_irq(pdev, 0);
2136 if (irq < 0)
2137 return irq;
2138
2139 nfc->ahb_clk = devm_clk_get(dev, "ahb");
2140 if (IS_ERR(nfc->ahb_clk)) {
2141 dev_err(dev, "failed to retrieve ahb clk\n");
2142 return PTR_ERR(nfc->ahb_clk);
2143 }
2144
2145 ret = clk_prepare_enable(nfc->ahb_clk);
2146 if (ret)
2147 return ret;
2148
2149 nfc->mod_clk = devm_clk_get(dev, "mod");
2150 if (IS_ERR(nfc->mod_clk)) {
2151 dev_err(dev, "failed to retrieve mod clk\n");
2152 ret = PTR_ERR(nfc->mod_clk);
2153 goto out_ahb_clk_unprepare;
2154 }
2155
2156 ret = clk_prepare_enable(nfc->mod_clk);
2157 if (ret)
2158 goto out_ahb_clk_unprepare;
2159
2160 nfc->reset = devm_reset_control_get_optional_exclusive(dev, "ahb");
2161 if (IS_ERR(nfc->reset)) {
2162 ret = PTR_ERR(nfc->reset);
2163 goto out_mod_clk_unprepare;
2164 }
2165
2166 ret = reset_control_deassert(nfc->reset);
2167 if (ret) {
2168 dev_err(dev, "reset err %d\n", ret);
2169 goto out_mod_clk_unprepare;
2170 }
2171
2172 nfc->caps = of_device_get_match_data(&pdev->dev);
2173 if (!nfc->caps) {
2174 ret = -EINVAL;
2175 goto out_ahb_reset_reassert;
2176 }
2177
2178 ret = sunxi_nfc_rst(nfc);
2179 if (ret)
2180 goto out_ahb_reset_reassert;
2181
2182 writel(0, nfc->regs + NFC_REG_INT);
2183 ret = devm_request_irq(dev, irq, sunxi_nfc_interrupt,
2184 0, "sunxi-nand", nfc);
2185 if (ret)
2186 goto out_ahb_reset_reassert;
2187
2188 ret = sunxi_nfc_dma_init(nfc, r);
2189
2190 if (ret)
2191 goto out_ahb_reset_reassert;
2192
2193 platform_set_drvdata(pdev, nfc);
2194
2195 ret = sunxi_nand_chips_init(dev, nfc);
2196 if (ret) {
2197 dev_err(dev, "failed to init nand chips\n");
2198 goto out_release_dmac;
2199 }
2200
2201 return 0;
2202
2203 out_release_dmac:
2204 if (nfc->dmac)
2205 dma_release_channel(nfc->dmac);
2206 out_ahb_reset_reassert:
2207 reset_control_assert(nfc->reset);
2208 out_mod_clk_unprepare:
2209 clk_disable_unprepare(nfc->mod_clk);
2210 out_ahb_clk_unprepare:
2211 clk_disable_unprepare(nfc->ahb_clk);
2212
2213 return ret;
2214 }
2215
2216 static int sunxi_nfc_remove(struct platform_device *pdev)
2217 {
2218 struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
2219
2220 sunxi_nand_chips_cleanup(nfc);
2221
2222 reset_control_assert(nfc->reset);
2223
2224 if (nfc->dmac)
2225 dma_release_channel(nfc->dmac);
2226 clk_disable_unprepare(nfc->mod_clk);
2227 clk_disable_unprepare(nfc->ahb_clk);
2228
2229 return 0;
2230 }
2231
2232 static const struct sunxi_nfc_caps sunxi_nfc_a10_caps = {
2233 .reg_io_data = NFC_REG_A10_IO_DATA,
2234 .dma_maxburst = 4,
2235 };
2236
2237 static const struct sunxi_nfc_caps sunxi_nfc_a23_caps = {
2238 .has_mdma = true,
2239 .reg_io_data = NFC_REG_A23_IO_DATA,
2240 .dma_maxburst = 8,
2241 };
2242
2243 static const struct of_device_id sunxi_nfc_ids[] = {
2244 {
2245 .compatible = "allwinner,sun4i-a10-nand",
2246 .data = &sunxi_nfc_a10_caps,
2247 },
2248 {
2249 .compatible = "allwinner,sun8i-a23-nand-controller",
2250 .data = &sunxi_nfc_a23_caps,
2251 },
2252 { /* sentinel */ }
2253 };
2254 MODULE_DEVICE_TABLE(of, sunxi_nfc_ids);
2255
2256 static struct platform_driver sunxi_nfc_driver = {
2257 .driver = {
2258 .name = "sunxi_nand",
2259 .of_match_table = sunxi_nfc_ids,
2260 },
2261 .probe = sunxi_nfc_probe,
2262 .remove = sunxi_nfc_remove,
2263 };
2264 module_platform_driver(sunxi_nfc_driver);
2265
2266 MODULE_LICENSE("GPL");
2267 MODULE_AUTHOR("Boris BREZILLON");
2268 MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver");
2269 MODULE_ALIAS("platform:sunxi_nand");
Linux with added FPC-III support