EDAC: i7core, sb_edac: Don't return NOTIFY_BAD from mce_decoder callback

[linux/fpc-iii.git] / drivers / mtd / nand / qcom_nandc.c

/*
 * Copyright (c) 2016, The Linux Foundation. All rights reserved.
 *
 * This software is licensed under the terms of the GNU General Public
 * License version 2, as published by the Free Software Foundation, and
 * may be copied, distributed, and modified under those terms.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */

#include <linux/clk.h>
#include <linux/slab.h>
#include <linux/bitops.h>
#include <linux/dma-mapping.h>
#include <linux/dmaengine.h>
#include <linux/module.h>
#include <linux/mtd/nand.h>
#include <linux/mtd/partitions.h>
#include <linux/of.h>
#include <linux/of_device.h>
#include <linux/of_mtd.h>
#include <linux/delay.h>

/* NANDc reg offsets */
#define NAND_FLASH_CMD                  0x00
#define NAND_ADDR0                      0x04
#define NAND_ADDR1                      0x08
#define NAND_FLASH_CHIP_SELECT          0x0c
#define NAND_EXEC_CMD                   0x10
#define NAND_FLASH_STATUS               0x14
#define NAND_BUFFER_STATUS              0x18
#define NAND_DEV0_CFG0                  0x20
#define NAND_DEV0_CFG1                  0x24
#define NAND_DEV0_ECC_CFG               0x28
#define NAND_DEV1_ECC_CFG               0x2c
#define NAND_DEV1_CFG0                  0x30
#define NAND_DEV1_CFG1                  0x34
#define NAND_READ_ID                    0x40
#define NAND_READ_STATUS                0x44
#define NAND_DEV_CMD0                   0xa0
#define NAND_DEV_CMD1                   0xa4
#define NAND_DEV_CMD2                   0xa8
#define NAND_DEV_CMD_VLD                0xac
#define SFLASHC_BURST_CFG               0xe0
#define NAND_ERASED_CW_DETECT_CFG       0xe8
#define NAND_ERASED_CW_DETECT_STATUS    0xec
#define NAND_EBI2_ECC_BUF_CFG           0xf0
#define FLASH_BUF_ACC                   0x100

#define NAND_CTRL                       0xf00
#define NAND_VERSION                    0xf08
#define NAND_READ_LOCATION_0            0xf20
#define NAND_READ_LOCATION_1            0xf24

/* dummy register offsets, used by write_reg_dma */
#define NAND_DEV_CMD1_RESTORE           0xdead
#define NAND_DEV_CMD_VLD_RESTORE        0xbeef

/* NAND_FLASH_CMD bits */
#define PAGE_ACC                        BIT(4)
#define LAST_PAGE                       BIT(5)

/* NAND_FLASH_CHIP_SELECT bits */
#define NAND_DEV_SEL                    0
#define DM_EN                           BIT(2)

/* NAND_FLASH_STATUS bits */
#define FS_OP_ERR                       BIT(4)
#define FS_READY_BSY_N                  BIT(5)
#define FS_MPU_ERR                      BIT(8)
#define FS_DEVICE_STS_ERR               BIT(16)
#define FS_DEVICE_WP                    BIT(23)

/* NAND_BUFFER_STATUS bits */
#define BS_UNCORRECTABLE_BIT            BIT(8)
#define BS_CORRECTABLE_ERR_MSK          0x1f

/* NAND_DEVn_CFG0 bits */
#define DISABLE_STATUS_AFTER_WRITE      4
#define CW_PER_PAGE                     6
#define UD_SIZE_BYTES                   9
#define ECC_PARITY_SIZE_BYTES_RS        19
#define SPARE_SIZE_BYTES                23
#define NUM_ADDR_CYCLES                 27
#define STATUS_BFR_READ                 30
#define SET_RD_MODE_AFTER_STATUS        31

/* NAND_DEVn_CFG0 bits */
#define DEV0_CFG1_ECC_DISABLE           0
#define WIDE_FLASH                      1
#define NAND_RECOVERY_CYCLES            2
#define CS_ACTIVE_BSY                   5
#define BAD_BLOCK_BYTE_NUM              6
#define BAD_BLOCK_IN_SPARE_AREA         16
#define WR_RD_BSY_GAP                   17
#define ENABLE_BCH_ECC                  27

/* NAND_DEV0_ECC_CFG bits */
#define ECC_CFG_ECC_DISABLE             0
#define ECC_SW_RESET                    1
#define ECC_MODE                        4
#define ECC_PARITY_SIZE_BYTES_BCH       8
#define ECC_NUM_DATA_BYTES              16
#define ECC_FORCE_CLK_OPEN              30

/* NAND_DEV_CMD1 bits */
#define READ_ADDR                       0

/* NAND_DEV_CMD_VLD bits */
#define READ_START_VLD                  0

/* NAND_EBI2_ECC_BUF_CFG bits */
#define NUM_STEPS                       0

/* NAND_ERASED_CW_DETECT_CFG bits */
#define ERASED_CW_ECC_MASK              1
#define AUTO_DETECT_RES                 0
#define MASK_ECC                        (1 << ERASED_CW_ECC_MASK)
#define RESET_ERASED_DET                (1 << AUTO_DETECT_RES)
#define ACTIVE_ERASED_DET               (0 << AUTO_DETECT_RES)
#define CLR_ERASED_PAGE_DET             (RESET_ERASED_DET | MASK_ECC)
#define SET_ERASED_PAGE_DET             (ACTIVE_ERASED_DET | MASK_ECC)

/* NAND_ERASED_CW_DETECT_STATUS bits */
#define PAGE_ALL_ERASED                 BIT(7)
#define CODEWORD_ALL_ERASED             BIT(6)
#define PAGE_ERASED                     BIT(5)
#define CODEWORD_ERASED                 BIT(4)
#define ERASED_PAGE                     (PAGE_ALL_ERASED | PAGE_ERASED)
#define ERASED_CW                       (CODEWORD_ALL_ERASED | CODEWORD_ERASED)

/* Version Mask */
#define NAND_VERSION_MAJOR_MASK         0xf0000000
#define NAND_VERSION_MAJOR_SHIFT        28
#define NAND_VERSION_MINOR_MASK         0x0fff0000
#define NAND_VERSION_MINOR_SHIFT        16

/* NAND OP_CMDs */
#define PAGE_READ                       0x2
#define PAGE_READ_WITH_ECC              0x3
#define PAGE_READ_WITH_ECC_SPARE        0x4
#define PROGRAM_PAGE                    0x6
#define PAGE_PROGRAM_WITH_ECC           0x7
#define PROGRAM_PAGE_SPARE              0x9
#define BLOCK_ERASE                     0xa
#define FETCH_ID                        0xb
#define RESET_DEVICE                    0xd

/*
 * the NAND controller performs reads/writes with ECC in 516 byte chunks.
 * the driver calls the chunks 'step' or 'codeword' interchangeably
 */
#define NANDC_STEP_SIZE                 512

/*
 * the largest page size we support is 8K, this will have 16 steps/codewords
 * of 512 bytes each
 */
#define MAX_NUM_STEPS                   (SZ_8K / NANDC_STEP_SIZE)

/* we read at most 3 registers per codeword scan */
#define MAX_REG_RD                      (3 * MAX_NUM_STEPS)

/* ECC modes supported by the controller */
#define ECC_NONE        BIT(0)
#define ECC_RS_4BIT     BIT(1)
#define ECC_BCH_4BIT    BIT(2)
#define ECC_BCH_8BIT    BIT(3)

struct desc_info {
        struct list_head node;

        enum dma_data_direction dir;
        struct scatterlist sgl;
        struct dma_async_tx_descriptor *dma_desc;
};

/*
 * holds the current register values that we want to write. acts as a contiguous
 * chunk of memory which we use to write the controller registers through DMA.
 */
struct nandc_regs {
        __le32 cmd;
        __le32 addr0;
        __le32 addr1;
        __le32 chip_sel;
        __le32 exec;

        __le32 cfg0;
        __le32 cfg1;
        __le32 ecc_bch_cfg;

        __le32 clrflashstatus;
        __le32 clrreadstatus;

        __le32 cmd1;
        __le32 vld;

        __le32 orig_cmd1;
        __le32 orig_vld;

        __le32 ecc_buf_cfg;
};

/*
 * NAND controller data struct
 *
 * @controller:                 base controller structure
 * @host_list:                  list containing all the chips attached to the
 *                              controller
 * @dev:                        parent device
 * @base:                       MMIO base
 * @base_dma:                   physical base address of controller registers
 * @core_clk:                   controller clock
 * @aon_clk:                    another controller clock
 *
 * @chan:                       dma channel
 * @cmd_crci:                   ADM DMA CRCI for command flow control
 * @data_crci:                  ADM DMA CRCI for data flow control
 * @desc_list:                  DMA descriptor list (list of desc_infos)
 *
 * @data_buffer:                our local DMA buffer for page read/writes,
 *                              used when we can't use the buffer provided
 *                              by upper layers directly
 * @buf_size/count/start:       markers for chip->read_buf/write_buf functions
 * @reg_read_buf:               local buffer for reading back registers via DMA
 * @reg_read_pos:               marker for data read in reg_read_buf
 *
 * @regs:                       a contiguous chunk of memory for DMA register
 *                              writes. contains the register values to be
 *                              written to controller
 * @cmd1/vld:                   some fixed controller register values
 * @ecc_modes:                  supported ECC modes by the current controller,
 *                              initialized via DT match data
 */
struct qcom_nand_controller {
        struct nand_hw_control controller;
        struct list_head host_list;

        struct device *dev;

        void __iomem *base;
        dma_addr_t base_dma;

        struct clk *core_clk;
        struct clk *aon_clk;

        struct dma_chan *chan;
        unsigned int cmd_crci;
        unsigned int data_crci;
        struct list_head desc_list;

        u8              *data_buffer;
        int             buf_size;
        int             buf_count;
        int             buf_start;

        __le32 *reg_read_buf;
        int reg_read_pos;

        struct nandc_regs *regs;

        u32 cmd1, vld;
        u32 ecc_modes;
};

/*
 * NAND chip structure
 *
 * @chip:                       base NAND chip structure
 * @node:                       list node to add itself to host_list in
 *                              qcom_nand_controller
 *
 * @cs:                         chip select value for this chip
 * @cw_size:                    the number of bytes in a single step/codeword
 *                              of a page, consisting of all data, ecc, spare
 *                              and reserved bytes
 * @cw_data:                    the number of bytes within a codeword protected
 *                              by ECC
 * @use_ecc:                    request the controller to use ECC for the
 *                              upcoming read/write
 * @bch_enabled:                flag to tell whether BCH ECC mode is used
 * @ecc_bytes_hw:               ECC bytes used by controller hardware for this
 *                              chip
 * @status:                     value to be returned if NAND_CMD_STATUS command
 *                              is executed
 * @last_command:               keeps track of last command on this chip. used
 *                              for reading correct status
 *
 * @cfg0, cfg1, cfg0_raw..:     NANDc register configurations needed for
 *                              ecc/non-ecc mode for the current nand flash
 *                              device
 */
struct qcom_nand_host {
        struct nand_chip chip;
        struct list_head node;

        int cs;
        int cw_size;
        int cw_data;
        bool use_ecc;
        bool bch_enabled;
        int ecc_bytes_hw;
        int spare_bytes;
        int bbm_size;
        u8 status;
        int last_command;

        u32 cfg0, cfg1;
        u32 cfg0_raw, cfg1_raw;
        u32 ecc_buf_cfg;
        u32 ecc_bch_cfg;
        u32 clrflashstatus;
        u32 clrreadstatus;
};

static inline struct qcom_nand_host *to_qcom_nand_host(struct nand_chip *chip)
{
        return container_of(chip, struct qcom_nand_host, chip);
}

static inline struct qcom_nand_controller *
get_qcom_nand_controller(struct nand_chip *chip)
{
        return container_of(chip->controller, struct qcom_nand_controller,
                            controller);
}

static inline u32 nandc_read(struct qcom_nand_controller *nandc, int offset)
{
        return ioread32(nandc->base + offset);
}

static inline void nandc_write(struct qcom_nand_controller *nandc, int offset,
                               u32 val)
{
        iowrite32(val, nandc->base + offset);
}

static __le32 *offset_to_nandc_reg(struct nandc_regs *regs, int offset)
{
        switch (offset) {
        case NAND_FLASH_CMD:
                return &regs->cmd;
        case NAND_ADDR0:
                return &regs->addr0;
        case NAND_ADDR1:
                return &regs->addr1;
        case NAND_FLASH_CHIP_SELECT:
                return &regs->chip_sel;
        case NAND_EXEC_CMD:
                return &regs->exec;
        case NAND_FLASH_STATUS:
                return &regs->clrflashstatus;
        case NAND_DEV0_CFG0:
                return &regs->cfg0;
        case NAND_DEV0_CFG1:
                return &regs->cfg1;
        case NAND_DEV0_ECC_CFG:
                return &regs->ecc_bch_cfg;
        case NAND_READ_STATUS:
                return &regs->clrreadstatus;
        case NAND_DEV_CMD1:
                return &regs->cmd1;
        case NAND_DEV_CMD1_RESTORE:
                return &regs->orig_cmd1;
        case NAND_DEV_CMD_VLD:
                return &regs->vld;
        case NAND_DEV_CMD_VLD_RESTORE:
                return &regs->orig_vld;
        case NAND_EBI2_ECC_BUF_CFG:
                return &regs->ecc_buf_cfg;
        default:
                return NULL;
        }
}

static void nandc_set_reg(struct qcom_nand_controller *nandc, int offset,
                          u32 val)
{
        struct nandc_regs *regs = nandc->regs;
        __le32 *reg;

        reg = offset_to_nandc_reg(regs, offset);

        if (reg)
                *reg = cpu_to_le32(val);
}

/* helper to configure address register values */
static void set_address(struct qcom_nand_host *host, u16 column, int page)
{
        struct nand_chip *chip = &host->chip;
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);

        if (chip->options & NAND_BUSWIDTH_16)
                column >>= 1;

        nandc_set_reg(nandc, NAND_ADDR0, page << 16 | column);
        nandc_set_reg(nandc, NAND_ADDR1, page >> 16 & 0xff);
}

/*
 * update_rw_regs:      set up read/write register values, these will be
 *                      written to the NAND controller registers via DMA
 *
 * @num_cw:             number of steps for the read/write operation
 * @read:               read or write operation
 */
static void update_rw_regs(struct qcom_nand_host *host, int num_cw, bool read)
{
        struct nand_chip *chip = &host->chip;
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
        u32 cmd, cfg0, cfg1, ecc_bch_cfg;

        if (read) {
                if (host->use_ecc)
                        cmd = PAGE_READ_WITH_ECC | PAGE_ACC | LAST_PAGE;
                else
                        cmd = PAGE_READ | PAGE_ACC | LAST_PAGE;
        } else {
                        cmd = PROGRAM_PAGE | PAGE_ACC | LAST_PAGE;
        }

        if (host->use_ecc) {
                cfg0 = (host->cfg0 & ~(7U << CW_PER_PAGE)) |
                                (num_cw - 1) << CW_PER_PAGE;

                cfg1 = host->cfg1;
                ecc_bch_cfg = host->ecc_bch_cfg;
        } else {
                cfg0 = (host->cfg0_raw & ~(7U << CW_PER_PAGE)) |
                                (num_cw - 1) << CW_PER_PAGE;

                cfg1 = host->cfg1_raw;
                ecc_bch_cfg = 1 << ECC_CFG_ECC_DISABLE;
        }

        nandc_set_reg(nandc, NAND_FLASH_CMD, cmd);
        nandc_set_reg(nandc, NAND_DEV0_CFG0, cfg0);
        nandc_set_reg(nandc, NAND_DEV0_CFG1, cfg1);
        nandc_set_reg(nandc, NAND_DEV0_ECC_CFG, ecc_bch_cfg);
        nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, host->ecc_buf_cfg);
        nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus);
        nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus);
        nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
}

static int prep_dma_desc(struct qcom_nand_controller *nandc, bool read,
                         int reg_off, const void *vaddr, int size,
                         bool flow_control)
{
        struct desc_info *desc;
        struct dma_async_tx_descriptor *dma_desc;
        struct scatterlist *sgl;
        struct dma_slave_config slave_conf;
        enum dma_transfer_direction dir_eng;
        int ret;

        desc = kzalloc(sizeof(*desc), GFP_KERNEL);
        if (!desc)
                return -ENOMEM;

        sgl = &desc->sgl;

        sg_init_one(sgl, vaddr, size);

        if (read) {
                dir_eng = DMA_DEV_TO_MEM;
                desc->dir = DMA_FROM_DEVICE;
        } else {
                dir_eng = DMA_MEM_TO_DEV;
                desc->dir = DMA_TO_DEVICE;
        }

        ret = dma_map_sg(nandc->dev, sgl, 1, desc->dir);
        if (ret == 0) {
                ret = -ENOMEM;
                goto err;
        }

        memset(&slave_conf, 0x00, sizeof(slave_conf));

        slave_conf.device_fc = flow_control;
        if (read) {
                slave_conf.src_maxburst = 16;
                slave_conf.src_addr = nandc->base_dma + reg_off;
                slave_conf.slave_id = nandc->data_crci;
        } else {
                slave_conf.dst_maxburst = 16;
                slave_conf.dst_addr = nandc->base_dma + reg_off;
                slave_conf.slave_id = nandc->cmd_crci;
        }

        ret = dmaengine_slave_config(nandc->chan, &slave_conf);
        if (ret) {
                dev_err(nandc->dev, "failed to configure dma channel\n");
                goto err;
        }

        dma_desc = dmaengine_prep_slave_sg(nandc->chan, sgl, 1, dir_eng, 0);
        if (!dma_desc) {
                dev_err(nandc->dev, "failed to prepare desc\n");
                ret = -EINVAL;
                goto err;
        }

        desc->dma_desc = dma_desc;

        list_add_tail(&desc->node, &nandc->desc_list);

        return 0;
err:
        kfree(desc);

        return ret;
}

/*
 * read_reg_dma:        prepares a descriptor to read a given number of
 *                      contiguous registers to the reg_read_buf pointer
 *
 * @first:              offset of the first register in the contiguous block
 * @num_regs:           number of registers to read
 */
static int read_reg_dma(struct qcom_nand_controller *nandc, int first,
                        int num_regs)
{
        bool flow_control = false;
        void *vaddr;
        int size;

        if (first == NAND_READ_ID || first == NAND_FLASH_STATUS)
                flow_control = true;

        size = num_regs * sizeof(u32);
        vaddr = nandc->reg_read_buf + nandc->reg_read_pos;
        nandc->reg_read_pos += num_regs;

        return prep_dma_desc(nandc, true, first, vaddr, size, flow_control);
}

/*
 * write_reg_dma:       prepares a descriptor to write a given number of
 *                      contiguous registers
 *
 * @first:              offset of the first register in the contiguous block
 * @num_regs:           number of registers to write
 */
static int write_reg_dma(struct qcom_nand_controller *nandc, int first,
                         int num_regs)
{
        bool flow_control = false;
        struct nandc_regs *regs = nandc->regs;
        void *vaddr;
        int size;

        vaddr = offset_to_nandc_reg(regs, first);

        if (first == NAND_FLASH_CMD)
                flow_control = true;

        if (first == NAND_DEV_CMD1_RESTORE)
                first = NAND_DEV_CMD1;

        if (first == NAND_DEV_CMD_VLD_RESTORE)
                first = NAND_DEV_CMD_VLD;

        size = num_regs * sizeof(u32);

        return prep_dma_desc(nandc, false, first, vaddr, size, flow_control);
}

/*
 * read_data_dma:       prepares a DMA descriptor to transfer data from the
 *                      controller's internal buffer to the buffer 'vaddr'
 *
 * @reg_off:            offset within the controller's data buffer
 * @vaddr:              virtual address of the buffer we want to write to
 * @size:               DMA transaction size in bytes
 */
static int read_data_dma(struct qcom_nand_controller *nandc, int reg_off,
                         const u8 *vaddr, int size)
{
        return prep_dma_desc(nandc, true, reg_off, vaddr, size, false);
}

/*
 * write_data_dma:      prepares a DMA descriptor to transfer data from
 *                      'vaddr' to the controller's internal buffer
 *
 * @reg_off:            offset within the controller's data buffer
 * @vaddr:              virtual address of the buffer we want to read from
 * @size:               DMA transaction size in bytes
 */
static int write_data_dma(struct qcom_nand_controller *nandc, int reg_off,
                          const u8 *vaddr, int size)
{
        return prep_dma_desc(nandc, false, reg_off, vaddr, size, false);
}

/*
 * helper to prepare dma descriptors to configure registers needed for reading a
 * codeword/step in a page
 */
static void config_cw_read(struct qcom_nand_controller *nandc)
{
        write_reg_dma(nandc, NAND_FLASH_CMD, 3);
        write_reg_dma(nandc, NAND_DEV0_CFG0, 3);
        write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1);

        write_reg_dma(nandc, NAND_EXEC_CMD, 1);

        read_reg_dma(nandc, NAND_FLASH_STATUS, 2);
        read_reg_dma(nandc, NAND_ERASED_CW_DETECT_STATUS, 1);
}

/*
 * helpers to prepare dma descriptors used to configure registers needed for
 * writing a codeword/step in a page
 */
static void config_cw_write_pre(struct qcom_nand_controller *nandc)
{
        write_reg_dma(nandc, NAND_FLASH_CMD, 3);
        write_reg_dma(nandc, NAND_DEV0_CFG0, 3);
        write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1);
}

static void config_cw_write_post(struct qcom_nand_controller *nandc)
{
        write_reg_dma(nandc, NAND_EXEC_CMD, 1);

        read_reg_dma(nandc, NAND_FLASH_STATUS, 1);

        write_reg_dma(nandc, NAND_FLASH_STATUS, 1);
        write_reg_dma(nandc, NAND_READ_STATUS, 1);
}

/*
 * the following functions are used within chip->cmdfunc() to perform different
 * NAND_CMD_* commands
 */

/* sets up descriptors for NAND_CMD_PARAM */
static int nandc_param(struct qcom_nand_host *host)
{
        struct nand_chip *chip = &host->chip;
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);

        /*
         * NAND_CMD_PARAM is called before we know much about the FLASH chip
         * in use. we configure the controller to perform a raw read of 512
         * bytes to read onfi params
         */
        nandc_set_reg(nandc, NAND_FLASH_CMD, PAGE_READ | PAGE_ACC | LAST_PAGE);
        nandc_set_reg(nandc, NAND_ADDR0, 0);
        nandc_set_reg(nandc, NAND_ADDR1, 0);
        nandc_set_reg(nandc, NAND_DEV0_CFG0, 0 << CW_PER_PAGE
                                        | 512 << UD_SIZE_BYTES
                                        | 5 << NUM_ADDR_CYCLES
                                        | 0 << SPARE_SIZE_BYTES);
        nandc_set_reg(nandc, NAND_DEV0_CFG1, 7 << NAND_RECOVERY_CYCLES
                                        | 0 << CS_ACTIVE_BSY
                                        | 17 << BAD_BLOCK_BYTE_NUM
                                        | 1 << BAD_BLOCK_IN_SPARE_AREA
                                        | 2 << WR_RD_BSY_GAP
                                        | 0 << WIDE_FLASH
                                        | 1 << DEV0_CFG1_ECC_DISABLE);
        nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, 1 << ECC_CFG_ECC_DISABLE);

        /* configure CMD1 and VLD for ONFI param probing */
        nandc_set_reg(nandc, NAND_DEV_CMD_VLD,
                      (nandc->vld & ~(1 << READ_START_VLD))
                      | 0 << READ_START_VLD);
        nandc_set_reg(nandc, NAND_DEV_CMD1,
                      (nandc->cmd1 & ~(0xFF << READ_ADDR))
                      | NAND_CMD_PARAM << READ_ADDR);

        nandc_set_reg(nandc, NAND_EXEC_CMD, 1);

        nandc_set_reg(nandc, NAND_DEV_CMD1_RESTORE, nandc->cmd1);
        nandc_set_reg(nandc, NAND_DEV_CMD_VLD_RESTORE, nandc->vld);

        write_reg_dma(nandc, NAND_DEV_CMD_VLD, 1);
        write_reg_dma(nandc, NAND_DEV_CMD1, 1);

        nandc->buf_count = 512;
        memset(nandc->data_buffer, 0xff, nandc->buf_count);

        config_cw_read(nandc);

        read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer,
                      nandc->buf_count);

        /* restore CMD1 and VLD regs */
        write_reg_dma(nandc, NAND_DEV_CMD1_RESTORE, 1);
        write_reg_dma(nandc, NAND_DEV_CMD_VLD_RESTORE, 1);

        return 0;
}

/* sets up descriptors for NAND_CMD_ERASE1 */
static int erase_block(struct qcom_nand_host *host, int page_addr)
{
        struct nand_chip *chip = &host->chip;
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);

        nandc_set_reg(nandc, NAND_FLASH_CMD,
                      BLOCK_ERASE | PAGE_ACC | LAST_PAGE);
        nandc_set_reg(nandc, NAND_ADDR0, page_addr);
        nandc_set_reg(nandc, NAND_ADDR1, 0);
        nandc_set_reg(nandc, NAND_DEV0_CFG0,
                      host->cfg0_raw & ~(7 << CW_PER_PAGE));
        nandc_set_reg(nandc, NAND_DEV0_CFG1, host->cfg1_raw);
        nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
        nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus);
        nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus);

        write_reg_dma(nandc, NAND_FLASH_CMD, 3);
        write_reg_dma(nandc, NAND_DEV0_CFG0, 2);
        write_reg_dma(nandc, NAND_EXEC_CMD, 1);

        read_reg_dma(nandc, NAND_FLASH_STATUS, 1);

        write_reg_dma(nandc, NAND_FLASH_STATUS, 1);
        write_reg_dma(nandc, NAND_READ_STATUS, 1);

        return 0;
}

/* sets up descriptors for NAND_CMD_READID */
static int read_id(struct qcom_nand_host *host, int column)
{
        struct nand_chip *chip = &host->chip;
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);

        if (column == -1)
                return 0;

        nandc_set_reg(nandc, NAND_FLASH_CMD, FETCH_ID);
        nandc_set_reg(nandc, NAND_ADDR0, column);
        nandc_set_reg(nandc, NAND_ADDR1, 0);
        nandc_set_reg(nandc, NAND_FLASH_CHIP_SELECT, DM_EN);
        nandc_set_reg(nandc, NAND_EXEC_CMD, 1);

        write_reg_dma(nandc, NAND_FLASH_CMD, 4);
        write_reg_dma(nandc, NAND_EXEC_CMD, 1);

        read_reg_dma(nandc, NAND_READ_ID, 1);

        return 0;
}

/* sets up descriptors for NAND_CMD_RESET */
static int reset(struct qcom_nand_host *host)
{
        struct nand_chip *chip = &host->chip;
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);

        nandc_set_reg(nandc, NAND_FLASH_CMD, RESET_DEVICE);
        nandc_set_reg(nandc, NAND_EXEC_CMD, 1);

        write_reg_dma(nandc, NAND_FLASH_CMD, 1);
        write_reg_dma(nandc, NAND_EXEC_CMD, 1);

        read_reg_dma(nandc, NAND_FLASH_STATUS, 1);

        return 0;
}

/* helpers to submit/free our list of dma descriptors */
static int submit_descs(struct qcom_nand_controller *nandc)
{
        struct desc_info *desc;
        dma_cookie_t cookie = 0;

        list_for_each_entry(desc, &nandc->desc_list, node)
                cookie = dmaengine_submit(desc->dma_desc);

        if (dma_sync_wait(nandc->chan, cookie) != DMA_COMPLETE)
                return -ETIMEDOUT;

        return 0;
}

static void free_descs(struct qcom_nand_controller *nandc)
{
        struct desc_info *desc, *n;

        list_for_each_entry_safe(desc, n, &nandc->desc_list, node) {
                list_del(&desc->node);
                dma_unmap_sg(nandc->dev, &desc->sgl, 1, desc->dir);
                kfree(desc);
        }
}

/* reset the register read buffer for next NAND operation */
static void clear_read_regs(struct qcom_nand_controller *nandc)
{
        nandc->reg_read_pos = 0;
        memset(nandc->reg_read_buf, 0,
               MAX_REG_RD * sizeof(*nandc->reg_read_buf));
}

static void pre_command(struct qcom_nand_host *host, int command)
{
        struct nand_chip *chip = &host->chip;
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);

        nandc->buf_count = 0;
        nandc->buf_start = 0;
        host->use_ecc = false;
        host->last_command = command;

        clear_read_regs(nandc);
}

/*
 * this is called after NAND_CMD_PAGEPROG and NAND_CMD_ERASE1 to set our
 * privately maintained status byte, this status byte can be read after
 * NAND_CMD_STATUS is called
 */
static void parse_erase_write_errors(struct qcom_nand_host *host, int command)
{
        struct nand_chip *chip = &host->chip;
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
        struct nand_ecc_ctrl *ecc = &chip->ecc;
        int num_cw;
        int i;

        num_cw = command == NAND_CMD_PAGEPROG ? ecc->steps : 1;

        for (i = 0; i < num_cw; i++) {
                u32 flash_status = le32_to_cpu(nandc->reg_read_buf[i]);

                if (flash_status & FS_MPU_ERR)
                        host->status &= ~NAND_STATUS_WP;

                if (flash_status & FS_OP_ERR || (i == (num_cw - 1) &&
                                                 (flash_status &
                                                  FS_DEVICE_STS_ERR)))
                        host->status |= NAND_STATUS_FAIL;
        }
}

static void post_command(struct qcom_nand_host *host, int command)
{
        struct nand_chip *chip = &host->chip;
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);

        switch (command) {
        case NAND_CMD_READID:
                memcpy(nandc->data_buffer, nandc->reg_read_buf,
                       nandc->buf_count);
                break;
        case NAND_CMD_PAGEPROG:
        case NAND_CMD_ERASE1:
                parse_erase_write_errors(host, command);
                break;
        default:
                break;
        }
}

/*
 * Implements chip->cmdfunc. It's  only used for a limited set of commands.
 * The rest of the commands wouldn't be called by upper layers. For example,
 * NAND_CMD_READOOB would never be called because we have our own versions
 * of read_oob ops for nand_ecc_ctrl.
 */
static void qcom_nandc_command(struct mtd_info *mtd, unsigned int command,
                               int column, int page_addr)
{
        struct nand_chip *chip = mtd_to_nand(mtd);
        struct qcom_nand_host *host = to_qcom_nand_host(chip);
        struct nand_ecc_ctrl *ecc = &chip->ecc;
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
        bool wait = false;
        int ret = 0;

        pre_command(host, command);

        switch (command) {
        case NAND_CMD_RESET:
                ret = reset(host);
                wait = true;
                break;

        case NAND_CMD_READID:
                nandc->buf_count = 4;
                ret = read_id(host, column);
                wait = true;
                break;

        case NAND_CMD_PARAM:
                ret = nandc_param(host);
                wait = true;
                break;

        case NAND_CMD_ERASE1:
                ret = erase_block(host, page_addr);
                wait = true;
                break;

        case NAND_CMD_READ0:
                /* we read the entire page for now */
                WARN_ON(column != 0);

                host->use_ecc = true;
                set_address(host, 0, page_addr);
                update_rw_regs(host, ecc->steps, true);
                break;

        case NAND_CMD_SEQIN:
                WARN_ON(column != 0);
                set_address(host, 0, page_addr);
                break;

        case NAND_CMD_PAGEPROG:
        case NAND_CMD_STATUS:
        case NAND_CMD_NONE:
        default:
                break;
        }

        if (ret) {
                dev_err(nandc->dev, "failure executing command %d\n",
                        command);
                free_descs(nandc);
                return;
        }

        if (wait) {
                ret = submit_descs(nandc);
                if (ret)
                        dev_err(nandc->dev,
                                "failure submitting descs for command %d\n",
                                command);
        }

        free_descs(nandc);

        post_command(host, command);
}

/*
 * when using BCH ECC, the HW flags an error in NAND_FLASH_STATUS if it read
 * an erased CW, and reports an erased CW in NAND_ERASED_CW_DETECT_STATUS.
 *
 * when using RS ECC, the HW reports the same erros when reading an erased CW,
 * but it notifies that it is an erased CW by placing special characters at
 * certain offsets in the buffer.
 *
 * verify if the page is erased or not, and fix up the page for RS ECC by
 * replacing the special characters with 0xff.
 */
static bool erased_chunk_check_and_fixup(u8 *data_buf, int data_len)
{
        u8 empty1, empty2;

        /*
         * an erased page flags an error in NAND_FLASH_STATUS, check if the page
         * is erased by looking for 0x54s at offsets 3 and 175 from the
         * beginning of each codeword
         */

        empty1 = data_buf[3];
        empty2 = data_buf[175];

        /*
         * if the erased codework markers, if they exist override them with
         * 0xffs
         */
        if ((empty1 == 0x54 && empty2 == 0xff) ||
            (empty1 == 0xff && empty2 == 0x54)) {
                data_buf[3] = 0xff;
                data_buf[175] = 0xff;
        }

        /*
         * check if the entire chunk contains 0xffs or not. if it doesn't, then
         * restore the original values at the special offsets
         */
        if (memchr_inv(data_buf, 0xff, data_len)) {
                data_buf[3] = empty1;
                data_buf[175] = empty2;

                return false;
        }

        return true;
}

struct read_stats {
        __le32 flash;
        __le32 buffer;
        __le32 erased_cw;
};

/*
 * reads back status registers set by the controller to notify page read
 * errors. this is equivalent to what 'ecc->correct()' would do.
 */
static int parse_read_errors(struct qcom_nand_host *host, u8 *data_buf,
                             u8 *oob_buf)
{
        struct nand_chip *chip = &host->chip;
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
        struct mtd_info *mtd = nand_to_mtd(chip);
        struct nand_ecc_ctrl *ecc = &chip->ecc;
        unsigned int max_bitflips = 0;
        struct read_stats *buf;
        int i;

        buf = (struct read_stats *)nandc->reg_read_buf;

        for (i = 0; i < ecc->steps; i++, buf++) {
                u32 flash, buffer, erased_cw;
                int data_len, oob_len;

                if (i == (ecc->steps - 1)) {
                        data_len = ecc->size - ((ecc->steps - 1) << 2);
                        oob_len = ecc->steps << 2;
                } else {
                        data_len = host->cw_data;
                        oob_len = 0;
                }

                flash = le32_to_cpu(buf->flash);
                buffer = le32_to_cpu(buf->buffer);
                erased_cw = le32_to_cpu(buf->erased_cw);

                if (flash & (FS_OP_ERR | FS_MPU_ERR)) {
                        bool erased;

                        /* ignore erased codeword errors */
                        if (host->bch_enabled) {
                                erased = (erased_cw & ERASED_CW) == ERASED_CW ?
                                         true : false;
                        } else {
                                erased = erased_chunk_check_and_fixup(data_buf,
                                                                      data_len);
                        }

                        if (erased) {
                                data_buf += data_len;
                                if (oob_buf)
                                        oob_buf += oob_len + ecc->bytes;
                                continue;
                        }

                        if (buffer & BS_UNCORRECTABLE_BIT) {
                                int ret, ecclen, extraooblen;
                                void *eccbuf;

                                eccbuf = oob_buf ? oob_buf + oob_len : NULL;
                                ecclen = oob_buf ? host->ecc_bytes_hw : 0;
                                extraooblen = oob_buf ? oob_len : 0;

                                /*
                                 * make sure it isn't an erased page reported
                                 * as not-erased by HW because of a few bitflips
                                 */
                                ret = nand_check_erased_ecc_chunk(data_buf,
                                        data_len, eccbuf, ecclen, oob_buf,
                                        extraooblen, ecc->strength);
                                if (ret < 0) {
                                        mtd->ecc_stats.failed++;
                                } else {
                                        mtd->ecc_stats.corrected += ret;
                                        max_bitflips =
                                                max_t(unsigned int, max_bitflips, ret);
                                }
                        }
                } else {
                        unsigned int stat;

                        stat = buffer & BS_CORRECTABLE_ERR_MSK;
                        mtd->ecc_stats.corrected += stat;
                        max_bitflips = max(max_bitflips, stat);
                }

                data_buf += data_len;
                if (oob_buf)
                        oob_buf += oob_len + ecc->bytes;
        }

        return max_bitflips;
}

/*
 * helper to perform the actual page read operation, used by ecc->read_page(),
 * ecc->read_oob()
 */
static int read_page_ecc(struct qcom_nand_host *host, u8 *data_buf,
                         u8 *oob_buf)
{
        struct nand_chip *chip = &host->chip;
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
        struct nand_ecc_ctrl *ecc = &chip->ecc;
        int i, ret;

        /* queue cmd descs for each codeword */
        for (i = 0; i < ecc->steps; i++) {
                int data_size, oob_size;

                if (i == (ecc->steps - 1)) {
                        data_size = ecc->size - ((ecc->steps - 1) << 2);
                        oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
                                   host->spare_bytes;
                } else {
                        data_size = host->cw_data;
                        oob_size = host->ecc_bytes_hw + host->spare_bytes;
                }

                config_cw_read(nandc);

                if (data_buf)
                        read_data_dma(nandc, FLASH_BUF_ACC, data_buf,
                                      data_size);

                /*
                 * when ecc is enabled, the controller doesn't read the real
                 * or dummy bad block markers in each chunk. To maintain a
                 * consistent layout across RAW and ECC reads, we just
                 * leave the real/dummy BBM offsets empty (i.e, filled with
                 * 0xffs)
                 */
                if (oob_buf) {
                        int j;

                        for (j = 0; j < host->bbm_size; j++)
                                *oob_buf++ = 0xff;

                        read_data_dma(nandc, FLASH_BUF_ACC + data_size,
                                      oob_buf, oob_size);
                }

                if (data_buf)
                        data_buf += data_size;
                if (oob_buf)
                        oob_buf += oob_size;
        }

        ret = submit_descs(nandc);
        if (ret)
                dev_err(nandc->dev, "failure to read page/oob\n");

        free_descs(nandc);

        return ret;
}

/*
 * a helper that copies the last step/codeword of a page (containing free oob)
 * into our local buffer
 */
static int copy_last_cw(struct qcom_nand_host *host, int page)
{
        struct nand_chip *chip = &host->chip;
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
        struct nand_ecc_ctrl *ecc = &chip->ecc;
        int size;
        int ret;

        clear_read_regs(nandc);

        size = host->use_ecc ? host->cw_data : host->cw_size;

        /* prepare a clean read buffer */
        memset(nandc->data_buffer, 0xff, size);

        set_address(host, host->cw_size * (ecc->steps - 1), page);
        update_rw_regs(host, 1, true);

        config_cw_read(nandc);

        read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, size);

        ret = submit_descs(nandc);
        if (ret)
                dev_err(nandc->dev, "failed to copy last codeword\n");

        free_descs(nandc);

        return ret;
}

/* implements ecc->read_page() */
static int qcom_nandc_read_page(struct mtd_info *mtd, struct nand_chip *chip,
                                uint8_t *buf, int oob_required, int page)
{
        struct qcom_nand_host *host = to_qcom_nand_host(chip);
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
        u8 *data_buf, *oob_buf = NULL;
        int ret;

        data_buf = buf;
        oob_buf = oob_required ? chip->oob_poi : NULL;

        ret = read_page_ecc(host, data_buf, oob_buf);
        if (ret) {
                dev_err(nandc->dev, "failure to read page\n");
                return ret;
        }

        return parse_read_errors(host, data_buf, oob_buf);
}

/* implements ecc->read_page_raw() */
static int qcom_nandc_read_page_raw(struct mtd_info *mtd,
                                    struct nand_chip *chip, uint8_t *buf,
                                    int oob_required, int page)
{
        struct qcom_nand_host *host = to_qcom_nand_host(chip);
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
        u8 *data_buf, *oob_buf;
        struct nand_ecc_ctrl *ecc = &chip->ecc;
        int i, ret;

        data_buf = buf;
        oob_buf = chip->oob_poi;

        host->use_ecc = false;
        update_rw_regs(host, ecc->steps, true);

        for (i = 0; i < ecc->steps; i++) {
                int data_size1, data_size2, oob_size1, oob_size2;
                int reg_off = FLASH_BUF_ACC;

                data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
                oob_size1 = host->bbm_size;

                if (i == (ecc->steps - 1)) {
                        data_size2 = ecc->size - data_size1 -
                                     ((ecc->steps - 1) << 2);
                        oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw +
                                    host->spare_bytes;
                } else {
                        data_size2 = host->cw_data - data_size1;
                        oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
                }

                config_cw_read(nandc);

                read_data_dma(nandc, reg_off, data_buf, data_size1);
                reg_off += data_size1;
                data_buf += data_size1;

                read_data_dma(nandc, reg_off, oob_buf, oob_size1);
                reg_off += oob_size1;
                oob_buf += oob_size1;

                read_data_dma(nandc, reg_off, data_buf, data_size2);
                reg_off += data_size2;
                data_buf += data_size2;

                read_data_dma(nandc, reg_off, oob_buf, oob_size2);
                oob_buf += oob_size2;
        }

        ret = submit_descs(nandc);
        if (ret)
                dev_err(nandc->dev, "failure to read raw page\n");

        free_descs(nandc);

        return 0;
}

/* implements ecc->read_oob() */
static int qcom_nandc_read_oob(struct mtd_info *mtd, struct nand_chip *chip,
                               int page)
{
        struct qcom_nand_host *host = to_qcom_nand_host(chip);
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
        struct nand_ecc_ctrl *ecc = &chip->ecc;
        int ret;

        clear_read_regs(nandc);

        host->use_ecc = true;
        set_address(host, 0, page);
        update_rw_regs(host, ecc->steps, true);

        ret = read_page_ecc(host, NULL, chip->oob_poi);
        if (ret)
                dev_err(nandc->dev, "failure to read oob\n");

        return ret;
}

/* implements ecc->write_page() */
static int qcom_nandc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
                                 const uint8_t *buf, int oob_required, int page)
{
        struct qcom_nand_host *host = to_qcom_nand_host(chip);
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
        struct nand_ecc_ctrl *ecc = &chip->ecc;
        u8 *data_buf, *oob_buf;
        int i, ret;

        clear_read_regs(nandc);

        data_buf = (u8 *)buf;
        oob_buf = chip->oob_poi;

        host->use_ecc = true;
        update_rw_regs(host, ecc->steps, false);

        for (i = 0; i < ecc->steps; i++) {
                int data_size, oob_size;

                if (i == (ecc->steps - 1)) {
                        data_size = ecc->size - ((ecc->steps - 1) << 2);
                        oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
                                   host->spare_bytes;
                } else {
                        data_size = host->cw_data;
                        oob_size = ecc->bytes;
                }

                config_cw_write_pre(nandc);

                write_data_dma(nandc, FLASH_BUF_ACC, data_buf, data_size);

                /*
                 * when ECC is enabled, we don't really need to write anything
                 * to oob for the first n - 1 codewords since these oob regions
                 * just contain ECC bytes that's written by the controller
                 * itself. For the last codeword, we skip the bbm positions and
                 * write to the free oob area.
                 */
                if (i == (ecc->steps - 1)) {
                        oob_buf += host->bbm_size;

                        write_data_dma(nandc, FLASH_BUF_ACC + data_size,
                                       oob_buf, oob_size);
                }

                config_cw_write_post(nandc);

                data_buf += data_size;
                oob_buf += oob_size;
        }

        ret = submit_descs(nandc);
        if (ret)
                dev_err(nandc->dev, "failure to write page\n");

        free_descs(nandc);

        return ret;
}

/* implements ecc->write_page_raw() */
static int qcom_nandc_write_page_raw(struct mtd_info *mtd,
                                     struct nand_chip *chip, const uint8_t *buf,
                                     int oob_required, int page)
{
        struct qcom_nand_host *host = to_qcom_nand_host(chip);
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
        struct nand_ecc_ctrl *ecc = &chip->ecc;
        u8 *data_buf, *oob_buf;
        int i, ret;

        clear_read_regs(nandc);

        data_buf = (u8 *)buf;
        oob_buf = chip->oob_poi;

        host->use_ecc = false;
        update_rw_regs(host, ecc->steps, false);

        for (i = 0; i < ecc->steps; i++) {
                int data_size1, data_size2, oob_size1, oob_size2;
                int reg_off = FLASH_BUF_ACC;

                data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
                oob_size1 = host->bbm_size;

                if (i == (ecc->steps - 1)) {
                        data_size2 = ecc->size - data_size1 -
                                     ((ecc->steps - 1) << 2);
                        oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw +
                                    host->spare_bytes;
                } else {
                        data_size2 = host->cw_data - data_size1;
                        oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
                }

                config_cw_write_pre(nandc);

                write_data_dma(nandc, reg_off, data_buf, data_size1);
                reg_off += data_size1;
                data_buf += data_size1;

                write_data_dma(nandc, reg_off, oob_buf, oob_size1);
                reg_off += oob_size1;
                oob_buf += oob_size1;

                write_data_dma(nandc, reg_off, data_buf, data_size2);
                reg_off += data_size2;
                data_buf += data_size2;

                write_data_dma(nandc, reg_off, oob_buf, oob_size2);
                oob_buf += oob_size2;

                config_cw_write_post(nandc);
        }

        ret = submit_descs(nandc);
        if (ret)
                dev_err(nandc->dev, "failure to write raw page\n");

        free_descs(nandc);

        return ret;
}

/*
 * implements ecc->write_oob()
 *
 * the NAND controller cannot write only data or only oob within a codeword,
 * since ecc is calculated for the combined codeword. we first copy the
 * entire contents for the last codeword(data + oob), replace the old oob
 * with the new one in chip->oob_poi, and then write the entire codeword.
 * this read-copy-write operation results in a slight performance loss.
 */
static int qcom_nandc_write_oob(struct mtd_info *mtd, struct nand_chip *chip,
                                int page)
{
        struct qcom_nand_host *host = to_qcom_nand_host(chip);
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
        struct nand_ecc_ctrl *ecc = &chip->ecc;
        u8 *oob = chip->oob_poi;
        int free_boff;
        int data_size, oob_size;
        int ret, status = 0;

        host->use_ecc = true;

        ret = copy_last_cw(host, page);
        if (ret)
                return ret;

        clear_read_regs(nandc);

        /* calculate the data and oob size for the last codeword/step */
        data_size = ecc->size - ((ecc->steps - 1) << 2);
        oob_size = ecc->steps << 2;

        free_boff = ecc->layout->oobfree[0].offset;

        /* override new oob content to last codeword */
        memcpy(nandc->data_buffer + data_size, oob + free_boff, oob_size);

        set_address(host, host->cw_size * (ecc->steps - 1), page);
        update_rw_regs(host, 1, false);

        config_cw_write_pre(nandc);
        write_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer,
                       data_size + oob_size);
        config_cw_write_post(nandc);

        ret = submit_descs(nandc);

        free_descs(nandc);

        if (ret) {
                dev_err(nandc->dev, "failure to write oob\n");
                return -EIO;
        }

        chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);

        status = chip->waitfunc(mtd, chip);

        return status & NAND_STATUS_FAIL ? -EIO : 0;
}

static int qcom_nandc_block_bad(struct mtd_info *mtd, loff_t ofs)
{
        struct nand_chip *chip = mtd_to_nand(mtd);
        struct qcom_nand_host *host = to_qcom_nand_host(chip);
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
        struct nand_ecc_ctrl *ecc = &chip->ecc;
        int page, ret, bbpos, bad = 0;
        u32 flash_status;

        page = (int)(ofs >> chip->page_shift) & chip->pagemask;

        /*
         * configure registers for a raw sub page read, the address is set to
         * the beginning of the last codeword, we don't care about reading ecc
         * portion of oob. we just want the first few bytes from this codeword
         * that contains the BBM
         */
        host->use_ecc = false;

        ret = copy_last_cw(host, page);
        if (ret)
                goto err;

        flash_status = le32_to_cpu(nandc->reg_read_buf[0]);

        if (flash_status & (FS_OP_ERR | FS_MPU_ERR)) {
                dev_warn(nandc->dev, "error when trying to read BBM\n");
                goto err;
        }

        bbpos = mtd->writesize - host->cw_size * (ecc->steps - 1);

        bad = nandc->data_buffer[bbpos] != 0xff;

        if (chip->options & NAND_BUSWIDTH_16)
                bad = bad || (nandc->data_buffer[bbpos + 1] != 0xff);
err:
        return bad;
}

static int qcom_nandc_block_markbad(struct mtd_info *mtd, loff_t ofs)
{
        struct nand_chip *chip = mtd_to_nand(mtd);
        struct qcom_nand_host *host = to_qcom_nand_host(chip);
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
        struct nand_ecc_ctrl *ecc = &chip->ecc;
        int page, ret, status = 0;

        clear_read_regs(nandc);

        /*
         * to mark the BBM as bad, we flash the entire last codeword with 0s.
         * we don't care about the rest of the content in the codeword since
         * we aren't going to use this block again
         */
        memset(nandc->data_buffer, 0x00, host->cw_size);

        page = (int)(ofs >> chip->page_shift) & chip->pagemask;

        /* prepare write */
        host->use_ecc = false;
        set_address(host, host->cw_size * (ecc->steps - 1), page);
        update_rw_regs(host, 1, false);

        config_cw_write_pre(nandc);
        write_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, host->cw_size);
        config_cw_write_post(nandc);

        ret = submit_descs(nandc);

        free_descs(nandc);

        if (ret) {
                dev_err(nandc->dev, "failure to update BBM\n");
                return -EIO;
        }

        chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);

        status = chip->waitfunc(mtd, chip);

        return status & NAND_STATUS_FAIL ? -EIO : 0;
}

/*
 * the three functions below implement chip->read_byte(), chip->read_buf()
 * and chip->write_buf() respectively. these aren't used for
 * reading/writing page data, they are used for smaller data like reading
 * id, status etc
 */
static uint8_t qcom_nandc_read_byte(struct mtd_info *mtd)
{
        struct nand_chip *chip = mtd_to_nand(mtd);
        struct qcom_nand_host *host = to_qcom_nand_host(chip);
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
        u8 *buf = nandc->data_buffer;
        u8 ret = 0x0;

        if (host->last_command == NAND_CMD_STATUS) {
                ret = host->status;

                host->status = NAND_STATUS_READY | NAND_STATUS_WP;

                return ret;
        }

        if (nandc->buf_start < nandc->buf_count)
                ret = buf[nandc->buf_start++];

        return ret;
}

static void qcom_nandc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
{
        struct nand_chip *chip = mtd_to_nand(mtd);
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
        int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);

        memcpy(buf, nandc->data_buffer + nandc->buf_start, real_len);
        nandc->buf_start += real_len;
}

static void qcom_nandc_write_buf(struct mtd_info *mtd, const uint8_t *buf,
                                 int len)
{
        struct nand_chip *chip = mtd_to_nand(mtd);
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
        int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);

        memcpy(nandc->data_buffer + nandc->buf_start, buf, real_len);

        nandc->buf_start += real_len;
}

/* we support only one external chip for now */
static void qcom_nandc_select_chip(struct mtd_info *mtd, int chipnr)
{
        struct nand_chip *chip = mtd_to_nand(mtd);
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);

        if (chipnr <= 0)
                return;

        dev_warn(nandc->dev, "invalid chip select\n");
}

/*
 * NAND controller page layout info
 *
 * Layout with ECC enabled:
 *
 * |----------------------|  |---------------------------------|
 * |           xx.......yy|  |             *********xx.......yy|
 * |    DATA   xx..ECC..yy|  |    DATA     **SPARE**xx..ECC..yy|
 * |   (516)   xx.......yy|  |  (516-n*4)  **(n*4)**xx.......yy|
 * |           xx.......yy|  |             *********xx.......yy|
 * |----------------------|  |---------------------------------|
 *     codeword 1,2..n-1                  codeword n
 *  <---(528/532 Bytes)-->    <-------(528/532 Bytes)--------->
 *
 * n = Number of codewords in the page
 * . = ECC bytes
 * * = Spare/free bytes
 * x = Unused byte(s)
 * y = Reserved byte(s)
 *
 * 2K page: n = 4, spare = 16 bytes
 * 4K page: n = 8, spare = 32 bytes
 * 8K page: n = 16, spare = 64 bytes
 *
 * the qcom nand controller operates at a sub page/codeword level. each
 * codeword is 528 and 532 bytes for 4 bit and 8 bit ECC modes respectively.
 * the number of ECC bytes vary based on the ECC strength and the bus width.
 *
 * the first n - 1 codewords contains 516 bytes of user data, the remaining
 * 12/16 bytes consist of ECC and reserved data. The nth codeword contains
 * both user data and spare(oobavail) bytes that sum up to 516 bytes.
 *
 * When we access a page with ECC enabled, the reserved bytes(s) are not
 * accessible at all. When reading, we fill up these unreadable positions
 * with 0xffs. When writing, the controller skips writing the inaccessible
 * bytes.
 *
 * Layout with ECC disabled:
 *
 * |------------------------------|  |---------------------------------------|
 * |         yy          xx.......|  |         bb          *********xx.......|
 * |  DATA1  yy  DATA2   xx..ECC..|  |  DATA1  bb  DATA2   **SPARE**xx..ECC..|
 * | (size1) yy (size2)  xx.......|  | (size1) bb (size2)  **(n*4)**xx.......|
 * |         yy          xx.......|  |         bb          *********xx.......|
 * |------------------------------|  |---------------------------------------|
 *         codeword 1,2..n-1                        codeword n
 *  <-------(528/532 Bytes)------>    <-----------(528/532 Bytes)----------->
 *
 * n = Number of codewords in the page
 * . = ECC bytes
 * * = Spare/free bytes
 * x = Unused byte(s)
 * y = Dummy Bad Bock byte(s)
 * b = Real Bad Block byte(s)
 * size1/size2 = function of codeword size and 'n'
 *
 * when the ECC block is disabled, one reserved byte (or two for 16 bit bus
 * width) is now accessible. For the first n - 1 codewords, these are dummy Bad
 * Block Markers. In the last codeword, this position contains the real BBM
 *
 * In order to have a consistent layout between RAW and ECC modes, we assume
 * the following OOB layout arrangement:
 *
 * |-----------|  |--------------------|
 * |yyxx.......|  |bb*********xx.......|
 * |yyxx..ECC..|  |bb*FREEOOB*xx..ECC..|
 * |yyxx.......|  |bb*********xx.......|
 * |yyxx.......|  |bb*********xx.......|
 * |-----------|  |--------------------|
 *  first n - 1       nth OOB region
 *  OOB regions
 *
 * n = Number of codewords in the page
 * . = ECC bytes
 * * = FREE OOB bytes
 * y = Dummy bad block byte(s) (inaccessible when ECC enabled)
 * x = Unused byte(s)
 * b = Real bad block byte(s) (inaccessible when ECC enabled)
 *
 * This layout is read as is when ECC is disabled. When ECC is enabled, the
 * inaccessible Bad Block byte(s) are ignored when we write to a page/oob,
 * and assumed as 0xffs when we read a page/oob. The ECC, unused and
 * dummy/real bad block bytes are grouped as ecc bytes in nand_ecclayout (i.e,
 * ecc->bytes is the sum of the three).
 */

static struct nand_ecclayout *
qcom_nand_create_layout(struct qcom_nand_host *host)
{
        struct nand_chip *chip = &host->chip;
        struct mtd_info *mtd = nand_to_mtd(chip);
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
        struct nand_ecc_ctrl *ecc = &chip->ecc;
        struct nand_ecclayout *layout;
        int i, j, steps, pos = 0, shift = 0;

        layout = devm_kzalloc(nandc->dev, sizeof(*layout), GFP_KERNEL);
        if (!layout)
                return NULL;

        steps = mtd->writesize / ecc->size;
        layout->eccbytes = steps * ecc->bytes;

        layout->oobfree[0].offset = (steps - 1) * ecc->bytes + host->bbm_size;
        layout->oobfree[0].length = steps << 2;

        /*
         * the oob bytes in the first n - 1 codewords are all grouped together
         * in the format:
         * DUMMY_BBM + UNUSED + ECC
         */
        for (i = 0; i < steps - 1; i++) {
                for (j = 0; j < ecc->bytes; j++)
                        layout->eccpos[pos++] = i * ecc->bytes + j;
        }

        /*
         * the oob bytes in the last codeword are grouped in the format:
         * BBM + FREE OOB + UNUSED + ECC
         */

        /* fill up the bbm positions */
        for (j = 0; j < host->bbm_size; j++)
                layout->eccpos[pos++] = i * ecc->bytes + j;

        /*
         * fill up the ecc and reserved positions, their indices are offseted
         * by the free oob region
         */
        shift = layout->oobfree[0].length + host->bbm_size;

        for (j = 0; j < (host->ecc_bytes_hw + host->spare_bytes); j++)
                layout->eccpos[pos++] = i * ecc->bytes + shift + j;

        return layout;
}

static int qcom_nand_host_setup(struct qcom_nand_host *host)
{
        struct nand_chip *chip = &host->chip;
        struct mtd_info *mtd = nand_to_mtd(chip);
        struct nand_ecc_ctrl *ecc = &chip->ecc;
        struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
        int cwperpage, bad_block_byte;
        bool wide_bus;
        int ecc_mode = 1;

        /*
         * the controller requires each step consists of 512 bytes of data.
         * bail out if DT has populated a wrong step size.
         */
        if (ecc->size != NANDC_STEP_SIZE) {
                dev_err(nandc->dev, "invalid ecc size\n");
                return -EINVAL;
        }

        wide_bus = chip->options & NAND_BUSWIDTH_16 ? true : false;

        if (ecc->strength >= 8) {
                /* 8 bit ECC defaults to BCH ECC on all platforms */
                host->bch_enabled = true;
                ecc_mode = 1;

                if (wide_bus) {
                        host->ecc_bytes_hw = 14;
                        host->spare_bytes = 0;
                        host->bbm_size = 2;
                } else {
                        host->ecc_bytes_hw = 13;
                        host->spare_bytes = 2;
                        host->bbm_size = 1;
                }
        } else {
                /*
                 * if the controller supports BCH for 4 bit ECC, the controller
                 * uses lesser bytes for ECC. If RS is used, the ECC bytes is
                 * always 10 bytes
                 */
                if (nandc->ecc_modes & ECC_BCH_4BIT) {
                        /* BCH */
                        host->bch_enabled = true;
                        ecc_mode = 0;

                        if (wide_bus) {
                                host->ecc_bytes_hw = 8;
                                host->spare_bytes = 2;
                                host->bbm_size = 2;
                        } else {
                                host->ecc_bytes_hw = 7;
                                host->spare_bytes = 4;
                                host->bbm_size = 1;
                        }
                } else {
                        /* RS */
                        host->ecc_bytes_hw = 10;

                        if (wide_bus) {
                                host->spare_bytes = 0;
                                host->bbm_size = 2;
                        } else {
                                host->spare_bytes = 1;
                                host->bbm_size = 1;
                        }
                }
        }

        /*
         * we consider ecc->bytes as the sum of all the non-data content in a
         * step. It gives us a clean representation of the oob area (even if
         * all the bytes aren't used for ECC).It is always 16 bytes for 8 bit
         * ECC and 12 bytes for 4 bit ECC
         */
        ecc->bytes = host->ecc_bytes_hw + host->spare_bytes + host->bbm_size;

        ecc->read_page          = qcom_nandc_read_page;
        ecc->read_page_raw      = qcom_nandc_read_page_raw;
        ecc->read_oob           = qcom_nandc_read_oob;
        ecc->write_page         = qcom_nandc_write_page;
        ecc->write_page_raw     = qcom_nandc_write_page_raw;
        ecc->write_oob          = qcom_nandc_write_oob;

        ecc->mode = NAND_ECC_HW;

        ecc->layout = qcom_nand_create_layout(host);
        if (!ecc->layout)
                return -ENOMEM;

        cwperpage = mtd->writesize / ecc->size;

        /*
         * DATA_UD_BYTES varies based on whether the read/write command protects
         * spare data with ECC too. We protect spare data by default, so we set
         * it to main + spare data, which are 512 and 4 bytes respectively.
         */
        host->cw_data = 516;

        /*
         * total bytes in a step, either 528 bytes for 4 bit ECC, or 532 bytes
         * for 8 bit ECC
         */
        host->cw_size = host->cw_data + ecc->bytes;

        if (ecc->bytes * (mtd->writesize / ecc->size) > mtd->oobsize) {
                dev_err(nandc->dev, "ecc data doesn't fit in OOB area\n");
                return -EINVAL;
        }

        bad_block_byte = mtd->writesize - host->cw_size * (cwperpage - 1) + 1;

        host->cfg0 = (cwperpage - 1) << CW_PER_PAGE
                                | host->cw_data << UD_SIZE_BYTES
                                | 0 << DISABLE_STATUS_AFTER_WRITE
                                | 5 << NUM_ADDR_CYCLES
                                | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_RS
                                | 0 << STATUS_BFR_READ
                                | 1 << SET_RD_MODE_AFTER_STATUS
                                | host->spare_bytes << SPARE_SIZE_BYTES;

        host->cfg1 = 7 << NAND_RECOVERY_CYCLES
                                | 0 <<  CS_ACTIVE_BSY
                                | bad_block_byte << BAD_BLOCK_BYTE_NUM
                                | 0 << BAD_BLOCK_IN_SPARE_AREA
                                | 2 << WR_RD_BSY_GAP
                                | wide_bus << WIDE_FLASH
                                | host->bch_enabled << ENABLE_BCH_ECC;

        host->cfg0_raw = (cwperpage - 1) << CW_PER_PAGE
                                | host->cw_size << UD_SIZE_BYTES
                                | 5 << NUM_ADDR_CYCLES
                                | 0 << SPARE_SIZE_BYTES;

        host->cfg1_raw = 7 << NAND_RECOVERY_CYCLES
                                | 0 << CS_ACTIVE_BSY
                                | 17 << BAD_BLOCK_BYTE_NUM
                                | 1 << BAD_BLOCK_IN_SPARE_AREA
                                | 2 << WR_RD_BSY_GAP
                                | wide_bus << WIDE_FLASH
                                | 1 << DEV0_CFG1_ECC_DISABLE;

        host->ecc_bch_cfg = host->bch_enabled << ECC_CFG_ECC_DISABLE
                                | 0 << ECC_SW_RESET
                                | host->cw_data << ECC_NUM_DATA_BYTES
                                | 1 << ECC_FORCE_CLK_OPEN
                                | ecc_mode << ECC_MODE
                                | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_BCH;

        host->ecc_buf_cfg = 0x203 << NUM_STEPS;

        host->clrflashstatus = FS_READY_BSY_N;
        host->clrreadstatus = 0xc0;

        dev_dbg(nandc->dev,
                "cfg0 %x cfg1 %x ecc_buf_cfg %x ecc_bch cfg %x cw_size %d cw_data %d strength %d parity_bytes %d steps %d\n",
                host->cfg0, host->cfg1, host->ecc_buf_cfg, host->ecc_bch_cfg,
                host->cw_size, host->cw_data, ecc->strength, ecc->bytes,
                cwperpage);

        return 0;
}

static int qcom_nandc_alloc(struct qcom_nand_controller *nandc)
{
        int ret;

        ret = dma_set_coherent_mask(nandc->dev, DMA_BIT_MASK(32));
        if (ret) {
                dev_err(nandc->dev, "failed to set DMA mask\n");
                return ret;
        }

        /*
         * we use the internal buffer for reading ONFI params, reading small
         * data like ID and status, and preforming read-copy-write operations
         * when writing to a codeword partially. 532 is the maximum possible
         * size of a codeword for our nand controller
         */
        nandc->buf_size = 532;

        nandc->data_buffer = devm_kzalloc(nandc->dev, nandc->buf_size,
                                        GFP_KERNEL);
        if (!nandc->data_buffer)
                return -ENOMEM;

        nandc->regs = devm_kzalloc(nandc->dev, sizeof(*nandc->regs),
                                        GFP_KERNEL);
        if (!nandc->regs)
                return -ENOMEM;

        nandc->reg_read_buf = devm_kzalloc(nandc->dev,
                                MAX_REG_RD * sizeof(*nandc->reg_read_buf),
                                GFP_KERNEL);
        if (!nandc->reg_read_buf)
                return -ENOMEM;

        nandc->chan = dma_request_slave_channel(nandc->dev, "rxtx");
        if (!nandc->chan) {
                dev_err(nandc->dev, "failed to request slave channel\n");
                return -ENODEV;
        }

        INIT_LIST_HEAD(&nandc->desc_list);
        INIT_LIST_HEAD(&nandc->host_list);

        spin_lock_init(&nandc->controller.lock);
        init_waitqueue_head(&nandc->controller.wq);

        return 0;
}

static void qcom_nandc_unalloc(struct qcom_nand_controller *nandc)
{
        dma_release_channel(nandc->chan);
}

/* one time setup of a few nand controller registers */
static int qcom_nandc_setup(struct qcom_nand_controller *nandc)
{
        /* kill onenand */
        nandc_write(nandc, SFLASHC_BURST_CFG, 0);

        /* enable ADM DMA */
        nandc_write(nandc, NAND_FLASH_CHIP_SELECT, DM_EN);

        /* save the original values of these registers */
        nandc->cmd1 = nandc_read(nandc, NAND_DEV_CMD1);
        nandc->vld = nandc_read(nandc, NAND_DEV_CMD_VLD);

        return 0;
}

static int qcom_nand_host_init(struct qcom_nand_controller *nandc,
                               struct qcom_nand_host *host,
                               struct device_node *dn)
{
        struct nand_chip *chip = &host->chip;
        struct mtd_info *mtd = nand_to_mtd(chip);
        struct device *dev = nandc->dev;
        int ret;

        ret = of_property_read_u32(dn, "reg", &host->cs);
        if (ret) {
                dev_err(dev, "can't get chip-select\n");
                return -ENXIO;
        }

        nand_set_flash_node(chip, dn);
        mtd->name = devm_kasprintf(dev, GFP_KERNEL, "qcom_nand.%d", host->cs);
        mtd->owner = THIS_MODULE;
        mtd->dev.parent = dev;

        chip->cmdfunc           = qcom_nandc_command;
        chip->select_chip       = qcom_nandc_select_chip;
        chip->read_byte         = qcom_nandc_read_byte;
        chip->read_buf          = qcom_nandc_read_buf;
        chip->write_buf         = qcom_nandc_write_buf;

        /*
         * the bad block marker is readable only when we read the last codeword
         * of a page with ECC disabled. currently, the nand_base and nand_bbt
         * helpers don't allow us to read BB from a nand chip with ECC
         * disabled (MTD_OPS_PLACE_OOB is set by default). use the block_bad
         * and block_markbad helpers until we permanently switch to using
         * MTD_OPS_RAW for all drivers (with the help of badblockbits)
         */
        chip->block_bad         = qcom_nandc_block_bad;
        chip->block_markbad     = qcom_nandc_block_markbad;

        chip->controller = &nandc->controller;
        chip->options |= NAND_NO_SUBPAGE_WRITE | NAND_USE_BOUNCE_BUFFER |
                         NAND_SKIP_BBTSCAN;

        /* set up initial status value */
        host->status = NAND_STATUS_READY | NAND_STATUS_WP;

        ret = nand_scan_ident(mtd, 1, NULL);
        if (ret)
                return ret;

        ret = qcom_nand_host_setup(host);
        if (ret)
                return ret;

        ret = nand_scan_tail(mtd);
        if (ret)
                return ret;

        return mtd_device_register(mtd, NULL, 0);
}

/* parse custom DT properties here */
static int qcom_nandc_parse_dt(struct platform_device *pdev)
{
        struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
        struct device_node *np = nandc->dev->of_node;
        int ret;

        ret = of_property_read_u32(np, "qcom,cmd-crci", &nandc->cmd_crci);
        if (ret) {
                dev_err(nandc->dev, "command CRCI unspecified\n");
                return ret;
        }

        ret = of_property_read_u32(np, "qcom,data-crci", &nandc->data_crci);
        if (ret) {
                dev_err(nandc->dev, "data CRCI unspecified\n");
                return ret;
        }

        return 0;
}

static int qcom_nandc_probe(struct platform_device *pdev)
{
        struct qcom_nand_controller *nandc;
        struct qcom_nand_host *host;
        const void *dev_data;
        struct device *dev = &pdev->dev;
        struct device_node *dn = dev->of_node, *child;
        struct resource *res;
        int ret;

        nandc = devm_kzalloc(&pdev->dev, sizeof(*nandc), GFP_KERNEL);
        if (!nandc)
                return -ENOMEM;

        platform_set_drvdata(pdev, nandc);
        nandc->dev = dev;

        dev_data = of_device_get_match_data(dev);
        if (!dev_data) {
                dev_err(&pdev->dev, "failed to get device data\n");
                return -ENODEV;
        }

        nandc->ecc_modes = (unsigned long)dev_data;

        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
        nandc->base = devm_ioremap_resource(dev, res);
        if (IS_ERR(nandc->base))
                return PTR_ERR(nandc->base);

        nandc->base_dma = phys_to_dma(dev, (phys_addr_t)res->start);

        nandc->core_clk = devm_clk_get(dev, "core");
        if (IS_ERR(nandc->core_clk))
                return PTR_ERR(nandc->core_clk);

        nandc->aon_clk = devm_clk_get(dev, "aon");
        if (IS_ERR(nandc->aon_clk))
                return PTR_ERR(nandc->aon_clk);

        ret = qcom_nandc_parse_dt(pdev);
        if (ret)
                return ret;

        ret = qcom_nandc_alloc(nandc);
        if (ret)
                return ret;

        ret = clk_prepare_enable(nandc->core_clk);
        if (ret)
                goto err_core_clk;

        ret = clk_prepare_enable(nandc->aon_clk);
        if (ret)
                goto err_aon_clk;

        ret = qcom_nandc_setup(nandc);
        if (ret)
                goto err_setup;

        for_each_available_child_of_node(dn, child) {
                if (of_device_is_compatible(child, "qcom,nandcs")) {
                        host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
                        if (!host) {
                                of_node_put(child);
                                ret = -ENOMEM;
                                goto err_cs_init;
                        }

                        ret = qcom_nand_host_init(nandc, host, child);
                        if (ret) {
                                devm_kfree(dev, host);
                                continue;
                        }

                        list_add_tail(&host->node, &nandc->host_list);
                }
        }

        if (list_empty(&nandc->host_list)) {
                ret = -ENODEV;
                goto err_cs_init;
        }

        return 0;

err_cs_init:
        list_for_each_entry(host, &nandc->host_list, node)
                nand_release(nand_to_mtd(&host->chip));
err_setup:
        clk_disable_unprepare(nandc->aon_clk);
err_aon_clk:
        clk_disable_unprepare(nandc->core_clk);
err_core_clk:
        qcom_nandc_unalloc(nandc);

        return ret;
}

static int qcom_nandc_remove(struct platform_device *pdev)
{
        struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
        struct qcom_nand_host *host;

        list_for_each_entry(host, &nandc->host_list, node)
                nand_release(nand_to_mtd(&host->chip));

        qcom_nandc_unalloc(nandc);

        clk_disable_unprepare(nandc->aon_clk);
        clk_disable_unprepare(nandc->core_clk);

        return 0;
}

#define EBI2_NANDC_ECC_MODES    (ECC_RS_4BIT | ECC_BCH_8BIT)

/*
 * data will hold a struct pointer containing more differences once we support
 * more controller variants
 */
static const struct of_device_id qcom_nandc_of_match[] = {
        {       .compatible = "qcom,ipq806x-nand",
                .data = (void *)EBI2_NANDC_ECC_MODES,
        },
        {}
};
MODULE_DEVICE_TABLE(of, qcom_nandc_of_match);

static struct platform_driver qcom_nandc_driver = {
        .driver = {
                .name = "qcom-nandc",
                .of_match_table = qcom_nandc_of_match,
        },
        .probe   = qcom_nandc_probe,
        .remove  = qcom_nandc_remove,
};
module_platform_driver(qcom_nandc_driver);

MODULE_AUTHOR("Archit Taneja <architt@codeaurora.org>");
MODULE_DESCRIPTION("Qualcomm NAND Controller driver");
MODULE_LICENSE("GPL v2");