target/linux/mediatek/patches-5.15/120-12-spi-add-driver-for-MTK-SPI-NAND-Flash-Interface.patch

   1 From 8170bafa8936e9fbfdce992932a63bd20eca3bc3 Mon Sep 17 00:00:00 2001
   2 From: Chuanhong Guo <gch981213@gmail.com>
   3 Date: Sat, 2 Apr 2022 10:16:11 +0800
   4 Subject: [PATCH v6 2/5] spi: add driver for MTK SPI NAND Flash Interface
   5
   6 This driver implements support for the SPI-NAND mode of MTK NAND Flash
   7 Interface as a SPI-MEM controller with pipelined ECC capability.
   8
   9 Signed-off-by: Chuanhong Guo <gch981213@gmail.com>
  10 Tested-by: Daniel Golle <daniel@makrotopia.org>
  11 ---
  12 Change since v1:
  13   fix CI warnings
  14
  15 Changes since v2:
  16  use streamed DMA api to avoid an extra memory copy during read
  17  make ECC engine config a per-nand context
  18  take user-requested ECC strength into account
  19
  20 Change since v3: none
  21 Changes since v4:
  22  fix missing OOB write
  23  print page format with dev_dbg
  24  replace uint*_t copied from vendor driver with u*
  25
  26 Changes since v5:
  27  add missing nfi mode register configuration in probe
  28  fix an off-by-one bug in mtk_snand_mac_io
  29
  30  drivers/spi/Kconfig        |   10 +
  31  drivers/spi/Makefile       |    1 +
  32  drivers/spi/spi-mtk-snfi.c | 1470 ++++++++++++++++++++++++++++++++++++
  33  3 files changed, 1481 insertions(+)
  34  create mode 100644 drivers/spi/spi-mtk-snfi.c
  35
  36 diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig
  37 index 6c28ca232444..83c3c2b57a22 100644
  38 --- a/drivers/spi/Kconfig
  39 +++ b/drivers/spi/Kconfig
  40 @@ -590,6 +590,16 @@ config SPI_MTK_NOR
  41           SPI interface as well as several SPI NOR specific instructions
  42           via SPI MEM interface.
  43
  44 +config SPI_MTK_SNFI
  45 +       tristate "MediaTek SPI NAND Flash Interface"
  46 +       depends on ARCH_MEDIATEK || COMPILE_TEST
  47 +       depends on MTD_NAND_ECC_MEDIATEK
  48 +       help
  49 +         This enables support for SPI-NAND mode on the MediaTek NAND
  50 +         Flash Interface found on MediaTek ARM SoCs. This controller
  51 +         is implemented as a SPI-MEM controller with pipelined ECC
  52 +         capcability.
  53 +
  54  config SPI_NPCM_FIU
  55         tristate "Nuvoton NPCM FLASH Interface Unit"
  56         depends on ARCH_NPCM || COMPILE_TEST
  57 diff --git a/drivers/spi/Makefile b/drivers/spi/Makefile
  58 index 3aa28ed3f761..51541ff17e67 100644
  59 --- a/drivers/spi/Makefile
  60 +++ b/drivers/spi/Makefile
  61 @@ -76,6 +76,7 @@ obj-$(CONFIG_SPI_MPC52xx)             += spi-mpc52xx.o
  62  obj-$(CONFIG_SPI_MT65XX)                += spi-mt65xx.o
  63  obj-$(CONFIG_SPI_MT7621)               += spi-mt7621.o
  64  obj-$(CONFIG_SPI_MTK_NOR)              += spi-mtk-nor.o
  65 +obj-$(CONFIG_SPI_MTK_SNFI)             += spi-mtk-snfi.o
  66  obj-$(CONFIG_SPI_MXIC)                 += spi-mxic.o
  67  obj-$(CONFIG_SPI_MXS)                  += spi-mxs.o
  68  obj-$(CONFIG_SPI_NPCM_FIU)             += spi-npcm-fiu.o
  69 diff --git a/drivers/spi/spi-mtk-snfi.c b/drivers/spi/spi-mtk-snfi.c
  70 new file mode 100644
  71 index 000000000000..2c556e304673
  72 --- /dev/null
  73 +++ b/drivers/spi/spi-mtk-snfi.c
  74 @@ -0,0 +1,1470 @@
  75 +// SPDX-License-Identifier: GPL-2.0
  76 +//
  77 +// Driver for the SPI-NAND mode of Mediatek NAND Flash Interface
  78 +//
  79 +// Copyright (c) 2022 Chuanhong Guo <gch981213@gmail.com>
  80 +//
  81 +// This driver is based on the SPI-NAND mtd driver from Mediatek SDK:
  82 +//
  83 +// Copyright (C) 2020 MediaTek Inc.
  84 +// Author: Weijie Gao <weijie.gao@mediatek.com>
  85 +//
  86 +// This controller organize the page data as several interleaved sectors
  87 +// like the following: (sizeof(FDM + ECC) = snf->nfi_cfg.spare_size)
  88 +// +---------+------+------+---------+------+------+-----+
  89 +// | Sector1 | FDM1 | ECC1 | Sector2 | FDM2 | ECC2 | ... |
  90 +// +---------+------+------+---------+------+------+-----+
  91 +// With auto-format turned on, DMA only returns this part:
  92 +// +---------+---------+-----+
  93 +// | Sector1 | Sector2 | ... |
  94 +// +---------+---------+-----+
  95 +// The FDM data will be filled to the registers, and ECC parity data isn't
  96 +// accessible.
  97 +// With auto-format off, all ((Sector+FDM+ECC)*nsectors) will be read over DMA
  98 +// in it's original order shown in the first table. ECC can't be turned on when
  99 +// auto-format is off.
 100 +//
 101 +// However, Linux SPI-NAND driver expects the data returned as:
 102 +// +------+-----+
 103 +// | Page | OOB |
 104 +// +------+-----+
 105 +// where the page data is continuously stored instead of interleaved.
 106 +// So we assume all instructions matching the page_op template between ECC
 107 +// prepare_io_req and finish_io_req are for page cache r/w.
 108 +// Here's how this spi-mem driver operates when reading:
 109 +//  1. Always set snf->autofmt = true in prepare_io_req (even when ECC is off).
 110 +//  2. Perform page ops and let the controller fill the DMA bounce buffer with
 111 +//     de-interleaved sector data and set FDM registers.
 112 +//  3. Return the data as:
 113 +//     +---------+---------+-----+------+------+-----+
 114 +//     | Sector1 | Sector2 | ... | FDM1 | FDM2 | ... |
 115 +//     +---------+---------+-----+------+------+-----+
 116 +//  4. For other matching spi_mem ops outside a prepare/finish_io_req pair,
 117 +//     read the data with auto-format off into the bounce buffer and copy
 118 +//     needed data to the buffer specified in the request.
 119 +//
 120 +// Write requests operates in a similar manner.
 121 +// As a limitation of this strategy, we won't be able to access any ECC parity
 122 +// data at all in Linux.
 123 +//
 124 +// Here's the bad block mark situation on MTK chips:
 125 +// In older chips like mt7622, MTK uses the first FDM byte in the first sector
 126 +// as the bad block mark. After de-interleaving, this byte appears at [pagesize]
 127 +// in the returned data, which is the BBM position expected by kernel. However,
 128 +// the conventional bad block mark is the first byte of the OOB, which is part
 129 +// of the last sector data in the interleaved layout. Instead of fixing their
 130 +// hardware, MTK decided to address this inconsistency in software. On these
 131 +// later chips, the BootROM expects the following:
 132 +// 1. The [pagesize] byte on a nand page is used as BBM, which will appear at
 133 +//    (page_size - (nsectors - 1) * spare_size) in the DMA buffer.
 134 +// 2. The original byte stored at that position in the DMA buffer will be stored
 135 +//    as the first byte of the FDM section in the last sector.
 136 +// We can't disagree with the BootROM, so after de-interleaving, we need to
 137 +// perform the following swaps in read:
 138 +// 1. Store the BBM at [page_size - (nsectors - 1) * spare_size] to [page_size],
 139 +//    which is the expected BBM position by kernel.
 140 +// 2. Store the page data byte at [pagesize + (nsectors-1) * fdm] back to
 141 +//    [page_size - (nsectors - 1) * spare_size]
 142 +// Similarly, when writing, we need to perform swaps in the other direction.
 143 +
 144 +#include <linux/kernel.h>
 145 +#include <linux/module.h>
 146 +#include <linux/init.h>
 147 +#include <linux/device.h>
 148 +#include <linux/mutex.h>
 149 +#include <linux/clk.h>
 150 +#include <linux/interrupt.h>
 151 +#include <linux/dma-mapping.h>
 152 +#include <linux/iopoll.h>
 153 +#include <linux/of_platform.h>
 154 +#include <linux/mtd/nand-ecc-mtk.h>
 155 +#include <linux/spi/spi.h>
 156 +#include <linux/spi/spi-mem.h>
 157 +#include <linux/mtd/nand.h>
 158 +
 159 +// NFI registers
 160 +#define NFI_CNFG 0x000
 161 +#define CNFG_OP_MODE_S 12
 162 +#define CNFG_OP_MODE_CUST 6
 163 +#define CNFG_OP_MODE_PROGRAM 3
 164 +#define CNFG_AUTO_FMT_EN BIT(9)
 165 +#define CNFG_HW_ECC_EN BIT(8)
 166 +#define CNFG_DMA_BURST_EN BIT(2)
 167 +#define CNFG_READ_MODE BIT(1)
 168 +#define CNFG_DMA_MODE BIT(0)
 169 +
 170 +#define NFI_PAGEFMT 0x0004
 171 +#define NFI_SPARE_SIZE_LS_S 16
 172 +#define NFI_FDM_ECC_NUM_S 12
 173 +#define NFI_FDM_NUM_S 8
 174 +#define NFI_SPARE_SIZE_S 4
 175 +#define NFI_SEC_SEL_512 BIT(2)
 176 +#define NFI_PAGE_SIZE_S 0
 177 +#define NFI_PAGE_SIZE_512_2K 0
 178 +#define NFI_PAGE_SIZE_2K_4K 1
 179 +#define NFI_PAGE_SIZE_4K_8K 2
 180 +#define NFI_PAGE_SIZE_8K_16K 3
 181 +
 182 +#define NFI_CON 0x008
 183 +#define CON_SEC_NUM_S 12
 184 +#define CON_BWR BIT(9)
 185 +#define CON_BRD BIT(8)
 186 +#define CON_NFI_RST BIT(1)
 187 +#define CON_FIFO_FLUSH BIT(0)
 188 +
 189 +#define NFI_INTR_EN 0x010
 190 +#define NFI_INTR_STA 0x014
 191 +#define NFI_IRQ_INTR_EN BIT(31)
 192 +#define NFI_IRQ_CUS_READ BIT(8)
 193 +#define NFI_IRQ_CUS_PG BIT(7)
 194 +
 195 +#define NFI_CMD 0x020
 196 +#define NFI_CMD_DUMMY_READ 0x00
 197 +#define NFI_CMD_DUMMY_WRITE 0x80
 198 +
 199 +#define NFI_STRDATA 0x040
 200 +#define STR_DATA BIT(0)
 201 +
 202 +#define NFI_STA 0x060
 203 +#define NFI_NAND_FSM GENMASK(28, 24)
 204 +#define NFI_FSM GENMASK(19, 16)
 205 +#define READ_EMPTY BIT(12)
 206 +
 207 +#define NFI_FIFOSTA 0x064
 208 +#define FIFO_WR_REMAIN_S 8
 209 +#define FIFO_RD_REMAIN_S 0
 210 +
 211 +#define NFI_ADDRCNTR 0x070
 212 +#define SEC_CNTR GENMASK(16, 12)
 213 +#define SEC_CNTR_S 12
 214 +#define NFI_SEC_CNTR(val) (((val)&SEC_CNTR) >> SEC_CNTR_S)
 215 +
 216 +#define NFI_STRADDR 0x080
 217 +
 218 +#define NFI_BYTELEN 0x084
 219 +#define BUS_SEC_CNTR(val) (((val)&SEC_CNTR) >> SEC_CNTR_S)
 220 +
 221 +#define NFI_FDM0L 0x0a0
 222 +#define NFI_FDM0M 0x0a4
 223 +#define NFI_FDML(n) (NFI_FDM0L + (n)*8)
 224 +#define NFI_FDMM(n) (NFI_FDM0M + (n)*8)
 225 +
 226 +#define NFI_DEBUG_CON1 0x220
 227 +#define WBUF_EN BIT(2)
 228 +
 229 +#define NFI_MASTERSTA 0x224
 230 +#define MAS_ADDR GENMASK(11, 9)
 231 +#define MAS_RD GENMASK(8, 6)
 232 +#define MAS_WR GENMASK(5, 3)
 233 +#define MAS_RDDLY GENMASK(2, 0)
 234 +#define NFI_MASTERSTA_MASK_7622 (MAS_ADDR | MAS_RD | MAS_WR | MAS_RDDLY)
 235 +
 236 +// SNFI registers
 237 +#define SNF_MAC_CTL 0x500
 238 +#define MAC_XIO_SEL BIT(4)
 239 +#define SF_MAC_EN BIT(3)
 240 +#define SF_TRIG BIT(2)
 241 +#define WIP_READY BIT(1)
 242 +#define WIP BIT(0)
 243 +
 244 +#define SNF_MAC_OUTL 0x504
 245 +#define SNF_MAC_INL 0x508
 246 +
 247 +#define SNF_RD_CTL2 0x510
 248 +#define DATA_READ_DUMMY_S 8
 249 +#define DATA_READ_MAX_DUMMY 0xf
 250 +#define DATA_READ_CMD_S 0
 251 +
 252 +#define SNF_RD_CTL3 0x514
 253 +
 254 +#define SNF_PG_CTL1 0x524
 255 +#define PG_LOAD_CMD_S 8
 256 +
 257 +#define SNF_PG_CTL2 0x528
 258 +
 259 +#define SNF_MISC_CTL 0x538
 260 +#define SW_RST BIT(28)
 261 +#define FIFO_RD_LTC_S 25
 262 +#define PG_LOAD_X4_EN BIT(20)
 263 +#define DATA_READ_MODE_S 16
 264 +#define DATA_READ_MODE GENMASK(18, 16)
 265 +#define DATA_READ_MODE_X1 0
 266 +#define DATA_READ_MODE_X2 1
 267 +#define DATA_READ_MODE_X4 2
 268 +#define DATA_READ_MODE_DUAL 5
 269 +#define DATA_READ_MODE_QUAD 6
 270 +#define PG_LOAD_CUSTOM_EN BIT(7)
 271 +#define DATARD_CUSTOM_EN BIT(6)
 272 +#define CS_DESELECT_CYC_S 0
 273 +
 274 +#define SNF_MISC_CTL2 0x53c
 275 +#define PROGRAM_LOAD_BYTE_NUM_S 16
 276 +#define READ_DATA_BYTE_NUM_S 11
 277 +
 278 +#define SNF_DLY_CTL3 0x548
 279 +#define SFCK_SAM_DLY_S 0
 280 +
 281 +#define SNF_STA_CTL1 0x550
 282 +#define CUS_PG_DONE BIT(28)
 283 +#define CUS_READ_DONE BIT(27)
 284 +#define SPI_STATE_S 0
 285 +#define SPI_STATE GENMASK(3, 0)
 286 +
 287 +#define SNF_CFG 0x55c
 288 +#define SPI_MODE BIT(0)
 289 +
 290 +#define SNF_GPRAM 0x800
 291 +#define SNF_GPRAM_SIZE 0xa0
 292 +
 293 +#define SNFI_POLL_INTERVAL 1000000
 294 +
 295 +static const u8 mt7622_spare_sizes[] = { 16, 26, 27, 28 };
 296 +
 297 +struct mtk_snand_caps {
 298 +       u16 sector_size;
 299 +       u16 max_sectors;
 300 +       u16 fdm_size;
 301 +       u16 fdm_ecc_size;
 302 +       u16 fifo_size;
 303 +
 304 +       bool bbm_swap;
 305 +       bool empty_page_check;
 306 +       u32 mastersta_mask;
 307 +
 308 +       const u8 *spare_sizes;
 309 +       u32 num_spare_size;
 310 +};
 311 +
 312 +static const struct mtk_snand_caps mt7622_snand_caps = {
 313 +       .sector_size = 512,
 314 +       .max_sectors = 8,
 315 +       .fdm_size = 8,
 316 +       .fdm_ecc_size = 1,
 317 +       .fifo_size = 32,
 318 +       .bbm_swap = false,
 319 +       .empty_page_check = false,
 320 +       .mastersta_mask = NFI_MASTERSTA_MASK_7622,
 321 +       .spare_sizes = mt7622_spare_sizes,
 322 +       .num_spare_size = ARRAY_SIZE(mt7622_spare_sizes)
 323 +};
 324 +
 325 +static const struct mtk_snand_caps mt7629_snand_caps = {
 326 +       .sector_size = 512,
 327 +       .max_sectors = 8,
 328 +       .fdm_size = 8,
 329 +       .fdm_ecc_size = 1,
 330 +       .fifo_size = 32,
 331 +       .bbm_swap = true,
 332 +       .empty_page_check = false,
 333 +       .mastersta_mask = NFI_MASTERSTA_MASK_7622,
 334 +       .spare_sizes = mt7622_spare_sizes,
 335 +       .num_spare_size = ARRAY_SIZE(mt7622_spare_sizes)
 336 +};
 337 +
 338 +struct mtk_snand_conf {
 339 +       size_t page_size;
 340 +       size_t oob_size;
 341 +       u8 nsectors;
 342 +       u8 spare_size;
 343 +};
 344 +
 345 +struct mtk_snand {
 346 +       struct spi_controller *ctlr;
 347 +       struct device *dev;
 348 +       struct clk *nfi_clk;
 349 +       struct clk *pad_clk;
 350 +       void __iomem *nfi_base;
 351 +       int irq;
 352 +       struct completion op_done;
 353 +       const struct mtk_snand_caps *caps;
 354 +       struct mtk_ecc_config *ecc_cfg;
 355 +       struct mtk_ecc *ecc;
 356 +       struct mtk_snand_conf nfi_cfg;
 357 +       struct mtk_ecc_stats ecc_stats;
 358 +       struct nand_ecc_engine ecc_eng;
 359 +       bool autofmt;
 360 +       u8 *buf;
 361 +       size_t buf_len;
 362 +};
 363 +
 364 +static struct mtk_snand *nand_to_mtk_snand(struct nand_device *nand)
 365 +{
 366 +       struct nand_ecc_engine *eng = nand->ecc.engine;
 367 +
 368 +       return container_of(eng, struct mtk_snand, ecc_eng);
 369 +}
 370 +
 371 +static inline int snand_prepare_bouncebuf(struct mtk_snand *snf, size_t size)
 372 +{
 373 +       if (snf->buf_len >= size)
 374 +               return 0;
 375 +       kfree(snf->buf);
 376 +       snf->buf = kmalloc(size, GFP_KERNEL);
 377 +       if (!snf->buf)
 378 +               return -ENOMEM;
 379 +       snf->buf_len = size;
 380 +       memset(snf->buf, 0xff, snf->buf_len);
 381 +       return 0;
 382 +}
 383 +
 384 +static inline u32 nfi_read32(struct mtk_snand *snf, u32 reg)
 385 +{
 386 +       return readl(snf->nfi_base + reg);
 387 +}
 388 +
 389 +static inline void nfi_write32(struct mtk_snand *snf, u32 reg, u32 val)
 390 +{
 391 +       writel(val, snf->nfi_base + reg);
 392 +}
 393 +
 394 +static inline void nfi_write16(struct mtk_snand *snf, u32 reg, u16 val)
 395 +{
 396 +       writew(val, snf->nfi_base + reg);
 397 +}
 398 +
 399 +static inline void nfi_rmw32(struct mtk_snand *snf, u32 reg, u32 clr, u32 set)
 400 +{
 401 +       u32 val;
 402 +
 403 +       val = readl(snf->nfi_base + reg);
 404 +       val &= ~clr;
 405 +       val |= set;
 406 +       writel(val, snf->nfi_base + reg);
 407 +}
 408 +
 409 +static void nfi_read_data(struct mtk_snand *snf, u32 reg, u8 *data, u32 len)
 410 +{
 411 +       u32 i, val = 0, es = sizeof(u32);
 412 +
 413 +       for (i = reg; i < reg + len; i++) {
 414 +               if (i == reg || i % es == 0)
 415 +                       val = nfi_read32(snf, i & ~(es - 1));
 416 +
 417 +               *data++ = (u8)(val >> (8 * (i % es)));
 418 +       }
 419 +}
 420 +
 421 +static int mtk_nfi_reset(struct mtk_snand *snf)
 422 +{
 423 +       u32 val, fifo_mask;
 424 +       int ret;
 425 +
 426 +       nfi_write32(snf, NFI_CON, CON_FIFO_FLUSH | CON_NFI_RST);
 427 +
 428 +       ret = readw_poll_timeout(snf->nfi_base + NFI_MASTERSTA, val,
 429 +                                !(val & snf->caps->mastersta_mask), 0,
 430 +                                SNFI_POLL_INTERVAL);
 431 +       if (ret) {
 432 +               dev_err(snf->dev, "NFI master is still busy after reset\n");
 433 +               return ret;
 434 +       }
 435 +
 436 +       ret = readl_poll_timeout(snf->nfi_base + NFI_STA, val,
 437 +                                !(val & (NFI_FSM | NFI_NAND_FSM)), 0,
 438 +                                SNFI_POLL_INTERVAL);
 439 +       if (ret) {
 440 +               dev_err(snf->dev, "Failed to reset NFI\n");
 441 +               return ret;
 442 +       }
 443 +
 444 +       fifo_mask = ((snf->caps->fifo_size - 1) << FIFO_RD_REMAIN_S) |
 445 +                   ((snf->caps->fifo_size - 1) << FIFO_WR_REMAIN_S);
 446 +       ret = readw_poll_timeout(snf->nfi_base + NFI_FIFOSTA, val,
 447 +                                !(val & fifo_mask), 0, SNFI_POLL_INTERVAL);
 448 +       if (ret) {
 449 +               dev_err(snf->dev, "NFI FIFOs are not empty\n");
 450 +               return ret;
 451 +       }
 452 +
 453 +       return 0;
 454 +}
 455 +
 456 +static int mtk_snand_mac_reset(struct mtk_snand *snf)
 457 +{
 458 +       int ret;
 459 +       u32 val;
 460 +
 461 +       nfi_rmw32(snf, SNF_MISC_CTL, 0, SW_RST);
 462 +
 463 +       ret = readl_poll_timeout(snf->nfi_base + SNF_STA_CTL1, val,
 464 +                                !(val & SPI_STATE), 0, SNFI_POLL_INTERVAL);
 465 +       if (ret)
 466 +               dev_err(snf->dev, "Failed to reset SNFI MAC\n");
 467 +
 468 +       nfi_write32(snf, SNF_MISC_CTL,
 469 +                   (2 << FIFO_RD_LTC_S) | (10 << CS_DESELECT_CYC_S));
 470 +
 471 +       return ret;
 472 +}
 473 +
 474 +static int mtk_snand_mac_trigger(struct mtk_snand *snf, u32 outlen, u32 inlen)
 475 +{
 476 +       int ret;
 477 +       u32 val;
 478 +
 479 +       nfi_write32(snf, SNF_MAC_CTL, SF_MAC_EN);
 480 +       nfi_write32(snf, SNF_MAC_OUTL, outlen);
 481 +       nfi_write32(snf, SNF_MAC_INL, inlen);
 482 +
 483 +       nfi_write32(snf, SNF_MAC_CTL, SF_MAC_EN | SF_TRIG);
 484 +
 485 +       ret = readl_poll_timeout(snf->nfi_base + SNF_MAC_CTL, val,
 486 +                                val & WIP_READY, 0, SNFI_POLL_INTERVAL);
 487 +       if (ret) {
 488 +               dev_err(snf->dev, "Timed out waiting for WIP_READY\n");
 489 +               goto cleanup;
 490 +       }
 491 +
 492 +       ret = readl_poll_timeout(snf->nfi_base + SNF_MAC_CTL, val, !(val & WIP),
 493 +                                0, SNFI_POLL_INTERVAL);
 494 +       if (ret)
 495 +               dev_err(snf->dev, "Timed out waiting for WIP cleared\n");
 496 +
 497 +cleanup:
 498 +       nfi_write32(snf, SNF_MAC_CTL, 0);
 499 +
 500 +       return ret;
 501 +}
 502 +
 503 +static int mtk_snand_mac_io(struct mtk_snand *snf, const struct spi_mem_op *op)
 504 +{
 505 +       u32 rx_len = 0;
 506 +       u32 reg_offs = 0;
 507 +       u32 val = 0;
 508 +       const u8 *tx_buf = NULL;
 509 +       u8 *rx_buf = NULL;
 510 +       int i, ret;
 511 +       u8 b;
 512 +
 513 +       if (op->data.dir == SPI_MEM_DATA_IN) {
 514 +               rx_len = op->data.nbytes;
 515 +               rx_buf = op->data.buf.in;
 516 +       } else {
 517 +               tx_buf = op->data.buf.out;
 518 +       }
 519 +
 520 +       mtk_snand_mac_reset(snf);
 521 +
 522 +       for (i = 0; i < op->cmd.nbytes; i++, reg_offs++) {
 523 +               b = (op->cmd.opcode >> ((op->cmd.nbytes - i - 1) * 8)) & 0xff;
 524 +               val |= b << (8 * (reg_offs % 4));
 525 +               if (reg_offs % 4 == 3) {
 526 +                       nfi_write32(snf, SNF_GPRAM + reg_offs - 3, val);
 527 +                       val = 0;
 528 +               }
 529 +       }
 530 +
 531 +       for (i = 0; i < op->addr.nbytes; i++, reg_offs++) {
 532 +               b = (op->addr.val >> ((op->addr.nbytes - i - 1) * 8)) & 0xff;
 533 +               val |= b << (8 * (reg_offs % 4));
 534 +               if (reg_offs % 4 == 3) {
 535 +                       nfi_write32(snf, SNF_GPRAM + reg_offs - 3, val);
 536 +                       val = 0;
 537 +               }
 538 +       }
 539 +
 540 +       for (i = 0; i < op->dummy.nbytes; i++, reg_offs++) {
 541 +               if (reg_offs % 4 == 3) {
 542 +                       nfi_write32(snf, SNF_GPRAM + reg_offs - 3, val);
 543 +                       val = 0;
 544 +               }
 545 +       }
 546 +
 547 +       if (op->data.dir == SPI_MEM_DATA_OUT) {
 548 +               for (i = 0; i < op->data.nbytes; i++, reg_offs++) {
 549 +                       val |= tx_buf[i] << (8 * (reg_offs % 4));
 550 +                       if (reg_offs % 4 == 3) {
 551 +                               nfi_write32(snf, SNF_GPRAM + reg_offs - 3, val);
 552 +                               val = 0;
 553 +                       }
 554 +               }
 555 +       }
 556 +
 557 +       if (reg_offs % 4)
 558 +               nfi_write32(snf, SNF_GPRAM + (reg_offs & ~3), val);
 559 +
 560 +       for (i = 0; i < reg_offs; i += 4)
 561 +               dev_dbg(snf->dev, "%d: %08X", i,
 562 +                       nfi_read32(snf, SNF_GPRAM + i));
 563 +
 564 +       dev_dbg(snf->dev, "SNF TX: %u RX: %u", reg_offs, rx_len);
 565 +
 566 +       ret = mtk_snand_mac_trigger(snf, reg_offs, rx_len);
 567 +       if (ret)
 568 +               return ret;
 569 +
 570 +       if (!rx_len)
 571 +               return 0;
 572 +
 573 +       nfi_read_data(snf, SNF_GPRAM + reg_offs, rx_buf, rx_len);
 574 +       return 0;
 575 +}
 576 +
 577 +static int mtk_snand_setup_pagefmt(struct mtk_snand *snf, u32 page_size,
 578 +                                  u32 oob_size)
 579 +{
 580 +       int spare_idx = -1;
 581 +       u32 spare_size, spare_size_shift, pagesize_idx;
 582 +       u32 sector_size_512;
 583 +       u8 nsectors;
 584 +       int i;
 585 +
 586 +       // skip if it's already configured as required.
 587 +       if (snf->nfi_cfg.page_size == page_size &&
 588 +           snf->nfi_cfg.oob_size == oob_size)
 589 +               return 0;
 590 +
 591 +       nsectors = page_size / snf->caps->sector_size;
 592 +       if (nsectors > snf->caps->max_sectors) {
 593 +               dev_err(snf->dev, "too many sectors required.\n");
 594 +               goto err;
 595 +       }
 596 +
 597 +       if (snf->caps->sector_size == 512) {
 598 +               sector_size_512 = NFI_SEC_SEL_512;
 599 +               spare_size_shift = NFI_SPARE_SIZE_S;
 600 +       } else {
 601 +               sector_size_512 = 0;
 602 +               spare_size_shift = NFI_SPARE_SIZE_LS_S;
 603 +       }
 604 +
 605 +       switch (page_size) {
 606 +       case SZ_512:
 607 +               pagesize_idx = NFI_PAGE_SIZE_512_2K;
 608 +               break;
 609 +       case SZ_2K:
 610 +               if (snf->caps->sector_size == 512)
 611 +                       pagesize_idx = NFI_PAGE_SIZE_2K_4K;
 612 +               else
 613 +                       pagesize_idx = NFI_PAGE_SIZE_512_2K;
 614 +               break;
 615 +       case SZ_4K:
 616 +               if (snf->caps->sector_size == 512)
 617 +                       pagesize_idx = NFI_PAGE_SIZE_4K_8K;
 618 +               else
 619 +                       pagesize_idx = NFI_PAGE_SIZE_2K_4K;
 620 +               break;
 621 +       case SZ_8K:
 622 +               if (snf->caps->sector_size == 512)
 623 +                       pagesize_idx = NFI_PAGE_SIZE_8K_16K;
 624 +               else
 625 +                       pagesize_idx = NFI_PAGE_SIZE_4K_8K;
 626 +               break;
 627 +       case SZ_16K:
 628 +               pagesize_idx = NFI_PAGE_SIZE_8K_16K;
 629 +               break;
 630 +       default:
 631 +               dev_err(snf->dev, "unsupported page size.\n");
 632 +               goto err;
 633 +       }
 634 +
 635 +       spare_size = oob_size / nsectors;
 636 +       // If we're using the 1KB sector size, HW will automatically double the
 637 +       // spare size. We should only use half of the value in this case.
 638 +       if (snf->caps->sector_size == 1024)
 639 +               spare_size /= 2;
 640 +
 641 +       for (i = snf->caps->num_spare_size - 1; i >= 0; i--) {
 642 +               if (snf->caps->spare_sizes[i] <= spare_size) {
 643 +                       spare_size = snf->caps->spare_sizes[i];
 644 +                       if (snf->caps->sector_size == 1024)
 645 +                               spare_size *= 2;
 646 +                       spare_idx = i;
 647 +                       break;
 648 +               }
 649 +       }
 650 +
 651 +       if (spare_idx < 0) {
 652 +               dev_err(snf->dev, "unsupported spare size: %u\n", spare_size);
 653 +               goto err;
 654 +       }
 655 +
 656 +       nfi_write32(snf, NFI_PAGEFMT,
 657 +                   (snf->caps->fdm_ecc_size << NFI_FDM_ECC_NUM_S) |
 658 +                           (snf->caps->fdm_size << NFI_FDM_NUM_S) |
 659 +                           (spare_idx << spare_size_shift) |
 660 +                           (pagesize_idx << NFI_PAGE_SIZE_S) |
 661 +                           sector_size_512);
 662 +
 663 +       snf->nfi_cfg.page_size = page_size;
 664 +       snf->nfi_cfg.oob_size = oob_size;
 665 +       snf->nfi_cfg.nsectors = nsectors;
 666 +       snf->nfi_cfg.spare_size = spare_size;
 667 +
 668 +       dev_dbg(snf->dev, "page format: (%u + %u) * %u\n",
 669 +               snf->caps->sector_size, spare_size, nsectors);
 670 +       return snand_prepare_bouncebuf(snf, page_size + oob_size);
 671 +err:
 672 +       dev_err(snf->dev, "page size %u + %u is not supported\n", page_size,
 673 +               oob_size);
 674 +       return -EOPNOTSUPP;
 675 +}
 676 +
 677 +static int mtk_snand_ooblayout_ecc(struct mtd_info *mtd, int section,
 678 +                                  struct mtd_oob_region *oobecc)
 679 +{
 680 +       // ECC area is not accessible
 681 +       return -ERANGE;
 682 +}
 683 +
 684 +static int mtk_snand_ooblayout_free(struct mtd_info *mtd, int section,
 685 +                                   struct mtd_oob_region *oobfree)
 686 +{
 687 +       struct nand_device *nand = mtd_to_nanddev(mtd);
 688 +       struct mtk_snand *ms = nand_to_mtk_snand(nand);
 689 +
 690 +       if (section >= ms->nfi_cfg.nsectors)
 691 +               return -ERANGE;
 692 +
 693 +       oobfree->length = ms->caps->fdm_size - 1;
 694 +       oobfree->offset = section * ms->caps->fdm_size + 1;
 695 +       return 0;
 696 +}
 697 +
 698 +static const struct mtd_ooblayout_ops mtk_snand_ooblayout = {
 699 +       .ecc = mtk_snand_ooblayout_ecc,
 700 +       .free = mtk_snand_ooblayout_free,
 701 +};
 702 +
 703 +static int mtk_snand_ecc_init_ctx(struct nand_device *nand)
 704 +{
 705 +       struct mtk_snand *snf = nand_to_mtk_snand(nand);
 706 +       struct nand_ecc_props *conf = &nand->ecc.ctx.conf;
 707 +       struct nand_ecc_props *reqs = &nand->ecc.requirements;
 708 +       struct nand_ecc_props *user = &nand->ecc.user_conf;
 709 +       struct mtd_info *mtd = nanddev_to_mtd(nand);
 710 +       int step_size = 0, strength = 0, desired_correction = 0, steps;
 711 +       bool ecc_user = false;
 712 +       int ret;
 713 +       u32 parity_bits, max_ecc_bytes;
 714 +       struct mtk_ecc_config *ecc_cfg;
 715 +
 716 +       ret = mtk_snand_setup_pagefmt(snf, nand->memorg.pagesize,
 717 +                                     nand->memorg.oobsize);
 718 +       if (ret)
 719 +               return ret;
 720 +
 721 +       ecc_cfg = kzalloc(sizeof(*ecc_cfg), GFP_KERNEL);
 722 +       if (!ecc_cfg)
 723 +               return -ENOMEM;
 724 +
 725 +       nand->ecc.ctx.priv = ecc_cfg;
 726 +
 727 +       if (user->step_size && user->strength) {
 728 +               step_size = user->step_size;
 729 +               strength = user->strength;
 730 +               ecc_user = true;
 731 +       } else if (reqs->step_size && reqs->strength) {
 732 +               step_size = reqs->step_size;
 733 +               strength = reqs->strength;
 734 +       }
 735 +
 736 +       if (step_size && strength) {
 737 +               steps = mtd->writesize / step_size;
 738 +               desired_correction = steps * strength;
 739 +               strength = desired_correction / snf->nfi_cfg.nsectors;
 740 +       }
 741 +
 742 +       ecc_cfg->mode = ECC_NFI_MODE;
 743 +       ecc_cfg->sectors = snf->nfi_cfg.nsectors;
 744 +       ecc_cfg->len = snf->caps->sector_size + snf->caps->fdm_ecc_size;
 745 +
 746 +       // calculate the max possible strength under current page format
 747 +       parity_bits = mtk_ecc_get_parity_bits(snf->ecc);
 748 +       max_ecc_bytes = snf->nfi_cfg.spare_size - snf->caps->fdm_size;
 749 +       ecc_cfg->strength = max_ecc_bytes * 8 / parity_bits;
 750 +       mtk_ecc_adjust_strength(snf->ecc, &ecc_cfg->strength);
 751 +
 752 +       // if there's a user requested strength, find the minimum strength that
 753 +       // meets the requirement. Otherwise use the maximum strength which is
 754 +       // expected by BootROM.
 755 +       if (ecc_user && strength) {
 756 +               u32 s_next = ecc_cfg->strength - 1;
 757 +
 758 +               while (1) {
 759 +                       mtk_ecc_adjust_strength(snf->ecc, &s_next);
 760 +                       if (s_next >= ecc_cfg->strength)
 761 +                               break;
 762 +                       if (s_next < strength)
 763 +                               break;
 764 +                       s_next = ecc_cfg->strength - 1;
 765 +               }
 766 +       }
 767 +
 768 +       mtd_set_ooblayout(mtd, &mtk_snand_ooblayout);
 769 +
 770 +       conf->step_size = snf->caps->sector_size;
 771 +       conf->strength = ecc_cfg->strength;
 772 +
 773 +       if (ecc_cfg->strength < strength)
 774 +               dev_warn(snf->dev, "unable to fulfill ECC of %u bits.\n",
 775 +                        strength);
 776 +       dev_info(snf->dev, "ECC strength: %u bits per %u bytes\n",
 777 +                ecc_cfg->strength, snf->caps->sector_size);
 778 +
 779 +       return 0;
 780 +}
 781 +
 782 +static void mtk_snand_ecc_cleanup_ctx(struct nand_device *nand)
 783 +{
 784 +       struct mtk_ecc_config *ecc_cfg = nand_to_ecc_ctx(nand);
 785 +
 786 +       kfree(ecc_cfg);
 787 +}
 788 +
 789 +static int mtk_snand_ecc_prepare_io_req(struct nand_device *nand,
 790 +                                       struct nand_page_io_req *req)
 791 +{
 792 +       struct mtk_snand *snf = nand_to_mtk_snand(nand);
 793 +       struct mtk_ecc_config *ecc_cfg = nand_to_ecc_ctx(nand);
 794 +       int ret;
 795 +
 796 +       ret = mtk_snand_setup_pagefmt(snf, nand->memorg.pagesize,
 797 +                                     nand->memorg.oobsize);
 798 +       if (ret)
 799 +               return ret;
 800 +       snf->autofmt = true;
 801 +       snf->ecc_cfg = ecc_cfg;
 802 +       return 0;
 803 +}
 804 +
 805 +static int mtk_snand_ecc_finish_io_req(struct nand_device *nand,
 806 +                                      struct nand_page_io_req *req)
 807 +{
 808 +       struct mtk_snand *snf = nand_to_mtk_snand(nand);
 809 +       struct mtd_info *mtd = nanddev_to_mtd(nand);
 810 +
 811 +       snf->ecc_cfg = NULL;
 812 +       snf->autofmt = false;
 813 +       if ((req->mode == MTD_OPS_RAW) || (req->type != NAND_PAGE_READ))
 814 +               return 0;
 815 +
 816 +       if (snf->ecc_stats.failed)
 817 +               mtd->ecc_stats.failed += snf->ecc_stats.failed;
 818 +       mtd->ecc_stats.corrected += snf->ecc_stats.corrected;
 819 +       return snf->ecc_stats.failed ? -EBADMSG : snf->ecc_stats.bitflips;
 820 +}
 821 +
 822 +static struct nand_ecc_engine_ops mtk_snfi_ecc_engine_ops = {
 823 +       .init_ctx = mtk_snand_ecc_init_ctx,
 824 +       .cleanup_ctx = mtk_snand_ecc_cleanup_ctx,
 825 +       .prepare_io_req = mtk_snand_ecc_prepare_io_req,
 826 +       .finish_io_req = mtk_snand_ecc_finish_io_req,
 827 +};
 828 +
 829 +static void mtk_snand_read_fdm(struct mtk_snand *snf, u8 *buf)
 830 +{
 831 +       u32 vall, valm;
 832 +       u8 *oobptr = buf;
 833 +       int i, j;
 834 +
 835 +       for (i = 0; i < snf->nfi_cfg.nsectors; i++) {
 836 +               vall = nfi_read32(snf, NFI_FDML(i));
 837 +               valm = nfi_read32(snf, NFI_FDMM(i));
 838 +
 839 +               for (j = 0; j < snf->caps->fdm_size; j++)
 840 +                       oobptr[j] = (j >= 4 ? valm : vall) >> ((j % 4) * 8);
 841 +
 842 +               oobptr += snf->caps->fdm_size;
 843 +       }
 844 +}
 845 +
 846 +static void mtk_snand_write_fdm(struct mtk_snand *snf, const u8 *buf)
 847 +{
 848 +       u32 fdm_size = snf->caps->fdm_size;
 849 +       const u8 *oobptr = buf;
 850 +       u32 vall, valm;
 851 +       int i, j;
 852 +
 853 +       for (i = 0; i < snf->nfi_cfg.nsectors; i++) {
 854 +               vall = 0;
 855 +               valm = 0;
 856 +
 857 +               for (j = 0; j < 8; j++) {
 858 +                       if (j < 4)
 859 +                               vall |= (j < fdm_size ? oobptr[j] : 0xff)
 860 +                                       << (j * 8);
 861 +                       else
 862 +                               valm |= (j < fdm_size ? oobptr[j] : 0xff)
 863 +                                       << ((j - 4) * 8);
 864 +               }
 865 +
 866 +               nfi_write32(snf, NFI_FDML(i), vall);
 867 +               nfi_write32(snf, NFI_FDMM(i), valm);
 868 +
 869 +               oobptr += fdm_size;
 870 +       }
 871 +}
 872 +
 873 +static void mtk_snand_bm_swap(struct mtk_snand *snf, u8 *buf)
 874 +{
 875 +       u32 buf_bbm_pos, fdm_bbm_pos;
 876 +
 877 +       if (!snf->caps->bbm_swap || snf->nfi_cfg.nsectors == 1)
 878 +               return;
 879 +
 880 +       // swap [pagesize] byte on nand with the first fdm byte
 881 +       // in the last sector.
 882 +       buf_bbm_pos = snf->nfi_cfg.page_size -
 883 +                     (snf->nfi_cfg.nsectors - 1) * snf->nfi_cfg.spare_size;
 884 +       fdm_bbm_pos = snf->nfi_cfg.page_size +
 885 +                     (snf->nfi_cfg.nsectors - 1) * snf->caps->fdm_size;
 886 +
 887 +       swap(snf->buf[fdm_bbm_pos], buf[buf_bbm_pos]);
 888 +}
 889 +
 890 +static void mtk_snand_fdm_bm_swap(struct mtk_snand *snf)
 891 +{
 892 +       u32 fdm_bbm_pos1, fdm_bbm_pos2;
 893 +
 894 +       if (!snf->caps->bbm_swap || snf->nfi_cfg.nsectors == 1)
 895 +               return;
 896 +
 897 +       // swap the first fdm byte in the first and the last sector.
 898 +       fdm_bbm_pos1 = snf->nfi_cfg.page_size;
 899 +       fdm_bbm_pos2 = snf->nfi_cfg.page_size +
 900 +                      (snf->nfi_cfg.nsectors - 1) * snf->caps->fdm_size;
 901 +       swap(snf->buf[fdm_bbm_pos1], snf->buf[fdm_bbm_pos2]);
 902 +}
 903 +
 904 +static int mtk_snand_read_page_cache(struct mtk_snand *snf,
 905 +                                    const struct spi_mem_op *op)
 906 +{
 907 +       u8 *buf = snf->buf;
 908 +       u8 *buf_fdm = buf + snf->nfi_cfg.page_size;
 909 +       // the address part to be sent by the controller
 910 +       u32 op_addr = op->addr.val;
 911 +       // where to start copying data from bounce buffer
 912 +       u32 rd_offset = 0;
 913 +       u32 dummy_clk = (op->dummy.nbytes * BITS_PER_BYTE / op->dummy.buswidth);
 914 +       u32 op_mode = 0;
 915 +       u32 dma_len = snf->buf_len;
 916 +       int ret = 0;
 917 +       u32 rd_mode, rd_bytes, val;
 918 +       dma_addr_t buf_dma;
 919 +
 920 +       if (snf->autofmt) {
 921 +               u32 last_bit;
 922 +               u32 mask;
 923 +
 924 +               dma_len = snf->nfi_cfg.page_size;
 925 +               op_mode = CNFG_AUTO_FMT_EN;
 926 +               if (op->data.ecc)
 927 +                       op_mode |= CNFG_HW_ECC_EN;
 928 +               // extract the plane bit:
 929 +               // Find the highest bit set in (pagesize+oobsize).
 930 +               // Bits higher than that in op->addr are kept and sent over SPI
 931 +               // Lower bits are used as an offset for copying data from DMA
 932 +               // bounce buffer.
 933 +               last_bit = fls(snf->nfi_cfg.page_size + snf->nfi_cfg.oob_size);
 934 +               mask = (1 << last_bit) - 1;
 935 +               rd_offset = op_addr & mask;
 936 +               op_addr &= ~mask;
 937 +
 938 +               // check if we can dma to the caller memory
 939 +               if (rd_offset == 0 && op->data.nbytes >= snf->nfi_cfg.page_size)
 940 +                       buf = op->data.buf.in;
 941 +       }
 942 +       mtk_snand_mac_reset(snf);
 943 +       mtk_nfi_reset(snf);
 944 +
 945 +       // command and dummy cycles
 946 +       nfi_write32(snf, SNF_RD_CTL2,
 947 +                   (dummy_clk << DATA_READ_DUMMY_S) |
 948 +                           (op->cmd.opcode << DATA_READ_CMD_S));
 949 +
 950 +       // read address
 951 +       nfi_write32(snf, SNF_RD_CTL3, op_addr);
 952 +
 953 +       // Set read op_mode
 954 +       if (op->data.buswidth == 4)
 955 +               rd_mode = op->addr.buswidth == 4 ? DATA_READ_MODE_QUAD :
 956 +                                                  DATA_READ_MODE_X4;
 957 +       else if (op->data.buswidth == 2)
 958 +               rd_mode = op->addr.buswidth == 2 ? DATA_READ_MODE_DUAL :
 959 +                                                  DATA_READ_MODE_X2;
 960 +       else
 961 +               rd_mode = DATA_READ_MODE_X1;
 962 +       rd_mode <<= DATA_READ_MODE_S;
 963 +       nfi_rmw32(snf, SNF_MISC_CTL, DATA_READ_MODE,
 964 +                 rd_mode | DATARD_CUSTOM_EN);
 965 +
 966 +       // Set bytes to read
 967 +       rd_bytes = (snf->nfi_cfg.spare_size + snf->caps->sector_size) *
 968 +                  snf->nfi_cfg.nsectors;
 969 +       nfi_write32(snf, SNF_MISC_CTL2,
 970 +                   (rd_bytes << PROGRAM_LOAD_BYTE_NUM_S) | rd_bytes);
 971 +
 972 +       // NFI read prepare
 973 +       nfi_write16(snf, NFI_CNFG,
 974 +                   (CNFG_OP_MODE_CUST << CNFG_OP_MODE_S) | CNFG_DMA_BURST_EN |
 975 +                           CNFG_READ_MODE | CNFG_DMA_MODE | op_mode);
 976 +
 977 +       nfi_write32(snf, NFI_CON, (snf->nfi_cfg.nsectors << CON_SEC_NUM_S));
 978 +
 979 +       buf_dma = dma_map_single(snf->dev, buf, dma_len, DMA_FROM_DEVICE);
 980 +       if (dma_mapping_error(snf->dev, buf_dma)) {
 981 +               dev_err(snf->dev, "DMA mapping failed.\n");
 982 +               goto cleanup;
 983 +       }
 984 +       nfi_write32(snf, NFI_STRADDR, buf_dma);
 985 +       if (op->data.ecc) {
 986 +               snf->ecc_cfg->op = ECC_DECODE;
 987 +               ret = mtk_ecc_enable(snf->ecc, snf->ecc_cfg);
 988 +               if (ret)
 989 +                       goto cleanup_dma;
 990 +       }
 991 +       // Prepare for custom read interrupt
 992 +       nfi_write32(snf, NFI_INTR_EN, NFI_IRQ_INTR_EN | NFI_IRQ_CUS_READ);
 993 +       reinit_completion(&snf->op_done);
 994 +
 995 +       // Trigger NFI into custom mode
 996 +       nfi_write16(snf, NFI_CMD, NFI_CMD_DUMMY_READ);
 997 +
 998 +       // Start DMA read
 999 +       nfi_rmw32(snf, NFI_CON, 0, CON_BRD);
1000 +       nfi_write16(snf, NFI_STRDATA, STR_DATA);
1001 +
1002 +       if (!wait_for_completion_timeout(
1003 +                   &snf->op_done, usecs_to_jiffies(SNFI_POLL_INTERVAL))) {
1004 +               dev_err(snf->dev, "DMA timed out for reading from cache.\n");
1005 +               ret = -ETIMEDOUT;
1006 +               goto cleanup;
1007 +       }
1008 +
1009 +       // Wait for BUS_SEC_CNTR returning expected value
1010 +       ret = readl_poll_timeout(snf->nfi_base + NFI_BYTELEN, val,
1011 +                                BUS_SEC_CNTR(val) >= snf->nfi_cfg.nsectors, 0,
1012 +                                SNFI_POLL_INTERVAL);
1013 +       if (ret) {
1014 +               dev_err(snf->dev, "Timed out waiting for BUS_SEC_CNTR\n");
1015 +               goto cleanup2;
1016 +       }
1017 +
1018 +       // Wait for bus becoming idle
1019 +       ret = readl_poll_timeout(snf->nfi_base + NFI_MASTERSTA, val,
1020 +                                !(val & snf->caps->mastersta_mask), 0,
1021 +                                SNFI_POLL_INTERVAL);
1022 +       if (ret) {
1023 +               dev_err(snf->dev, "Timed out waiting for bus becoming idle\n");
1024 +               goto cleanup2;
1025 +       }
1026 +
1027 +       if (op->data.ecc) {
1028 +               ret = mtk_ecc_wait_done(snf->ecc, ECC_DECODE);
1029 +               if (ret) {
1030 +                       dev_err(snf->dev, "wait ecc done timeout\n");
1031 +                       goto cleanup2;
1032 +               }
1033 +               // save status before disabling ecc
1034 +               mtk_ecc_get_stats(snf->ecc, &snf->ecc_stats,
1035 +                                 snf->nfi_cfg.nsectors);
1036 +       }
1037 +
1038 +       dma_unmap_single(snf->dev, buf_dma, dma_len, DMA_FROM_DEVICE);
1039 +
1040 +       if (snf->autofmt) {
1041 +               mtk_snand_read_fdm(snf, buf_fdm);
1042 +               if (snf->caps->bbm_swap) {
1043 +                       mtk_snand_bm_swap(snf, buf);
1044 +                       mtk_snand_fdm_bm_swap(snf);
1045 +               }
1046 +       }
1047 +
1048 +       // copy data back
1049 +       if (nfi_read32(snf, NFI_STA) & READ_EMPTY) {
1050 +               memset(op->data.buf.in, 0xff, op->data.nbytes);
1051 +               snf->ecc_stats.bitflips = 0;
1052 +               snf->ecc_stats.failed = 0;
1053 +               snf->ecc_stats.corrected = 0;
1054 +       } else {
1055 +               if (buf == op->data.buf.in) {
1056 +                       u32 cap_len = snf->buf_len - snf->nfi_cfg.page_size;
1057 +                       u32 req_left = op->data.nbytes - snf->nfi_cfg.page_size;
1058 +
1059 +                       if (req_left)
1060 +                               memcpy(op->data.buf.in + snf->nfi_cfg.page_size,
1061 +                                      buf_fdm,
1062 +                                      cap_len < req_left ? cap_len : req_left);
1063 +               } else if (rd_offset < snf->buf_len) {
1064 +                       u32 cap_len = snf->buf_len - rd_offset;
1065 +
1066 +                       if (op->data.nbytes < cap_len)
1067 +                               cap_len = op->data.nbytes;
1068 +                       memcpy(op->data.buf.in, snf->buf + rd_offset, cap_len);
1069 +               }
1070 +       }
1071 +cleanup2:
1072 +       if (op->data.ecc)
1073 +               mtk_ecc_disable(snf->ecc);
1074 +cleanup_dma:
1075 +       // unmap dma only if any error happens. (otherwise it's done before
1076 +       // data copying)
1077 +       if (ret)
1078 +               dma_unmap_single(snf->dev, buf_dma, dma_len, DMA_FROM_DEVICE);
1079 +cleanup:
1080 +       // Stop read
1081 +       nfi_write32(snf, NFI_CON, 0);
1082 +       nfi_write16(snf, NFI_CNFG, 0);
1083 +
1084 +       // Clear SNF done flag
1085 +       nfi_rmw32(snf, SNF_STA_CTL1, 0, CUS_READ_DONE);
1086 +       nfi_write32(snf, SNF_STA_CTL1, 0);
1087 +
1088 +       // Disable interrupt
1089 +       nfi_read32(snf, NFI_INTR_STA);
1090 +       nfi_write32(snf, NFI_INTR_EN, 0);
1091 +
1092 +       nfi_rmw32(snf, SNF_MISC_CTL, DATARD_CUSTOM_EN, 0);
1093 +       return ret;
1094 +}
1095 +
1096 +static int mtk_snand_write_page_cache(struct mtk_snand *snf,
1097 +                                     const struct spi_mem_op *op)
1098 +{
1099 +       // the address part to be sent by the controller
1100 +       u32 op_addr = op->addr.val;
1101 +       // where to start copying data from bounce buffer
1102 +       u32 wr_offset = 0;
1103 +       u32 op_mode = 0;
1104 +       int ret = 0;
1105 +       u32 wr_mode = 0;
1106 +       u32 dma_len = snf->buf_len;
1107 +       u32 wr_bytes, val;
1108 +       size_t cap_len;
1109 +       dma_addr_t buf_dma;
1110 +
1111 +       if (snf->autofmt) {
1112 +               u32 last_bit;
1113 +               u32 mask;
1114 +
1115 +               dma_len = snf->nfi_cfg.page_size;
1116 +               op_mode = CNFG_AUTO_FMT_EN;
1117 +               if (op->data.ecc)
1118 +                       op_mode |= CNFG_HW_ECC_EN;
1119 +
1120 +               last_bit = fls(snf->nfi_cfg.page_size + snf->nfi_cfg.oob_size);
1121 +               mask = (1 << last_bit) - 1;
1122 +               wr_offset = op_addr & mask;
1123 +               op_addr &= ~mask;
1124 +       }
1125 +       mtk_snand_mac_reset(snf);
1126 +       mtk_nfi_reset(snf);
1127 +
1128 +       if (wr_offset)
1129 +               memset(snf->buf, 0xff, wr_offset);
1130 +
1131 +       cap_len = snf->buf_len - wr_offset;
1132 +       if (op->data.nbytes < cap_len)
1133 +               cap_len = op->data.nbytes;
1134 +       memcpy(snf->buf + wr_offset, op->data.buf.out, cap_len);
1135 +       if (snf->autofmt) {
1136 +               if (snf->caps->bbm_swap) {
1137 +                       mtk_snand_fdm_bm_swap(snf);
1138 +                       mtk_snand_bm_swap(snf, snf->buf);
1139 +               }
1140 +               mtk_snand_write_fdm(snf, snf->buf + snf->nfi_cfg.page_size);
1141 +       }
1142 +
1143 +       // Command
1144 +       nfi_write32(snf, SNF_PG_CTL1, (op->cmd.opcode << PG_LOAD_CMD_S));
1145 +
1146 +       // write address
1147 +       nfi_write32(snf, SNF_PG_CTL2, op_addr);
1148 +
1149 +       // Set read op_mode
1150 +       if (op->data.buswidth == 4)
1151 +               wr_mode = PG_LOAD_X4_EN;
1152 +
1153 +       nfi_rmw32(snf, SNF_MISC_CTL, PG_LOAD_X4_EN,
1154 +                 wr_mode | PG_LOAD_CUSTOM_EN);
1155 +
1156 +       // Set bytes to write
1157 +       wr_bytes = (snf->nfi_cfg.spare_size + snf->caps->sector_size) *
1158 +                  snf->nfi_cfg.nsectors;
1159 +       nfi_write32(snf, SNF_MISC_CTL2,
1160 +                   (wr_bytes << PROGRAM_LOAD_BYTE_NUM_S) | wr_bytes);
1161 +
1162 +       // NFI write prepare
1163 +       nfi_write16(snf, NFI_CNFG,
1164 +                   (CNFG_OP_MODE_PROGRAM << CNFG_OP_MODE_S) |
1165 +                           CNFG_DMA_BURST_EN | CNFG_DMA_MODE | op_mode);
1166 +
1167 +       nfi_write32(snf, NFI_CON, (snf->nfi_cfg.nsectors << CON_SEC_NUM_S));
1168 +       buf_dma = dma_map_single(snf->dev, snf->buf, dma_len, DMA_TO_DEVICE);
1169 +       if (dma_mapping_error(snf->dev, buf_dma)) {
1170 +               dev_err(snf->dev, "DMA mapping failed.\n");
1171 +               goto cleanup;
1172 +       }
1173 +       nfi_write32(snf, NFI_STRADDR, buf_dma);
1174 +       if (op->data.ecc) {
1175 +               snf->ecc_cfg->op = ECC_ENCODE;
1176 +               ret = mtk_ecc_enable(snf->ecc, snf->ecc_cfg);
1177 +               if (ret)
1178 +                       goto cleanup_dma;
1179 +       }
1180 +       // Prepare for custom write interrupt
1181 +       nfi_write32(snf, NFI_INTR_EN, NFI_IRQ_INTR_EN | NFI_IRQ_CUS_PG);
1182 +       reinit_completion(&snf->op_done);
1183 +       ;
1184 +
1185 +       // Trigger NFI into custom mode
1186 +       nfi_write16(snf, NFI_CMD, NFI_CMD_DUMMY_WRITE);
1187 +
1188 +       // Start DMA write
1189 +       nfi_rmw32(snf, NFI_CON, 0, CON_BWR);
1190 +       nfi_write16(snf, NFI_STRDATA, STR_DATA);
1191 +
1192 +       if (!wait_for_completion_timeout(
1193 +                   &snf->op_done, usecs_to_jiffies(SNFI_POLL_INTERVAL))) {
1194 +               dev_err(snf->dev, "DMA timed out for program load.\n");
1195 +               ret = -ETIMEDOUT;
1196 +               goto cleanup_ecc;
1197 +       }
1198 +
1199 +       // Wait for NFI_SEC_CNTR returning expected value
1200 +       ret = readl_poll_timeout(snf->nfi_base + NFI_ADDRCNTR, val,
1201 +                                NFI_SEC_CNTR(val) >= snf->nfi_cfg.nsectors, 0,
1202 +                                SNFI_POLL_INTERVAL);
1203 +       if (ret)
1204 +               dev_err(snf->dev, "Timed out waiting for NFI_SEC_CNTR\n");
1205 +
1206 +cleanup_ecc:
1207 +       if (op->data.ecc)
1208 +               mtk_ecc_disable(snf->ecc);
1209 +cleanup_dma:
1210 +       dma_unmap_single(snf->dev, buf_dma, dma_len, DMA_TO_DEVICE);
1211 +cleanup:
1212 +       // Stop write
1213 +       nfi_write32(snf, NFI_CON, 0);
1214 +       nfi_write16(snf, NFI_CNFG, 0);
1215 +
1216 +       // Clear SNF done flag
1217 +       nfi_rmw32(snf, SNF_STA_CTL1, 0, CUS_PG_DONE);
1218 +       nfi_write32(snf, SNF_STA_CTL1, 0);
1219 +
1220 +       // Disable interrupt
1221 +       nfi_read32(snf, NFI_INTR_STA);
1222 +       nfi_write32(snf, NFI_INTR_EN, 0);
1223 +
1224 +       nfi_rmw32(snf, SNF_MISC_CTL, PG_LOAD_CUSTOM_EN, 0);
1225 +
1226 +       return ret;
1227 +}
1228 +
1229 +/**
1230 + * mtk_snand_is_page_ops() - check if the op is a controller supported page op.
1231 + * @op spi-mem op to check
1232 + *
1233 + * Check whether op can be executed with read_from_cache or program_load
1234 + * mode in the controller.
1235 + * This controller can execute typical Read From Cache and Program Load
1236 + * instructions found on SPI-NAND with 2-byte address.
1237 + * DTR and cmd buswidth & nbytes should be checked before calling this.
1238 + *
1239 + * Return: true if the op matches the instruction template
1240 + */
1241 +static bool mtk_snand_is_page_ops(const struct spi_mem_op *op)
1242 +{
1243 +       if (op->addr.nbytes != 2)
1244 +               return false;
1245 +
1246 +       if (op->addr.buswidth != 1 && op->addr.buswidth != 2 &&
1247 +           op->addr.buswidth != 4)
1248 +               return false;
1249 +
1250 +       // match read from page instructions
1251 +       if (op->data.dir == SPI_MEM_DATA_IN) {
1252 +               // check dummy cycle first
1253 +               if (op->dummy.nbytes * BITS_PER_BYTE / op->dummy.buswidth >
1254 +                   DATA_READ_MAX_DUMMY)
1255 +                       return false;
1256 +               // quad io / quad out
1257 +               if ((op->addr.buswidth == 4 || op->addr.buswidth == 1) &&
1258 +                   op->data.buswidth == 4)
1259 +                       return true;
1260 +
1261 +               // dual io / dual out
1262 +               if ((op->addr.buswidth == 2 || op->addr.buswidth == 1) &&
1263 +                   op->data.buswidth == 2)
1264 +                       return true;
1265 +
1266 +               // standard spi
1267 +               if (op->addr.buswidth == 1 && op->data.buswidth == 1)
1268 +                       return true;
1269 +       } else if (op->data.dir == SPI_MEM_DATA_OUT) {
1270 +               // check dummy cycle first
1271 +               if (op->dummy.nbytes)
1272 +                       return false;
1273 +               // program load quad out
1274 +               if (op->addr.buswidth == 1 && op->data.buswidth == 4)
1275 +                       return true;
1276 +               // standard spi
1277 +               if (op->addr.buswidth == 1 && op->data.buswidth == 1)
1278 +                       return true;
1279 +       }
1280 +       return false;
1281 +}
1282 +
1283 +static bool mtk_snand_supports_op(struct spi_mem *mem,
1284 +                                 const struct spi_mem_op *op)
1285 +{
1286 +       if (!spi_mem_default_supports_op(mem, op))
1287 +               return false;
1288 +       if (op->cmd.nbytes != 1 || op->cmd.buswidth != 1)
1289 +               return false;
1290 +       if (mtk_snand_is_page_ops(op))
1291 +               return true;
1292 +       return ((op->addr.nbytes == 0 || op->addr.buswidth == 1) &&
1293 +               (op->dummy.nbytes == 0 || op->dummy.buswidth == 1) &&
1294 +               (op->data.nbytes == 0 || op->data.buswidth == 1));
1295 +}
1296 +
1297 +static int mtk_snand_adjust_op_size(struct spi_mem *mem, struct spi_mem_op *op)
1298 +{
1299 +       struct mtk_snand *ms = spi_controller_get_devdata(mem->spi->master);
1300 +       // page ops transfer size must be exactly ((sector_size + spare_size) *
1301 +       // nsectors). Limit the op size if the caller requests more than that.
1302 +       // exec_op will read more than needed and discard the leftover if the
1303 +       // caller requests less data.
1304 +       if (mtk_snand_is_page_ops(op)) {
1305 +               size_t l;
1306 +               // skip adjust_op_size for page ops
1307 +               if (ms->autofmt)
1308 +                       return 0;
1309 +               l = ms->caps->sector_size + ms->nfi_cfg.spare_size;
1310 +               l *= ms->nfi_cfg.nsectors;
1311 +               if (op->data.nbytes > l)
1312 +                       op->data.nbytes = l;
1313 +       } else {
1314 +               size_t hl = op->cmd.nbytes + op->addr.nbytes + op->dummy.nbytes;
1315 +
1316 +               if (hl >= SNF_GPRAM_SIZE)
1317 +                       return -EOPNOTSUPP;
1318 +               if (op->data.nbytes > SNF_GPRAM_SIZE - hl)
1319 +                       op->data.nbytes = SNF_GPRAM_SIZE - hl;
1320 +       }
1321 +       return 0;
1322 +}
1323 +
1324 +static int mtk_snand_exec_op(struct spi_mem *mem, const struct spi_mem_op *op)
1325 +{
1326 +       struct mtk_snand *ms = spi_controller_get_devdata(mem->spi->master);
1327 +
1328 +       dev_dbg(ms->dev, "OP %02x ADDR %08llX@%d:%u DATA %d:%u", op->cmd.opcode,
1329 +               op->addr.val, op->addr.buswidth, op->addr.nbytes,
1330 +               op->data.buswidth, op->data.nbytes);
1331 +       if (mtk_snand_is_page_ops(op)) {
1332 +               if (op->data.dir == SPI_MEM_DATA_IN)
1333 +                       return mtk_snand_read_page_cache(ms, op);
1334 +               else
1335 +                       return mtk_snand_write_page_cache(ms, op);
1336 +       } else {
1337 +               return mtk_snand_mac_io(ms, op);
1338 +       }
1339 +}
1340 +
1341 +static const struct spi_controller_mem_ops mtk_snand_mem_ops = {
1342 +       .adjust_op_size = mtk_snand_adjust_op_size,
1343 +       .supports_op = mtk_snand_supports_op,
1344 +       .exec_op = mtk_snand_exec_op,
1345 +};
1346 +
1347 +static const struct spi_controller_mem_caps mtk_snand_mem_caps = {
1348 +       .ecc = true,
1349 +};
1350 +
1351 +static irqreturn_t mtk_snand_irq(int irq, void *id)
1352 +{
1353 +       struct mtk_snand *snf = id;
1354 +       u32 sta, ien;
1355 +
1356 +       sta = nfi_read32(snf, NFI_INTR_STA);
1357 +       ien = nfi_read32(snf, NFI_INTR_EN);
1358 +
1359 +       if (!(sta & ien))
1360 +               return IRQ_NONE;
1361 +
1362 +       nfi_write32(snf, NFI_INTR_EN, 0);
1363 +       complete(&snf->op_done);
1364 +       return IRQ_HANDLED;
1365 +}
1366 +
1367 +static const struct of_device_id mtk_snand_ids[] = {
1368 +       { .compatible = "mediatek,mt7622-snand", .data = &mt7622_snand_caps },
1369 +       { .compatible = "mediatek,mt7629-snand", .data = &mt7629_snand_caps },
1370 +       {},
1371 +};
1372 +
1373 +MODULE_DEVICE_TABLE(of, mtk_snand_ids);
1374 +
1375 +static int mtk_snand_enable_clk(struct mtk_snand *ms)
1376 +{
1377 +       int ret;
1378 +
1379 +       ret = clk_prepare_enable(ms->nfi_clk);
1380 +       if (ret) {
1381 +               dev_err(ms->dev, "unable to enable nfi clk\n");
1382 +               return ret;
1383 +       }
1384 +       ret = clk_prepare_enable(ms->pad_clk);
1385 +       if (ret) {
1386 +               dev_err(ms->dev, "unable to enable pad clk\n");
1387 +               goto err1;
1388 +       }
1389 +       return 0;
1390 +err1:
1391 +       clk_disable_unprepare(ms->nfi_clk);
1392 +       return ret;
1393 +}
1394 +
1395 +static void mtk_snand_disable_clk(struct mtk_snand *ms)
1396 +{
1397 +       clk_disable_unprepare(ms->pad_clk);
1398 +       clk_disable_unprepare(ms->nfi_clk);
1399 +}
1400 +
1401 +static int mtk_snand_probe(struct platform_device *pdev)
1402 +{
1403 +       struct device_node *np = pdev->dev.of_node;
1404 +       const struct of_device_id *dev_id;
1405 +       struct spi_controller *ctlr;
1406 +       struct mtk_snand *ms;
1407 +       int ret;
1408 +
1409 +       dev_id = of_match_node(mtk_snand_ids, np);
1410 +       if (!dev_id)
1411 +               return -EINVAL;
1412 +
1413 +       ctlr = devm_spi_alloc_master(&pdev->dev, sizeof(*ms));
1414 +       if (!ctlr)
1415 +               return -ENOMEM;
1416 +       platform_set_drvdata(pdev, ctlr);
1417 +
1418 +       ms = spi_controller_get_devdata(ctlr);
1419 +
1420 +       ms->ctlr = ctlr;
1421 +       ms->caps = dev_id->data;
1422 +
1423 +       ms->ecc = of_mtk_ecc_get(np);
1424 +       if (IS_ERR(ms->ecc))
1425 +               return PTR_ERR(ms->ecc);
1426 +       else if (!ms->ecc)
1427 +               return -ENODEV;
1428 +
1429 +       ms->nfi_base = devm_platform_ioremap_resource(pdev, 0);
1430 +       if (IS_ERR(ms->nfi_base)) {
1431 +               ret = PTR_ERR(ms->nfi_base);
1432 +               goto release_ecc;
1433 +       }
1434 +
1435 +       ms->dev = &pdev->dev;
1436 +
1437 +       ms->nfi_clk = devm_clk_get(&pdev->dev, "nfi_clk");
1438 +       if (IS_ERR(ms->nfi_clk)) {
1439 +               ret = PTR_ERR(ms->nfi_clk);
1440 +               dev_err(&pdev->dev, "unable to get nfi_clk, err = %d\n", ret);
1441 +               goto release_ecc;
1442 +       }
1443 +
1444 +       ms->pad_clk = devm_clk_get(&pdev->dev, "pad_clk");
1445 +       if (IS_ERR(ms->pad_clk)) {
1446 +               ret = PTR_ERR(ms->pad_clk);
1447 +               dev_err(&pdev->dev, "unable to get pad_clk, err = %d\n", ret);
1448 +               goto release_ecc;
1449 +       }
1450 +
1451 +       ret = mtk_snand_enable_clk(ms);
1452 +       if (ret)
1453 +               goto release_ecc;
1454 +
1455 +       init_completion(&ms->op_done);
1456 +
1457 +       ms->irq = platform_get_irq(pdev, 0);
1458 +       if (ms->irq < 0) {
1459 +               ret = ms->irq;
1460 +               goto disable_clk;
1461 +       }
1462 +       ret = devm_request_irq(ms->dev, ms->irq, mtk_snand_irq, 0x0,
1463 +                              "mtk-snand", ms);
1464 +       if (ret) {
1465 +               dev_err(ms->dev, "failed to request snfi irq\n");
1466 +               goto disable_clk;
1467 +       }
1468 +
1469 +       ret = dma_set_mask(ms->dev, DMA_BIT_MASK(32));
1470 +       if (ret) {
1471 +               dev_err(ms->dev, "failed to set dma mask\n");
1472 +               goto disable_clk;
1473 +       }
1474 +
1475 +       // switch to SNFI mode
1476 +       nfi_write32(ms, SNF_CFG, SPI_MODE);
1477 +
1478 +       // setup an initial page format for ops matching page_cache_op template
1479 +       // before ECC is called.
1480 +       ret = mtk_snand_setup_pagefmt(ms, ms->caps->sector_size,
1481 +                                     ms->caps->spare_sizes[0]);
1482 +       if (ret) {
1483 +               dev_err(ms->dev, "failed to set initial page format\n");
1484 +               goto disable_clk;
1485 +       }
1486 +
1487 +       // setup ECC engine
1488 +       ms->ecc_eng.dev = &pdev->dev;
1489 +       ms->ecc_eng.integration = NAND_ECC_ENGINE_INTEGRATION_PIPELINED;
1490 +       ms->ecc_eng.ops = &mtk_snfi_ecc_engine_ops;
1491 +       ms->ecc_eng.priv = ms;
1492 +
1493 +       ret = nand_ecc_register_on_host_hw_engine(&ms->ecc_eng);
1494 +       if (ret) {
1495 +               dev_err(&pdev->dev, "failed to register ecc engine.\n");
1496 +               goto disable_clk;
1497 +       }
1498 +
1499 +       ctlr->num_chipselect = 1;
1500 +       ctlr->mem_ops = &mtk_snand_mem_ops;
1501 +       ctlr->mem_caps = &mtk_snand_mem_caps;
1502 +       ctlr->bits_per_word_mask = SPI_BPW_MASK(8);
1503 +       ctlr->mode_bits = SPI_RX_DUAL | SPI_RX_QUAD | SPI_TX_DUAL | SPI_TX_QUAD;
1504 +       ctlr->dev.of_node = pdev->dev.of_node;
1505 +       ret = spi_register_controller(ctlr);
1506 +       if (ret) {
1507 +               dev_err(&pdev->dev, "spi_register_controller failed.\n");
1508 +               goto disable_clk;
1509 +       }
1510 +
1511 +       return 0;
1512 +disable_clk:
1513 +       mtk_snand_disable_clk(ms);
1514 +release_ecc:
1515 +       mtk_ecc_release(ms->ecc);
1516 +       return ret;
1517 +}
1518 +
1519 +static int mtk_snand_remove(struct platform_device *pdev)
1520 +{
1521 +       struct spi_controller *ctlr = platform_get_drvdata(pdev);
1522 +       struct mtk_snand *ms = spi_controller_get_devdata(ctlr);
1523 +
1524 +       spi_unregister_controller(ctlr);
1525 +       mtk_snand_disable_clk(ms);
1526 +       mtk_ecc_release(ms->ecc);
1527 +       kfree(ms->buf);
1528 +       return 0;
1529 +}
1530 +
1531 +static struct platform_driver mtk_snand_driver = {
1532 +       .probe = mtk_snand_probe,
1533 +       .remove = mtk_snand_remove,
1534 +       .driver = {
1535 +               .name = "mtk-snand",
1536 +               .of_match_table = mtk_snand_ids,
1537 +       },
1538 +};
1539 +
1540 +module_platform_driver(mtk_snand_driver);
1541 +
1542 +MODULE_LICENSE("GPL");
1543 +MODULE_AUTHOR("Chuanhong Guo <gch981213@gmail.com>");
1544 +MODULE_DESCRIPTION("MeidaTek SPI-NAND Flash Controller Driver");
1545 --
1546 2.35.1
1547