drivers/cache/sifive_ccache.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * SiFive composable cache controller Driver
   4  *
   5  * Copyright (C) 2018-2022 SiFive, Inc.
   6  *
   7  */
   8
   9 #define pr_fmt(fmt) "CCACHE: " fmt
  10
  11 #include <linux/align.h>
  12 #include <linux/debugfs.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/of_irq.h>
  15 #include <linux/of_address.h>
  16 #include <linux/device.h>
  17 #include <linux/bitfield.h>
  18 #include <linux/platform_device.h>
  19 #include <linux/property.h>
  20 #include <asm/cacheflush.h>
  21 #include <asm/cacheinfo.h>
  22 #include <asm/dma-noncoherent.h>
  23 #include <soc/sifive/sifive_ccache.h>
  24
  25 #define SIFIVE_CCACHE_DIRECCFIX_LOW 0x100
  26 #define SIFIVE_CCACHE_DIRECCFIX_HIGH 0x104
  27 #define SIFIVE_CCACHE_DIRECCFIX_COUNT 0x108
  28
  29 #define SIFIVE_CCACHE_DIRECCFAIL_LOW 0x120
  30 #define SIFIVE_CCACHE_DIRECCFAIL_HIGH 0x124
  31 #define SIFIVE_CCACHE_DIRECCFAIL_COUNT 0x128
  32
  33 #define SIFIVE_CCACHE_DATECCFIX_LOW 0x140
  34 #define SIFIVE_CCACHE_DATECCFIX_HIGH 0x144
  35 #define SIFIVE_CCACHE_DATECCFIX_COUNT 0x148
  36
  37 #define SIFIVE_CCACHE_DATECCFAIL_LOW 0x160
  38 #define SIFIVE_CCACHE_DATECCFAIL_HIGH 0x164
  39 #define SIFIVE_CCACHE_DATECCFAIL_COUNT 0x168
  40
  41 #define SIFIVE_CCACHE_CONFIG 0x00
  42 #define SIFIVE_CCACHE_CONFIG_BANK_MASK GENMASK_ULL(7, 0)
  43 #define SIFIVE_CCACHE_CONFIG_WAYS_MASK GENMASK_ULL(15, 8)
  44 #define SIFIVE_CCACHE_CONFIG_SETS_MASK GENMASK_ULL(23, 16)
  45 #define SIFIVE_CCACHE_CONFIG_BLKS_MASK GENMASK_ULL(31, 24)
  46
  47 #define SIFIVE_CCACHE_FLUSH64 0x200
  48 #define SIFIVE_CCACHE_FLUSH32 0x240
  49
  50 #define SIFIVE_CCACHE_WAYENABLE 0x08
  51 #define SIFIVE_CCACHE_ECCINJECTERR 0x40
  52
  53 #define SIFIVE_CCACHE_MAX_ECCINTR 4
  54 #define SIFIVE_CCACHE_LINE_SIZE 64
  55
  56 static void __iomem *ccache_base;
  57 static int g_irq[SIFIVE_CCACHE_MAX_ECCINTR];
  58 static struct riscv_cacheinfo_ops ccache_cache_ops;
  59 static int level;
  60
  61 enum {
  62         DIR_CORR = 0,
  63         DATA_CORR,
  64         DATA_UNCORR,
  65         DIR_UNCORR,
  66 };
  67
  68 enum {
  69         QUIRK_NONSTANDARD_CACHE_OPS     = BIT(0),
  70         QUIRK_BROKEN_DATA_UNCORR        = BIT(1),
  71 };
  72
  73 #ifdef CONFIG_DEBUG_FS
  74 static struct dentry *sifive_test;
  75
  76 static ssize_t ccache_write(struct file *file, const char __user *data,
  77                             size_t count, loff_t *ppos)
  78 {
  79         unsigned int val;
  80
  81         if (kstrtouint_from_user(data, count, 0, &val))
  82                 return -EINVAL;
  83         if ((val < 0xFF) || (val >= 0x10000 && val < 0x100FF))
  84                 writel(val, ccache_base + SIFIVE_CCACHE_ECCINJECTERR);
  85         else
  86                 return -EINVAL;
  87         return count;
  88 }
  89
  90 static const struct file_operations ccache_fops = {
  91         .owner = THIS_MODULE,
  92         .open = simple_open,
  93         .write = ccache_write
  94 };
  95
  96 static void setup_sifive_debug(void)
  97 {
  98         sifive_test = debugfs_create_dir("sifive_ccache_cache", NULL);
  99
 100         debugfs_create_file("sifive_debug_inject_error", 0200,
 101                             sifive_test, NULL, &ccache_fops);
 102 }
 103 #endif
 104
 105 static void ccache_config_read(void)
 106 {
 107         u32 cfg;
 108
 109         cfg = readl(ccache_base + SIFIVE_CCACHE_CONFIG);
 110         pr_info("%llu banks, %llu ways, sets/bank=%llu, bytes/block=%llu\n",
 111                 FIELD_GET(SIFIVE_CCACHE_CONFIG_BANK_MASK, cfg),
 112                 FIELD_GET(SIFIVE_CCACHE_CONFIG_WAYS_MASK, cfg),
 113                 BIT_ULL(FIELD_GET(SIFIVE_CCACHE_CONFIG_SETS_MASK, cfg)),
 114                 BIT_ULL(FIELD_GET(SIFIVE_CCACHE_CONFIG_BLKS_MASK, cfg)));
 115
 116         cfg = readl(ccache_base + SIFIVE_CCACHE_WAYENABLE);
 117         pr_info("Index of the largest way enabled: %u\n", cfg);
 118 }
 119
 120 static const struct of_device_id sifive_ccache_ids[] = {
 121         { .compatible = "sifive,fu540-c000-ccache" },
 122         { .compatible = "sifive,fu740-c000-ccache" },
 123         { .compatible = "starfive,jh7100-ccache",
 124           .data = (void *)(QUIRK_NONSTANDARD_CACHE_OPS | QUIRK_BROKEN_DATA_UNCORR) },
 125         { .compatible = "sifive,ccache0" },
 126         { /* end of table */ }
 127 };
 128
 129 static ATOMIC_NOTIFIER_HEAD(ccache_err_chain);
 130
 131 int register_sifive_ccache_error_notifier(struct notifier_block *nb)
 132 {
 133         return atomic_notifier_chain_register(&ccache_err_chain, nb);
 134 }
 135 EXPORT_SYMBOL_GPL(register_sifive_ccache_error_notifier);
 136
 137 int unregister_sifive_ccache_error_notifier(struct notifier_block *nb)
 138 {
 139         return atomic_notifier_chain_unregister(&ccache_err_chain, nb);
 140 }
 141 EXPORT_SYMBOL_GPL(unregister_sifive_ccache_error_notifier);
 142
 143 #ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS
 144 static void ccache_flush_range(phys_addr_t start, size_t len)
 145 {
 146         phys_addr_t end = start + len;
 147         phys_addr_t line;
 148
 149         if (!len)
 150                 return;
 151
 152         mb();
 153         for (line = ALIGN_DOWN(start, SIFIVE_CCACHE_LINE_SIZE); line < end;
 154                         line += SIFIVE_CCACHE_LINE_SIZE) {
 155 #ifdef CONFIG_32BIT
 156                 writel(line >> 4, ccache_base + SIFIVE_CCACHE_FLUSH32);
 157 #else
 158                 writeq(line, ccache_base + SIFIVE_CCACHE_FLUSH64);
 159 #endif
 160                 mb();
 161         }
 162 }
 163
 164 static const struct riscv_nonstd_cache_ops ccache_mgmt_ops __initconst = {
 165         .wback = &ccache_flush_range,
 166         .inv = &ccache_flush_range,
 167         .wback_inv = &ccache_flush_range,
 168 };
 169 #endif /* CONFIG_RISCV_NONSTANDARD_CACHE_OPS */
 170
 171 static int ccache_largest_wayenabled(void)
 172 {
 173         return readl(ccache_base + SIFIVE_CCACHE_WAYENABLE) & 0xFF;
 174 }
 175
 176 static ssize_t number_of_ways_enabled_show(struct device *dev,
 177                                            struct device_attribute *attr,
 178                                            char *buf)
 179 {
 180         return sprintf(buf, "%u\n", ccache_largest_wayenabled());
 181 }
 182
 183 static DEVICE_ATTR_RO(number_of_ways_enabled);
 184
 185 static struct attribute *priv_attrs[] = {
 186         &dev_attr_number_of_ways_enabled.attr,
 187         NULL,
 188 };
 189
 190 static const struct attribute_group priv_attr_group = {
 191         .attrs = priv_attrs,
 192 };
 193
 194 static const struct attribute_group *ccache_get_priv_group(struct cacheinfo
 195                                                            *this_leaf)
 196 {
 197         /* We want to use private group for composable cache only */
 198         if (this_leaf->level == level)
 199                 return &priv_attr_group;
 200         else
 201                 return NULL;
 202 }
 203
 204 static irqreturn_t ccache_int_handler(int irq, void *device)
 205 {
 206         unsigned int add_h, add_l;
 207
 208         if (irq == g_irq[DIR_CORR]) {
 209                 add_h = readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_HIGH);
 210                 add_l = readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_LOW);
 211                 pr_err("DirError @ 0x%08X.%08X\n", add_h, add_l);
 212                 /* Reading this register clears the DirError interrupt sig */
 213                 readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_COUNT);
 214                 atomic_notifier_call_chain(&ccache_err_chain,
 215                                            SIFIVE_CCACHE_ERR_TYPE_CE,
 216                                            "DirECCFix");
 217         }
 218         if (irq == g_irq[DIR_UNCORR]) {
 219                 add_h = readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_HIGH);
 220                 add_l = readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_LOW);
 221                 /* Reading this register clears the DirFail interrupt sig */
 222                 readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_COUNT);
 223                 atomic_notifier_call_chain(&ccache_err_chain,
 224                                            SIFIVE_CCACHE_ERR_TYPE_UE,
 225                                            "DirECCFail");
 226                 panic("CCACHE: DirFail @ 0x%08X.%08X\n", add_h, add_l);
 227         }
 228         if (irq == g_irq[DATA_CORR]) {
 229                 add_h = readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_HIGH);
 230                 add_l = readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_LOW);
 231                 pr_err("DataError @ 0x%08X.%08X\n", add_h, add_l);
 232                 /* Reading this register clears the DataError interrupt sig */
 233                 readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_COUNT);
 234                 atomic_notifier_call_chain(&ccache_err_chain,
 235                                            SIFIVE_CCACHE_ERR_TYPE_CE,
 236                                            "DatECCFix");
 237         }
 238         if (irq == g_irq[DATA_UNCORR]) {
 239                 add_h = readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_HIGH);
 240                 add_l = readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_LOW);
 241                 pr_err("DataFail @ 0x%08X.%08X\n", add_h, add_l);
 242                 /* Reading this register clears the DataFail interrupt sig */
 243                 readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_COUNT);
 244                 atomic_notifier_call_chain(&ccache_err_chain,
 245                                            SIFIVE_CCACHE_ERR_TYPE_UE,
 246                                            "DatECCFail");
 247         }
 248
 249         return IRQ_HANDLED;
 250 }
 251
 252 static int sifive_ccache_probe(struct platform_device *pdev)
 253 {
 254         struct device *dev = &pdev->dev;
 255         unsigned long quirks;
 256         int intr_num, rc;
 257
 258         quirks = (unsigned long)device_get_match_data(dev);
 259
 260         intr_num = platform_irq_count(pdev);
 261         if (!intr_num)
 262                 return dev_err_probe(dev, -ENODEV, "No interrupts property\n");
 263
 264         for (int i = 0; i < intr_num; i++) {
 265                 if (i == DATA_UNCORR && (quirks & QUIRK_BROKEN_DATA_UNCORR))
 266                         continue;
 267
 268                 g_irq[i] = platform_get_irq(pdev, i);
 269                 if (g_irq[i] < 0)
 270                         return g_irq[i];
 271
 272                 rc = devm_request_irq(dev, g_irq[i], ccache_int_handler, 0, "ccache_ecc", NULL);
 273                 if (rc)
 274                         return dev_err_probe(dev, rc, "Could not request IRQ %d\n", g_irq[i]);
 275         }
 276
 277         return 0;
 278 }
 279
 280 static struct platform_driver sifive_ccache_driver = {
 281         .probe  = sifive_ccache_probe,
 282         .driver = {
 283                 .name           = "sifive_ccache",
 284                 .of_match_table = sifive_ccache_ids,
 285         },
 286 };
 287
 288 static int __init sifive_ccache_init(void)
 289 {
 290         struct device_node *np;
 291         struct resource res;
 292         const struct of_device_id *match;
 293         unsigned long quirks __maybe_unused;
 294         int rc;
 295
 296         np = of_find_matching_node_and_match(NULL, sifive_ccache_ids, &match);
 297         if (!np)
 298                 return -ENODEV;
 299
 300         quirks = (uintptr_t)match->data;
 301
 302         if (of_address_to_resource(np, 0, &res)) {
 303                 rc = -ENODEV;
 304                 goto err_node_put;
 305         }
 306
 307         ccache_base = ioremap(res.start, resource_size(&res));
 308         if (!ccache_base) {
 309                 rc = -ENOMEM;
 310                 goto err_node_put;
 311         }
 312
 313         if (of_property_read_u32(np, "cache-level", &level)) {
 314                 rc = -ENOENT;
 315                 goto err_unmap;
 316         }
 317
 318 #ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS
 319         if (quirks & QUIRK_NONSTANDARD_CACHE_OPS) {
 320                 riscv_cbom_block_size = SIFIVE_CCACHE_LINE_SIZE;
 321                 riscv_noncoherent_supported();
 322                 riscv_noncoherent_register_cache_ops(&ccache_mgmt_ops);
 323         }
 324 #endif
 325
 326         ccache_config_read();
 327
 328         ccache_cache_ops.get_priv_group = ccache_get_priv_group;
 329         riscv_set_cacheinfo_ops(&ccache_cache_ops);
 330
 331 #ifdef CONFIG_DEBUG_FS
 332         setup_sifive_debug();
 333 #endif
 334
 335         rc = platform_driver_register(&sifive_ccache_driver);
 336         if (rc)
 337                 goto err_unmap;
 338
 339         of_node_put(np);
 340
 341         return 0;
 342
 343 err_unmap:
 344         iounmap(ccache_base);
 345 err_node_put:
 346         of_node_put(np);
 347         return rc;
 348 }
 349
 350 arch_initcall(sifive_ccache_init);