drivers/devfreq/imx8m-ddrc.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright 2019 NXP
   4  */
   5
   6 #include <linux/module.h>
   7 #include <linux/device.h>
   8 #include <linux/of_device.h>
   9 #include <linux/platform_device.h>
  10 #include <linux/devfreq.h>
  11 #include <linux/pm_opp.h>
  12 #include <linux/clk.h>
  13 #include <linux/clk-provider.h>
  14 #include <linux/arm-smccc.h>
  15
  16 #define IMX_SIP_DDR_DVFS                        0xc2000004
  17
  18 /* Query available frequencies. */
  19 #define IMX_SIP_DDR_DVFS_GET_FREQ_COUNT         0x10
  20 #define IMX_SIP_DDR_DVFS_GET_FREQ_INFO          0x11
  21
  22 /*
  23  * This should be in a 1:1 mapping with devicetree OPPs but
  24  * firmware provides additional info.
  25  */
  26 struct imx8m_ddrc_freq {
  27         unsigned long rate;
  28         unsigned long smcarg;
  29         int dram_core_parent_index;
  30         int dram_alt_parent_index;
  31         int dram_apb_parent_index;
  32 };
  33
  34 /* Hardware limitation */
  35 #define IMX8M_DDRC_MAX_FREQ_COUNT 4
  36
  37 /*
  38  * i.MX8M DRAM Controller clocks have the following structure (abridged):
  39  *
  40  * +----------+       |\            +------+
  41  * | dram_pll |-------|M| dram_core |      |
  42  * +----------+       |U|---------->| D    |
  43  *                 /--|X|           |  D   |
  44  *   dram_alt_root |  |/            |   R  |
  45  *                 |                |    C |
  46  *            +---------+           |      |
  47  *            |FIX DIV/4|           |      |
  48  *            +---------+           |      |
  49  *  composite:     |                |      |
  50  * +----------+    |                |      |
  51  * | dram_alt |----/                |      |
  52  * +----------+                     |      |
  53  * | dram_apb |-------------------->|      |
  54  * +----------+                     +------+
  55  *
  56  * The dram_pll is used for higher rates and dram_alt is used for lower rates.
  57  *
  58  * Frequency switching is implemented in TF-A (via SMC call) and can change the
  59  * configuration of the clocks, including mux parents. The dram_alt and
  60  * dram_apb clocks are "imx composite" and their parent can change too.
  61  *
  62  * We need to prepare/enable the new mux parents head of switching and update
  63  * their information afterwards.
  64  */
  65 struct imx8m_ddrc {
  66         struct devfreq_dev_profile profile;
  67         struct devfreq *devfreq;
  68
  69         /* For frequency switching: */
  70         struct clk *dram_core;
  71         struct clk *dram_pll;
  72         struct clk *dram_alt;
  73         struct clk *dram_apb;
  74
  75         int freq_count;
  76         struct imx8m_ddrc_freq freq_table[IMX8M_DDRC_MAX_FREQ_COUNT];
  77 };
  78
  79 static struct imx8m_ddrc_freq *imx8m_ddrc_find_freq(struct imx8m_ddrc *priv,
  80                                                     unsigned long rate)
  81 {
  82         struct imx8m_ddrc_freq *freq;
  83         int i;
  84
  85         /*
  86          * Firmware reports values in MT/s, so we round-down from Hz
  87          * Rounding is extra generous to ensure a match.
  88          */
  89         rate = DIV_ROUND_CLOSEST(rate, 250000);
  90         for (i = 0; i < priv->freq_count; ++i) {
  91                 freq = &priv->freq_table[i];
  92                 if (freq->rate == rate ||
  93                                 freq->rate + 1 == rate ||
  94                                 freq->rate - 1 == rate)
  95                         return freq;
  96         }
  97
  98         return NULL;
  99 }
 100
 101 static void imx8m_ddrc_smc_set_freq(int target_freq)
 102 {
 103         struct arm_smccc_res res;
 104         u32 online_cpus = 0;
 105         int cpu;
 106
 107         local_irq_disable();
 108
 109         for_each_online_cpu(cpu)
 110                 online_cpus |= (1 << (cpu * 8));
 111
 112         /* change the ddr freqency */
 113         arm_smccc_smc(IMX_SIP_DDR_DVFS, target_freq, online_cpus,
 114                         0, 0, 0, 0, 0, &res);
 115
 116         local_irq_enable();
 117 }
 118
 119 static struct clk *clk_get_parent_by_index(struct clk *clk, int index)
 120 {
 121         struct clk_hw *hw;
 122
 123         hw = clk_hw_get_parent_by_index(__clk_get_hw(clk), index);
 124
 125         return hw ? hw->clk : NULL;
 126 }
 127
 128 static int imx8m_ddrc_set_freq(struct device *dev, struct imx8m_ddrc_freq *freq)
 129 {
 130         struct imx8m_ddrc *priv = dev_get_drvdata(dev);
 131         struct clk *new_dram_core_parent;
 132         struct clk *new_dram_alt_parent;
 133         struct clk *new_dram_apb_parent;
 134         int ret;
 135
 136         /*
 137          * Fetch new parents
 138          *
 139          * new_dram_alt_parent and new_dram_apb_parent are optional but
 140          * new_dram_core_parent is not.
 141          */
 142         new_dram_core_parent = clk_get_parent_by_index(
 143                         priv->dram_core, freq->dram_core_parent_index - 1);
 144         if (!new_dram_core_parent) {
 145                 dev_err(dev, "failed to fetch new dram_core parent\n");
 146                 return -EINVAL;
 147         }
 148         if (freq->dram_alt_parent_index) {
 149                 new_dram_alt_parent = clk_get_parent_by_index(
 150                                 priv->dram_alt,
 151                                 freq->dram_alt_parent_index - 1);
 152                 if (!new_dram_alt_parent) {
 153                         dev_err(dev, "failed to fetch new dram_alt parent\n");
 154                         return -EINVAL;
 155                 }
 156         } else
 157                 new_dram_alt_parent = NULL;
 158
 159         if (freq->dram_apb_parent_index) {
 160                 new_dram_apb_parent = clk_get_parent_by_index(
 161                                 priv->dram_apb,
 162                                 freq->dram_apb_parent_index - 1);
 163                 if (!new_dram_apb_parent) {
 164                         dev_err(dev, "failed to fetch new dram_apb parent\n");
 165                         return -EINVAL;
 166                 }
 167         } else
 168                 new_dram_apb_parent = NULL;
 169
 170         /* increase reference counts and ensure clks are ON before switch */
 171         ret = clk_prepare_enable(new_dram_core_parent);
 172         if (ret) {
 173                 dev_err(dev, "failed to enable new dram_core parent: %d\n",
 174                         ret);
 175                 goto out;
 176         }
 177         ret = clk_prepare_enable(new_dram_alt_parent);
 178         if (ret) {
 179                 dev_err(dev, "failed to enable new dram_alt parent: %d\n",
 180                         ret);
 181                 goto out_disable_core_parent;
 182         }
 183         ret = clk_prepare_enable(new_dram_apb_parent);
 184         if (ret) {
 185                 dev_err(dev, "failed to enable new dram_apb parent: %d\n",
 186                         ret);
 187                 goto out_disable_alt_parent;
 188         }
 189
 190         imx8m_ddrc_smc_set_freq(freq->smcarg);
 191
 192         /* update parents in clk tree after switch. */
 193         ret = clk_set_parent(priv->dram_core, new_dram_core_parent);
 194         if (ret)
 195                 dev_warn(dev, "failed to set dram_core parent: %d\n", ret);
 196         if (new_dram_alt_parent) {
 197                 ret = clk_set_parent(priv->dram_alt, new_dram_alt_parent);
 198                 if (ret)
 199                         dev_warn(dev, "failed to set dram_alt parent: %d\n",
 200                                  ret);
 201         }
 202         if (new_dram_apb_parent) {
 203                 ret = clk_set_parent(priv->dram_apb, new_dram_apb_parent);
 204                 if (ret)
 205                         dev_warn(dev, "failed to set dram_apb parent: %d\n",
 206                                  ret);
 207         }
 208
 209         /*
 210          * Explicitly refresh dram PLL rate.
 211          *
 212          * Even if it's marked with CLK_GET_RATE_NOCACHE the rate will not be
 213          * automatically refreshed when clk_get_rate is called on children.
 214          */
 215         clk_get_rate(priv->dram_pll);
 216
 217         /*
 218          * clk_set_parent transfer the reference count from old parent.
 219          * now we drop extra reference counts used during the switch
 220          */
 221         clk_disable_unprepare(new_dram_apb_parent);
 222 out_disable_alt_parent:
 223         clk_disable_unprepare(new_dram_alt_parent);
 224 out_disable_core_parent:
 225         clk_disable_unprepare(new_dram_core_parent);
 226 out:
 227         return ret;
 228 }
 229
 230 static int imx8m_ddrc_target(struct device *dev, unsigned long *freq, u32 flags)
 231 {
 232         struct imx8m_ddrc *priv = dev_get_drvdata(dev);
 233         struct imx8m_ddrc_freq *freq_info;
 234         struct dev_pm_opp *new_opp;
 235         unsigned long old_freq, new_freq;
 236         int ret;
 237
 238         new_opp = devfreq_recommended_opp(dev, freq, flags);
 239         if (IS_ERR(new_opp)) {
 240                 ret = PTR_ERR(new_opp);
 241                 dev_err(dev, "failed to get recommended opp: %d\n", ret);
 242                 return ret;
 243         }
 244         dev_pm_opp_put(new_opp);
 245
 246         old_freq = clk_get_rate(priv->dram_core);
 247         if (*freq == old_freq)
 248                 return 0;
 249
 250         freq_info = imx8m_ddrc_find_freq(priv, *freq);
 251         if (!freq_info)
 252                 return -EINVAL;
 253
 254         /*
 255          * Read back the clk rate to verify switch was correct and so that
 256          * we can report it on all error paths.
 257          */
 258         ret = imx8m_ddrc_set_freq(dev, freq_info);
 259
 260         new_freq = clk_get_rate(priv->dram_core);
 261         if (ret)
 262                 dev_err(dev, "ddrc failed freq switch to %lu from %lu: error %d. now at %lu\n",
 263                         *freq, old_freq, ret, new_freq);
 264         else if (*freq != new_freq)
 265                 dev_err(dev, "ddrc failed freq update to %lu from %lu, now at %lu\n",
 266                         *freq, old_freq, new_freq);
 267         else
 268                 dev_dbg(dev, "ddrc freq set to %lu (was %lu)\n",
 269                         *freq, old_freq);
 270
 271         return ret;
 272 }
 273
 274 static int imx8m_ddrc_get_cur_freq(struct device *dev, unsigned long *freq)
 275 {
 276         struct imx8m_ddrc *priv = dev_get_drvdata(dev);
 277
 278         *freq = clk_get_rate(priv->dram_core);
 279
 280         return 0;
 281 }
 282
 283 static int imx8m_ddrc_get_dev_status(struct device *dev,
 284                                      struct devfreq_dev_status *stat)
 285 {
 286         struct imx8m_ddrc *priv = dev_get_drvdata(dev);
 287
 288         stat->busy_time = 0;
 289         stat->total_time = 0;
 290         stat->current_frequency = clk_get_rate(priv->dram_core);
 291
 292         return 0;
 293 }
 294
 295 static int imx8m_ddrc_init_freq_info(struct device *dev)
 296 {
 297         struct imx8m_ddrc *priv = dev_get_drvdata(dev);
 298         struct arm_smccc_res res;
 299         int index;
 300
 301         /* An error here means DDR DVFS API not supported by firmware */
 302         arm_smccc_smc(IMX_SIP_DDR_DVFS, IMX_SIP_DDR_DVFS_GET_FREQ_COUNT,
 303                         0, 0, 0, 0, 0, 0, &res);
 304         priv->freq_count = res.a0;
 305         if (priv->freq_count <= 0 ||
 306                         priv->freq_count > IMX8M_DDRC_MAX_FREQ_COUNT)
 307                 return -ENODEV;
 308
 309         for (index = 0; index < priv->freq_count; ++index) {
 310                 struct imx8m_ddrc_freq *freq = &priv->freq_table[index];
 311
 312                 arm_smccc_smc(IMX_SIP_DDR_DVFS, IMX_SIP_DDR_DVFS_GET_FREQ_INFO,
 313                               index, 0, 0, 0, 0, 0, &res);
 314                 /* Result should be strictly positive */
 315                 if ((long)res.a0 <= 0)
 316                         return -ENODEV;
 317
 318                 freq->rate = res.a0;
 319                 freq->smcarg = index;
 320                 freq->dram_core_parent_index = res.a1;
 321                 freq->dram_alt_parent_index = res.a2;
 322                 freq->dram_apb_parent_index = res.a3;
 323
 324                 /* dram_core has 2 options: dram_pll or dram_alt_root */
 325                 if (freq->dram_core_parent_index != 1 &&
 326                                 freq->dram_core_parent_index != 2)
 327                         return -ENODEV;
 328                 /* dram_apb and dram_alt have exactly 8 possible parents */
 329                 if (freq->dram_alt_parent_index > 8 ||
 330                                 freq->dram_apb_parent_index > 8)
 331                         return -ENODEV;
 332                 /* dram_core from alt requires explicit dram_alt parent */
 333                 if (freq->dram_core_parent_index == 2 &&
 334                                 freq->dram_alt_parent_index == 0)
 335                         return -ENODEV;
 336         }
 337
 338         return 0;
 339 }
 340
 341 static int imx8m_ddrc_check_opps(struct device *dev)
 342 {
 343         struct imx8m_ddrc *priv = dev_get_drvdata(dev);
 344         struct imx8m_ddrc_freq *freq_info;
 345         struct dev_pm_opp *opp;
 346         unsigned long freq;
 347         int i, opp_count;
 348
 349         /* Enumerate DT OPPs and disable those not supported by firmware */
 350         opp_count = dev_pm_opp_get_opp_count(dev);
 351         if (opp_count < 0)
 352                 return opp_count;
 353         for (i = 0, freq = 0; i < opp_count; ++i, ++freq) {
 354                 opp = dev_pm_opp_find_freq_ceil(dev, &freq);
 355                 if (IS_ERR(opp)) {
 356                         dev_err(dev, "Failed enumerating OPPs: %ld\n",
 357                                 PTR_ERR(opp));
 358                         return PTR_ERR(opp);
 359                 }
 360                 dev_pm_opp_put(opp);
 361
 362                 freq_info = imx8m_ddrc_find_freq(priv, freq);
 363                 if (!freq_info) {
 364                         dev_info(dev, "Disable unsupported OPP %luHz %luMT/s\n",
 365                                         freq, DIV_ROUND_CLOSEST(freq, 250000));
 366                         dev_pm_opp_disable(dev, freq);
 367                 }
 368         }
 369
 370         return 0;
 371 }
 372
 373 static void imx8m_ddrc_exit(struct device *dev)
 374 {
 375         dev_pm_opp_of_remove_table(dev);
 376 }
 377
 378 static int imx8m_ddrc_probe(struct platform_device *pdev)
 379 {
 380         struct device *dev = &pdev->dev;
 381         struct imx8m_ddrc *priv;
 382         const char *gov = DEVFREQ_GOV_USERSPACE;
 383         int ret;
 384
 385         priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
 386         if (!priv)
 387                 return -ENOMEM;
 388
 389         platform_set_drvdata(pdev, priv);
 390
 391         ret = imx8m_ddrc_init_freq_info(dev);
 392         if (ret) {
 393                 dev_err(dev, "failed to init firmware freq info: %d\n", ret);
 394                 return ret;
 395         }
 396
 397         priv->dram_core = devm_clk_get(dev, "core");
 398         if (IS_ERR(priv->dram_core)) {
 399                 ret = PTR_ERR(priv->dram_core);
 400                 dev_err(dev, "failed to fetch core clock: %d\n", ret);
 401                 return ret;
 402         }
 403         priv->dram_pll = devm_clk_get(dev, "pll");
 404         if (IS_ERR(priv->dram_pll)) {
 405                 ret = PTR_ERR(priv->dram_pll);
 406                 dev_err(dev, "failed to fetch pll clock: %d\n", ret);
 407                 return ret;
 408         }
 409         priv->dram_alt = devm_clk_get(dev, "alt");
 410         if (IS_ERR(priv->dram_alt)) {
 411                 ret = PTR_ERR(priv->dram_alt);
 412                 dev_err(dev, "failed to fetch alt clock: %d\n", ret);
 413                 return ret;
 414         }
 415         priv->dram_apb = devm_clk_get(dev, "apb");
 416         if (IS_ERR(priv->dram_apb)) {
 417                 ret = PTR_ERR(priv->dram_apb);
 418                 dev_err(dev, "failed to fetch apb clock: %d\n", ret);
 419                 return ret;
 420         }
 421
 422         ret = dev_pm_opp_of_add_table(dev);
 423         if (ret < 0) {
 424                 dev_err(dev, "failed to get OPP table\n");
 425                 return ret;
 426         }
 427
 428         ret = imx8m_ddrc_check_opps(dev);
 429         if (ret < 0)
 430                 goto err;
 431
 432         priv->profile.polling_ms = 1000;
 433         priv->profile.target = imx8m_ddrc_target;
 434         priv->profile.get_dev_status = imx8m_ddrc_get_dev_status;
 435         priv->profile.exit = imx8m_ddrc_exit;
 436         priv->profile.get_cur_freq = imx8m_ddrc_get_cur_freq;
 437         priv->profile.initial_freq = clk_get_rate(priv->dram_core);
 438
 439         priv->devfreq = devm_devfreq_add_device(dev, &priv->profile,
 440                                                 gov, NULL);
 441         if (IS_ERR(priv->devfreq)) {
 442                 ret = PTR_ERR(priv->devfreq);
 443                 dev_err(dev, "failed to add devfreq device: %d\n", ret);
 444                 goto err;
 445         }
 446
 447         return 0;
 448
 449 err:
 450         dev_pm_opp_of_remove_table(dev);
 451         return ret;
 452 }
 453
 454 static const struct of_device_id imx8m_ddrc_of_match[] = {
 455         { .compatible = "fsl,imx8m-ddrc", },
 456         { /* sentinel */ },
 457 };
 458 MODULE_DEVICE_TABLE(of, imx8m_ddrc_of_match);
 459
 460 static struct platform_driver imx8m_ddrc_platdrv = {
 461         .probe          = imx8m_ddrc_probe,
 462         .driver = {
 463                 .name   = "imx8m-ddrc-devfreq",
 464                 .of_match_table = of_match_ptr(imx8m_ddrc_of_match),
 465         },
 466 };
 467 module_platform_driver(imx8m_ddrc_platdrv);
 468
 469 MODULE_DESCRIPTION("i.MX8M DDR Controller frequency driver");
 470 MODULE_AUTHOR("Leonard Crestez <leonard.crestez@nxp.com>");
 471 MODULE_LICENSE("GPL v2");