drivers/cpufreq/apple-soc-cpufreq.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Apple SoC CPU cluster performance state driver
   4  *
   5  * Copyright The Asahi Linux Contributors
   6  *
   7  * Based on scpi-cpufreq.c
   8  */
   9
  10 #include <linux/bitfield.h>
  11 #include <linux/bitops.h>
  12 #include <linux/cpu.h>
  13 #include <linux/cpufreq.h>
  14 #include <linux/cpumask.h>
  15 #include <linux/delay.h>
  16 #include <linux/err.h>
  17 #include <linux/io.h>
  18 #include <linux/iopoll.h>
  19 #include <linux/module.h>
  20 #include <linux/of.h>
  21 #include <linux/of_address.h>
  22 #include <linux/pm_opp.h>
  23 #include <linux/slab.h>
  24
  25 #define APPLE_DVFS_CMD                  0x20
  26 #define APPLE_DVFS_CMD_BUSY             BIT(31)
  27 #define APPLE_DVFS_CMD_SET              BIT(25)
  28 #define APPLE_DVFS_CMD_PS2              GENMASK(16, 12)
  29 #define APPLE_DVFS_CMD_PS1              GENMASK(4, 0)
  30
  31 /* Same timebase as CPU counter (24MHz) */
  32 #define APPLE_DVFS_LAST_CHG_TIME        0x38
  33
  34 /*
  35  * Apple ran out of bits and had to shift this in T8112...
  36  */
  37 #define APPLE_DVFS_STATUS                       0x50
  38 #define APPLE_DVFS_STATUS_CUR_PS_T8103          GENMASK(7, 4)
  39 #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103    4
  40 #define APPLE_DVFS_STATUS_TGT_PS_T8103          GENMASK(3, 0)
  41 #define APPLE_DVFS_STATUS_CUR_PS_T8112          GENMASK(9, 5)
  42 #define APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112    5
  43 #define APPLE_DVFS_STATUS_TGT_PS_T8112          GENMASK(4, 0)
  44
  45 /*
  46  * Div is +1, base clock is 12MHz on existing SoCs.
  47  * For documentation purposes. We use the OPP table to
  48  * get the frequency.
  49  */
  50 #define APPLE_DVFS_PLL_STATUS           0xc0
  51 #define APPLE_DVFS_PLL_FACTOR           0xc8
  52 #define APPLE_DVFS_PLL_FACTOR_MULT      GENMASK(31, 16)
  53 #define APPLE_DVFS_PLL_FACTOR_DIV       GENMASK(15, 0)
  54
  55 #define APPLE_DVFS_TRANSITION_TIMEOUT 100
  56
  57 struct apple_soc_cpufreq_info {
  58         u64 max_pstate;
  59         u64 cur_pstate_mask;
  60         u64 cur_pstate_shift;
  61 };
  62
  63 struct apple_cpu_priv {
  64         struct device *cpu_dev;
  65         void __iomem *reg_base;
  66         const struct apple_soc_cpufreq_info *info;
  67 };
  68
  69 static struct cpufreq_driver apple_soc_cpufreq_driver;
  70
  71 static const struct apple_soc_cpufreq_info soc_t8103_info = {
  72         .max_pstate = 15,
  73         .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8103,
  74         .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8103,
  75 };
  76
  77 static const struct apple_soc_cpufreq_info soc_t8112_info = {
  78         .max_pstate = 31,
  79         .cur_pstate_mask = APPLE_DVFS_STATUS_CUR_PS_T8112,
  80         .cur_pstate_shift = APPLE_DVFS_STATUS_CUR_PS_SHIFT_T8112,
  81 };
  82
  83 static const struct apple_soc_cpufreq_info soc_default_info = {
  84         .max_pstate = 15,
  85         .cur_pstate_mask = 0, /* fallback */
  86 };
  87
  88 static const struct of_device_id apple_soc_cpufreq_of_match[] __maybe_unused = {
  89         {
  90                 .compatible = "apple,t8103-cluster-cpufreq",
  91                 .data = &soc_t8103_info,
  92         },
  93         {
  94                 .compatible = "apple,t8112-cluster-cpufreq",
  95                 .data = &soc_t8112_info,
  96         },
  97         {
  98                 .compatible = "apple,cluster-cpufreq",
  99                 .data = &soc_default_info,
 100         },
 101         {}
 102 };
 103
 104 static unsigned int apple_soc_cpufreq_get_rate(unsigned int cpu)
 105 {
 106         struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
 107         struct apple_cpu_priv *priv = policy->driver_data;
 108         struct cpufreq_frequency_table *p;
 109         unsigned int pstate;
 110
 111         if (priv->info->cur_pstate_mask) {
 112                 u64 reg = readq_relaxed(priv->reg_base + APPLE_DVFS_STATUS);
 113
 114                 pstate = (reg & priv->info->cur_pstate_mask) >>  priv->info->cur_pstate_shift;
 115         } else {
 116                 /*
 117                  * For the fallback case we might not know the layout of DVFS_STATUS,
 118                  * so just use the command register value (which ignores boost limitations).
 119                  */
 120                 u64 reg = readq_relaxed(priv->reg_base + APPLE_DVFS_CMD);
 121
 122                 pstate = FIELD_GET(APPLE_DVFS_CMD_PS1, reg);
 123         }
 124
 125         cpufreq_for_each_valid_entry(p, policy->freq_table)
 126                 if (p->driver_data == pstate)
 127                         return p->frequency;
 128
 129         dev_err(priv->cpu_dev, "could not find frequency for pstate %d\n",
 130                 pstate);
 131         return 0;
 132 }
 133
 134 static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy,
 135                                         unsigned int index)
 136 {
 137         struct apple_cpu_priv *priv = policy->driver_data;
 138         unsigned int pstate = policy->freq_table[index].driver_data;
 139         u64 reg;
 140
 141         /* Fallback for newer SoCs */
 142         if (index > priv->info->max_pstate)
 143                 index = priv->info->max_pstate;
 144
 145         if (readq_poll_timeout_atomic(priv->reg_base + APPLE_DVFS_CMD, reg,
 146                                       !(reg & APPLE_DVFS_CMD_BUSY), 2,
 147                                       APPLE_DVFS_TRANSITION_TIMEOUT)) {
 148                 return -EIO;
 149         }
 150
 151         reg &= ~(APPLE_DVFS_CMD_PS1 | APPLE_DVFS_CMD_PS2);
 152         reg |= FIELD_PREP(APPLE_DVFS_CMD_PS1, pstate);
 153         reg |= FIELD_PREP(APPLE_DVFS_CMD_PS2, pstate);
 154         reg |= APPLE_DVFS_CMD_SET;
 155
 156         writeq_relaxed(reg, priv->reg_base + APPLE_DVFS_CMD);
 157
 158         return 0;
 159 }
 160
 161 static unsigned int apple_soc_cpufreq_fast_switch(struct cpufreq_policy *policy,
 162                                                   unsigned int target_freq)
 163 {
 164         if (apple_soc_cpufreq_set_target(policy, policy->cached_resolved_idx) < 0)
 165                 return 0;
 166
 167         return policy->freq_table[policy->cached_resolved_idx].frequency;
 168 }
 169
 170 static int apple_soc_cpufreq_find_cluster(struct cpufreq_policy *policy,
 171                                           void __iomem **reg_base,
 172                                           const struct apple_soc_cpufreq_info **info)
 173 {
 174         struct of_phandle_args args;
 175         const struct of_device_id *match;
 176         int ret = 0;
 177
 178         ret = of_perf_domain_get_sharing_cpumask(policy->cpu, "performance-domains",
 179                                                  "#performance-domain-cells",
 180                                                  policy->cpus, &args);
 181         if (ret < 0)
 182                 return ret;
 183
 184         match = of_match_node(apple_soc_cpufreq_of_match, args.np);
 185         of_node_put(args.np);
 186         if (!match)
 187                 return -ENODEV;
 188
 189         *info = match->data;
 190
 191         *reg_base = of_iomap(args.np, 0);
 192         if (!*reg_base)
 193                 return -ENOMEM;
 194
 195         return 0;
 196 }
 197
 198 static struct freq_attr *apple_soc_cpufreq_hw_attr[] = {
 199         &cpufreq_freq_attr_scaling_available_freqs,
 200         NULL, /* Filled in below if boost is enabled */
 201         NULL,
 202 };
 203
 204 static int apple_soc_cpufreq_init(struct cpufreq_policy *policy)
 205 {
 206         int ret, i;
 207         unsigned int transition_latency;
 208         void __iomem *reg_base;
 209         struct device *cpu_dev;
 210         struct apple_cpu_priv *priv;
 211         const struct apple_soc_cpufreq_info *info;
 212         struct cpufreq_frequency_table *freq_table;
 213
 214         cpu_dev = get_cpu_device(policy->cpu);
 215         if (!cpu_dev) {
 216                 pr_err("failed to get cpu%d device\n", policy->cpu);
 217                 return -ENODEV;
 218         }
 219
 220         ret = dev_pm_opp_of_add_table(cpu_dev);
 221         if (ret < 0) {
 222                 dev_err(cpu_dev, "%s: failed to add OPP table: %d\n", __func__, ret);
 223                 return ret;
 224         }
 225
 226         ret = apple_soc_cpufreq_find_cluster(policy, &reg_base, &info);
 227         if (ret) {
 228                 dev_err(cpu_dev, "%s: failed to get cluster info: %d\n", __func__, ret);
 229                 return ret;
 230         }
 231
 232         ret = dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus);
 233         if (ret) {
 234                 dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", __func__, ret);
 235                 goto out_iounmap;
 236         }
 237
 238         ret = dev_pm_opp_get_opp_count(cpu_dev);
 239         if (ret <= 0) {
 240                 dev_dbg(cpu_dev, "OPP table is not ready, deferring probe\n");
 241                 ret = -EPROBE_DEFER;
 242                 goto out_free_opp;
 243         }
 244
 245         priv = kzalloc(sizeof(*priv), GFP_KERNEL);
 246         if (!priv) {
 247                 ret = -ENOMEM;
 248                 goto out_free_opp;
 249         }
 250
 251         ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table);
 252         if (ret) {
 253                 dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret);
 254                 goto out_free_priv;
 255         }
 256
 257         /* Get OPP levels (p-state indexes) and stash them in driver_data */
 258         for (i = 0; freq_table[i].frequency != CPUFREQ_TABLE_END; i++) {
 259                 unsigned long rate = freq_table[i].frequency * 1000 + 999;
 260                 struct dev_pm_opp *opp = dev_pm_opp_find_freq_floor(cpu_dev, &rate);
 261
 262                 if (IS_ERR(opp)) {
 263                         ret = PTR_ERR(opp);
 264                         goto out_free_cpufreq_table;
 265                 }
 266                 freq_table[i].driver_data = dev_pm_opp_get_level(opp);
 267                 dev_pm_opp_put(opp);
 268         }
 269
 270         priv->cpu_dev = cpu_dev;
 271         priv->reg_base = reg_base;
 272         priv->info = info;
 273         policy->driver_data = priv;
 274         policy->freq_table = freq_table;
 275
 276         transition_latency = dev_pm_opp_get_max_transition_latency(cpu_dev);
 277         if (!transition_latency)
 278                 transition_latency = CPUFREQ_ETERNAL;
 279
 280         policy->cpuinfo.transition_latency = transition_latency;
 281         policy->dvfs_possible_from_any_cpu = true;
 282         policy->fast_switch_possible = true;
 283         policy->suspend_freq = freq_table[0].frequency;
 284
 285         if (policy_has_boost_freq(policy)) {
 286                 ret = cpufreq_enable_boost_support();
 287                 if (ret) {
 288                         dev_warn(cpu_dev, "failed to enable boost: %d\n", ret);
 289                 } else {
 290                         apple_soc_cpufreq_hw_attr[1] = &cpufreq_freq_attr_scaling_boost_freqs;
 291                         apple_soc_cpufreq_driver.boost_enabled = true;
 292                 }
 293         }
 294
 295         return 0;
 296
 297 out_free_cpufreq_table:
 298         dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table);
 299 out_free_priv:
 300         kfree(priv);
 301 out_free_opp:
 302         dev_pm_opp_remove_all_dynamic(cpu_dev);
 303 out_iounmap:
 304         iounmap(reg_base);
 305         return ret;
 306 }
 307
 308 static void apple_soc_cpufreq_exit(struct cpufreq_policy *policy)
 309 {
 310         struct apple_cpu_priv *priv = policy->driver_data;
 311
 312         dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table);
 313         dev_pm_opp_remove_all_dynamic(priv->cpu_dev);
 314         iounmap(priv->reg_base);
 315         kfree(priv);
 316 }
 317
 318 static struct cpufreq_driver apple_soc_cpufreq_driver = {
 319         .name           = "apple-cpufreq",
 320         .flags          = CPUFREQ_HAVE_GOVERNOR_PER_POLICY |
 321                           CPUFREQ_NEED_INITIAL_FREQ_CHECK | CPUFREQ_IS_COOLING_DEV,
 322         .verify         = cpufreq_generic_frequency_table_verify,
 323         .get            = apple_soc_cpufreq_get_rate,
 324         .init           = apple_soc_cpufreq_init,
 325         .exit           = apple_soc_cpufreq_exit,
 326         .target_index   = apple_soc_cpufreq_set_target,
 327         .fast_switch    = apple_soc_cpufreq_fast_switch,
 328         .register_em    = cpufreq_register_em_with_opp,
 329         .attr           = apple_soc_cpufreq_hw_attr,
 330         .suspend        = cpufreq_generic_suspend,
 331 };
 332
 333 static int __init apple_soc_cpufreq_module_init(void)
 334 {
 335         if (!of_machine_is_compatible("apple,arm-platform"))
 336                 return -ENODEV;
 337
 338         return cpufreq_register_driver(&apple_soc_cpufreq_driver);
 339 }
 340 module_init(apple_soc_cpufreq_module_init);
 341
 342 static void __exit apple_soc_cpufreq_module_exit(void)
 343 {
 344         cpufreq_unregister_driver(&apple_soc_cpufreq_driver);
 345 }
 346 module_exit(apple_soc_cpufreq_module_exit);
 347
 348 MODULE_DEVICE_TABLE(of, apple_soc_cpufreq_of_match);
 349 MODULE_AUTHOR("Hector Martin <marcan@marcan.st>");
 350 MODULE_DESCRIPTION("Apple SoC CPU cluster DVFS driver");
 351 MODULE_LICENSE("GPL");