arch/s390/kernel/topology.c

   1 /*
   2  *    Copyright IBM Corp. 2007, 2011
   3  *    Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>
   4  */
   5
   6 #define KMSG_COMPONENT "cpu"
   7 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
   8
   9 #include <linux/workqueue.h>
  10 #include <linux/bootmem.h>
  11 #include <linux/cpuset.h>
  12 #include <linux/device.h>
  13 #include <linux/export.h>
  14 #include <linux/kernel.h>
  15 #include <linux/sched.h>
  16 #include <linux/init.h>
  17 #include <linux/delay.h>
  18 #include <linux/cpu.h>
  19 #include <linux/smp.h>
  20 #include <linux/mm.h>
  21 #include <asm/sysinfo.h>
  22
  23 #define PTF_HORIZONTAL  (0UL)
  24 #define PTF_VERTICAL    (1UL)
  25 #define PTF_CHECK       (2UL)
  26
  27 struct mask_info {
  28         struct mask_info *next;
  29         unsigned char id;
  30         cpumask_t mask;
  31 };
  32
  33 static void set_topology_timer(void);
  34 static void topology_work_fn(struct work_struct *work);
  35 static struct sysinfo_15_1_x *tl_info;
  36
  37 static int topology_enabled = 1;
  38 static DECLARE_WORK(topology_work, topology_work_fn);
  39
  40 /* topology_lock protects the socket and book linked lists */
  41 static DEFINE_SPINLOCK(topology_lock);
  42 static struct mask_info socket_info;
  43 static struct mask_info book_info;
  44
  45 struct cpu_topology_s390 cpu_topology[NR_CPUS];
  46 EXPORT_SYMBOL_GPL(cpu_topology);
  47
  48 static cpumask_t cpu_group_map(struct mask_info *info, unsigned int cpu)
  49 {
  50         cpumask_t mask;
  51
  52         cpumask_copy(&mask, cpumask_of(cpu));
  53         if (!topology_enabled || !MACHINE_HAS_TOPOLOGY)
  54                 return mask;
  55         for (; info; info = info->next) {
  56                 if (cpumask_test_cpu(cpu, &info->mask))
  57                         return info->mask;
  58         }
  59         return mask;
  60 }
  61
  62 static struct mask_info *add_cpus_to_mask(struct topology_cpu *tl_cpu,
  63                                           struct mask_info *book,
  64                                           struct mask_info *socket,
  65                                           int one_socket_per_cpu)
  66 {
  67         unsigned int cpu;
  68
  69         for_each_set_bit(cpu, &tl_cpu->mask[0], TOPOLOGY_CPU_BITS) {
  70                 unsigned int rcpu;
  71                 int lcpu;
  72
  73                 rcpu = TOPOLOGY_CPU_BITS - 1 - cpu + tl_cpu->origin;
  74                 lcpu = smp_find_processor_id(rcpu);
  75                 if (lcpu < 0)
  76                         continue;
  77                 cpumask_set_cpu(lcpu, &book->mask);
  78                 cpu_topology[lcpu].book_id = book->id;
  79                 cpumask_set_cpu(lcpu, &socket->mask);
  80                 cpu_topology[lcpu].core_id = rcpu;
  81                 if (one_socket_per_cpu) {
  82                         cpu_topology[lcpu].socket_id = rcpu;
  83                         socket = socket->next;
  84                 } else {
  85                         cpu_topology[lcpu].socket_id = socket->id;
  86                 }
  87                 smp_cpu_set_polarization(lcpu, tl_cpu->pp);
  88         }
  89         return socket;
  90 }
  91
  92 static void clear_masks(void)
  93 {
  94         struct mask_info *info;
  95
  96         info = &socket_info;
  97         while (info) {
  98                 cpumask_clear(&info->mask);
  99                 info = info->next;
 100         }
 101         info = &book_info;
 102         while (info) {
 103                 cpumask_clear(&info->mask);
 104                 info = info->next;
 105         }
 106 }
 107
 108 static union topology_entry *next_tle(union topology_entry *tle)
 109 {
 110         if (!tle->nl)
 111                 return (union topology_entry *)((struct topology_cpu *)tle + 1);
 112         return (union topology_entry *)((struct topology_container *)tle + 1);
 113 }
 114
 115 static void __tl_to_masks_generic(struct sysinfo_15_1_x *info)
 116 {
 117         struct mask_info *socket = &socket_info;
 118         struct mask_info *book = &book_info;
 119         union topology_entry *tle, *end;
 120
 121         tle = info->tle;
 122         end = (union topology_entry *)((unsigned long)info + info->length);
 123         while (tle < end) {
 124                 switch (tle->nl) {
 125                 case 2:
 126                         book = book->next;
 127                         book->id = tle->container.id;
 128                         break;
 129                 case 1:
 130                         socket = socket->next;
 131                         socket->id = tle->container.id;
 132                         break;
 133                 case 0:
 134                         add_cpus_to_mask(&tle->cpu, book, socket, 0);
 135                         break;
 136                 default:
 137                         clear_masks();
 138                         return;
 139                 }
 140                 tle = next_tle(tle);
 141         }
 142 }
 143
 144 static void __tl_to_masks_z10(struct sysinfo_15_1_x *info)
 145 {
 146         struct mask_info *socket = &socket_info;
 147         struct mask_info *book = &book_info;
 148         union topology_entry *tle, *end;
 149
 150         tle = info->tle;
 151         end = (union topology_entry *)((unsigned long)info + info->length);
 152         while (tle < end) {
 153                 switch (tle->nl) {
 154                 case 1:
 155                         book = book->next;
 156                         book->id = tle->container.id;
 157                         break;
 158                 case 0:
 159                         socket = add_cpus_to_mask(&tle->cpu, book, socket, 1);
 160                         break;
 161                 default:
 162                         clear_masks();
 163                         return;
 164                 }
 165                 tle = next_tle(tle);
 166         }
 167 }
 168
 169 static void tl_to_masks(struct sysinfo_15_1_x *info)
 170 {
 171         struct cpuid cpu_id;
 172
 173         spin_lock_irq(&topology_lock);
 174         get_cpu_id(&cpu_id);
 175         clear_masks();
 176         switch (cpu_id.machine) {
 177         case 0x2097:
 178         case 0x2098:
 179                 __tl_to_masks_z10(info);
 180                 break;
 181         default:
 182                 __tl_to_masks_generic(info);
 183         }
 184         spin_unlock_irq(&topology_lock);
 185 }
 186
 187 static void topology_update_polarization_simple(void)
 188 {
 189         int cpu;
 190
 191         mutex_lock(&smp_cpu_state_mutex);
 192         for_each_possible_cpu(cpu)
 193                 smp_cpu_set_polarization(cpu, POLARIZATION_HRZ);
 194         mutex_unlock(&smp_cpu_state_mutex);
 195 }
 196
 197 static int ptf(unsigned long fc)
 198 {
 199         int rc;
 200
 201         asm volatile(
 202                 "       .insn   rre,0xb9a20000,%1,%1\n"
 203                 "       ipm     %0\n"
 204                 "       srl     %0,28\n"
 205                 : "=d" (rc)
 206                 : "d" (fc)  : "cc");
 207         return rc;
 208 }
 209
 210 int topology_set_cpu_management(int fc)
 211 {
 212         int cpu, rc;
 213
 214         if (!MACHINE_HAS_TOPOLOGY)
 215                 return -EOPNOTSUPP;
 216         if (fc)
 217                 rc = ptf(PTF_VERTICAL);
 218         else
 219                 rc = ptf(PTF_HORIZONTAL);
 220         if (rc)
 221                 return -EBUSY;
 222         for_each_possible_cpu(cpu)
 223                 smp_cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
 224         return rc;
 225 }
 226
 227 static void update_cpu_masks(void)
 228 {
 229         unsigned long flags;
 230         int cpu;
 231
 232         spin_lock_irqsave(&topology_lock, flags);
 233         for_each_possible_cpu(cpu) {
 234                 cpu_topology[cpu].core_mask = cpu_group_map(&socket_info, cpu);
 235                 cpu_topology[cpu].book_mask = cpu_group_map(&book_info, cpu);
 236                 if (!MACHINE_HAS_TOPOLOGY) {
 237                         cpu_topology[cpu].core_id = cpu;
 238                         cpu_topology[cpu].socket_id = cpu;
 239                         cpu_topology[cpu].book_id = cpu;
 240                 }
 241         }
 242         spin_unlock_irqrestore(&topology_lock, flags);
 243 }
 244
 245 void store_topology(struct sysinfo_15_1_x *info)
 246 {
 247         if (topology_max_mnest >= 3)
 248                 stsi(info, 15, 1, 3);
 249         else
 250                 stsi(info, 15, 1, 2);
 251 }
 252
 253 int arch_update_cpu_topology(void)
 254 {
 255         struct sysinfo_15_1_x *info = tl_info;
 256         struct device *dev;
 257         int cpu;
 258
 259         if (!MACHINE_HAS_TOPOLOGY) {
 260                 update_cpu_masks();
 261                 topology_update_polarization_simple();
 262                 return 0;
 263         }
 264         store_topology(info);
 265         tl_to_masks(info);
 266         update_cpu_masks();
 267         for_each_online_cpu(cpu) {
 268                 dev = get_cpu_device(cpu);
 269                 kobject_uevent(&dev->kobj, KOBJ_CHANGE);
 270         }
 271         return 1;
 272 }
 273
 274 static void topology_work_fn(struct work_struct *work)
 275 {
 276         rebuild_sched_domains();
 277 }
 278
 279 void topology_schedule_update(void)
 280 {
 281         schedule_work(&topology_work);
 282 }
 283
 284 static void topology_timer_fn(unsigned long ignored)
 285 {
 286         if (ptf(PTF_CHECK))
 287                 topology_schedule_update();
 288         set_topology_timer();
 289 }
 290
 291 static struct timer_list topology_timer =
 292         TIMER_DEFERRED_INITIALIZER(topology_timer_fn, 0, 0);
 293
 294 static atomic_t topology_poll = ATOMIC_INIT(0);
 295
 296 static void set_topology_timer(void)
 297 {
 298         if (atomic_add_unless(&topology_poll, -1, 0))
 299                 mod_timer(&topology_timer, jiffies + HZ / 10);
 300         else
 301                 mod_timer(&topology_timer, jiffies + HZ * 60);
 302 }
 303
 304 void topology_expect_change(void)
 305 {
 306         if (!MACHINE_HAS_TOPOLOGY)
 307                 return;
 308         /* This is racy, but it doesn't matter since it is just a heuristic.
 309          * Worst case is that we poll in a higher frequency for a bit longer.
 310          */
 311         if (atomic_read(&topology_poll) > 60)
 312                 return;
 313         atomic_add(60, &topology_poll);
 314         set_topology_timer();
 315 }
 316
 317 static int __init early_parse_topology(char *p)
 318 {
 319         if (strncmp(p, "off", 3))
 320                 return 0;
 321         topology_enabled = 0;
 322         return 0;
 323 }
 324 early_param("topology", early_parse_topology);
 325
 326 static void __init alloc_masks(struct sysinfo_15_1_x *info,
 327                                struct mask_info *mask, int offset)
 328 {
 329         int i, nr_masks;
 330
 331         nr_masks = info->mag[TOPOLOGY_NR_MAG - offset];
 332         for (i = 0; i < info->mnest - offset; i++)
 333                 nr_masks *= info->mag[TOPOLOGY_NR_MAG - offset - 1 - i];
 334         nr_masks = max(nr_masks, 1);
 335         for (i = 0; i < nr_masks; i++) {
 336                 mask->next = alloc_bootmem_align(
 337                         roundup_pow_of_two(sizeof(struct mask_info)),
 338                         roundup_pow_of_two(sizeof(struct mask_info)));
 339                 mask = mask->next;
 340         }
 341 }
 342
 343 void __init s390_init_cpu_topology(void)
 344 {
 345         struct sysinfo_15_1_x *info;
 346         int i;
 347
 348         if (!MACHINE_HAS_TOPOLOGY)
 349                 return;
 350         tl_info = alloc_bootmem_pages(PAGE_SIZE);
 351         info = tl_info;
 352         store_topology(info);
 353         pr_info("The CPU configuration topology of the machine is:");
 354         for (i = 0; i < TOPOLOGY_NR_MAG; i++)
 355                 printk(KERN_CONT " %d", info->mag[i]);
 356         printk(KERN_CONT " / %d\n", info->mnest);
 357         alloc_masks(info, &socket_info, 1);
 358         alloc_masks(info, &book_info, 2);
 359 }
 360
 361 static int cpu_management;
 362
 363 static ssize_t dispatching_show(struct device *dev,
 364                                 struct device_attribute *attr,
 365                                 char *buf)
 366 {
 367         ssize_t count;
 368
 369         mutex_lock(&smp_cpu_state_mutex);
 370         count = sprintf(buf, "%d\n", cpu_management);
 371         mutex_unlock(&smp_cpu_state_mutex);
 372         return count;
 373 }
 374
 375 static ssize_t dispatching_store(struct device *dev,
 376                                  struct device_attribute *attr,
 377                                  const char *buf,
 378                                  size_t count)
 379 {
 380         int val, rc;
 381         char delim;
 382
 383         if (sscanf(buf, "%d %c", &val, &delim) != 1)
 384                 return -EINVAL;
 385         if (val != 0 && val != 1)
 386                 return -EINVAL;
 387         rc = 0;
 388         get_online_cpus();
 389         mutex_lock(&smp_cpu_state_mutex);
 390         if (cpu_management == val)
 391                 goto out;
 392         rc = topology_set_cpu_management(val);
 393         if (rc)
 394                 goto out;
 395         cpu_management = val;
 396         topology_expect_change();
 397 out:
 398         mutex_unlock(&smp_cpu_state_mutex);
 399         put_online_cpus();
 400         return rc ? rc : count;
 401 }
 402 static DEVICE_ATTR(dispatching, 0644, dispatching_show,
 403                          dispatching_store);
 404
 405 static ssize_t cpu_polarization_show(struct device *dev,
 406                                      struct device_attribute *attr, char *buf)
 407 {
 408         int cpu = dev->id;
 409         ssize_t count;
 410
 411         mutex_lock(&smp_cpu_state_mutex);
 412         switch (smp_cpu_get_polarization(cpu)) {
 413         case POLARIZATION_HRZ:
 414                 count = sprintf(buf, "horizontal\n");
 415                 break;
 416         case POLARIZATION_VL:
 417                 count = sprintf(buf, "vertical:low\n");
 418                 break;
 419         case POLARIZATION_VM:
 420                 count = sprintf(buf, "vertical:medium\n");
 421                 break;
 422         case POLARIZATION_VH:
 423                 count = sprintf(buf, "vertical:high\n");
 424                 break;
 425         default:
 426                 count = sprintf(buf, "unknown\n");
 427                 break;
 428         }
 429         mutex_unlock(&smp_cpu_state_mutex);
 430         return count;
 431 }
 432 static DEVICE_ATTR(polarization, 0444, cpu_polarization_show, NULL);
 433
 434 static struct attribute *topology_cpu_attrs[] = {
 435         &dev_attr_polarization.attr,
 436         NULL,
 437 };
 438
 439 static struct attribute_group topology_cpu_attr_group = {
 440         .attrs = topology_cpu_attrs,
 441 };
 442
 443 int topology_cpu_init(struct cpu *cpu)
 444 {
 445         return sysfs_create_group(&cpu->dev.kobj, &topology_cpu_attr_group);
 446 }
 447
 448 const struct cpumask *cpu_coregroup_mask(int cpu)
 449 {
 450         return &cpu_topology[cpu].core_mask;
 451 }
 452
 453 static const struct cpumask *cpu_book_mask(int cpu)
 454 {
 455         return &cpu_topology[cpu].book_mask;
 456 }
 457
 458 static struct sched_domain_topology_level s390_topology[] = {
 459         { cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) },
 460         { cpu_book_mask, SD_INIT_NAME(BOOK) },
 461         { cpu_cpu_mask, SD_INIT_NAME(DIE) },
 462         { NULL, },
 463 };
 464
 465 static int __init topology_init(void)
 466 {
 467         if (MACHINE_HAS_TOPOLOGY)
 468                 set_topology_timer();
 469         else
 470                 topology_update_polarization_simple();
 471         return device_create_file(cpu_subsys.dev_root, &dev_attr_dispatching);
 472 }
 473 device_initcall(topology_init);
 474
 475 static int __init early_topology_init(void)
 476 {
 477         set_sched_topology(s390_topology);
 478         return 0;
 479 }
 480 early_initcall(early_topology_init);