drivers/firmware/arm_sdei.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 // Copyright (C) 2017 Arm Ltd.
   3 #define pr_fmt(fmt) "sdei: " fmt
   4
   5 #include <linux/acpi.h>
   6 #include <linux/arm_sdei.h>
   7 #include <linux/arm-smccc.h>
   8 #include <linux/atomic.h>
   9 #include <linux/bitops.h>
  10 #include <linux/compiler.h>
  11 #include <linux/cpuhotplug.h>
  12 #include <linux/cpu.h>
  13 #include <linux/cpu_pm.h>
  14 #include <linux/errno.h>
  15 #include <linux/hardirq.h>
  16 #include <linux/kernel.h>
  17 #include <linux/kprobes.h>
  18 #include <linux/kvm_host.h>
  19 #include <linux/list.h>
  20 #include <linux/mutex.h>
  21 #include <linux/notifier.h>
  22 #include <linux/of.h>
  23 #include <linux/of_platform.h>
  24 #include <linux/percpu.h>
  25 #include <linux/platform_device.h>
  26 #include <linux/pm.h>
  27 #include <linux/ptrace.h>
  28 #include <linux/preempt.h>
  29 #include <linux/reboot.h>
  30 #include <linux/slab.h>
  31 #include <linux/smp.h>
  32 #include <linux/spinlock.h>
  33 #include <linux/uaccess.h>
  34
  35 /*
  36  * The call to use to reach the firmware.
  37  */
  38 static asmlinkage void (*sdei_firmware_call)(unsigned long function_id,
  39                       unsigned long arg0, unsigned long arg1,
  40                       unsigned long arg2, unsigned long arg3,
  41                       unsigned long arg4, struct arm_smccc_res *res);
  42
  43 /* entry point from firmware to arch asm code */
  44 static unsigned long sdei_entry_point;
  45
  46 struct sdei_event {
  47         /* These three are protected by the sdei_list_lock */
  48         struct list_head        list;
  49         bool                    reregister;
  50         bool                    reenable;
  51
  52         u32                     event_num;
  53         u8                      type;
  54         u8                      priority;
  55
  56         /* This pointer is handed to firmware as the event argument. */
  57         union {
  58                 /* Shared events */
  59                 struct sdei_registered_event *registered;
  60
  61                 /* CPU private events */
  62                 struct sdei_registered_event __percpu *private_registered;
  63         };
  64 };
  65
  66 /* Take the mutex for any API call or modification. Take the mutex first. */
  67 static DEFINE_MUTEX(sdei_events_lock);
  68
  69 /* and then hold this when modifying the list */
  70 static DEFINE_SPINLOCK(sdei_list_lock);
  71 static LIST_HEAD(sdei_list);
  72
  73 /* Private events are registered/enabled via IPI passing one of these */
  74 struct sdei_crosscall_args {
  75         struct sdei_event *event;
  76         atomic_t errors;
  77         int first_error;
  78 };
  79
  80 #define CROSSCALL_INIT(arg, event)      (arg.event = event, \
  81                                          arg.first_error = 0, \
  82                                          atomic_set(&arg.errors, 0))
  83
  84 static inline int sdei_do_cross_call(void *fn, struct sdei_event * event)
  85 {
  86         struct sdei_crosscall_args arg;
  87
  88         CROSSCALL_INIT(arg, event);
  89         on_each_cpu(fn, &arg, true);
  90
  91         return arg.first_error;
  92 }
  93
  94 static inline void
  95 sdei_cross_call_return(struct sdei_crosscall_args *arg, int err)
  96 {
  97         if (err && (atomic_inc_return(&arg->errors) == 1))
  98                 arg->first_error = err;
  99 }
 100
 101 static int sdei_to_linux_errno(unsigned long sdei_err)
 102 {
 103         switch (sdei_err) {
 104         case SDEI_NOT_SUPPORTED:
 105                 return -EOPNOTSUPP;
 106         case SDEI_INVALID_PARAMETERS:
 107                 return -EINVAL;
 108         case SDEI_DENIED:
 109                 return -EPERM;
 110         case SDEI_PENDING:
 111                 return -EINPROGRESS;
 112         case SDEI_OUT_OF_RESOURCE:
 113                 return -ENOMEM;
 114         }
 115
 116         /* Not an error value ... */
 117         return sdei_err;
 118 }
 119
 120 /*
 121  * If x0 is any of these values, then the call failed, use sdei_to_linux_errno()
 122  * to translate.
 123  */
 124 static int sdei_is_err(struct arm_smccc_res *res)
 125 {
 126         switch (res->a0) {
 127         case SDEI_NOT_SUPPORTED:
 128         case SDEI_INVALID_PARAMETERS:
 129         case SDEI_DENIED:
 130         case SDEI_PENDING:
 131         case SDEI_OUT_OF_RESOURCE:
 132                 return true;
 133         }
 134
 135         return false;
 136 }
 137
 138 static int invoke_sdei_fn(unsigned long function_id, unsigned long arg0,
 139                           unsigned long arg1, unsigned long arg2,
 140                           unsigned long arg3, unsigned long arg4,
 141                           u64 *result)
 142 {
 143         int err = 0;
 144         struct arm_smccc_res res;
 145
 146         if (sdei_firmware_call) {
 147                 sdei_firmware_call(function_id, arg0, arg1, arg2, arg3, arg4,
 148                                    &res);
 149                 if (sdei_is_err(&res))
 150                         err = sdei_to_linux_errno(res.a0);
 151         } else {
 152                 /*
 153                  * !sdei_firmware_call means we failed to probe or called
 154                  * sdei_mark_interface_broken(). -EIO is not an error returned
 155                  * by sdei_to_linux_errno() and is used to suppress messages
 156                  * from this driver.
 157                  */
 158                 err = -EIO;
 159                 res.a0 = SDEI_NOT_SUPPORTED;
 160         }
 161
 162         if (result)
 163                 *result = res.a0;
 164
 165         return err;
 166 }
 167
 168 static struct sdei_event *sdei_event_find(u32 event_num)
 169 {
 170         struct sdei_event *e, *found = NULL;
 171
 172         lockdep_assert_held(&sdei_events_lock);
 173
 174         spin_lock(&sdei_list_lock);
 175         list_for_each_entry(e, &sdei_list, list) {
 176                 if (e->event_num == event_num) {
 177                         found = e;
 178                         break;
 179                 }
 180         }
 181         spin_unlock(&sdei_list_lock);
 182
 183         return found;
 184 }
 185
 186 int sdei_api_event_context(u32 query, u64 *result)
 187 {
 188         return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_CONTEXT, query, 0, 0, 0, 0,
 189                               result);
 190 }
 191 NOKPROBE_SYMBOL(sdei_api_event_context);
 192
 193 static int sdei_api_event_get_info(u32 event, u32 info, u64 *result)
 194 {
 195         return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_GET_INFO, event, info, 0,
 196                               0, 0, result);
 197 }
 198
 199 static struct sdei_event *sdei_event_create(u32 event_num,
 200                                             sdei_event_callback *cb,
 201                                             void *cb_arg)
 202 {
 203         int err;
 204         u64 result;
 205         struct sdei_event *event;
 206         struct sdei_registered_event *reg;
 207
 208         lockdep_assert_held(&sdei_events_lock);
 209
 210         event = kzalloc(sizeof(*event), GFP_KERNEL);
 211         if (!event)
 212                 return ERR_PTR(-ENOMEM);
 213
 214         INIT_LIST_HEAD(&event->list);
 215         event->event_num = event_num;
 216
 217         err = sdei_api_event_get_info(event_num, SDEI_EVENT_INFO_EV_PRIORITY,
 218                                       &result);
 219         if (err) {
 220                 kfree(event);
 221                 return ERR_PTR(err);
 222         }
 223         event->priority = result;
 224
 225         err = sdei_api_event_get_info(event_num, SDEI_EVENT_INFO_EV_TYPE,
 226                                       &result);
 227         if (err) {
 228                 kfree(event);
 229                 return ERR_PTR(err);
 230         }
 231         event->type = result;
 232
 233         if (event->type == SDEI_EVENT_TYPE_SHARED) {
 234                 reg = kzalloc(sizeof(*reg), GFP_KERNEL);
 235                 if (!reg) {
 236                         kfree(event);
 237                         return ERR_PTR(-ENOMEM);
 238                 }
 239
 240                 reg->event_num = event_num;
 241                 reg->priority = event->priority;
 242
 243                 reg->callback = cb;
 244                 reg->callback_arg = cb_arg;
 245                 event->registered = reg;
 246         } else {
 247                 int cpu;
 248                 struct sdei_registered_event __percpu *regs;
 249
 250                 regs = alloc_percpu(struct sdei_registered_event);
 251                 if (!regs) {
 252                         kfree(event);
 253                         return ERR_PTR(-ENOMEM);
 254                 }
 255
 256                 for_each_possible_cpu(cpu) {
 257                         reg = per_cpu_ptr(regs, cpu);
 258
 259                         reg->event_num = event->event_num;
 260                         reg->priority = event->priority;
 261                         reg->callback = cb;
 262                         reg->callback_arg = cb_arg;
 263                 }
 264
 265                 event->private_registered = regs;
 266         }
 267
 268         if (sdei_event_find(event_num)) {
 269                 kfree(event->registered);
 270                 kfree(event);
 271                 event = ERR_PTR(-EBUSY);
 272         } else {
 273                 spin_lock(&sdei_list_lock);
 274                 list_add(&event->list, &sdei_list);
 275                 spin_unlock(&sdei_list_lock);
 276         }
 277
 278         return event;
 279 }
 280
 281 static void sdei_event_destroy(struct sdei_event *event)
 282 {
 283         lockdep_assert_held(&sdei_events_lock);
 284
 285         spin_lock(&sdei_list_lock);
 286         list_del(&event->list);
 287         spin_unlock(&sdei_list_lock);
 288
 289         if (event->type == SDEI_EVENT_TYPE_SHARED)
 290                 kfree(event->registered);
 291         else
 292                 free_percpu(event->private_registered);
 293
 294         kfree(event);
 295 }
 296
 297 static int sdei_api_get_version(u64 *version)
 298 {
 299         return invoke_sdei_fn(SDEI_1_0_FN_SDEI_VERSION, 0, 0, 0, 0, 0, version);
 300 }
 301
 302 int sdei_mask_local_cpu(void)
 303 {
 304         int err;
 305
 306         WARN_ON_ONCE(preemptible());
 307
 308         err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PE_MASK, 0, 0, 0, 0, 0, NULL);
 309         if (err && err != -EIO) {
 310                 pr_warn_once("failed to mask CPU[%u]: %d\n",
 311                               smp_processor_id(), err);
 312                 return err;
 313         }
 314
 315         return 0;
 316 }
 317
 318 static void _ipi_mask_cpu(void *ignored)
 319 {
 320         sdei_mask_local_cpu();
 321 }
 322
 323 int sdei_unmask_local_cpu(void)
 324 {
 325         int err;
 326
 327         WARN_ON_ONCE(preemptible());
 328
 329         err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PE_UNMASK, 0, 0, 0, 0, 0, NULL);
 330         if (err && err != -EIO) {
 331                 pr_warn_once("failed to unmask CPU[%u]: %d\n",
 332                              smp_processor_id(), err);
 333                 return err;
 334         }
 335
 336         return 0;
 337 }
 338
 339 static void _ipi_unmask_cpu(void *ignored)
 340 {
 341         sdei_unmask_local_cpu();
 342 }
 343
 344 static void _ipi_private_reset(void *ignored)
 345 {
 346         int err;
 347
 348         err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PRIVATE_RESET, 0, 0, 0, 0, 0,
 349                              NULL);
 350         if (err && err != -EIO)
 351                 pr_warn_once("failed to reset CPU[%u]: %d\n",
 352                              smp_processor_id(), err);
 353 }
 354
 355 static int sdei_api_shared_reset(void)
 356 {
 357         return invoke_sdei_fn(SDEI_1_0_FN_SDEI_SHARED_RESET, 0, 0, 0, 0, 0,
 358                               NULL);
 359 }
 360
 361 static void sdei_mark_interface_broken(void)
 362 {
 363         pr_err("disabling SDEI firmware interface\n");
 364         on_each_cpu(&_ipi_mask_cpu, NULL, true);
 365         sdei_firmware_call = NULL;
 366 }
 367
 368 static int sdei_platform_reset(void)
 369 {
 370         int err;
 371
 372         on_each_cpu(&_ipi_private_reset, NULL, true);
 373         err = sdei_api_shared_reset();
 374         if (err) {
 375                 pr_err("Failed to reset platform: %d\n", err);
 376                 sdei_mark_interface_broken();
 377         }
 378
 379         return err;
 380 }
 381
 382 static int sdei_api_event_enable(u32 event_num)
 383 {
 384         return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_ENABLE, event_num, 0, 0, 0,
 385                               0, NULL);
 386 }
 387
 388 /* Called directly by the hotplug callbacks */
 389 static void _local_event_enable(void *data)
 390 {
 391         int err;
 392         struct sdei_crosscall_args *arg = data;
 393
 394         WARN_ON_ONCE(preemptible());
 395
 396         err = sdei_api_event_enable(arg->event->event_num);
 397
 398         sdei_cross_call_return(arg, err);
 399 }
 400
 401 int sdei_event_enable(u32 event_num)
 402 {
 403         int err = -EINVAL;
 404         struct sdei_event *event;
 405
 406         mutex_lock(&sdei_events_lock);
 407         event = sdei_event_find(event_num);
 408         if (!event) {
 409                 mutex_unlock(&sdei_events_lock);
 410                 return -ENOENT;
 411         }
 412
 413         spin_lock(&sdei_list_lock);
 414         event->reenable = true;
 415         spin_unlock(&sdei_list_lock);
 416
 417         if (event->type == SDEI_EVENT_TYPE_SHARED)
 418                 err = sdei_api_event_enable(event->event_num);
 419         else
 420                 err = sdei_do_cross_call(_local_event_enable, event);
 421         mutex_unlock(&sdei_events_lock);
 422
 423         return err;
 424 }
 425 EXPORT_SYMBOL(sdei_event_enable);
 426
 427 static int sdei_api_event_disable(u32 event_num)
 428 {
 429         return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_DISABLE, event_num, 0, 0,
 430                               0, 0, NULL);
 431 }
 432
 433 static void _ipi_event_disable(void *data)
 434 {
 435         int err;
 436         struct sdei_crosscall_args *arg = data;
 437
 438         err = sdei_api_event_disable(arg->event->event_num);
 439
 440         sdei_cross_call_return(arg, err);
 441 }
 442
 443 int sdei_event_disable(u32 event_num)
 444 {
 445         int err = -EINVAL;
 446         struct sdei_event *event;
 447
 448         mutex_lock(&sdei_events_lock);
 449         event = sdei_event_find(event_num);
 450         if (!event) {
 451                 mutex_unlock(&sdei_events_lock);
 452                 return -ENOENT;
 453         }
 454
 455         spin_lock(&sdei_list_lock);
 456         event->reenable = false;
 457         spin_unlock(&sdei_list_lock);
 458
 459         if (event->type == SDEI_EVENT_TYPE_SHARED)
 460                 err = sdei_api_event_disable(event->event_num);
 461         else
 462                 err = sdei_do_cross_call(_ipi_event_disable, event);
 463         mutex_unlock(&sdei_events_lock);
 464
 465         return err;
 466 }
 467 EXPORT_SYMBOL(sdei_event_disable);
 468
 469 static int sdei_api_event_unregister(u32 event_num)
 470 {
 471         return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_UNREGISTER, event_num, 0,
 472                               0, 0, 0, NULL);
 473 }
 474
 475 /* Called directly by the hotplug callbacks */
 476 static void _local_event_unregister(void *data)
 477 {
 478         int err;
 479         struct sdei_crosscall_args *arg = data;
 480
 481         WARN_ON_ONCE(preemptible());
 482
 483         err = sdei_api_event_unregister(arg->event->event_num);
 484
 485         sdei_cross_call_return(arg, err);
 486 }
 487
 488 static int _sdei_event_unregister(struct sdei_event *event)
 489 {
 490         lockdep_assert_held(&sdei_events_lock);
 491
 492         if (event->type == SDEI_EVENT_TYPE_SHARED)
 493                 return sdei_api_event_unregister(event->event_num);
 494
 495         return sdei_do_cross_call(_local_event_unregister, event);
 496 }
 497
 498 int sdei_event_unregister(u32 event_num)
 499 {
 500         int err;
 501         struct sdei_event *event;
 502
 503         WARN_ON(in_nmi());
 504
 505         mutex_lock(&sdei_events_lock);
 506         event = sdei_event_find(event_num);
 507         do {
 508                 if (!event) {
 509                         pr_warn("Event %u not registered\n", event_num);
 510                         err = -ENOENT;
 511                         break;
 512                 }
 513
 514                 spin_lock(&sdei_list_lock);
 515                 event->reregister = false;
 516                 event->reenable = false;
 517                 spin_unlock(&sdei_list_lock);
 518
 519                 err = _sdei_event_unregister(event);
 520                 if (err)
 521                         break;
 522
 523                 sdei_event_destroy(event);
 524         } while (0);
 525         mutex_unlock(&sdei_events_lock);
 526
 527         return err;
 528 }
 529 EXPORT_SYMBOL(sdei_event_unregister);
 530
 531 /*
 532  * unregister events, but don't destroy them as they are re-registered by
 533  * sdei_reregister_shared().
 534  */
 535 static int sdei_unregister_shared(void)
 536 {
 537         int err = 0;
 538         struct sdei_event *event;
 539
 540         mutex_lock(&sdei_events_lock);
 541         spin_lock(&sdei_list_lock);
 542         list_for_each_entry(event, &sdei_list, list) {
 543                 if (event->type != SDEI_EVENT_TYPE_SHARED)
 544                         continue;
 545
 546                 err = _sdei_event_unregister(event);
 547                 if (err)
 548                         break;
 549         }
 550         spin_unlock(&sdei_list_lock);
 551         mutex_unlock(&sdei_events_lock);
 552
 553         return err;
 554 }
 555
 556 static int sdei_api_event_register(u32 event_num, unsigned long entry_point,
 557                                    void *arg, u64 flags, u64 affinity)
 558 {
 559         return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_REGISTER, event_num,
 560                               (unsigned long)entry_point, (unsigned long)arg,
 561                               flags, affinity, NULL);
 562 }
 563
 564 /* Called directly by the hotplug callbacks */
 565 static void _local_event_register(void *data)
 566 {
 567         int err;
 568         struct sdei_registered_event *reg;
 569         struct sdei_crosscall_args *arg = data;
 570
 571         WARN_ON(preemptible());
 572
 573         reg = per_cpu_ptr(arg->event->private_registered, smp_processor_id());
 574         err = sdei_api_event_register(arg->event->event_num, sdei_entry_point,
 575                                       reg, 0, 0);
 576
 577         sdei_cross_call_return(arg, err);
 578 }
 579
 580 static int _sdei_event_register(struct sdei_event *event)
 581 {
 582         int err;
 583
 584         lockdep_assert_held(&sdei_events_lock);
 585
 586         if (event->type == SDEI_EVENT_TYPE_SHARED)
 587                 return sdei_api_event_register(event->event_num,
 588                                                sdei_entry_point,
 589                                                event->registered,
 590                                                SDEI_EVENT_REGISTER_RM_ANY, 0);
 591
 592         err = sdei_do_cross_call(_local_event_register, event);
 593         if (err)
 594                 sdei_do_cross_call(_local_event_unregister, event);
 595
 596         return err;
 597 }
 598
 599 int sdei_event_register(u32 event_num, sdei_event_callback *cb, void *arg)
 600 {
 601         int err;
 602         struct sdei_event *event;
 603
 604         WARN_ON(in_nmi());
 605
 606         mutex_lock(&sdei_events_lock);
 607         do {
 608                 if (sdei_event_find(event_num)) {
 609                         pr_warn("Event %u already registered\n", event_num);
 610                         err = -EBUSY;
 611                         break;
 612                 }
 613
 614                 event = sdei_event_create(event_num, cb, arg);
 615                 if (IS_ERR(event)) {
 616                         err = PTR_ERR(event);
 617                         pr_warn("Failed to create event %u: %d\n", event_num,
 618                                 err);
 619                         break;
 620                 }
 621
 622                 spin_lock(&sdei_list_lock);
 623                 event->reregister = true;
 624                 spin_unlock(&sdei_list_lock);
 625
 626                 err = _sdei_event_register(event);
 627                 if (err) {
 628                         spin_lock(&sdei_list_lock);
 629                         event->reregister = false;
 630                         event->reenable = false;
 631                         spin_unlock(&sdei_list_lock);
 632
 633                         sdei_event_destroy(event);
 634                         pr_warn("Failed to register event %u: %d\n", event_num,
 635                                 err);
 636                 }
 637         } while (0);
 638         mutex_unlock(&sdei_events_lock);
 639
 640         return err;
 641 }
 642 EXPORT_SYMBOL(sdei_event_register);
 643
 644 static int sdei_reregister_event(struct sdei_event *event)
 645 {
 646         int err;
 647
 648         lockdep_assert_held(&sdei_events_lock);
 649
 650         err = _sdei_event_register(event);
 651         if (err) {
 652                 pr_err("Failed to re-register event %u\n", event->event_num);
 653                 sdei_event_destroy(event);
 654                 return err;
 655         }
 656
 657         if (event->reenable) {
 658                 if (event->type == SDEI_EVENT_TYPE_SHARED)
 659                         err = sdei_api_event_enable(event->event_num);
 660                 else
 661                         err = sdei_do_cross_call(_local_event_enable, event);
 662         }
 663
 664         if (err)
 665                 pr_err("Failed to re-enable event %u\n", event->event_num);
 666
 667         return err;
 668 }
 669
 670 static int sdei_reregister_shared(void)
 671 {
 672         int err = 0;
 673         struct sdei_event *event;
 674
 675         mutex_lock(&sdei_events_lock);
 676         spin_lock(&sdei_list_lock);
 677         list_for_each_entry(event, &sdei_list, list) {
 678                 if (event->type != SDEI_EVENT_TYPE_SHARED)
 679                         continue;
 680
 681                 if (event->reregister) {
 682                         err = sdei_reregister_event(event);
 683                         if (err)
 684                                 break;
 685                 }
 686         }
 687         spin_unlock(&sdei_list_lock);
 688         mutex_unlock(&sdei_events_lock);
 689
 690         return err;
 691 }
 692
 693 static int sdei_cpuhp_down(unsigned int cpu)
 694 {
 695         struct sdei_event *event;
 696         struct sdei_crosscall_args arg;
 697
 698         /* un-register private events */
 699         spin_lock(&sdei_list_lock);
 700         list_for_each_entry(event, &sdei_list, list) {
 701                 if (event->type == SDEI_EVENT_TYPE_SHARED)
 702                         continue;
 703
 704                 CROSSCALL_INIT(arg, event);
 705                 /* call the cross-call function locally... */
 706                 _local_event_unregister(&arg);
 707                 if (arg.first_error)
 708                         pr_err("Failed to unregister event %u: %d\n",
 709                                event->event_num, arg.first_error);
 710         }
 711         spin_unlock(&sdei_list_lock);
 712
 713         return sdei_mask_local_cpu();
 714 }
 715
 716 static int sdei_cpuhp_up(unsigned int cpu)
 717 {
 718         struct sdei_event *event;
 719         struct sdei_crosscall_args arg;
 720
 721         /* re-register/enable private events */
 722         spin_lock(&sdei_list_lock);
 723         list_for_each_entry(event, &sdei_list, list) {
 724                 if (event->type == SDEI_EVENT_TYPE_SHARED)
 725                         continue;
 726
 727                 if (event->reregister) {
 728                         CROSSCALL_INIT(arg, event);
 729                         /* call the cross-call function locally... */
 730                         _local_event_register(&arg);
 731                         if (arg.first_error)
 732                                 pr_err("Failed to re-register event %u: %d\n",
 733                                        event->event_num, arg.first_error);
 734                 }
 735
 736                 if (event->reenable) {
 737                         CROSSCALL_INIT(arg, event);
 738                         _local_event_enable(&arg);
 739                         if (arg.first_error)
 740                                 pr_err("Failed to re-enable event %u: %d\n",
 741                                        event->event_num, arg.first_error);
 742                 }
 743         }
 744         spin_unlock(&sdei_list_lock);
 745
 746         return sdei_unmask_local_cpu();
 747 }
 748
 749 /* When entering idle, mask/unmask events for this cpu */
 750 static int sdei_pm_notifier(struct notifier_block *nb, unsigned long action,
 751                             void *data)
 752 {
 753         int rv;
 754
 755         switch (action) {
 756         case CPU_PM_ENTER:
 757                 rv = sdei_mask_local_cpu();
 758                 break;
 759         case CPU_PM_EXIT:
 760         case CPU_PM_ENTER_FAILED:
 761                 rv = sdei_unmask_local_cpu();
 762                 break;
 763         default:
 764                 return NOTIFY_DONE;
 765         }
 766
 767         if (rv)
 768                 return notifier_from_errno(rv);
 769
 770         return NOTIFY_OK;
 771 }
 772
 773 static struct notifier_block sdei_pm_nb = {
 774         .notifier_call = sdei_pm_notifier,
 775 };
 776
 777 static int sdei_device_suspend(struct device *dev)
 778 {
 779         on_each_cpu(_ipi_mask_cpu, NULL, true);
 780
 781         return 0;
 782 }
 783
 784 static int sdei_device_resume(struct device *dev)
 785 {
 786         on_each_cpu(_ipi_unmask_cpu, NULL, true);
 787
 788         return 0;
 789 }
 790
 791 /*
 792  * We need all events to be reregistered when we resume from hibernate.
 793  *
 794  * The sequence is freeze->thaw. Reboot. freeze->restore. We unregister
 795  * events during freeze, then re-register and re-enable them during thaw
 796  * and restore.
 797  */
 798 static int sdei_device_freeze(struct device *dev)
 799 {
 800         int err;
 801
 802         /* unregister private events */
 803         cpuhp_remove_state(CPUHP_AP_ARM_SDEI_STARTING);
 804
 805         err = sdei_unregister_shared();
 806         if (err)
 807                 return err;
 808
 809         return 0;
 810 }
 811
 812 static int sdei_device_thaw(struct device *dev)
 813 {
 814         int err;
 815
 816         /* re-register shared events */
 817         err = sdei_reregister_shared();
 818         if (err) {
 819                 pr_warn("Failed to re-register shared events...\n");
 820                 sdei_mark_interface_broken();
 821                 return err;
 822         }
 823
 824         err = cpuhp_setup_state(CPUHP_AP_ARM_SDEI_STARTING, "SDEI",
 825                                 &sdei_cpuhp_up, &sdei_cpuhp_down);
 826         if (err)
 827                 pr_warn("Failed to re-register CPU hotplug notifier...\n");
 828
 829         return err;
 830 }
 831
 832 static int sdei_device_restore(struct device *dev)
 833 {
 834         int err;
 835
 836         err = sdei_platform_reset();
 837         if (err)
 838                 return err;
 839
 840         return sdei_device_thaw(dev);
 841 }
 842
 843 static const struct dev_pm_ops sdei_pm_ops = {
 844         .suspend = sdei_device_suspend,
 845         .resume = sdei_device_resume,
 846         .freeze = sdei_device_freeze,
 847         .thaw = sdei_device_thaw,
 848         .restore = sdei_device_restore,
 849 };
 850
 851 /*
 852  * Mask all CPUs and unregister all events on panic, reboot or kexec.
 853  */
 854 static int sdei_reboot_notifier(struct notifier_block *nb, unsigned long action,
 855                                 void *data)
 856 {
 857         /*
 858          * We are going to reset the interface, after this there is no point
 859          * doing work when we take CPUs offline.
 860          */
 861         cpuhp_remove_state(CPUHP_AP_ARM_SDEI_STARTING);
 862
 863         sdei_platform_reset();
 864
 865         return NOTIFY_OK;
 866 }
 867
 868 static struct notifier_block sdei_reboot_nb = {
 869         .notifier_call = sdei_reboot_notifier,
 870 };
 871
 872 static void sdei_smccc_smc(unsigned long function_id,
 873                            unsigned long arg0, unsigned long arg1,
 874                            unsigned long arg2, unsigned long arg3,
 875                            unsigned long arg4, struct arm_smccc_res *res)
 876 {
 877         arm_smccc_smc(function_id, arg0, arg1, arg2, arg3, arg4, 0, 0, res);
 878 }
 879
 880 static void sdei_smccc_hvc(unsigned long function_id,
 881                            unsigned long arg0, unsigned long arg1,
 882                            unsigned long arg2, unsigned long arg3,
 883                            unsigned long arg4, struct arm_smccc_res *res)
 884 {
 885         arm_smccc_hvc(function_id, arg0, arg1, arg2, arg3, arg4, 0, 0, res);
 886 }
 887
 888 static int sdei_get_conduit(struct platform_device *pdev)
 889 {
 890         const char *method;
 891         struct device_node *np = pdev->dev.of_node;
 892
 893         sdei_firmware_call = NULL;
 894         if (np) {
 895                 if (of_property_read_string(np, "method", &method)) {
 896                         pr_warn("missing \"method\" property\n");
 897                         return CONDUIT_INVALID;
 898                 }
 899
 900                 if (!strcmp("hvc", method)) {
 901                         sdei_firmware_call = &sdei_smccc_hvc;
 902                         return CONDUIT_HVC;
 903                 } else if (!strcmp("smc", method)) {
 904                         sdei_firmware_call = &sdei_smccc_smc;
 905                         return CONDUIT_SMC;
 906                 }
 907
 908                 pr_warn("invalid \"method\" property: %s\n", method);
 909         } else if (IS_ENABLED(CONFIG_ACPI) && !acpi_disabled) {
 910                 if (acpi_psci_use_hvc()) {
 911                         sdei_firmware_call = &sdei_smccc_hvc;
 912                         return CONDUIT_HVC;
 913                 } else {
 914                         sdei_firmware_call = &sdei_smccc_smc;
 915                         return CONDUIT_SMC;
 916                 }
 917         }
 918
 919         return CONDUIT_INVALID;
 920 }
 921
 922 static int sdei_probe(struct platform_device *pdev)
 923 {
 924         int err;
 925         u64 ver = 0;
 926         int conduit;
 927
 928         conduit = sdei_get_conduit(pdev);
 929         if (!sdei_firmware_call)
 930                 return 0;
 931
 932         err = sdei_api_get_version(&ver);
 933         if (err == -EOPNOTSUPP)
 934                 pr_err("advertised but not implemented in platform firmware\n");
 935         if (err) {
 936                 pr_err("Failed to get SDEI version: %d\n", err);
 937                 sdei_mark_interface_broken();
 938                 return err;
 939         }
 940
 941         pr_info("SDEIv%d.%d (0x%x) detected in firmware.\n",
 942                 (int)SDEI_VERSION_MAJOR(ver), (int)SDEI_VERSION_MINOR(ver),
 943                 (int)SDEI_VERSION_VENDOR(ver));
 944
 945         if (SDEI_VERSION_MAJOR(ver) != 1) {
 946                 pr_warn("Conflicting SDEI version detected.\n");
 947                 sdei_mark_interface_broken();
 948                 return -EINVAL;
 949         }
 950
 951         err = sdei_platform_reset();
 952         if (err)
 953                 return err;
 954
 955         sdei_entry_point = sdei_arch_get_entry_point(conduit);
 956         if (!sdei_entry_point) {
 957                 /* Not supported due to hardware or boot configuration */
 958                 sdei_mark_interface_broken();
 959                 return 0;
 960         }
 961
 962         err = cpu_pm_register_notifier(&sdei_pm_nb);
 963         if (err) {
 964                 pr_warn("Failed to register CPU PM notifier...\n");
 965                 goto error;
 966         }
 967
 968         err = register_reboot_notifier(&sdei_reboot_nb);
 969         if (err) {
 970                 pr_warn("Failed to register reboot notifier...\n");
 971                 goto remove_cpupm;
 972         }
 973
 974         err = cpuhp_setup_state(CPUHP_AP_ARM_SDEI_STARTING, "SDEI",
 975                                 &sdei_cpuhp_up, &sdei_cpuhp_down);
 976         if (err) {
 977                 pr_warn("Failed to register CPU hotplug notifier...\n");
 978                 goto remove_reboot;
 979         }
 980
 981         return 0;
 982
 983 remove_reboot:
 984         unregister_reboot_notifier(&sdei_reboot_nb);
 985
 986 remove_cpupm:
 987         cpu_pm_unregister_notifier(&sdei_pm_nb);
 988
 989 error:
 990         sdei_mark_interface_broken();
 991         return err;
 992 }
 993
 994 static const struct of_device_id sdei_of_match[] = {
 995         { .compatible = "arm,sdei-1.0" },
 996         {}
 997 };
 998
 999 static struct platform_driver sdei_driver = {
1000         .driver         = {
1001                 .name                   = "sdei",
1002                 .pm                     = &sdei_pm_ops,
1003                 .of_match_table         = sdei_of_match,
1004         },
1005         .probe          = sdei_probe,
1006 };
1007
1008 static bool __init sdei_present_dt(void)
1009 {
1010         struct device_node *np, *fw_np;
1011
1012         fw_np = of_find_node_by_name(NULL, "firmware");
1013         if (!fw_np)
1014                 return false;
1015
1016         np = of_find_matching_node(fw_np, sdei_of_match);
1017         if (!np)
1018                 return false;
1019         of_node_put(np);
1020
1021         return true;
1022 }
1023
1024 static bool __init sdei_present_acpi(void)
1025 {
1026         acpi_status status;
1027         struct platform_device *pdev;
1028         struct acpi_table_header *sdei_table_header;
1029
1030         if (acpi_disabled)
1031                 return false;
1032
1033         status = acpi_get_table(ACPI_SIG_SDEI, 0, &sdei_table_header);
1034         if (ACPI_FAILURE(status) && status != AE_NOT_FOUND) {
1035                 const char *msg = acpi_format_exception(status);
1036
1037                 pr_info("Failed to get ACPI:SDEI table, %s\n", msg);
1038         }
1039         if (ACPI_FAILURE(status))
1040                 return false;
1041
1042         pdev = platform_device_register_simple(sdei_driver.driver.name, 0, NULL,
1043                                                0);
1044         if (IS_ERR(pdev))
1045                 return false;
1046
1047         return true;
1048 }
1049
1050 static int __init sdei_init(void)
1051 {
1052         if (sdei_present_dt() || sdei_present_acpi())
1053                 platform_driver_register(&sdei_driver);
1054
1055         return 0;
1056 }
1057
1058 /*
1059  * On an ACPI system SDEI needs to be ready before HEST:GHES tries to register
1060  * its events. ACPI is initialised from a subsys_initcall(), GHES is initialised
1061  * by device_initcall(). We want to be called in the middle.
1062  */
1063 subsys_initcall_sync(sdei_init);
1064
1065 int sdei_event_handler(struct pt_regs *regs,
1066                        struct sdei_registered_event *arg)
1067 {
1068         int err;
1069         mm_segment_t orig_addr_limit;
1070         u32 event_num = arg->event_num;
1071
1072         orig_addr_limit = get_fs();
1073         set_fs(USER_DS);
1074
1075         err = arg->callback(event_num, regs, arg->callback_arg);
1076         if (err)
1077                 pr_err_ratelimited("event %u on CPU %u failed with error: %d\n",
1078                                    event_num, smp_processor_id(), err);
1079
1080         set_fs(orig_addr_limit);
1081
1082         return err;
1083 }
1084 NOKPROBE_SYMBOL(sdei_event_handler);