1 // SPDX-License-Identifier: GPL-2.0-only
3 * SBI initialilization and all extension implementation.
5 * Copyright (c) 2020 Western Digital Corporation or its affiliates.
8 #include <linux/bits.h>
9 #include <linux/init.h>
12 #include <linux/reboot.h>
15 #include <asm/tlbflush.h>
17 /* default SBI version is 0.1 */
18 unsigned long sbi_spec_version __ro_after_init
= SBI_SPEC_VERSION_DEFAULT
;
19 EXPORT_SYMBOL(sbi_spec_version
);
21 static void (*__sbi_set_timer
)(uint64_t stime
) __ro_after_init
;
22 static void (*__sbi_send_ipi
)(unsigned int cpu
) __ro_after_init
;
23 static int (*__sbi_rfence
)(int fid
, const struct cpumask
*cpu_mask
,
24 unsigned long start
, unsigned long size
,
25 unsigned long arg4
, unsigned long arg5
) __ro_after_init
;
27 #ifdef CONFIG_RISCV_SBI_V01
28 static unsigned long __sbi_v01_cpumask_to_hartmask(const struct cpumask
*cpu_mask
)
30 unsigned long cpuid
, hartid
;
31 unsigned long hmask
= 0;
34 * There is no maximum hartid concept in RISC-V and NR_CPUS must not be
35 * associated with hartid. As SBI v0.1 is only kept for backward compatibility
36 * and will be removed in the future, there is no point in supporting hartid
37 * greater than BITS_PER_LONG (32 for RV32 and 64 for RV64). Ideally, SBI v0.2
38 * should be used for platforms with hartid greater than BITS_PER_LONG.
40 for_each_cpu(cpuid
, cpu_mask
) {
41 hartid
= cpuid_to_hartid_map(cpuid
);
42 if (hartid
>= BITS_PER_LONG
) {
43 pr_warn("Unable to send any request to hartid > BITS_PER_LONG for SBI v0.1\n");
53 * sbi_console_putchar() - Writes given character to the console device.
54 * @ch: The data to be written to the console.
58 void sbi_console_putchar(int ch
)
60 sbi_ecall(SBI_EXT_0_1_CONSOLE_PUTCHAR
, 0, ch
, 0, 0, 0, 0, 0);
62 EXPORT_SYMBOL(sbi_console_putchar
);
65 * sbi_console_getchar() - Reads a byte from console device.
67 * Returns the value read from console.
69 int sbi_console_getchar(void)
73 ret
= sbi_ecall(SBI_EXT_0_1_CONSOLE_GETCHAR
, 0, 0, 0, 0, 0, 0, 0);
77 EXPORT_SYMBOL(sbi_console_getchar
);
80 * sbi_shutdown() - Remove all the harts from executing supervisor code.
84 void sbi_shutdown(void)
86 sbi_ecall(SBI_EXT_0_1_SHUTDOWN
, 0, 0, 0, 0, 0, 0, 0);
88 EXPORT_SYMBOL(sbi_shutdown
);
91 * __sbi_set_timer_v01() - Program the timer for next timer event.
92 * @stime_value: The value after which next timer event should fire.
96 static void __sbi_set_timer_v01(uint64_t stime_value
)
98 #if __riscv_xlen == 32
99 sbi_ecall(SBI_EXT_0_1_SET_TIMER
, 0, stime_value
,
100 stime_value
>> 32, 0, 0, 0, 0);
102 sbi_ecall(SBI_EXT_0_1_SET_TIMER
, 0, stime_value
, 0, 0, 0, 0, 0);
106 static void __sbi_send_ipi_v01(unsigned int cpu
)
108 unsigned long hart_mask
=
109 __sbi_v01_cpumask_to_hartmask(cpumask_of(cpu
));
110 sbi_ecall(SBI_EXT_0_1_SEND_IPI
, 0, (unsigned long)(&hart_mask
),
114 static int __sbi_rfence_v01(int fid
, const struct cpumask
*cpu_mask
,
115 unsigned long start
, unsigned long size
,
116 unsigned long arg4
, unsigned long arg5
)
119 unsigned long hart_mask
;
121 if (!cpu_mask
|| cpumask_empty(cpu_mask
))
122 cpu_mask
= cpu_online_mask
;
123 hart_mask
= __sbi_v01_cpumask_to_hartmask(cpu_mask
);
125 /* v0.2 function IDs are equivalent to v0.1 extension IDs */
127 case SBI_EXT_RFENCE_REMOTE_FENCE_I
:
128 sbi_ecall(SBI_EXT_0_1_REMOTE_FENCE_I
, 0,
129 (unsigned long)&hart_mask
, 0, 0, 0, 0, 0);
131 case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA
:
132 sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA
, 0,
133 (unsigned long)&hart_mask
, start
, size
,
136 case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID
:
137 sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA_ASID
, 0,
138 (unsigned long)&hart_mask
, start
, size
,
142 pr_err("SBI call [%d]not supported in SBI v0.1\n", fid
);
149 static void sbi_set_power_off(void)
151 pm_power_off
= sbi_shutdown
;
154 static void __sbi_set_timer_v01(uint64_t stime_value
)
156 pr_warn("Timer extension is not available in SBI v%lu.%lu\n",
157 sbi_major_version(), sbi_minor_version());
160 static void __sbi_send_ipi_v01(unsigned int cpu
)
162 pr_warn("IPI extension is not available in SBI v%lu.%lu\n",
163 sbi_major_version(), sbi_minor_version());
166 static int __sbi_rfence_v01(int fid
, const struct cpumask
*cpu_mask
,
167 unsigned long start
, unsigned long size
,
168 unsigned long arg4
, unsigned long arg5
)
170 pr_warn("remote fence extension is not available in SBI v%lu.%lu\n",
171 sbi_major_version(), sbi_minor_version());
176 static void sbi_set_power_off(void) {}
177 #endif /* CONFIG_RISCV_SBI_V01 */
179 static void __sbi_set_timer_v02(uint64_t stime_value
)
181 #if __riscv_xlen == 32
182 sbi_ecall(SBI_EXT_TIME
, SBI_EXT_TIME_SET_TIMER
, stime_value
,
183 stime_value
>> 32, 0, 0, 0, 0);
185 sbi_ecall(SBI_EXT_TIME
, SBI_EXT_TIME_SET_TIMER
, stime_value
, 0,
190 static void __sbi_send_ipi_v02(unsigned int cpu
)
193 struct sbiret ret
= {0};
195 ret
= sbi_ecall(SBI_EXT_IPI
, SBI_EXT_IPI_SEND_IPI
,
196 1UL, cpuid_to_hartid_map(cpu
), 0, 0, 0, 0);
198 result
= sbi_err_map_linux_errno(ret
.error
);
199 pr_err("%s: hbase = [%lu] failed (error [%d])\n",
200 __func__
, cpuid_to_hartid_map(cpu
), result
);
204 static int __sbi_rfence_v02_call(unsigned long fid
, unsigned long hmask
,
205 unsigned long hbase
, unsigned long start
,
206 unsigned long size
, unsigned long arg4
,
209 struct sbiret ret
= {0};
210 int ext
= SBI_EXT_RFENCE
;
214 case SBI_EXT_RFENCE_REMOTE_FENCE_I
:
215 ret
= sbi_ecall(ext
, fid
, hmask
, hbase
, 0, 0, 0, 0);
217 case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA
:
218 ret
= sbi_ecall(ext
, fid
, hmask
, hbase
, start
,
221 case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID
:
222 ret
= sbi_ecall(ext
, fid
, hmask
, hbase
, start
,
226 case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA
:
227 ret
= sbi_ecall(ext
, fid
, hmask
, hbase
, start
,
230 case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID
:
231 ret
= sbi_ecall(ext
, fid
, hmask
, hbase
, start
,
234 case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA
:
235 ret
= sbi_ecall(ext
, fid
, hmask
, hbase
, start
,
238 case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID
:
239 ret
= sbi_ecall(ext
, fid
, hmask
, hbase
, start
,
243 pr_err("unknown function ID [%lu] for SBI extension [%d]\n",
249 result
= sbi_err_map_linux_errno(ret
.error
);
250 pr_err("%s: hbase = [%lu] hmask = [0x%lx] failed (error [%d])\n",
251 __func__
, hbase
, hmask
, result
);
257 static int __sbi_rfence_v02(int fid
, const struct cpumask
*cpu_mask
,
258 unsigned long start
, unsigned long size
,
259 unsigned long arg4
, unsigned long arg5
)
261 unsigned long hartid
, cpuid
, hmask
= 0, hbase
= 0, htop
= 0;
264 if (!cpu_mask
|| cpumask_empty(cpu_mask
))
265 cpu_mask
= cpu_online_mask
;
267 for_each_cpu(cpuid
, cpu_mask
) {
268 hartid
= cpuid_to_hartid_map(cpuid
);
270 if (hartid
+ BITS_PER_LONG
<= htop
||
271 hbase
+ BITS_PER_LONG
<= hartid
) {
272 result
= __sbi_rfence_v02_call(fid
, hmask
,
273 hbase
, start
, size
, arg4
, arg5
);
277 } else if (hartid
< hbase
) {
278 /* shift the mask to fit lower hartid */
279 hmask
<<= hbase
- hartid
;
286 } else if (hartid
> htop
) {
289 hmask
|= BIT(hartid
- hbase
);
293 result
= __sbi_rfence_v02_call(fid
, hmask
, hbase
,
294 start
, size
, arg4
, arg5
);
303 * sbi_set_timer() - Program the timer for next timer event.
304 * @stime_value: The value after which next timer event should fire.
308 void sbi_set_timer(uint64_t stime_value
)
310 __sbi_set_timer(stime_value
);
314 * sbi_send_ipi() - Send an IPI to any hart.
315 * @cpu: Logical id of the target CPU.
317 void sbi_send_ipi(unsigned int cpu
)
321 EXPORT_SYMBOL(sbi_send_ipi
);
324 * sbi_remote_fence_i() - Execute FENCE.I instruction on given remote harts.
325 * @cpu_mask: A cpu mask containing all the target harts.
327 * Return: 0 on success, appropriate linux error code otherwise.
329 int sbi_remote_fence_i(const struct cpumask
*cpu_mask
)
331 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_FENCE_I
,
332 cpu_mask
, 0, 0, 0, 0);
334 EXPORT_SYMBOL(sbi_remote_fence_i
);
337 * sbi_remote_sfence_vma_asid() - Execute SFENCE.VMA instructions on given
338 * remote harts for a virtual address range belonging to a specific ASID or not.
340 * @cpu_mask: A cpu mask containing all the target harts.
341 * @start: Start of the virtual address
342 * @size: Total size of the virtual address range.
343 * @asid: The value of address space identifier (ASID), or FLUSH_TLB_NO_ASID
344 * for flushing all address spaces.
346 * Return: 0 on success, appropriate linux error code otherwise.
348 int sbi_remote_sfence_vma_asid(const struct cpumask
*cpu_mask
,
353 if (asid
== FLUSH_TLB_NO_ASID
)
354 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA
,
355 cpu_mask
, start
, size
, 0, 0);
357 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID
,
358 cpu_mask
, start
, size
, asid
, 0);
360 EXPORT_SYMBOL(sbi_remote_sfence_vma_asid
);
363 * sbi_remote_hfence_gvma() - Execute HFENCE.GVMA instructions on given remote
364 * harts for the specified guest physical address range.
365 * @cpu_mask: A cpu mask containing all the target harts.
366 * @start: Start of the guest physical address
367 * @size: Total size of the guest physical address range.
371 int sbi_remote_hfence_gvma(const struct cpumask
*cpu_mask
,
375 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA
,
376 cpu_mask
, start
, size
, 0, 0);
378 EXPORT_SYMBOL_GPL(sbi_remote_hfence_gvma
);
381 * sbi_remote_hfence_gvma_vmid() - Execute HFENCE.GVMA instructions on given
382 * remote harts for a guest physical address range belonging to a specific VMID.
384 * @cpu_mask: A cpu mask containing all the target harts.
385 * @start: Start of the guest physical address
386 * @size: Total size of the guest physical address range.
387 * @vmid: The value of guest ID (VMID).
389 * Return: 0 if success, Error otherwise.
391 int sbi_remote_hfence_gvma_vmid(const struct cpumask
*cpu_mask
,
396 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID
,
397 cpu_mask
, start
, size
, vmid
, 0);
399 EXPORT_SYMBOL(sbi_remote_hfence_gvma_vmid
);
402 * sbi_remote_hfence_vvma() - Execute HFENCE.VVMA instructions on given remote
403 * harts for the current guest virtual address range.
404 * @cpu_mask: A cpu mask containing all the target harts.
405 * @start: Start of the current guest virtual address
406 * @size: Total size of the current guest virtual address range.
410 int sbi_remote_hfence_vvma(const struct cpumask
*cpu_mask
,
414 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA
,
415 cpu_mask
, start
, size
, 0, 0);
417 EXPORT_SYMBOL(sbi_remote_hfence_vvma
);
420 * sbi_remote_hfence_vvma_asid() - Execute HFENCE.VVMA instructions on given
421 * remote harts for current guest virtual address range belonging to a specific
424 * @cpu_mask: A cpu mask containing all the target harts.
425 * @start: Start of the current guest virtual address
426 * @size: Total size of the current guest virtual address range.
427 * @asid: The value of address space identifier (ASID).
431 int sbi_remote_hfence_vvma_asid(const struct cpumask
*cpu_mask
,
436 return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID
,
437 cpu_mask
, start
, size
, asid
, 0);
439 EXPORT_SYMBOL(sbi_remote_hfence_vvma_asid
);
441 static void sbi_srst_reset(unsigned long type
, unsigned long reason
)
443 sbi_ecall(SBI_EXT_SRST
, SBI_EXT_SRST_RESET
, type
, reason
,
445 pr_warn("%s: type=0x%lx reason=0x%lx failed\n",
446 __func__
, type
, reason
);
449 static int sbi_srst_reboot(struct notifier_block
*this,
450 unsigned long mode
, void *cmd
)
452 sbi_srst_reset((mode
== REBOOT_WARM
|| mode
== REBOOT_SOFT
) ?
453 SBI_SRST_RESET_TYPE_WARM_REBOOT
:
454 SBI_SRST_RESET_TYPE_COLD_REBOOT
,
455 SBI_SRST_RESET_REASON_NONE
);
459 static struct notifier_block sbi_srst_reboot_nb
;
461 static void sbi_srst_power_off(void)
463 sbi_srst_reset(SBI_SRST_RESET_TYPE_SHUTDOWN
,
464 SBI_SRST_RESET_REASON_NONE
);
468 * sbi_probe_extension() - Check if an SBI extension ID is supported or not.
469 * @extid: The extension ID to be probed.
471 * Return: 1 or an extension specific nonzero value if yes, 0 otherwise.
473 long sbi_probe_extension(int extid
)
477 ret
= sbi_ecall(SBI_EXT_BASE
, SBI_EXT_BASE_PROBE_EXT
, extid
,
484 EXPORT_SYMBOL(sbi_probe_extension
);
486 static inline long sbi_get_spec_version(void)
488 return __sbi_base_ecall(SBI_EXT_BASE_GET_SPEC_VERSION
);
491 static inline long sbi_get_firmware_id(void)
493 return __sbi_base_ecall(SBI_EXT_BASE_GET_IMP_ID
);
496 static inline long sbi_get_firmware_version(void)
498 return __sbi_base_ecall(SBI_EXT_BASE_GET_IMP_VERSION
);
501 long sbi_get_mvendorid(void)
503 return __sbi_base_ecall(SBI_EXT_BASE_GET_MVENDORID
);
505 EXPORT_SYMBOL_GPL(sbi_get_mvendorid
);
507 long sbi_get_marchid(void)
509 return __sbi_base_ecall(SBI_EXT_BASE_GET_MARCHID
);
511 EXPORT_SYMBOL_GPL(sbi_get_marchid
);
513 long sbi_get_mimpid(void)
515 return __sbi_base_ecall(SBI_EXT_BASE_GET_MIMPID
);
517 EXPORT_SYMBOL_GPL(sbi_get_mimpid
);
519 bool sbi_debug_console_available
;
521 int sbi_debug_console_write(const char *bytes
, unsigned int num_bytes
)
523 phys_addr_t base_addr
;
526 if (!sbi_debug_console_available
)
529 if (is_vmalloc_addr(bytes
))
530 base_addr
= page_to_phys(vmalloc_to_page(bytes
)) +
531 offset_in_page(bytes
);
533 base_addr
= __pa(bytes
);
534 if (PAGE_SIZE
< (offset_in_page(bytes
) + num_bytes
))
535 num_bytes
= PAGE_SIZE
- offset_in_page(bytes
);
537 if (IS_ENABLED(CONFIG_32BIT
))
538 ret
= sbi_ecall(SBI_EXT_DBCN
, SBI_EXT_DBCN_CONSOLE_WRITE
,
539 num_bytes
, lower_32_bits(base_addr
),
540 upper_32_bits(base_addr
), 0, 0, 0);
542 ret
= sbi_ecall(SBI_EXT_DBCN
, SBI_EXT_DBCN_CONSOLE_WRITE
,
543 num_bytes
, base_addr
, 0, 0, 0, 0);
545 if (ret
.error
== SBI_ERR_FAILURE
)
547 return ret
.error
? sbi_err_map_linux_errno(ret
.error
) : ret
.value
;
550 int sbi_debug_console_read(char *bytes
, unsigned int num_bytes
)
552 phys_addr_t base_addr
;
555 if (!sbi_debug_console_available
)
558 if (is_vmalloc_addr(bytes
))
559 base_addr
= page_to_phys(vmalloc_to_page(bytes
)) +
560 offset_in_page(bytes
);
562 base_addr
= __pa(bytes
);
563 if (PAGE_SIZE
< (offset_in_page(bytes
) + num_bytes
))
564 num_bytes
= PAGE_SIZE
- offset_in_page(bytes
);
566 if (IS_ENABLED(CONFIG_32BIT
))
567 ret
= sbi_ecall(SBI_EXT_DBCN
, SBI_EXT_DBCN_CONSOLE_READ
,
568 num_bytes
, lower_32_bits(base_addr
),
569 upper_32_bits(base_addr
), 0, 0, 0);
571 ret
= sbi_ecall(SBI_EXT_DBCN
, SBI_EXT_DBCN_CONSOLE_READ
,
572 num_bytes
, base_addr
, 0, 0, 0, 0);
574 if (ret
.error
== SBI_ERR_FAILURE
)
576 return ret
.error
? sbi_err_map_linux_errno(ret
.error
) : ret
.value
;
579 void __init
sbi_init(void)
584 ret
= sbi_get_spec_version();
586 sbi_spec_version
= ret
;
588 pr_info("SBI specification v%lu.%lu detected\n",
589 sbi_major_version(), sbi_minor_version());
591 if (!sbi_spec_is_0_1()) {
592 pr_info("SBI implementation ID=0x%lx Version=0x%lx\n",
593 sbi_get_firmware_id(), sbi_get_firmware_version());
594 if (sbi_probe_extension(SBI_EXT_TIME
)) {
595 __sbi_set_timer
= __sbi_set_timer_v02
;
596 pr_info("SBI TIME extension detected\n");
598 __sbi_set_timer
= __sbi_set_timer_v01
;
600 if (sbi_probe_extension(SBI_EXT_IPI
)) {
601 __sbi_send_ipi
= __sbi_send_ipi_v02
;
602 pr_info("SBI IPI extension detected\n");
604 __sbi_send_ipi
= __sbi_send_ipi_v01
;
606 if (sbi_probe_extension(SBI_EXT_RFENCE
)) {
607 __sbi_rfence
= __sbi_rfence_v02
;
608 pr_info("SBI RFENCE extension detected\n");
610 __sbi_rfence
= __sbi_rfence_v01
;
612 if ((sbi_spec_version
>= sbi_mk_version(0, 3)) &&
613 sbi_probe_extension(SBI_EXT_SRST
)) {
614 pr_info("SBI SRST extension detected\n");
615 pm_power_off
= sbi_srst_power_off
;
616 sbi_srst_reboot_nb
.notifier_call
= sbi_srst_reboot
;
617 sbi_srst_reboot_nb
.priority
= 192;
618 register_restart_handler(&sbi_srst_reboot_nb
);
620 if ((sbi_spec_version
>= sbi_mk_version(2, 0)) &&
621 (sbi_probe_extension(SBI_EXT_DBCN
) > 0)) {
622 pr_info("SBI DBCN extension detected\n");
623 sbi_debug_console_available
= true;
626 __sbi_set_timer
= __sbi_set_timer_v01
;
627 __sbi_send_ipi
= __sbi_send_ipi_v01
;
628 __sbi_rfence
= __sbi_rfence_v01
;