arch/arm/common/mcpm_head.S

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * arch/arm/common/mcpm_head.S -- kernel entry point for multi-cluster PM
   4  *
   5  * Created by:  Nicolas Pitre, March 2012
   6  * Copyright:   (C) 2012-2013  Linaro Limited
   7  *
   8  * Refer to Documentation/arm/cluster-pm-race-avoidance.rst
   9  * for details of the synchronisation algorithms used here.
  10  */
  11
  12 #include <linux/linkage.h>
  13 #include <asm/mcpm.h>
  14 #include <asm/assembler.h>
  15
  16 #include "vlock.h"
  17
  18 .if MCPM_SYNC_CLUSTER_CPUS
  19 .error "cpus must be the first member of struct mcpm_sync_struct"
  20 .endif
  21
  22         .macro  pr_dbg  string
  23 #if defined(CONFIG_DEBUG_LL) && defined(DEBUG)
  24         b       1901f
  25 1902:   .asciz  "CPU"
  26 1903:   .asciz  " cluster"
  27 1904:   .asciz  ": \string"
  28         .align
  29 1901:   adr     r0, 1902b
  30         bl      printascii
  31         mov     r0, r9
  32         bl      printhex2
  33         adr     r0, 1903b
  34         bl      printascii
  35         mov     r0, r10
  36         bl      printhex2
  37         adr     r0, 1904b
  38         bl      printascii
  39 #endif
  40         .endm
  41
  42         .arm
  43         .align
  44
  45 ENTRY(mcpm_entry_point)
  46
  47  ARM_BE8(setend        be)
  48  THUMB( badr    r12, 1f         )
  49  THUMB( bx      r12             )
  50  THUMB( .thumb                  )
  51 1:
  52         mrc     p15, 0, r0, c0, c0, 5           @ MPIDR
  53         ubfx    r9, r0, #0, #8                  @ r9 = cpu
  54         ubfx    r10, r0, #8, #8                 @ r10 = cluster
  55         mov     r3, #MAX_CPUS_PER_CLUSTER
  56         mla     r4, r3, r10, r9                 @ r4 = canonical CPU index
  57         cmp     r4, #(MAX_CPUS_PER_CLUSTER * MAX_NR_CLUSTERS)
  58         blo     2f
  59
  60         /* We didn't expect this CPU.  Try to cheaply make it quiet. */
  61 1:      wfi
  62         wfe
  63         b       1b
  64
  65 2:      pr_dbg  "kernel mcpm_entry_point\n"
  66
  67         /*
  68          * MMU is off so we need to get to various variables in a
  69          * position independent way.
  70          */
  71         adr     r5, 3f
  72         ldmia   r5, {r0, r6, r7, r8, r11}
  73         add     r0, r5, r0                      @ r0 = mcpm_entry_early_pokes
  74         add     r6, r5, r6                      @ r6 = mcpm_entry_vectors
  75         ldr     r7, [r5, r7]                    @ r7 = mcpm_power_up_setup_phys
  76         add     r8, r5, r8                      @ r8 = mcpm_sync
  77         add     r11, r5, r11                    @ r11 = first_man_locks
  78
  79         @ Perform an early poke, if any
  80         add     r0, r0, r4, lsl #3
  81         ldmia   r0, {r0, r1}
  82         teq     r0, #0
  83         strne   r1, [r0]
  84
  85         mov     r0, #MCPM_SYNC_CLUSTER_SIZE
  86         mla     r8, r0, r10, r8                 @ r8 = sync cluster base
  87
  88         @ Signal that this CPU is coming UP:
  89         mov     r0, #CPU_COMING_UP
  90         mov     r5, #MCPM_SYNC_CPU_SIZE
  91         mla     r5, r9, r5, r8                  @ r5 = sync cpu address
  92         strb    r0, [r5]
  93
  94         @ At this point, the cluster cannot unexpectedly enter the GOING_DOWN
  95         @ state, because there is at least one active CPU (this CPU).
  96
  97         mov     r0, #VLOCK_SIZE
  98         mla     r11, r0, r10, r11               @ r11 = cluster first man lock
  99         mov     r0, r11
 100         mov     r1, r9                          @ cpu
 101         bl      vlock_trylock                   @ implies DMB
 102
 103         cmp     r0, #0                          @ failed to get the lock?
 104         bne     mcpm_setup_wait         @ wait for cluster setup if so
 105
 106         ldrb    r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
 107         cmp     r0, #CLUSTER_UP                 @ cluster already up?
 108         bne     mcpm_setup                      @ if not, set up the cluster
 109
 110         @ Otherwise, release the first man lock and skip setup:
 111         mov     r0, r11
 112         bl      vlock_unlock
 113         b       mcpm_setup_complete
 114
 115 mcpm_setup:
 116         @ Control dependency implies strb not observable before previous ldrb.
 117
 118         @ Signal that the cluster is being brought up:
 119         mov     r0, #INBOUND_COMING_UP
 120         strb    r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND]
 121         dmb
 122
 123         @ Any CPU trying to take the cluster into CLUSTER_GOING_DOWN from this
 124         @ point onwards will observe INBOUND_COMING_UP and abort.
 125
 126         @ Wait for any previously-pending cluster teardown operations to abort
 127         @ or complete:
 128 mcpm_teardown_wait:
 129         ldrb    r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
 130         cmp     r0, #CLUSTER_GOING_DOWN
 131         bne     first_man_setup
 132         wfe
 133         b       mcpm_teardown_wait
 134
 135 first_man_setup:
 136         dmb
 137
 138         @ If the outbound gave up before teardown started, skip cluster setup:
 139
 140         cmp     r0, #CLUSTER_UP
 141         beq     mcpm_setup_leave
 142
 143         @ power_up_setup is now responsible for setting up the cluster:
 144
 145         cmp     r7, #0
 146         mov     r0, #1          @ second (cluster) affinity level
 147         blxne   r7              @ Call power_up_setup if defined
 148         dmb
 149
 150         mov     r0, #CLUSTER_UP
 151         strb    r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
 152         dmb
 153
 154 mcpm_setup_leave:
 155         @ Leave the cluster setup critical section:
 156
 157         mov     r0, #INBOUND_NOT_COMING_UP
 158         strb    r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND]
 159         dsb     st
 160         sev
 161
 162         mov     r0, r11
 163         bl      vlock_unlock    @ implies DMB
 164         b       mcpm_setup_complete
 165
 166         @ In the contended case, non-first men wait here for cluster setup
 167         @ to complete:
 168 mcpm_setup_wait:
 169         ldrb    r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
 170         cmp     r0, #CLUSTER_UP
 171         wfene
 172         bne     mcpm_setup_wait
 173         dmb
 174
 175 mcpm_setup_complete:
 176         @ If a platform-specific CPU setup hook is needed, it is
 177         @ called from here.
 178
 179         cmp     r7, #0
 180         mov     r0, #0          @ first (CPU) affinity level
 181         blxne   r7              @ Call power_up_setup if defined
 182         dmb
 183
 184         @ Mark the CPU as up:
 185
 186         mov     r0, #CPU_UP
 187         strb    r0, [r5]
 188
 189         @ Observability order of CPU_UP and opening of the gate does not matter.
 190
 191 mcpm_entry_gated:
 192         ldr     r5, [r6, r4, lsl #2]            @ r5 = CPU entry vector
 193         cmp     r5, #0
 194         wfeeq
 195         beq     mcpm_entry_gated
 196         dmb
 197
 198         pr_dbg  "released\n"
 199         bx      r5
 200
 201         .align  2
 202
 203 3:      .word   mcpm_entry_early_pokes - .
 204         .word   mcpm_entry_vectors - 3b
 205         .word   mcpm_power_up_setup_phys - 3b
 206         .word   mcpm_sync - 3b
 207         .word   first_man_locks - 3b
 208
 209 ENDPROC(mcpm_entry_point)
 210
 211         .bss
 212
 213         .align  CACHE_WRITEBACK_ORDER
 214         .type   first_man_locks, #object
 215 first_man_locks:
 216         .space  VLOCK_SIZE * MAX_NR_CLUSTERS
 217         .align  CACHE_WRITEBACK_ORDER
 218
 219         .type   mcpm_entry_vectors, #object
 220 ENTRY(mcpm_entry_vectors)
 221         .space  4 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER
 222
 223         .type   mcpm_entry_early_pokes, #object
 224 ENTRY(mcpm_entry_early_pokes)
 225         .space  8 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER
 226
 227         .type   mcpm_power_up_setup_phys, #object
 228 ENTRY(mcpm_power_up_setup_phys)
 229         .space  4               @ set by mcpm_sync_init()