arch/arm/common/mcpm_head.S

   1 /*
   2  * arch/arm/common/mcpm_head.S -- kernel entry point for multi-cluster PM
   3  *
   4  * Created by:  Nicolas Pitre, March 2012
   5  * Copyright:   (C) 2012-2013  Linaro Limited
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  *
  11  *
  12  * Refer to Documentation/arm/cluster-pm-race-avoidance.txt
  13  * for details of the synchronisation algorithms used here.
  14  */
  15
  16 #include <linux/linkage.h>
  17 #include <asm/mcpm.h>
  18
  19 #include "vlock.h"
  20
  21 .if MCPM_SYNC_CLUSTER_CPUS
  22 .error "cpus must be the first member of struct mcpm_sync_struct"
  23 .endif
  24
  25         .macro  pr_dbg  string
  26 #if defined(CONFIG_DEBUG_LL) && defined(DEBUG)
  27         b       1901f
  28 1902:   .asciz  "CPU"
  29 1903:   .asciz  " cluster"
  30 1904:   .asciz  ": \string"
  31         .align
  32 1901:   adr     r0, 1902b
  33         bl      printascii
  34         mov     r0, r9
  35         bl      printhex2
  36         adr     r0, 1903b
  37         bl      printascii
  38         mov     r0, r10
  39         bl      printhex2
  40         adr     r0, 1904b
  41         bl      printascii
  42 #endif
  43         .endm
  44
  45         .arm
  46         .align
  47
  48 ENTRY(mcpm_entry_point)
  49
  50  THUMB( adr     r12, BSYM(1f)   )
  51  THUMB( bx      r12             )
  52  THUMB( .thumb                  )
  53 1:
  54         mrc     p15, 0, r0, c0, c0, 5           @ MPIDR
  55         ubfx    r9, r0, #0, #8                  @ r9 = cpu
  56         ubfx    r10, r0, #8, #8                 @ r10 = cluster
  57         mov     r3, #MAX_CPUS_PER_CLUSTER
  58         mla     r4, r3, r10, r9                 @ r4 = canonical CPU index
  59         cmp     r4, #(MAX_CPUS_PER_CLUSTER * MAX_NR_CLUSTERS)
  60         blo     2f
  61
  62         /* We didn't expect this CPU.  Try to cheaply make it quiet. */
  63 1:      wfi
  64         wfe
  65         b       1b
  66
  67 2:      pr_dbg  "kernel mcpm_entry_point\n"
  68
  69         /*
  70          * MMU is off so we need to get to various variables in a
  71          * position independent way.
  72          */
  73         adr     r5, 3f
  74         ldmia   r5, {r6, r7, r8, r11}
  75         add     r6, r5, r6                      @ r6 = mcpm_entry_vectors
  76         ldr     r7, [r5, r7]                    @ r7 = mcpm_power_up_setup_phys
  77         add     r8, r5, r8                      @ r8 = mcpm_sync
  78         add     r11, r5, r11                    @ r11 = first_man_locks
  79
  80         mov     r0, #MCPM_SYNC_CLUSTER_SIZE
  81         mla     r8, r0, r10, r8                 @ r8 = sync cluster base
  82
  83         @ Signal that this CPU is coming UP:
  84         mov     r0, #CPU_COMING_UP
  85         mov     r5, #MCPM_SYNC_CPU_SIZE
  86         mla     r5, r9, r5, r8                  @ r5 = sync cpu address
  87         strb    r0, [r5]
  88
  89         @ At this point, the cluster cannot unexpectedly enter the GOING_DOWN
  90         @ state, because there is at least one active CPU (this CPU).
  91
  92         mov     r0, #VLOCK_SIZE
  93         mla     r11, r0, r10, r11               @ r11 = cluster first man lock
  94         mov     r0, r11
  95         mov     r1, r9                          @ cpu
  96         bl      vlock_trylock                   @ implies DMB
  97
  98         cmp     r0, #0                          @ failed to get the lock?
  99         bne     mcpm_setup_wait         @ wait for cluster setup if so
 100
 101         ldrb    r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
 102         cmp     r0, #CLUSTER_UP                 @ cluster already up?
 103         bne     mcpm_setup                      @ if not, set up the cluster
 104
 105         @ Otherwise, release the first man lock and skip setup:
 106         mov     r0, r11
 107         bl      vlock_unlock
 108         b       mcpm_setup_complete
 109
 110 mcpm_setup:
 111         @ Control dependency implies strb not observable before previous ldrb.
 112
 113         @ Signal that the cluster is being brought up:
 114         mov     r0, #INBOUND_COMING_UP
 115         strb    r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND]
 116         dmb
 117
 118         @ Any CPU trying to take the cluster into CLUSTER_GOING_DOWN from this
 119         @ point onwards will observe INBOUND_COMING_UP and abort.
 120
 121         @ Wait for any previously-pending cluster teardown operations to abort
 122         @ or complete:
 123 mcpm_teardown_wait:
 124         ldrb    r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
 125         cmp     r0, #CLUSTER_GOING_DOWN
 126         bne     first_man_setup
 127         wfe
 128         b       mcpm_teardown_wait
 129
 130 first_man_setup:
 131         dmb
 132
 133         @ If the outbound gave up before teardown started, skip cluster setup:
 134
 135         cmp     r0, #CLUSTER_UP
 136         beq     mcpm_setup_leave
 137
 138         @ power_up_setup is now responsible for setting up the cluster:
 139
 140         cmp     r7, #0
 141         mov     r0, #1          @ second (cluster) affinity level
 142         blxne   r7              @ Call power_up_setup if defined
 143         dmb
 144
 145         mov     r0, #CLUSTER_UP
 146         strb    r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
 147         dmb
 148
 149 mcpm_setup_leave:
 150         @ Leave the cluster setup critical section:
 151
 152         mov     r0, #INBOUND_NOT_COMING_UP
 153         strb    r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND]
 154         dsb     st
 155         sev
 156
 157         mov     r0, r11
 158         bl      vlock_unlock    @ implies DMB
 159         b       mcpm_setup_complete
 160
 161         @ In the contended case, non-first men wait here for cluster setup
 162         @ to complete:
 163 mcpm_setup_wait:
 164         ldrb    r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER]
 165         cmp     r0, #CLUSTER_UP
 166         wfene
 167         bne     mcpm_setup_wait
 168         dmb
 169
 170 mcpm_setup_complete:
 171         @ If a platform-specific CPU setup hook is needed, it is
 172         @ called from here.
 173
 174         cmp     r7, #0
 175         mov     r0, #0          @ first (CPU) affinity level
 176         blxne   r7              @ Call power_up_setup if defined
 177         dmb
 178
 179         @ Mark the CPU as up:
 180
 181         mov     r0, #CPU_UP
 182         strb    r0, [r5]
 183
 184         @ Observability order of CPU_UP and opening of the gate does not matter.
 185
 186 mcpm_entry_gated:
 187         ldr     r5, [r6, r4, lsl #2]            @ r5 = CPU entry vector
 188         cmp     r5, #0
 189         wfeeq
 190         beq     mcpm_entry_gated
 191         dmb
 192
 193         pr_dbg  "released\n"
 194         bx      r5
 195
 196         .align  2
 197
 198 3:      .word   mcpm_entry_vectors - .
 199         .word   mcpm_power_up_setup_phys - 3b
 200         .word   mcpm_sync - 3b
 201         .word   first_man_locks - 3b
 202
 203 ENDPROC(mcpm_entry_point)
 204
 205         .bss
 206
 207         .align  CACHE_WRITEBACK_ORDER
 208         .type   first_man_locks, #object
 209 first_man_locks:
 210         .space  VLOCK_SIZE * MAX_NR_CLUSTERS
 211         .align  CACHE_WRITEBACK_ORDER
 212
 213         .type   mcpm_entry_vectors, #object
 214 ENTRY(mcpm_entry_vectors)
 215         .space  4 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER
 216
 217         .type   mcpm_power_up_setup_phys, #object
 218 ENTRY(mcpm_power_up_setup_phys)
 219         .space  4               @ set by mcpm_sync_init()