| blob | 39e3fb3db80123b1f276f811f8b26921e77564ee |
1 /*
2 * linux/arch/arm/mm/cache-v7.S
3 *
4 * Copyright (C) 2001 Deep Blue Solutions Ltd.
5 * Copyright (C) 2005 ARM Ltd.
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This is the "shell" of the ARMv7 processor support.
12 */
13 #include <linux/linkage.h>
14 #include <linux/init.h>
15 #include <asm/assembler.h>
16 #include <asm/errno.h>
17 #include <asm/unwind.h>
19 #include "proc-macros.S"
21 /*
22 * v7_flush_icache_all()
23 *
24 * Flush the whole I-cache.
25 *
26 * Registers:
27 * r0 - set to 0
28 */
29 ENTRY(v7_flush_icache_all)
30 mov r0, #0
31 ALT_SMP(mcr p15, 0, r0, c7, c1, 0) @ invalidate I-cache inner shareable
32 ALT_UP(mcr p15, 0, r0, c7, c5, 0) @ I+BTB cache invalidate
33 mov pc, lr
34 ENDPROC(v7_flush_icache_all)
36 /*
37 * v7_flush_dcache_all()
38 *
39 * Flush the whole D-cache.
40 *
41 * Corrupted registers: r0-r7, r9-r11 (r6 only in Thumb mode)
42 *
43 * - mm - mm_struct describing address space
44 */
45 ENTRY(v7_flush_dcache_all)
46 dmb @ ensure ordering with previous memory accesses
47 mrc p15, 1, r0, c0, c0, 1 @ read clidr
48 ands r3, r0, #0x7000000 @ extract loc from clidr
49 mov r3, r3, lsr #23 @ left align loc bit field
50 beq finished @ if loc is 0, then no need to clean
51 mov r10, #0 @ start clean at cache level 0
52 loop1:
53 add r2, r10, r10, lsr #1 @ work out 3x current cache level
54 mov r1, r0, lsr r2 @ extract cache type bits from clidr
55 and r1, r1, #7 @ mask of the bits for current cache only
56 cmp r1, #2 @ see what cache we have at this level
57 blt skip @ skip if no cache, or just i-cache
58 #ifdef CONFIG_PREEMPT
59 save_and_disable_irqs_notrace r9 @ make cssr&csidr read atomic
60 #endif
61 mcr p15, 2, r10, c0, c0, 0 @ select current cache level in cssr
62 isb @ isb to sych the new cssr&csidr
63 mrc p15, 1, r1, c0, c0, 0 @ read the new csidr
64 #ifdef CONFIG_PREEMPT
65 restore_irqs_notrace r9
66 #endif
67 and r2, r1, #7 @ extract the length of the cache lines
68 add r2, r2, #4 @ add 4 (line length offset)
69 ldr r4, =0x3ff
70 ands r4, r4, r1, lsr #3 @ find maximum number on the way size
71 clz r5, r4 @ find bit position of way size increment
72 ldr r7, =0x7fff
73 ands r7, r7, r1, lsr #13 @ extract max number of the index size
74 loop2:
75 mov r9, r4 @ create working copy of max way size
76 loop3:
77 ARM( orr r11, r10, r9, lsl r5 ) @ factor way and cache number into r11
78 THUMB( lsl r6, r9, r5 )
79 THUMB( orr r11, r10, r6 ) @ factor way and cache number into r11
80 ARM( orr r11, r11, r7, lsl r2 ) @ factor index number into r11
81 THUMB( lsl r6, r7, r2 )
82 THUMB( orr r11, r11, r6 ) @ factor index number into r11
83 mcr p15, 0, r11, c7, c14, 2 @ clean & invalidate by set/way
84 subs r9, r9, #1 @ decrement the way
85 bge loop3
86 subs r7, r7, #1 @ decrement the index
87 bge loop2
88 skip:
89 add r10, r10, #2 @ increment cache number
90 cmp r3, r10
91 bgt loop1
92 finished:
93 mov r10, #0 @ swith back to cache level 0
94 mcr p15, 2, r10, c0, c0, 0 @ select current cache level in cssr
95 dsb
96 isb
97 mov pc, lr
98 ENDPROC(v7_flush_dcache_all)
100 /*
101 * v7_flush_cache_all()
102 *
103 * Flush the entire cache system.
104 * The data cache flush is now achieved using atomic clean / invalidates
105 * working outwards from L1 cache. This is done using Set/Way based cache
106 * maintenance instructions.
107 * The instruction cache can still be invalidated back to the point of
108 * unification in a single instruction.
109 *
110 */
111 ENTRY(v7_flush_kern_cache_all)
112 ARM( stmfd sp!, {r4-r5, r7, r9-r11, lr} )
113 THUMB( stmfd sp!, {r4-r7, r9-r11, lr} )
114 bl v7_flush_dcache_all
115 mov r0, #0
116 ALT_SMP(mcr p15, 0, r0, c7, c1, 0) @ invalidate I-cache inner shareable
117 ALT_UP(mcr p15, 0, r0, c7, c5, 0) @ I+BTB cache invalidate
118 ARM( ldmfd sp!, {r4-r5, r7, r9-r11, lr} )
119 THUMB( ldmfd sp!, {r4-r7, r9-r11, lr} )
120 mov pc, lr
121 ENDPROC(v7_flush_kern_cache_all)
123 /*
124 * v7_flush_cache_all()
125 *
126 * Flush all TLB entries in a particular address space
127 *
128 * - mm - mm_struct describing address space
129 */
130 ENTRY(v7_flush_user_cache_all)
131 /*FALLTHROUGH*/
133 /*
134 * v7_flush_cache_range(start, end, flags)
135 *
136 * Flush a range of TLB entries in the specified address space.
137 *
138 * - start - start address (may not be aligned)
139 * - end - end address (exclusive, may not be aligned)
140 * - flags - vm_area_struct flags describing address space
141 *
142 * It is assumed that:
143 * - we have a VIPT cache.
144 */
145 ENTRY(v7_flush_user_cache_range)
146 mov pc, lr
147 ENDPROC(v7_flush_user_cache_all)
148 ENDPROC(v7_flush_user_cache_range)
150 /*
151 * v7_coherent_kern_range(start,end)
152 *
153 * Ensure that the I and D caches are coherent within specified
154 * region. This is typically used when code has been written to
155 * a memory region, and will be executed.
156 *
157 * - start - virtual start address of region
158 * - end - virtual end address of region
159 *
160 * It is assumed that:
161 * - the Icache does not read data from the write buffer
162 */
163 ENTRY(v7_coherent_kern_range)
164 /* FALLTHROUGH */
166 /*
167 * v7_coherent_user_range(start,end)
168 *
169 * Ensure that the I and D caches are coherent within specified
170 * region. This is typically used when code has been written to
171 * a memory region, and will be executed.
172 *
173 * - start - virtual start address of region
174 * - end - virtual end address of region
175 *
176 * It is assumed that:
177 * - the Icache does not read data from the write buffer
178 */
179 ENTRY(v7_coherent_user_range)
180 UNWIND(.fnstart )
181 dcache_line_size r2, r3
182 sub r3, r2, #1
183 bic r12, r0, r3
184 #ifdef CONFIG_ARM_ERRATA_764369
185 ALT_SMP(W(dsb))
186 ALT_UP(W(nop))
187 #endif
188 1:
189 USER( mcr p15, 0, r12, c7, c11, 1 ) @ clean D line to the point of unification
190 add r12, r12, r2
191 cmp r12, r1
192 blo 1b
193 dsb
194 icache_line_size r2, r3
195 sub r3, r2, #1
196 bic r12, r0, r3
197 2:
198 USER( mcr p15, 0, r12, c7, c5, 1 ) @ invalidate I line
199 add r12, r12, r2
200 cmp r12, r1
201 blo 2b
202 mov r0, #0
203 ALT_SMP(mcr p15, 0, r0, c7, c1, 6) @ invalidate BTB Inner Shareable
204 ALT_UP(mcr p15, 0, r0, c7, c5, 6) @ invalidate BTB
205 dsb
206 isb
207 mov pc, lr
209 /*
210 * Fault handling for the cache operation above. If the virtual address in r0
211 * isn't mapped, fail with -EFAULT.
212 */
213 9001:
214 mov r0, #-EFAULT
215 mov pc, lr
216 UNWIND(.fnend )
217 ENDPROC(v7_coherent_kern_range)
218 ENDPROC(v7_coherent_user_range)
220 /*
221 * v7_flush_kern_dcache_area(void *addr, size_t size)
222 *
223 * Ensure that the data held in the page kaddr is written back
224 * to the page in question.
225 *
226 * - addr - kernel address
227 * - size - region size
228 */
229 ENTRY(v7_flush_kern_dcache_area)
230 dcache_line_size r2, r3
231 add r1, r0, r1
232 sub r3, r2, #1
233 bic r0, r0, r3
234 #ifdef CONFIG_ARM_ERRATA_764369
235 ALT_SMP(W(dsb))
236 ALT_UP(W(nop))
237 #endif
238 1:
239 mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line / unified line
240 add r0, r0, r2
241 cmp r0, r1
242 blo 1b
243 dsb
244 mov pc, lr
245 ENDPROC(v7_flush_kern_dcache_area)
247 /*
248 * v7_dma_inv_range(start,end)
249 *
250 * Invalidate the data cache within the specified region; we will
251 * be performing a DMA operation in this region and we want to
252 * purge old data in the cache.
253 *
254 * - start - virtual start address of region
255 * - end - virtual end address of region
256 */
257 v7_dma_inv_range:
258 dcache_line_size r2, r3
259 sub r3, r2, #1
260 tst r0, r3
261 bic r0, r0, r3
262 #ifdef CONFIG_ARM_ERRATA_764369
263 ALT_SMP(W(dsb))
264 ALT_UP(W(nop))
265 #endif
266 mcrne p15, 0, r0, c7, c14, 1 @ clean & invalidate D / U line
268 tst r1, r3
269 bic r1, r1, r3
270 mcrne p15, 0, r1, c7, c14, 1 @ clean & invalidate D / U line
271 1:
272 mcr p15, 0, r0, c7, c6, 1 @ invalidate D / U line
273 add r0, r0, r2
274 cmp r0, r1
275 blo 1b
276 dsb
277 mov pc, lr
278 ENDPROC(v7_dma_inv_range)
280 /*
281 * v7_dma_clean_range(start,end)
282 * - start - virtual start address of region
283 * - end - virtual end address of region
284 */
285 v7_dma_clean_range:
286 dcache_line_size r2, r3
287 sub r3, r2, #1
288 bic r0, r0, r3
289 #ifdef CONFIG_ARM_ERRATA_764369
290 ALT_SMP(W(dsb))
291 ALT_UP(W(nop))
292 #endif
293 1:
294 mcr p15, 0, r0, c7, c10, 1 @ clean D / U line
295 add r0, r0, r2
296 cmp r0, r1
297 blo 1b
298 dsb
299 mov pc, lr
300 ENDPROC(v7_dma_clean_range)
302 /*
303 * v7_dma_flush_range(start,end)
304 * - start - virtual start address of region
305 * - end - virtual end address of region
306 */
307 ENTRY(v7_dma_flush_range)
308 dcache_line_size r2, r3
309 sub r3, r2, #1
310 bic r0, r0, r3
311 #ifdef CONFIG_ARM_ERRATA_764369
312 ALT_SMP(W(dsb))
313 ALT_UP(W(nop))
314 #endif
315 1:
316 mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D / U line
317 add r0, r0, r2
318 cmp r0, r1
319 blo 1b
320 dsb
321 mov pc, lr
322 ENDPROC(v7_dma_flush_range)
324 /*
325 * dma_map_area(start, size, dir)
326 * - start - kernel virtual start address
327 * - size - size of region
328 * - dir - DMA direction
329 */
330 ENTRY(v7_dma_map_area)
331 add r1, r1, r0
332 teq r2, #DMA_FROM_DEVICE
333 beq v7_dma_inv_range
334 b v7_dma_clean_range
335 ENDPROC(v7_dma_map_area)
337 /*
338 * dma_unmap_area(start, size, dir)
339 * - start - kernel virtual start address
340 * - size - size of region
341 * - dir - DMA direction
342 */
343 ENTRY(v7_dma_unmap_area)
344 add r1, r1, r0
345 teq r2, #DMA_TO_DEVICE
346 bne v7_dma_inv_range
347 mov pc, lr
348 ENDPROC(v7_dma_unmap_area)
350 __INITDATA
352 @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S)
353 define_cache_functions v7