?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 2.6.13-rc4
[linux-2.6/next.git] / arch / mips / mm / pg-r4k.c
blob9f8b16541577776180c15192a03a5ab6856f05d9
1 /*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2003, 04, 05 Ralf Baechle (ralf@linux-mips.org)
7 */
8 #include <linux/init.h>
9 #include <linux/kernel.h>
10 #include <linux/sched.h>
11 #include <linux/mm.h>
12 #include <linux/module.h>
13 #include <linux/proc_fs.h>
14
15 #include <asm/cacheops.h>
16 #include <asm/inst.h>
17 #include <asm/io.h>
18 #include <asm/page.h>
19 #include <asm/pgtable.h>
20 #include <asm/prefetch.h>
21 #include <asm/system.h>
22 #include <asm/bootinfo.h>
23 #include <asm/mipsregs.h>
24 #include <asm/mmu_context.h>
25 #include <asm/cpu.h>
26 #include <asm/war.h>
27
28 #define half_scache_line_size() (cpu_scache_line_size() >> 1)
29
30 /*
31 * Maximum sizes:
32 *
33 * R4000 128 bytes S-cache: 0x58 bytes
34 * R4600 v1.7: 0x5c bytes
35 * R4600 v2.0: 0x60 bytes
36 * With prefetching, 16 byte strides 0xa0 bytes
37 */
38
39 static unsigned int clear_page_array[0x130 / 4];
40
41 void clear_page(void * page) __attribute__((alias("clear_page_array")));
42
43 EXPORT_SYMBOL(clear_page);
44
45 /*
46 * Maximum sizes:
47 *
48 * R4000 128 bytes S-cache: 0x11c bytes
49 * R4600 v1.7: 0x080 bytes
50 * R4600 v2.0: 0x07c bytes
51 * With prefetching, 16 byte strides 0x0b8 bytes
52 */
53 static unsigned int copy_page_array[0x148 / 4];
54
55 void copy_page(void *to, void *from) __attribute__((alias("copy_page_array")));
56
57 EXPORT_SYMBOL(copy_page);
58
59 /*
60 * This is suboptimal for 32-bit kernels; we assume that R10000 is only used
61 * with 64-bit kernels. The prefetch offsets have been experimentally tuned
62 * an Origin 200.
63 */
64 static int pref_offset_clear __initdata = 512;
65 static int pref_offset_copy __initdata = 256;
66
67 static unsigned int pref_src_mode __initdata;
68 static unsigned int pref_dst_mode __initdata;
69
70 static int load_offset __initdata;
71 static int store_offset __initdata;
72
73 static unsigned int __initdata *dest, *epc;
74
75 static unsigned int instruction_pending;
76 static union mips_instruction delayed_mi;
77
78 static void __init emit_instruction(union mips_instruction mi)
79 {
80 if (instruction_pending)
81 *epc++ = delayed_mi.word;
82
83 instruction_pending = 1;
84 delayed_mi = mi;
85 }
86
87 static inline void flush_delay_slot_or_nop(void)
88 {
89 if (instruction_pending) {
90 *epc++ = delayed_mi.word;
91 instruction_pending = 0;
92 return;
93 }
94
95 *epc++ = 0;
96 }
97
98 static inline unsigned int *label(void)
99 {
100 if (instruction_pending) {
101 *epc++ = delayed_mi.word;
102 instruction_pending = 0;
103 }
104
105 return epc;
106 }
107
108 static inline void build_insn_word(unsigned int word)
109 {
110 union mips_instruction mi;
111
112 mi.word = word;
113
114 emit_instruction(mi);
115 }
116
117 static inline void build_nop(void)
118 {
119 build_insn_word(0); /* nop */
120 }
121
122 static inline void build_src_pref(int advance)
123 {
124 if (!(load_offset & (cpu_dcache_line_size() - 1))) {
125 union mips_instruction mi;
126
127 mi.i_format.opcode = pref_op;
128 mi.i_format.rs = 5; /* $a1 */
129 mi.i_format.rt = pref_src_mode;
130 mi.i_format.simmediate = load_offset + advance;
131
132 emit_instruction(mi);
133 }
134 }
135
136 static inline void __build_load_reg(int reg)
137 {
138 union mips_instruction mi;
139 unsigned int width;
140
141 if (cpu_has_64bit_gp_regs) {
142 mi.i_format.opcode = ld_op;
143 width = 8;
144 } else {
145 mi.i_format.opcode = lw_op;
146 width = 4;
147 }
148 mi.i_format.rs = 5; /* $a1 */
149 mi.i_format.rt = reg; /* $reg */
150 mi.i_format.simmediate = load_offset;
151
152 load_offset += width;
153 emit_instruction(mi);
154 }
155
156 static inline void build_load_reg(int reg)
157 {
158 if (cpu_has_prefetch)
159 build_src_pref(pref_offset_copy);
160
161 __build_load_reg(reg);
162 }
163
164 static inline void build_dst_pref(int advance)
165 {
166 if (!(store_offset & (cpu_dcache_line_size() - 1))) {
167 union mips_instruction mi;
168
169 mi.i_format.opcode = pref_op;
170 mi.i_format.rs = 4; /* $a0 */
171 mi.i_format.rt = pref_dst_mode;
172 mi.i_format.simmediate = store_offset + advance;
173
174 emit_instruction(mi);
175 }
176 }
177
178 static inline void build_cdex_s(void)
179 {
180 union mips_instruction mi;
181
182 if ((store_offset & (cpu_scache_line_size() - 1)))
183 return;
184
185 mi.c_format.opcode = cache_op;
186 mi.c_format.rs = 4; /* $a0 */
187 mi.c_format.c_op = 3; /* Create Dirty Exclusive */
188 mi.c_format.cache = 3; /* Secondary Data Cache */
189 mi.c_format.simmediate = store_offset;
190
191 emit_instruction(mi);
192 }
193
194 static inline void build_cdex_p(void)
195 {
196 union mips_instruction mi;
197
198 if (store_offset & (cpu_dcache_line_size() - 1))
199 return;
200
201 if (R4600_V1_HIT_CACHEOP_WAR && ((read_c0_prid() & 0xfff0) == 0x2010)) {
202 build_nop();
203 build_nop();
204 build_nop();
205 build_nop();
206 }
207
208 if (R4600_V2_HIT_CACHEOP_WAR && ((read_c0_prid() & 0xfff0) == 0x2020))
209 build_insn_word(0x8c200000); /* lw $zero, ($at) */
210
211 mi.c_format.opcode = cache_op;
212 mi.c_format.rs = 4; /* $a0 */
213 mi.c_format.c_op = 3; /* Create Dirty Exclusive */
214 mi.c_format.cache = 1; /* Data Cache */
215 mi.c_format.simmediate = store_offset;
216
217 emit_instruction(mi);
218 }
219
220 static void __init __build_store_reg(int reg)
221 {
222 union mips_instruction mi;
223 unsigned int width;
224
225 if (cpu_has_64bit_gp_regs ||
226 (cpu_has_64bit_zero_reg && reg == 0)) {
227 mi.i_format.opcode = sd_op;
228 width = 8;
229 } else {
230 mi.i_format.opcode = sw_op;
231 width = 4;
232 }
233 mi.i_format.rs = 4; /* $a0 */
234 mi.i_format.rt = reg; /* $reg */
235 mi.i_format.simmediate = store_offset;
236
237 store_offset += width;
238 emit_instruction(mi);
239 }
240
241 static inline void build_store_reg(int reg)
242 {
243 if (cpu_has_prefetch)
244 if (reg)
245 build_dst_pref(pref_offset_copy);
246 else
247 build_dst_pref(pref_offset_clear);
248 else if (cpu_has_cache_cdex_s)
249 build_cdex_s();
250 else if (cpu_has_cache_cdex_p)
251 build_cdex_p();
252
253 __build_store_reg(reg);
254 }
255
256 static inline void build_addiu_a2_a0(unsigned long offset)
257 {
258 union mips_instruction mi;
259
260 BUG_ON(offset > 0x7fff);
261
262 mi.i_format.opcode = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op;
263 mi.i_format.rs = 4; /* $a0 */
264 mi.i_format.rt = 6; /* $a2 */
265 mi.i_format.simmediate = offset;
266
267 emit_instruction(mi);
268 }
269
270 static inline void build_addiu_a1(unsigned long offset)
271 {
272 union mips_instruction mi;
273
274 BUG_ON(offset > 0x7fff);
275
276 mi.i_format.opcode = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op;
277 mi.i_format.rs = 5; /* $a1 */
278 mi.i_format.rt = 5; /* $a1 */
279 mi.i_format.simmediate = offset;
280
281 load_offset -= offset;
282
283 emit_instruction(mi);
284 }
285
286 static inline void build_addiu_a0(unsigned long offset)
287 {
288 union mips_instruction mi;
289
290 BUG_ON(offset > 0x7fff);
291
292 mi.i_format.opcode = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op;
293 mi.i_format.rs = 4; /* $a0 */
294 mi.i_format.rt = 4; /* $a0 */
295 mi.i_format.simmediate = offset;
296
297 store_offset -= offset;
298
299 emit_instruction(mi);
300 }
301
302 static inline void build_bne(unsigned int *dest)
303 {
304 union mips_instruction mi;
305
306 mi.i_format.opcode = bne_op;
307 mi.i_format.rs = 6; /* $a2 */
308 mi.i_format.rt = 4; /* $a0 */
309 mi.i_format.simmediate = dest - epc - 1;
310
311 *epc++ = mi.word;
312 flush_delay_slot_or_nop();
313 }
314
315 static inline void build_jr_ra(void)
316 {
317 union mips_instruction mi;
318
319 mi.r_format.opcode = spec_op;
320 mi.r_format.rs = 31;
321 mi.r_format.rt = 0;
322 mi.r_format.rd = 0;
323 mi.r_format.re = 0;
324 mi.r_format.func = jr_op;
325
326 *epc++ = mi.word;
327 flush_delay_slot_or_nop();
328 }
329
330 void __init build_clear_page(void)
331 {
332 unsigned int loop_start;
333
334 epc = (unsigned int *) &clear_page_array;
335 instruction_pending = 0;
336 store_offset = 0;
337
338 if (cpu_has_prefetch) {
339 switch (current_cpu_data.cputype) {
340 case CPU_RM9000:
341 /*
342 * As a workaround for erratum G105 which make the
343 * PrepareForStore hint unusable we fall back to
344 * StoreRetained on the RM9000. Once it is known which
345 * versions of the RM9000 we'll be able to condition-
346 * alize this.
347 */
348
349 case CPU_R10000:
350 case CPU_R12000:
351 pref_src_mode = Pref_LoadStreamed;
352 pref_dst_mode = Pref_StoreStreamed;
353 break;
354
355 default:
356 pref_src_mode = Pref_LoadStreamed;
357 pref_dst_mode = Pref_PrepareForStore;
358 break;
359 }
360 }
361
362 build_addiu_a2_a0(PAGE_SIZE - (cpu_has_prefetch ? pref_offset_clear : 0));
363
364 if (R4600_V2_HIT_CACHEOP_WAR && ((read_c0_prid() & 0xfff0) == 0x2020))
365 build_insn_word(0x3c01a000); /* lui $at, 0xa000 */
366
367 dest = label();
368 do {
369 build_store_reg(0);
370 build_store_reg(0);
371 build_store_reg(0);
372 build_store_reg(0);
373 } while (store_offset < half_scache_line_size());
374 build_addiu_a0(2 * store_offset);
375 loop_start = store_offset;
376 do {
377 build_store_reg(0);
378 build_store_reg(0);
379 build_store_reg(0);
380 build_store_reg(0);
381 } while ((store_offset - loop_start) < half_scache_line_size());
382 build_bne(dest);
383
384 if (cpu_has_prefetch && pref_offset_clear) {
385 build_addiu_a2_a0(pref_offset_clear);
386 dest = label();
387 loop_start = store_offset;
388 do {
389 __build_store_reg(0);
390 __build_store_reg(0);
391 __build_store_reg(0);
392 __build_store_reg(0);
393 } while ((store_offset - loop_start) < half_scache_line_size());
394 build_addiu_a0(2 * store_offset);
395 loop_start = store_offset;
396 do {
397 __build_store_reg(0);
398 __build_store_reg(0);
399 __build_store_reg(0);
400 __build_store_reg(0);
401 } while ((store_offset - loop_start) < half_scache_line_size());
402 build_bne(dest);
403 }
404
405 build_jr_ra();
406
407 flush_icache_range((unsigned long)&clear_page_array,
408 (unsigned long) epc);
409
410 BUG_ON(epc > clear_page_array + ARRAY_SIZE(clear_page_array));
411 }
412
413 void __init build_copy_page(void)
414 {
415 unsigned int loop_start;
416
417 epc = (unsigned int *) &copy_page_array;
418 store_offset = load_offset = 0;
419 instruction_pending = 0;
420
421 build_addiu_a2_a0(PAGE_SIZE - (cpu_has_prefetch ? pref_offset_copy : 0));
422
423 if (R4600_V2_HIT_CACHEOP_WAR && ((read_c0_prid() & 0xfff0) == 0x2020))
424 build_insn_word(0x3c01a000); /* lui $at, 0xa000 */
425
426 dest = label();
427 loop_start = store_offset;
428 do {
429 build_load_reg( 8);
430 build_load_reg( 9);
431 build_load_reg(10);
432 build_load_reg(11);
433 build_store_reg( 8);
434 build_store_reg( 9);
435 build_store_reg(10);
436 build_store_reg(11);
437 } while ((store_offset - loop_start) < half_scache_line_size());
438 build_addiu_a0(2 * store_offset);
439 build_addiu_a1(2 * load_offset);
440 loop_start = store_offset;
441 do {
442 build_load_reg( 8);
443 build_load_reg( 9);
444 build_load_reg(10);
445 build_load_reg(11);
446 build_store_reg( 8);
447 build_store_reg( 9);
448 build_store_reg(10);
449 build_store_reg(11);
450 } while ((store_offset - loop_start) < half_scache_line_size());
451 build_bne(dest);
452
453 if (cpu_has_prefetch && pref_offset_copy) {
454 build_addiu_a2_a0(pref_offset_copy);
455 dest = label();
456 loop_start = store_offset;
457 do {
458 __build_load_reg( 8);
459 __build_load_reg( 9);
460 __build_load_reg(10);
461 __build_load_reg(11);
462 __build_store_reg( 8);
463 __build_store_reg( 9);
464 __build_store_reg(10);
465 __build_store_reg(11);
466 } while ((store_offset - loop_start) < half_scache_line_size());
467 build_addiu_a0(2 * store_offset);
468 build_addiu_a1(2 * load_offset);
469 loop_start = store_offset;
470 do {
471 __build_load_reg( 8);
472 __build_load_reg( 9);
473 __build_load_reg(10);
474 __build_load_reg(11);
475 __build_store_reg( 8);
476 __build_store_reg( 9);
477 __build_store_reg(10);
478 __build_store_reg(11);
479 } while ((store_offset - loop_start) < half_scache_line_size());
480 build_bne(dest);
481 }
482
483 build_jr_ra();
484
485 flush_icache_range((unsigned long)&copy_page_array,
486 (unsigned long) epc);
487
488 BUG_ON(epc > copy_page_array + ARRAY_SIZE(copy_page_array));
489 }
linux-next