?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'v3.3.7' into 3.3/master
[zen-stable.git] / arch / x86 / include / asm / paravirt.h
bloba7d2db9a74fbc835dd7ed41ca2f7b5754b0263fe
1 #ifndef _ASM_X86_PARAVIRT_H
2 #define _ASM_X86_PARAVIRT_H
3 /* Various instructions on x86 need to be replaced for
4 * para-virtualization: those hooks are defined here. */
5
6 #ifdef CONFIG_PARAVIRT
7 #include <asm/pgtable_types.h>
8 #include <asm/asm.h>
9
10 #include <asm/paravirt_types.h>
11
12 #ifndef __ASSEMBLY__
13 #include <linux/types.h>
14 #include <linux/cpumask.h>
15
16 static inline int paravirt_enabled(void)
17 {
18 return pv_info.paravirt_enabled;
19 }
20
21 static inline void load_sp0(struct tss_struct *tss,
22 struct thread_struct *thread)
23 {
24 PVOP_VCALL2(pv_cpu_ops.load_sp0, tss, thread);
25 }
26
27 /* The paravirtualized CPUID instruction. */
28 static inline void __cpuid(unsigned int *eax, unsigned int *ebx,
29 unsigned int *ecx, unsigned int *edx)
30 {
31 PVOP_VCALL4(pv_cpu_ops.cpuid, eax, ebx, ecx, edx);
32 }
33
34 /*
35 * These special macros can be used to get or set a debugging register
36 */
37 static inline unsigned long paravirt_get_debugreg(int reg)
38 {
39 return PVOP_CALL1(unsigned long, pv_cpu_ops.get_debugreg, reg);
40 }
41 #define get_debugreg(var, reg) var = paravirt_get_debugreg(reg)
42 static inline void set_debugreg(unsigned long val, int reg)
43 {
44 PVOP_VCALL2(pv_cpu_ops.set_debugreg, reg, val);
45 }
46
47 static inline void clts(void)
48 {
49 PVOP_VCALL0(pv_cpu_ops.clts);
50 }
51
52 static inline unsigned long read_cr0(void)
53 {
54 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr0);
55 }
56
57 static inline void write_cr0(unsigned long x)
58 {
59 PVOP_VCALL1(pv_cpu_ops.write_cr0, x);
60 }
61
62 static inline unsigned long read_cr2(void)
63 {
64 return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr2);
65 }
66
67 static inline void write_cr2(unsigned long x)
68 {
69 PVOP_VCALL1(pv_mmu_ops.write_cr2, x);
70 }
71
72 static inline unsigned long read_cr3(void)
73 {
74 return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr3);
75 }
76
77 static inline void write_cr3(unsigned long x)
78 {
79 PVOP_VCALL1(pv_mmu_ops.write_cr3, x);
80 }
81
82 static inline unsigned long read_cr4(void)
83 {
84 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4);
85 }
86 static inline unsigned long read_cr4_safe(void)
87 {
88 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4_safe);
89 }
90
91 static inline void write_cr4(unsigned long x)
92 {
93 PVOP_VCALL1(pv_cpu_ops.write_cr4, x);
94 }
95
96 #ifdef CONFIG_X86_64
97 static inline unsigned long read_cr8(void)
98 {
99 return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr8);
100 }
101
102 static inline void write_cr8(unsigned long x)
103 {
104 PVOP_VCALL1(pv_cpu_ops.write_cr8, x);
105 }
106 #endif
107
108 static inline void arch_safe_halt(void)
109 {
110 PVOP_VCALL0(pv_irq_ops.safe_halt);
111 }
112
113 static inline void halt(void)
114 {
115 PVOP_VCALL0(pv_irq_ops.halt);
116 }
117
118 static inline void wbinvd(void)
119 {
120 PVOP_VCALL0(pv_cpu_ops.wbinvd);
121 }
122
123 #define get_kernel_rpl() (pv_info.kernel_rpl)
124
125 static inline u64 paravirt_read_msr(unsigned msr, int *err)
126 {
127 return PVOP_CALL2(u64, pv_cpu_ops.read_msr, msr, err);
128 }
129
130 static inline int paravirt_rdmsr_regs(u32 *regs)
131 {
132 return PVOP_CALL1(int, pv_cpu_ops.rdmsr_regs, regs);
133 }
134
135 static inline int paravirt_write_msr(unsigned msr, unsigned low, unsigned high)
136 {
137 return PVOP_CALL3(int, pv_cpu_ops.write_msr, msr, low, high);
138 }
139
140 static inline int paravirt_wrmsr_regs(u32 *regs)
141 {
142 return PVOP_CALL1(int, pv_cpu_ops.wrmsr_regs, regs);
143 }
144
145 /* These should all do BUG_ON(_err), but our headers are too tangled. */
146 #define rdmsr(msr, val1, val2) \
147 do { \
148 int _err; \
149 u64 _l = paravirt_read_msr(msr, &_err); \
150 val1 = (u32)_l; \
151 val2 = _l >> 32; \
152 } while (0)
153
154 #define wrmsr(msr, val1, val2) \
155 do { \
156 paravirt_write_msr(msr, val1, val2); \
157 } while (0)
158
159 #define rdmsrl(msr, val) \
160 do { \
161 int _err; \
162 val = paravirt_read_msr(msr, &_err); \
163 } while (0)
164
165 #define wrmsrl(msr, val) wrmsr(msr, (u32)((u64)(val)), ((u64)(val))>>32)
166 #define wrmsr_safe(msr, a, b) paravirt_write_msr(msr, a, b)
167
168 /* rdmsr with exception handling */
169 #define rdmsr_safe(msr, a, b) \
170 ({ \
171 int _err; \
172 u64 _l = paravirt_read_msr(msr, &_err); \
173 (*a) = (u32)_l; \
174 (*b) = _l >> 32; \
175 _err; \
176 })
177
178 #define rdmsr_safe_regs(regs) paravirt_rdmsr_regs(regs)
179 #define wrmsr_safe_regs(regs) paravirt_wrmsr_regs(regs)
180
181 static inline int rdmsrl_safe(unsigned msr, unsigned long long *p)
182 {
183 int err;
184
185 *p = paravirt_read_msr(msr, &err);
186 return err;
187 }
188 static inline int rdmsrl_amd_safe(unsigned msr, unsigned long long *p)
189 {
190 u32 gprs[8] = { 0 };
191 int err;
192
193 gprs[1] = msr;
194 gprs[7] = 0x9c5a203a;
195
196 err = paravirt_rdmsr_regs(gprs);
197
198 *p = gprs[0] | ((u64)gprs[2] << 32);
199
200 return err;
201 }
202
203 static inline int wrmsrl_amd_safe(unsigned msr, unsigned long long val)
204 {
205 u32 gprs[8] = { 0 };
206
207 gprs[0] = (u32)val;
208 gprs[1] = msr;
209 gprs[2] = val >> 32;
210 gprs[7] = 0x9c5a203a;
211
212 return paravirt_wrmsr_regs(gprs);
213 }
214
215 static inline u64 paravirt_read_tsc(void)
216 {
217 return PVOP_CALL0(u64, pv_cpu_ops.read_tsc);
218 }
219
220 #define rdtscl(low) \
221 do { \
222 u64 _l = paravirt_read_tsc(); \
223 low = (int)_l; \
224 } while (0)
225
226 #define rdtscll(val) (val = paravirt_read_tsc())
227
228 static inline unsigned long long paravirt_sched_clock(void)
229 {
230 return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock);
231 }
232
233 struct jump_label_key;
234 extern struct jump_label_key paravirt_steal_enabled;
235 extern struct jump_label_key paravirt_steal_rq_enabled;
236
237 static inline u64 paravirt_steal_clock(int cpu)
238 {
239 return PVOP_CALL1(u64, pv_time_ops.steal_clock, cpu);
240 }
241
242 static inline unsigned long long paravirt_read_pmc(int counter)
243 {
244 return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter);
245 }
246
247 #define rdpmc(counter, low, high) \
248 do { \
249 u64 _l = paravirt_read_pmc(counter); \
250 low = (u32)_l; \
251 high = _l >> 32; \
252 } while (0)
253
254 static inline unsigned long long paravirt_rdtscp(unsigned int *aux)
255 {
256 return PVOP_CALL1(u64, pv_cpu_ops.read_tscp, aux);
257 }
258
259 #define rdtscp(low, high, aux) \
260 do { \
261 int __aux; \
262 unsigned long __val = paravirt_rdtscp(&__aux); \
263 (low) = (u32)__val; \
264 (high) = (u32)(__val >> 32); \
265 (aux) = __aux; \
266 } while (0)
267
268 #define rdtscpll(val, aux) \
269 do { \
270 unsigned long __aux; \
271 val = paravirt_rdtscp(&__aux); \
272 (aux) = __aux; \
273 } while (0)
274
275 static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries)
276 {
277 PVOP_VCALL2(pv_cpu_ops.alloc_ldt, ldt, entries);
278 }
279
280 static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries)
281 {
282 PVOP_VCALL2(pv_cpu_ops.free_ldt, ldt, entries);
283 }
284
285 static inline void load_TR_desc(void)
286 {
287 PVOP_VCALL0(pv_cpu_ops.load_tr_desc);
288 }
289 static inline void load_gdt(const struct desc_ptr *dtr)
290 {
291 PVOP_VCALL1(pv_cpu_ops.load_gdt, dtr);
292 }
293 static inline void load_idt(const struct desc_ptr *dtr)
294 {
295 PVOP_VCALL1(pv_cpu_ops.load_idt, dtr);
296 }
297 static inline void set_ldt(const void *addr, unsigned entries)
298 {
299 PVOP_VCALL2(pv_cpu_ops.set_ldt, addr, entries);
300 }
301 static inline void store_gdt(struct desc_ptr *dtr)
302 {
303 PVOP_VCALL1(pv_cpu_ops.store_gdt, dtr);
304 }
305 static inline void store_idt(struct desc_ptr *dtr)
306 {
307 PVOP_VCALL1(pv_cpu_ops.store_idt, dtr);
308 }
309 static inline unsigned long paravirt_store_tr(void)
310 {
311 return PVOP_CALL0(unsigned long, pv_cpu_ops.store_tr);
312 }
313 #define store_tr(tr) ((tr) = paravirt_store_tr())
314 static inline void load_TLS(struct thread_struct *t, unsigned cpu)
315 {
316 PVOP_VCALL2(pv_cpu_ops.load_tls, t, cpu);
317 }
318
319 #ifdef CONFIG_X86_64
320 static inline void load_gs_index(unsigned int gs)
321 {
322 PVOP_VCALL1(pv_cpu_ops.load_gs_index, gs);
323 }
324 #endif
325
326 static inline void write_ldt_entry(struct desc_struct *dt, int entry,
327 const void *desc)
328 {
329 PVOP_VCALL3(pv_cpu_ops.write_ldt_entry, dt, entry, desc);
330 }
331
332 static inline void write_gdt_entry(struct desc_struct *dt, int entry,
333 void *desc, int type)
334 {
335 PVOP_VCALL4(pv_cpu_ops.write_gdt_entry, dt, entry, desc, type);
336 }
337
338 static inline void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g)
339 {
340 PVOP_VCALL3(pv_cpu_ops.write_idt_entry, dt, entry, g);
341 }
342 static inline void set_iopl_mask(unsigned mask)
343 {
344 PVOP_VCALL1(pv_cpu_ops.set_iopl_mask, mask);
345 }
346
347 /* The paravirtualized I/O functions */
348 static inline void slow_down_io(void)
349 {
350 pv_cpu_ops.io_delay();
351 #ifdef REALLY_SLOW_IO
352 pv_cpu_ops.io_delay();
353 pv_cpu_ops.io_delay();
354 pv_cpu_ops.io_delay();
355 #endif
356 }
357
358 #ifdef CONFIG_SMP
359 static inline void startup_ipi_hook(int phys_apicid, unsigned long start_eip,
360 unsigned long start_esp)
361 {
362 PVOP_VCALL3(pv_apic_ops.startup_ipi_hook,
363 phys_apicid, start_eip, start_esp);
364 }
365 #endif
366
367 static inline void paravirt_activate_mm(struct mm_struct *prev,
368 struct mm_struct *next)
369 {
370 PVOP_VCALL2(pv_mmu_ops.activate_mm, prev, next);
371 }
372
373 static inline void arch_dup_mmap(struct mm_struct *oldmm,
374 struct mm_struct *mm)
375 {
376 PVOP_VCALL2(pv_mmu_ops.dup_mmap, oldmm, mm);
377 }
378
379 static inline void arch_exit_mmap(struct mm_struct *mm)
380 {
381 PVOP_VCALL1(pv_mmu_ops.exit_mmap, mm);
382 }
383
384 static inline void __flush_tlb(void)
385 {
386 PVOP_VCALL0(pv_mmu_ops.flush_tlb_user);
387 }
388 static inline void __flush_tlb_global(void)
389 {
390 PVOP_VCALL0(pv_mmu_ops.flush_tlb_kernel);
391 }
392 static inline void __flush_tlb_single(unsigned long addr)
393 {
394 PVOP_VCALL1(pv_mmu_ops.flush_tlb_single, addr);
395 }
396
397 static inline void flush_tlb_others(const struct cpumask *cpumask,
398 struct mm_struct *mm,
399 unsigned long va)
400 {
401 PVOP_VCALL3(pv_mmu_ops.flush_tlb_others, cpumask, mm, va);
402 }
403
404 static inline int paravirt_pgd_alloc(struct mm_struct *mm)
405 {
406 return PVOP_CALL1(int, pv_mmu_ops.pgd_alloc, mm);
407 }
408
409 static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd)
410 {
411 PVOP_VCALL2(pv_mmu_ops.pgd_free, mm, pgd);
412 }
413
414 static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn)
415 {
416 PVOP_VCALL2(pv_mmu_ops.alloc_pte, mm, pfn);
417 }
418 static inline void paravirt_release_pte(unsigned long pfn)
419 {
420 PVOP_VCALL1(pv_mmu_ops.release_pte, pfn);
421 }
422
423 static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn)
424 {
425 PVOP_VCALL2(pv_mmu_ops.alloc_pmd, mm, pfn);
426 }
427
428 static inline void paravirt_release_pmd(unsigned long pfn)
429 {
430 PVOP_VCALL1(pv_mmu_ops.release_pmd, pfn);
431 }
432
433 static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn)
434 {
435 PVOP_VCALL2(pv_mmu_ops.alloc_pud, mm, pfn);
436 }
437 static inline void paravirt_release_pud(unsigned long pfn)
438 {
439 PVOP_VCALL1(pv_mmu_ops.release_pud, pfn);
440 }
441
442 static inline void pte_update(struct mm_struct *mm, unsigned long addr,
443 pte_t *ptep)
444 {
445 PVOP_VCALL3(pv_mmu_ops.pte_update, mm, addr, ptep);
446 }
447 static inline void pmd_update(struct mm_struct *mm, unsigned long addr,
448 pmd_t *pmdp)
449 {
450 PVOP_VCALL3(pv_mmu_ops.pmd_update, mm, addr, pmdp);
451 }
452
453 static inline void pte_update_defer(struct mm_struct *mm, unsigned long addr,
454 pte_t *ptep)
455 {
456 PVOP_VCALL3(pv_mmu_ops.pte_update_defer, mm, addr, ptep);
457 }
458
459 static inline void pmd_update_defer(struct mm_struct *mm, unsigned long addr,
460 pmd_t *pmdp)
461 {
462 PVOP_VCALL3(pv_mmu_ops.pmd_update_defer, mm, addr, pmdp);
463 }
464
465 static inline pte_t __pte(pteval_t val)
466 {
467 pteval_t ret;
468
469 if (sizeof(pteval_t) > sizeof(long))
470 ret = PVOP_CALLEE2(pteval_t,
471 pv_mmu_ops.make_pte,
472 val, (u64)val >> 32);
473 else
474 ret = PVOP_CALLEE1(pteval_t,
475 pv_mmu_ops.make_pte,
476 val);
477
478 return (pte_t) { .pte = ret };
479 }
480
481 static inline pteval_t pte_val(pte_t pte)
482 {
483 pteval_t ret;
484
485 if (sizeof(pteval_t) > sizeof(long))
486 ret = PVOP_CALLEE2(pteval_t, pv_mmu_ops.pte_val,
487 pte.pte, (u64)pte.pte >> 32);
488 else
489 ret = PVOP_CALLEE1(pteval_t, pv_mmu_ops.pte_val,
490 pte.pte);
491
492 return ret;
493 }
494
495 static inline pgd_t __pgd(pgdval_t val)
496 {
497 pgdval_t ret;
498
499 if (sizeof(pgdval_t) > sizeof(long))
500 ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.make_pgd,
501 val, (u64)val >> 32);
502 else
503 ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.make_pgd,
504 val);
505
506 return (pgd_t) { ret };
507 }
508
509 static inline pgdval_t pgd_val(pgd_t pgd)
510 {
511 pgdval_t ret;
512
513 if (sizeof(pgdval_t) > sizeof(long))
514 ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.pgd_val,
515 pgd.pgd, (u64)pgd.pgd >> 32);
516 else
517 ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.pgd_val,
518 pgd.pgd);
519
520 return ret;
521 }
522
523 #define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
524 static inline pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr,
525 pte_t *ptep)
526 {
527 pteval_t ret;
528
529 ret = PVOP_CALL3(pteval_t, pv_mmu_ops.ptep_modify_prot_start,
530 mm, addr, ptep);
531
532 return (pte_t) { .pte = ret };
533 }
534
535 static inline void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
536 pte_t *ptep, pte_t pte)
537 {
538 if (sizeof(pteval_t) > sizeof(long))
539 /* 5 arg words */
540 pv_mmu_ops.ptep_modify_prot_commit(mm, addr, ptep, pte);
541 else
542 PVOP_VCALL4(pv_mmu_ops.ptep_modify_prot_commit,
543 mm, addr, ptep, pte.pte);
544 }
545
546 static inline void set_pte(pte_t *ptep, pte_t pte)
547 {
548 if (sizeof(pteval_t) > sizeof(long))
549 PVOP_VCALL3(pv_mmu_ops.set_pte, ptep,
550 pte.pte, (u64)pte.pte >> 32);
551 else
552 PVOP_VCALL2(pv_mmu_ops.set_pte, ptep,
553 pte.pte);
554 }
555
556 static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
557 pte_t *ptep, pte_t pte)
558 {
559 if (sizeof(pteval_t) > sizeof(long))
560 /* 5 arg words */
561 pv_mmu_ops.set_pte_at(mm, addr, ptep, pte);
562 else
563 PVOP_VCALL4(pv_mmu_ops.set_pte_at, mm, addr, ptep, pte.pte);
564 }
565
566 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
567 static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
568 pmd_t *pmdp, pmd_t pmd)
569 {
570 if (sizeof(pmdval_t) > sizeof(long))
571 /* 5 arg words */
572 pv_mmu_ops.set_pmd_at(mm, addr, pmdp, pmd);
573 else
574 PVOP_VCALL4(pv_mmu_ops.set_pmd_at, mm, addr, pmdp,
575 native_pmd_val(pmd));
576 }
577 #endif
578
579 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
580 {
581 pmdval_t val = native_pmd_val(pmd);
582
583 if (sizeof(pmdval_t) > sizeof(long))
584 PVOP_VCALL3(pv_mmu_ops.set_pmd, pmdp, val, (u64)val >> 32);
585 else
586 PVOP_VCALL2(pv_mmu_ops.set_pmd, pmdp, val);
587 }
588
589 #if PAGETABLE_LEVELS >= 3
590 static inline pmd_t __pmd(pmdval_t val)
591 {
592 pmdval_t ret;
593
594 if (sizeof(pmdval_t) > sizeof(long))
595 ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.make_pmd,
596 val, (u64)val >> 32);
597 else
598 ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.make_pmd,
599 val);
600
601 return (pmd_t) { ret };
602 }
603
604 static inline pmdval_t pmd_val(pmd_t pmd)
605 {
606 pmdval_t ret;
607
608 if (sizeof(pmdval_t) > sizeof(long))
609 ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.pmd_val,
610 pmd.pmd, (u64)pmd.pmd >> 32);
611 else
612 ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.pmd_val,
613 pmd.pmd);
614
615 return ret;
616 }
617
618 static inline void set_pud(pud_t *pudp, pud_t pud)
619 {
620 pudval_t val = native_pud_val(pud);
621
622 if (sizeof(pudval_t) > sizeof(long))
623 PVOP_VCALL3(pv_mmu_ops.set_pud, pudp,
624 val, (u64)val >> 32);
625 else
626 PVOP_VCALL2(pv_mmu_ops.set_pud, pudp,
627 val);
628 }
629 #if PAGETABLE_LEVELS == 4
630 static inline pud_t __pud(pudval_t val)
631 {
632 pudval_t ret;
633
634 if (sizeof(pudval_t) > sizeof(long))
635 ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.make_pud,
636 val, (u64)val >> 32);
637 else
638 ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.make_pud,
639 val);
640
641 return (pud_t) { ret };
642 }
643
644 static inline pudval_t pud_val(pud_t pud)
645 {
646 pudval_t ret;
647
648 if (sizeof(pudval_t) > sizeof(long))
649 ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.pud_val,
650 pud.pud, (u64)pud.pud >> 32);
651 else
652 ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.pud_val,
653 pud.pud);
654
655 return ret;
656 }
657
658 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
659 {
660 pgdval_t val = native_pgd_val(pgd);
661
662 if (sizeof(pgdval_t) > sizeof(long))
663 PVOP_VCALL3(pv_mmu_ops.set_pgd, pgdp,
664 val, (u64)val >> 32);
665 else
666 PVOP_VCALL2(pv_mmu_ops.set_pgd, pgdp,
667 val);
668 }
669
670 static inline void pgd_clear(pgd_t *pgdp)
671 {
672 set_pgd(pgdp, __pgd(0));
673 }
674
675 static inline void pud_clear(pud_t *pudp)
676 {
677 set_pud(pudp, __pud(0));
678 }
679
680 #endif /* PAGETABLE_LEVELS == 4 */
681
682 #endif /* PAGETABLE_LEVELS >= 3 */
683
684 #ifdef CONFIG_X86_PAE
685 /* Special-case pte-setting operations for PAE, which can't update a
686 64-bit pte atomically */
687 static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
688 {
689 PVOP_VCALL3(pv_mmu_ops.set_pte_atomic, ptep,
690 pte.pte, pte.pte >> 32);
691 }
692
693 static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
694 pte_t *ptep)
695 {
696 PVOP_VCALL3(pv_mmu_ops.pte_clear, mm, addr, ptep);
697 }
698
699 static inline void pmd_clear(pmd_t *pmdp)
700 {
701 PVOP_VCALL1(pv_mmu_ops.pmd_clear, pmdp);
702 }
703 #else /* !CONFIG_X86_PAE */
704 static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
705 {
706 set_pte(ptep, pte);
707 }
708
709 static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
710 pte_t *ptep)
711 {
712 set_pte_at(mm, addr, ptep, __pte(0));
713 }
714
715 static inline void pmd_clear(pmd_t *pmdp)
716 {
717 set_pmd(pmdp, __pmd(0));
718 }
719 #endif /* CONFIG_X86_PAE */
720
721 #define __HAVE_ARCH_START_CONTEXT_SWITCH
722 static inline void arch_start_context_switch(struct task_struct *prev)
723 {
724 PVOP_VCALL1(pv_cpu_ops.start_context_switch, prev);
725 }
726
727 static inline void arch_end_context_switch(struct task_struct *next)
728 {
729 PVOP_VCALL1(pv_cpu_ops.end_context_switch, next);
730 }
731
732 #define __HAVE_ARCH_ENTER_LAZY_MMU_MODE
733 static inline void arch_enter_lazy_mmu_mode(void)
734 {
735 PVOP_VCALL0(pv_mmu_ops.lazy_mode.enter);
736 }
737
738 static inline void arch_leave_lazy_mmu_mode(void)
739 {
740 PVOP_VCALL0(pv_mmu_ops.lazy_mode.leave);
741 }
742
743 void arch_flush_lazy_mmu_mode(void);
744
745 static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx,
746 phys_addr_t phys, pgprot_t flags)
747 {
748 pv_mmu_ops.set_fixmap(idx, phys, flags);
749 }
750
751 #if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS)
752
753 static inline int arch_spin_is_locked(struct arch_spinlock *lock)
754 {
755 return PVOP_CALL1(int, pv_lock_ops.spin_is_locked, lock);
756 }
757
758 static inline int arch_spin_is_contended(struct arch_spinlock *lock)
759 {
760 return PVOP_CALL1(int, pv_lock_ops.spin_is_contended, lock);
761 }
762 #define arch_spin_is_contended arch_spin_is_contended
763
764 static __always_inline void arch_spin_lock(struct arch_spinlock *lock)
765 {
766 PVOP_VCALL1(pv_lock_ops.spin_lock, lock);
767 }
768
769 static __always_inline void arch_spin_lock_flags(struct arch_spinlock *lock,
770 unsigned long flags)
771 {
772 PVOP_VCALL2(pv_lock_ops.spin_lock_flags, lock, flags);
773 }
774
775 static __always_inline int arch_spin_trylock(struct arch_spinlock *lock)
776 {
777 return PVOP_CALL1(int, pv_lock_ops.spin_trylock, lock);
778 }
779
780 static __always_inline void arch_spin_unlock(struct arch_spinlock *lock)
781 {
782 PVOP_VCALL1(pv_lock_ops.spin_unlock, lock);
783 }
784
785 #endif
786
787 #ifdef CONFIG_X86_32
788 #define PV_SAVE_REGS "pushl %ecx; pushl %edx;"
789 #define PV_RESTORE_REGS "popl %edx; popl %ecx;"
790
791 /* save and restore all caller-save registers, except return value */
792 #define PV_SAVE_ALL_CALLER_REGS "pushl %ecx;"
793 #define PV_RESTORE_ALL_CALLER_REGS "popl %ecx;"
794
795 #define PV_FLAGS_ARG "0"
796 #define PV_EXTRA_CLOBBERS
797 #define PV_VEXTRA_CLOBBERS
798 #else
799 /* save and restore all caller-save registers, except return value */
800 #define PV_SAVE_ALL_CALLER_REGS \
801 "push %rcx;" \
802 "push %rdx;" \
803 "push %rsi;" \
804 "push %rdi;" \
805 "push %r8;" \
806 "push %r9;" \
807 "push %r10;" \
808 "push %r11;"
809 #define PV_RESTORE_ALL_CALLER_REGS \
810 "pop %r11;" \
811 "pop %r10;" \
812 "pop %r9;" \
813 "pop %r8;" \
814 "pop %rdi;" \
815 "pop %rsi;" \
816 "pop %rdx;" \
817 "pop %rcx;"
818
819 /* We save some registers, but all of them, that's too much. We clobber all
820 * caller saved registers but the argument parameter */
821 #define PV_SAVE_REGS "pushq %%rdi;"
822 #define PV_RESTORE_REGS "popq %%rdi;"
823 #define PV_EXTRA_CLOBBERS EXTRA_CLOBBERS, "rcx" , "rdx", "rsi"
824 #define PV_VEXTRA_CLOBBERS EXTRA_CLOBBERS, "rdi", "rcx" , "rdx", "rsi"
825 #define PV_FLAGS_ARG "D"
826 #endif
827
828 /*
829 * Generate a thunk around a function which saves all caller-save
830 * registers except for the return value. This allows C functions to
831 * be called from assembler code where fewer than normal registers are
832 * available. It may also help code generation around calls from C
833 * code if the common case doesn't use many registers.
834 *
835 * When a callee is wrapped in a thunk, the caller can assume that all
836 * arg regs and all scratch registers are preserved across the
837 * call. The return value in rax/eax will not be saved, even for void
838 * functions.
839 */
840 #define PV_CALLEE_SAVE_REGS_THUNK(func) \
841 extern typeof(func) __raw_callee_save_##func; \
842 static void *__##func##__ __used = func; \
843 \
844 asm(".pushsection .text;" \
845 "__raw_callee_save_" #func ": " \
846 PV_SAVE_ALL_CALLER_REGS \
847 "call " #func ";" \
848 PV_RESTORE_ALL_CALLER_REGS \
849 "ret;" \
850 ".popsection")
851
852 /* Get a reference to a callee-save function */
853 #define PV_CALLEE_SAVE(func) \
854 ((struct paravirt_callee_save) { __raw_callee_save_##func })
855
856 /* Promise that "func" already uses the right calling convention */
857 #define __PV_IS_CALLEE_SAVE(func) \
858 ((struct paravirt_callee_save) { func })
859
860 static inline notrace unsigned long arch_local_save_flags(void)
861 {
862 return PVOP_CALLEE0(unsigned long, pv_irq_ops.save_fl);
863 }
864
865 static inline notrace void arch_local_irq_restore(unsigned long f)
866 {
867 PVOP_VCALLEE1(pv_irq_ops.restore_fl, f);
868 }
869
870 static inline notrace void arch_local_irq_disable(void)
871 {
872 PVOP_VCALLEE0(pv_irq_ops.irq_disable);
873 }
874
875 static inline notrace void arch_local_irq_enable(void)
876 {
877 PVOP_VCALLEE0(pv_irq_ops.irq_enable);
878 }
879
880 static inline notrace unsigned long arch_local_irq_save(void)
881 {
882 unsigned long f;
883
884 f = arch_local_save_flags();
885 arch_local_irq_disable();
886 return f;
887 }
888
889
890 /* Make sure as little as possible of this mess escapes. */
891 #undef PARAVIRT_CALL
892 #undef __PVOP_CALL
893 #undef __PVOP_VCALL
894 #undef PVOP_VCALL0
895 #undef PVOP_CALL0
896 #undef PVOP_VCALL1
897 #undef PVOP_CALL1
898 #undef PVOP_VCALL2
899 #undef PVOP_CALL2
900 #undef PVOP_VCALL3
901 #undef PVOP_CALL3
902 #undef PVOP_VCALL4
903 #undef PVOP_CALL4
904
905 extern void default_banner(void);
906
907 #else /* __ASSEMBLY__ */
908
909 #define _PVSITE(ptype, clobbers, ops, word, algn) \
910 771:; \
911 ops; \
912 772:; \
913 .pushsection .parainstructions,"a"; \
914 .align algn; \
915 word 771b; \
916 .byte ptype; \
917 .byte 772b-771b; \
918 .short clobbers; \
919 .popsection
920
921
922 #define COND_PUSH(set, mask, reg) \
923 .if ((~(set)) & mask); push %reg; .endif
924 #define COND_POP(set, mask, reg) \
925 .if ((~(set)) & mask); pop %reg; .endif
926
927 #ifdef CONFIG_X86_64
928
929 #define PV_SAVE_REGS(set) \
930 COND_PUSH(set, CLBR_RAX, rax); \
931 COND_PUSH(set, CLBR_RCX, rcx); \
932 COND_PUSH(set, CLBR_RDX, rdx); \
933 COND_PUSH(set, CLBR_RSI, rsi); \
934 COND_PUSH(set, CLBR_RDI, rdi); \
935 COND_PUSH(set, CLBR_R8, r8); \
936 COND_PUSH(set, CLBR_R9, r9); \
937 COND_PUSH(set, CLBR_R10, r10); \
938 COND_PUSH(set, CLBR_R11, r11)
939 #define PV_RESTORE_REGS(set) \
940 COND_POP(set, CLBR_R11, r11); \
941 COND_POP(set, CLBR_R10, r10); \
942 COND_POP(set, CLBR_R9, r9); \
943 COND_POP(set, CLBR_R8, r8); \
944 COND_POP(set, CLBR_RDI, rdi); \
945 COND_POP(set, CLBR_RSI, rsi); \
946 COND_POP(set, CLBR_RDX, rdx); \
947 COND_POP(set, CLBR_RCX, rcx); \
948 COND_POP(set, CLBR_RAX, rax)
949
950 #define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 8)
951 #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .quad, 8)
952 #define PARA_INDIRECT(addr) *addr(%rip)
953 #else
954 #define PV_SAVE_REGS(set) \
955 COND_PUSH(set, CLBR_EAX, eax); \
956 COND_PUSH(set, CLBR_EDI, edi); \
957 COND_PUSH(set, CLBR_ECX, ecx); \
958 COND_PUSH(set, CLBR_EDX, edx)
959 #define PV_RESTORE_REGS(set) \
960 COND_POP(set, CLBR_EDX, edx); \
961 COND_POP(set, CLBR_ECX, ecx); \
962 COND_POP(set, CLBR_EDI, edi); \
963 COND_POP(set, CLBR_EAX, eax)
964
965 #define PARA_PATCH(struct, off) ((PARAVIRT_PATCH_##struct + (off)) / 4)
966 #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4)
967 #define PARA_INDIRECT(addr) *%cs:addr
968 #endif
969
970 #define INTERRUPT_RETURN \
971 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_iret), CLBR_NONE, \
972 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_iret))
973
974 #define DISABLE_INTERRUPTS(clobbers) \
975 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \
976 PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \
977 call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_disable); \
978 PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
979
980 #define ENABLE_INTERRUPTS(clobbers) \
981 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers, \
982 PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \
983 call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_enable); \
984 PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
985
986 #define USERGS_SYSRET32 \
987 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret32), \
988 CLBR_NONE, \
989 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret32))
990
991 #ifdef CONFIG_X86_32
992 #define GET_CR0_INTO_EAX \
993 push %ecx; push %edx; \
994 call PARA_INDIRECT(pv_cpu_ops+PV_CPU_read_cr0); \
995 pop %edx; pop %ecx
996
997 #define ENABLE_INTERRUPTS_SYSEXIT \
998 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_sysexit), \
999 CLBR_NONE, \
1000 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_irq_enable_sysexit))
1001
1002
1003 #else /* !CONFIG_X86_32 */
1004
1005 /*
1006 * If swapgs is used while the userspace stack is still current,
1007 * there's no way to call a pvop. The PV replacement *must* be
1008 * inlined, or the swapgs instruction must be trapped and emulated.
1009 */
1010 #define SWAPGS_UNSAFE_STACK \
1011 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \
1012 swapgs)
1013
1014 /*
1015 * Note: swapgs is very special, and in practise is either going to be
1016 * implemented with a single "swapgs" instruction or something very
1017 * special. Either way, we don't need to save any registers for
1018 * it.
1019 */
1020 #define SWAPGS \
1021 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE, \
1022 call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs) \
1023 )
1024
1025 #define GET_CR2_INTO_RCX \
1026 call PARA_INDIRECT(pv_mmu_ops+PV_MMU_read_cr2); \
1027 movq %rax, %rcx; \
1028 xorq %rax, %rax;
1029
1030 #define PARAVIRT_ADJUST_EXCEPTION_FRAME \
1031 PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_adjust_exception_frame), \
1032 CLBR_NONE, \
1033 call PARA_INDIRECT(pv_irq_ops+PV_IRQ_adjust_exception_frame))
1034
1035 #define USERGS_SYSRET64 \
1036 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret64), \
1037 CLBR_NONE, \
1038 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret64))
1039
1040 #define ENABLE_INTERRUPTS_SYSEXIT32 \
1041 PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_irq_enable_sysexit), \
1042 CLBR_NONE, \
1043 jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_irq_enable_sysexit))
1044 #endif /* CONFIG_X86_32 */
1045
1046 #endif /* __ASSEMBLY__ */
1047 #else /* CONFIG_PARAVIRT */
1048 # define default_banner x86_init_noop
1049 #endif /* !CONFIG_PARAVIRT */
1050 #endif /* _ASM_X86_PARAVIRT_H */