4 #include <linux/kernel.h>
5 #include <asm/segment.h>
6 #include <asm/cmpxchg.h>
11 #define STR(x) __STR(x)
13 #define __SAVE(reg,offset) "movq %%" #reg ",(14-" #offset ")*8(%%rsp)\n\t"
14 #define __RESTORE(reg,offset) "movq (14-" #offset ")*8(%%rsp),%%" #reg "\n\t"
16 /* frame pointer must be last for get_wchan */
17 #define SAVE_CONTEXT "pushf ; pushq %%rbp ; movq %%rsi,%%rbp\n\t"
18 #define RESTORE_CONTEXT "movq %%rbp,%%rsi ; popq %%rbp ; popf\t"
20 #define __EXTRA_CLOBBER \
21 ,"rcx","rbx","rdx","r8","r9","r10","r11","r12","r13","r14","r15"
23 /* Save restore flags to clear handle leaking NT */
24 #define switch_to(prev,next,last) \
25 asm volatile(SAVE_CONTEXT \
26 "movq %%rsp,%P[threadrsp](%[prev])\n\t" /* save RSP */ \
27 "movq %P[threadrsp](%[next]),%%rsp\n\t" /* restore RSP */ \
28 "call __switch_to\n\t" \
29 ".globl thread_return\n" \
30 "thread_return:\n\t" \
31 "movq %%gs:%P[pda_pcurrent],%%rsi\n\t" \
32 "movq %P[thread_info](%%rsi),%%r8\n\t" \
33 LOCK_PREFIX "btr %[tif_fork],%P[ti_flags](%%r8)\n\t" \
34 "movq %%rax,%%rdi\n\t" \
35 "jc ret_from_fork\n\t" \
38 : [next] "S" (next), [prev] "D" (prev), \
39 [threadrsp] "i" (offsetof(struct task_struct, thread.rsp)), \
40 [ti_flags] "i" (offsetof(struct thread_info, flags)),\
41 [tif_fork] "i" (TIF_FORK), \
42 [thread_info] "i" (offsetof(struct task_struct, stack)), \
43 [pda_pcurrent] "i" (offsetof(struct x8664_pda, pcurrent)) \
44 : "memory", "cc" __EXTRA_CLOBBER)
46 extern void load_gs_index(unsigned);
49 * Load a segment. Fall back on loading the zero
50 * segment if something goes wrong..
52 #define loadsegment(seg,value) \
55 "movl %k0,%%" #seg "\n" \
57 ".section .fixup,\"ax\"\n" \
59 "movl %1,%%" #seg "\n\t" \
62 ".section __ex_table,\"a\"\n\t" \
66 : :"r" (value), "r" (0))
69 * Clear and set 'TS' bit respectively
71 #define clts() __asm__ __volatile__ ("clts")
73 static inline unsigned long read_cr0(void)
76 asm volatile("movq %%cr0,%0" : "=r" (cr0
));
80 static inline void write_cr0(unsigned long val
)
82 asm volatile("movq %0,%%cr0" :: "r" (val
));
85 static inline unsigned long read_cr2(void)
88 asm("movq %%cr2,%0" : "=r" (cr2
));
92 static inline void write_cr2(unsigned long val
)
94 asm volatile("movq %0,%%cr2" :: "r" (val
));
97 static inline unsigned long read_cr3(void)
100 asm("movq %%cr3,%0" : "=r" (cr3
));
104 static inline void write_cr3(unsigned long val
)
106 asm volatile("movq %0,%%cr3" :: "r" (val
) : "memory");
109 static inline unsigned long read_cr4(void)
112 asm("movq %%cr4,%0" : "=r" (cr4
));
116 static inline void write_cr4(unsigned long val
)
118 asm volatile("movq %0,%%cr4" :: "r" (val
) : "memory");
121 static inline unsigned long read_cr8(void)
124 asm("movq %%cr8,%0" : "=r" (cr8
));
128 static inline void write_cr8(unsigned long val
)
130 asm volatile("movq %0,%%cr8" :: "r" (val
) : "memory");
133 #define stts() write_cr0(8 | read_cr0())
136 __asm__ __volatile__ ("wbinvd": : :"memory")
138 #endif /* __KERNEL__ */
140 #define nop() __asm__ __volatile__ ("nop")
143 #define smp_mb() mb()
144 #define smp_rmb() barrier()
145 #define smp_wmb() barrier()
146 #define smp_read_barrier_depends() do {} while(0)
148 #define smp_mb() barrier()
149 #define smp_rmb() barrier()
150 #define smp_wmb() barrier()
151 #define smp_read_barrier_depends() do {} while(0)
156 * Force strict CPU ordering.
157 * And yes, this is required on UP too when we're talking
160 #define mb() asm volatile("mfence":::"memory")
161 #define rmb() asm volatile("lfence":::"memory")
162 #define wmb() asm volatile("sfence" ::: "memory")
164 #define read_barrier_depends() do {} while(0)
165 #define set_mb(var, value) do { (void) xchg(&var, value); } while (0)
167 #define warn_if_not_ulong(x) do { unsigned long foo; (void) (&(x) == &foo); } while (0)
169 #include <linux/irqflags.h>
171 void cpu_idle_wait(void);
173 extern unsigned long arch_align_stack(unsigned long sp
);
174 extern void free_init_pages(char *what
, unsigned long begin
, unsigned long end
);