acpi_pad: build only on X86
[linux-2.6/linux-acpi-2.6.git] / arch / x86 / kernel / vsyscall_64.c
blob25ee06a80aad3cd116292227826f9a32fa4b3f7a
1 /*
2 * Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
3 * Copyright 2003 Andi Kleen, SuSE Labs.
5 * Thanks to hpa@transmeta.com for some useful hint.
6 * Special thanks to Ingo Molnar for his early experience with
7 * a different vsyscall implementation for Linux/IA32 and for the name.
9 * vsyscall 1 is located at -10Mbyte, vsyscall 2 is located
10 * at virtual address -10Mbyte+1024bytes etc... There are at max 4
11 * vsyscalls. One vsyscall can reserve more than 1 slot to avoid
12 * jumping out of line if necessary. We cannot add more with this
13 * mechanism because older kernels won't return -ENOSYS.
14 * If we want more than four we need a vDSO.
16 * Note: the concept clashes with user mode linux. If you use UML and
17 * want per guest time just set the kernel.vsyscall64 sysctl to 0.
20 /* Disable profiling for userspace code: */
21 #define DISABLE_BRANCH_PROFILING
23 #include <linux/time.h>
24 #include <linux/init.h>
25 #include <linux/kernel.h>
26 #include <linux/timer.h>
27 #include <linux/seqlock.h>
28 #include <linux/jiffies.h>
29 #include <linux/sysctl.h>
30 #include <linux/clocksource.h>
31 #include <linux/getcpu.h>
32 #include <linux/cpu.h>
33 #include <linux/smp.h>
34 #include <linux/notifier.h>
36 #include <asm/vsyscall.h>
37 #include <asm/pgtable.h>
38 #include <asm/page.h>
39 #include <asm/unistd.h>
40 #include <asm/fixmap.h>
41 #include <asm/errno.h>
42 #include <asm/io.h>
43 #include <asm/segment.h>
44 #include <asm/desc.h>
45 #include <asm/topology.h>
46 #include <asm/vgtod.h>
48 #define __vsyscall(nr) \
49 __attribute__ ((unused, __section__(".vsyscall_" #nr))) notrace
50 #define __syscall_clobber "r11","cx","memory"
53 * vsyscall_gtod_data contains data that is :
54 * - readonly from vsyscalls
55 * - written by timer interrupt or systcl (/proc/sys/kernel/vsyscall64)
56 * Try to keep this structure as small as possible to avoid cache line ping pongs
58 int __vgetcpu_mode __section_vgetcpu_mode;
60 struct vsyscall_gtod_data __vsyscall_gtod_data __section_vsyscall_gtod_data =
62 .lock = SEQLOCK_UNLOCKED,
63 .sysctl_enabled = 1,
66 void update_vsyscall_tz(void)
68 unsigned long flags;
70 write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
71 /* sys_tz has changed */
72 vsyscall_gtod_data.sys_tz = sys_tz;
73 write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
76 void update_vsyscall(struct timespec *wall_time, struct clocksource *clock)
78 unsigned long flags;
80 write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
81 /* copy vsyscall data */
82 vsyscall_gtod_data.clock.vread = clock->vread;
83 vsyscall_gtod_data.clock.cycle_last = clock->cycle_last;
84 vsyscall_gtod_data.clock.mask = clock->mask;
85 vsyscall_gtod_data.clock.mult = clock->mult;
86 vsyscall_gtod_data.clock.shift = clock->shift;
87 vsyscall_gtod_data.wall_time_sec = wall_time->tv_sec;
88 vsyscall_gtod_data.wall_time_nsec = wall_time->tv_nsec;
89 vsyscall_gtod_data.wall_to_monotonic = wall_to_monotonic;
90 write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
93 /* RED-PEN may want to readd seq locking, but then the variable should be
94 * write-once.
96 static __always_inline void do_get_tz(struct timezone * tz)
98 *tz = __vsyscall_gtod_data.sys_tz;
101 static __always_inline int gettimeofday(struct timeval *tv, struct timezone *tz)
103 int ret;
104 asm volatile("syscall"
105 : "=a" (ret)
106 : "0" (__NR_gettimeofday),"D" (tv),"S" (tz)
107 : __syscall_clobber );
108 return ret;
111 static __always_inline long time_syscall(long *t)
113 long secs;
114 asm volatile("syscall"
115 : "=a" (secs)
116 : "0" (__NR_time),"D" (t) : __syscall_clobber);
117 return secs;
120 static __always_inline void do_vgettimeofday(struct timeval * tv)
122 cycle_t now, base, mask, cycle_delta;
123 unsigned seq;
124 unsigned long mult, shift, nsec;
125 cycle_t (*vread)(void);
126 do {
127 seq = read_seqbegin(&__vsyscall_gtod_data.lock);
129 vread = __vsyscall_gtod_data.clock.vread;
130 if (unlikely(!__vsyscall_gtod_data.sysctl_enabled || !vread)) {
131 gettimeofday(tv,NULL);
132 return;
135 now = vread();
136 base = __vsyscall_gtod_data.clock.cycle_last;
137 mask = __vsyscall_gtod_data.clock.mask;
138 mult = __vsyscall_gtod_data.clock.mult;
139 shift = __vsyscall_gtod_data.clock.shift;
141 tv->tv_sec = __vsyscall_gtod_data.wall_time_sec;
142 nsec = __vsyscall_gtod_data.wall_time_nsec;
143 } while (read_seqretry(&__vsyscall_gtod_data.lock, seq));
145 /* calculate interval: */
146 cycle_delta = (now - base) & mask;
147 /* convert to nsecs: */
148 nsec += (cycle_delta * mult) >> shift;
150 while (nsec >= NSEC_PER_SEC) {
151 tv->tv_sec += 1;
152 nsec -= NSEC_PER_SEC;
154 tv->tv_usec = nsec / NSEC_PER_USEC;
157 int __vsyscall(0) vgettimeofday(struct timeval * tv, struct timezone * tz)
159 if (tv)
160 do_vgettimeofday(tv);
161 if (tz)
162 do_get_tz(tz);
163 return 0;
166 /* This will break when the xtime seconds get inaccurate, but that is
167 * unlikely */
168 time_t __vsyscall(1) vtime(time_t *t)
170 struct timeval tv;
171 time_t result;
172 if (unlikely(!__vsyscall_gtod_data.sysctl_enabled))
173 return time_syscall(t);
175 vgettimeofday(&tv, NULL);
176 result = tv.tv_sec;
177 if (t)
178 *t = result;
179 return result;
182 /* Fast way to get current CPU and node.
183 This helps to do per node and per CPU caches in user space.
184 The result is not guaranteed without CPU affinity, but usually
185 works out because the scheduler tries to keep a thread on the same
186 CPU.
188 tcache must point to a two element sized long array.
189 All arguments can be NULL. */
190 long __vsyscall(2)
191 vgetcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache)
193 unsigned int p;
194 unsigned long j = 0;
196 /* Fast cache - only recompute value once per jiffies and avoid
197 relatively costly rdtscp/cpuid otherwise.
198 This works because the scheduler usually keeps the process
199 on the same CPU and this syscall doesn't guarantee its
200 results anyways.
201 We do this here because otherwise user space would do it on
202 its own in a likely inferior way (no access to jiffies).
203 If you don't like it pass NULL. */
204 if (tcache && tcache->blob[0] == (j = __jiffies)) {
205 p = tcache->blob[1];
206 } else if (__vgetcpu_mode == VGETCPU_RDTSCP) {
207 /* Load per CPU data from RDTSCP */
208 native_read_tscp(&p);
209 } else {
210 /* Load per CPU data from GDT */
211 asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG));
213 if (tcache) {
214 tcache->blob[0] = j;
215 tcache->blob[1] = p;
217 if (cpu)
218 *cpu = p & 0xfff;
219 if (node)
220 *node = p >> 12;
221 return 0;
224 static long __vsyscall(3) venosys_1(void)
226 return -ENOSYS;
229 #ifdef CONFIG_SYSCTL
231 static int
232 vsyscall_sysctl_change(ctl_table *ctl, int write, struct file * filp,
233 void __user *buffer, size_t *lenp, loff_t *ppos)
235 return proc_dointvec(ctl, write, filp, buffer, lenp, ppos);
238 static ctl_table kernel_table2[] = {
239 { .procname = "vsyscall64",
240 .data = &vsyscall_gtod_data.sysctl_enabled, .maxlen = sizeof(int),
241 .mode = 0644,
242 .proc_handler = vsyscall_sysctl_change },
246 static ctl_table kernel_root_table2[] = {
247 { .ctl_name = CTL_KERN, .procname = "kernel", .mode = 0555,
248 .child = kernel_table2 },
251 #endif
253 /* Assume __initcall executes before all user space. Hopefully kmod
254 doesn't violate that. We'll find out if it does. */
255 static void __cpuinit vsyscall_set_cpu(int cpu)
257 unsigned long d;
258 unsigned long node = 0;
259 #ifdef CONFIG_NUMA
260 node = cpu_to_node(cpu);
261 #endif
262 if (cpu_has(&cpu_data(cpu), X86_FEATURE_RDTSCP))
263 write_rdtscp_aux((node << 12) | cpu);
265 /* Store cpu number in limit so that it can be loaded quickly
266 in user space in vgetcpu.
267 12 bits for the CPU and 8 bits for the node. */
268 d = 0x0f40000000000ULL;
269 d |= cpu;
270 d |= (node & 0xf) << 12;
271 d |= (node >> 4) << 48;
272 write_gdt_entry(get_cpu_gdt_table(cpu), GDT_ENTRY_PER_CPU, &d, DESCTYPE_S);
275 static void __cpuinit cpu_vsyscall_init(void *arg)
277 /* preemption should be already off */
278 vsyscall_set_cpu(raw_smp_processor_id());
281 static int __cpuinit
282 cpu_vsyscall_notifier(struct notifier_block *n, unsigned long action, void *arg)
284 long cpu = (long)arg;
285 if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN)
286 smp_call_function_single(cpu, cpu_vsyscall_init, NULL, 1);
287 return NOTIFY_DONE;
290 void __init map_vsyscall(void)
292 extern char __vsyscall_0;
293 unsigned long physaddr_page0 = __pa_symbol(&__vsyscall_0);
295 /* Note that VSYSCALL_MAPPED_PAGES must agree with the code below. */
296 __set_fixmap(VSYSCALL_FIRST_PAGE, physaddr_page0, PAGE_KERNEL_VSYSCALL);
299 static int __init vsyscall_init(void)
301 BUG_ON(((unsigned long) &vgettimeofday !=
302 VSYSCALL_ADDR(__NR_vgettimeofday)));
303 BUG_ON((unsigned long) &vtime != VSYSCALL_ADDR(__NR_vtime));
304 BUG_ON((VSYSCALL_ADDR(0) != __fix_to_virt(VSYSCALL_FIRST_PAGE)));
305 BUG_ON((unsigned long) &vgetcpu != VSYSCALL_ADDR(__NR_vgetcpu));
306 #ifdef CONFIG_SYSCTL
307 register_sysctl_table(kernel_root_table2);
308 #endif
309 on_each_cpu(cpu_vsyscall_init, NULL, 1);
310 hotcpu_notifier(cpu_vsyscall_notifier, 0);
311 return 0;
314 __initcall(vsyscall_init);