?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
ia64/pv_ops/xen: implement xen pv_time_ops.
[pv_ops_mirror.git] / include / asm-x86 / uaccess_64.h
blobb8a2f43399032e1ce617e38a35eb28919f08a287
1 #ifndef __X86_64_UACCESS_H
2 #define __X86_64_UACCESS_H
3
4 /*
5 * User space memory access functions
6 */
7 #include <linux/compiler.h>
8 #include <linux/errno.h>
9 #include <linux/prefetch.h>
10 #include <asm/page.h>
11
12 #define VERIFY_READ 0
13 #define VERIFY_WRITE 1
14
15 /*
16 * The fs value determines whether argument validity checking should be
17 * performed or not. If get_fs() == USER_DS, checking is performed, with
18 * get_fs() == KERNEL_DS, checking is bypassed.
19 *
20 * For historical reasons, these macros are grossly misnamed.
21 */
22
23 #define MAKE_MM_SEG(s) ((mm_segment_t) { (s) })
24
25 #define KERNEL_DS MAKE_MM_SEG(0xFFFFFFFFFFFFFFFFUL)
26 #define USER_DS MAKE_MM_SEG(PAGE_OFFSET)
27
28 #define get_ds() (KERNEL_DS)
29 #define get_fs() (current_thread_info()->addr_limit)
30 #define set_fs(x) (current_thread_info()->addr_limit = (x))
31
32 #define segment_eq(a, b) ((a).seg == (b).seg)
33
34 #define __addr_ok(addr) (!((unsigned long)(addr) & \
35 (current_thread_info()->addr_limit.seg)))
36
37 /*
38 * Uhhuh, this needs 65-bit arithmetic. We have a carry..
39 */
40 #define __range_not_ok(addr, size) \
41 ({ \
42 unsigned long flag, roksum; \
43 __chk_user_ptr(addr); \
44 asm("# range_ok\n\r" \
45 "addq %3,%1 ; sbbq %0,%0 ; cmpq %1,%4 ; sbbq $0,%0" \
46 : "=&r" (flag), "=r" (roksum) \
47 : "1" (addr), "g" ((long)(size)), \
48 "g" (current_thread_info()->addr_limit.seg)); \
49 flag; \
50 })
51
52 #define access_ok(type, addr, size) (__range_not_ok(addr, size) == 0)
53
54 /*
55 * The exception table consists of pairs of addresses: the first is the
56 * address of an instruction that is allowed to fault, and the second is
57 * the address at which the program should continue. No registers are
58 * modified, so it is entirely up to the continuation code to figure out
59 * what to do.
60 *
61 * All the routines below use bits of fixup code that are out of line
62 * with the main instruction path. This means when everything is well,
63 * we don't even have to jump over them. Further, they do not intrude
64 * on our cache or tlb entries.
65 */
66
67 struct exception_table_entry {
68 unsigned long insn, fixup;
69 };
70
71 extern int fixup_exception(struct pt_regs *regs);
72
73 #define ARCH_HAS_SEARCH_EXTABLE
74
75 /*
76 * These are the main single-value transfer routines. They automatically
77 * use the right size if we just have the right pointer type.
78 *
79 * This gets kind of ugly. We want to return _two_ values in "get_user()"
80 * and yet we don't want to do any pointers, because that is too much
81 * of a performance impact. Thus we have a few rather ugly macros here,
82 * and hide all the ugliness from the user.
83 *
84 * The "__xxx" versions of the user access functions are versions that
85 * do not verify the address space, that must have been done previously
86 * with a separate "access_ok()" call (this is used when we do multiple
87 * accesses to the same area of user memory).
88 */
89
90 #define __get_user_x(size, ret, x, ptr) \
91 asm volatile("call __get_user_" #size \
92 : "=a" (ret),"=d" (x) \
93 : "c" (ptr) \
94 : "r8")
95
96 /* Careful: we have to cast the result to the type of the pointer
97 * for sign reasons */
98
99 #define get_user(x, ptr) \
100 ({ \
101 unsigned long __val_gu; \
102 int __ret_gu; \
103 __chk_user_ptr(ptr); \
104 switch (sizeof(*(ptr))) { \
105 case 1: \
106 __get_user_x(1, __ret_gu, __val_gu, ptr); \
107 break; \
108 case 2: \
109 __get_user_x(2, __ret_gu, __val_gu, ptr); \
110 break; \
111 case 4: \
112 __get_user_x(4, __ret_gu, __val_gu, ptr); \
113 break; \
114 case 8: \
115 __get_user_x(8, __ret_gu, __val_gu, ptr); \
116 break; \
117 default: \
118 __get_user_bad(); \
119 break; \
120 } \
121 (x) = (__force typeof(*(ptr)))__val_gu; \
122 __ret_gu; \
123 })
124
125 extern void __put_user_1(void);
126 extern void __put_user_2(void);
127 extern void __put_user_4(void);
128 extern void __put_user_8(void);
129 extern void __put_user_bad(void);
130
131 #define __put_user_x(size, ret, x, ptr) \
132 asm volatile("call __put_user_" #size \
133 :"=a" (ret) \
134 :"c" (ptr),"d" (x) \
135 :"r8")
136
137 #define put_user(x, ptr) \
138 __put_user_check((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
139
140 #define __get_user(x, ptr) \
141 __get_user_nocheck((x), (ptr), sizeof(*(ptr)))
142 #define __put_user(x, ptr) \
143 __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
144
145 #define __get_user_unaligned __get_user
146 #define __put_user_unaligned __put_user
147
148 #define __put_user_nocheck(x, ptr, size) \
149 ({ \
150 int __pu_err; \
151 __put_user_size((x), (ptr), (size), __pu_err); \
152 __pu_err; \
153 })
154
155
156 #define __put_user_check(x, ptr, size) \
157 ({ \
158 int __pu_err; \
159 typeof(*(ptr)) __user *__pu_addr = (ptr); \
160 switch (size) { \
161 case 1: \
162 __put_user_x(1, __pu_err, x, __pu_addr); \
163 break; \
164 case 2: \
165 __put_user_x(2, __pu_err, x, __pu_addr); \
166 break; \
167 case 4: \
168 __put_user_x(4, __pu_err, x, __pu_addr); \
169 break; \
170 case 8: \
171 __put_user_x(8, __pu_err, x, __pu_addr); \
172 break; \
173 default: \
174 __put_user_bad(); \
175 } \
176 __pu_err; \
177 })
178
179 #define __put_user_size(x, ptr, size, retval) \
180 do { \
181 retval = 0; \
182 __chk_user_ptr(ptr); \
183 switch (size) { \
184 case 1: \
185 __put_user_asm(x, ptr, retval, "b", "b", "iq", -EFAULT);\
186 break; \
187 case 2: \
188 __put_user_asm(x, ptr, retval, "w", "w", "ir", -EFAULT);\
189 break; \
190 case 4: \
191 __put_user_asm(x, ptr, retval, "l", "k", "ir", -EFAULT);\
192 break; \
193 case 8: \
194 __put_user_asm(x, ptr, retval, "q", "", "Zr", -EFAULT); \
195 break; \
196 default: \
197 __put_user_bad(); \
198 } \
199 } while (0)
200
201 /* FIXME: this hack is definitely wrong -AK */
202 struct __large_struct { unsigned long buf[100]; };
203 #define __m(x) (*(struct __large_struct __user *)(x))
204
205 /*
206 * Tell gcc we read from memory instead of writing: this is because
207 * we do not write to any memory gcc knows about, so there are no
208 * aliasing issues.
209 */
210 #define __put_user_asm(x, addr, err, itype, rtype, ltype, errno) \
211 asm volatile("1: mov"itype" %"rtype"1,%2\n" \
212 "2:\n" \
213 ".section .fixup, \"ax\"\n" \
214 "3: mov %3,%0\n" \
215 " jmp 2b\n" \
216 ".previous\n" \
217 _ASM_EXTABLE(1b, 3b) \
218 : "=r"(err) \
219 : ltype (x), "m" (__m(addr)), "i" (errno), "0" (err))
220
221
222 #define __get_user_nocheck(x, ptr, size) \
223 ({ \
224 int __gu_err; \
225 unsigned long __gu_val; \
226 __get_user_size(__gu_val, (ptr), (size), __gu_err); \
227 (x) = (__force typeof(*(ptr)))__gu_val; \
228 __gu_err; \
229 })
230
231 extern int __get_user_1(void);
232 extern int __get_user_2(void);
233 extern int __get_user_4(void);
234 extern int __get_user_8(void);
235 extern int __get_user_bad(void);
236
237 #define __get_user_size(x, ptr, size, retval) \
238 do { \
239 retval = 0; \
240 __chk_user_ptr(ptr); \
241 switch (size) { \
242 case 1: \
243 __get_user_asm(x, ptr, retval, "b", "b", "=q", -EFAULT);\
244 break; \
245 case 2: \
246 __get_user_asm(x, ptr, retval, "w", "w", "=r", -EFAULT);\
247 break; \
248 case 4: \
249 __get_user_asm(x, ptr, retval, "l", "k", "=r", -EFAULT);\
250 break; \
251 case 8: \
252 __get_user_asm(x, ptr, retval, "q", "", "=r", -EFAULT); \
253 break; \
254 default: \
255 (x) = __get_user_bad(); \
256 } \
257 } while (0)
258
259 #define __get_user_asm(x, addr, err, itype, rtype, ltype, errno) \
260 asm volatile("1: mov"itype" %2,%"rtype"1\n" \
261 "2:\n" \
262 ".section .fixup, \"ax\"\n" \
263 "3: mov %3,%0\n" \
264 " xor"itype" %"rtype"1,%"rtype"1\n" \
265 " jmp 2b\n" \
266 ".previous\n" \
267 _ASM_EXTABLE(1b, 3b) \
268 : "=r" (err), ltype (x) \
269 : "m" (__m(addr)), "i"(errno), "0"(err))
270
271 /*
272 * Copy To/From Userspace
273 */
274
275 /* Handles exceptions in both to and from, but doesn't do access_ok */
276 __must_check unsigned long
277 copy_user_generic(void *to, const void *from, unsigned len);
278
279 __must_check unsigned long
280 copy_to_user(void __user *to, const void *from, unsigned len);
281 __must_check unsigned long
282 copy_from_user(void *to, const void __user *from, unsigned len);
283 __must_check unsigned long
284 copy_in_user(void __user *to, const void __user *from, unsigned len);
285
286 static __always_inline __must_check
287 int __copy_from_user(void *dst, const void __user *src, unsigned size)
288 {
289 int ret = 0;
290 if (!__builtin_constant_p(size))
291 return copy_user_generic(dst, (__force void *)src, size);
292 switch (size) {
293 case 1:__get_user_asm(*(u8 *)dst, (u8 __user *)src,
294 ret, "b", "b", "=q", 1);
295 return ret;
296 case 2:__get_user_asm(*(u16 *)dst, (u16 __user *)src,
297 ret, "w", "w", "=r", 2);
298 return ret;
299 case 4:__get_user_asm(*(u32 *)dst, (u32 __user *)src,
300 ret, "l", "k", "=r", 4);
301 return ret;
302 case 8:__get_user_asm(*(u64 *)dst, (u64 __user *)src,
303 ret, "q", "", "=r", 8);
304 return ret;
305 case 10:
306 __get_user_asm(*(u64 *)dst, (u64 __user *)src,
307 ret, "q", "", "=r", 16);
308 if (unlikely(ret))
309 return ret;
310 __get_user_asm(*(u16 *)(8 + (char *)dst),
311 (u16 __user *)(8 + (char __user *)src),
312 ret, "w", "w", "=r", 2);
313 return ret;
314 case 16:
315 __get_user_asm(*(u64 *)dst, (u64 __user *)src,
316 ret, "q", "", "=r", 16);
317 if (unlikely(ret))
318 return ret;
319 __get_user_asm(*(u64 *)(8 + (char *)dst),
320 (u64 __user *)(8 + (char __user *)src),
321 ret, "q", "", "=r", 8);
322 return ret;
323 default:
324 return copy_user_generic(dst, (__force void *)src, size);
325 }
326 }
327
328 static __always_inline __must_check
329 int __copy_to_user(void __user *dst, const void *src, unsigned size)
330 {
331 int ret = 0;
332 if (!__builtin_constant_p(size))
333 return copy_user_generic((__force void *)dst, src, size);
334 switch (size) {
335 case 1:__put_user_asm(*(u8 *)src, (u8 __user *)dst,
336 ret, "b", "b", "iq", 1);
337 return ret;
338 case 2:__put_user_asm(*(u16 *)src, (u16 __user *)dst,
339 ret, "w", "w", "ir", 2);
340 return ret;
341 case 4:__put_user_asm(*(u32 *)src, (u32 __user *)dst,
342 ret, "l", "k", "ir", 4);
343 return ret;
344 case 8:__put_user_asm(*(u64 *)src, (u64 __user *)dst,
345 ret, "q", "", "ir", 8);
346 return ret;
347 case 10:
348 __put_user_asm(*(u64 *)src, (u64 __user *)dst,
349 ret, "q", "", "ir", 10);
350 if (unlikely(ret))
351 return ret;
352 asm("":::"memory");
353 __put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst,
354 ret, "w", "w", "ir", 2);
355 return ret;
356 case 16:
357 __put_user_asm(*(u64 *)src, (u64 __user *)dst,
358 ret, "q", "", "ir", 16);
359 if (unlikely(ret))
360 return ret;
361 asm("":::"memory");
362 __put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst,
363 ret, "q", "", "ir", 8);
364 return ret;
365 default:
366 return copy_user_generic((__force void *)dst, src, size);
367 }
368 }
369
370 static __always_inline __must_check
371 int __copy_in_user(void __user *dst, const void __user *src, unsigned size)
372 {
373 int ret = 0;
374 if (!__builtin_constant_p(size))
375 return copy_user_generic((__force void *)dst,
376 (__force void *)src, size);
377 switch (size) {
378 case 1: {
379 u8 tmp;
380 __get_user_asm(tmp, (u8 __user *)src,
381 ret, "b", "b", "=q", 1);
382 if (likely(!ret))
383 __put_user_asm(tmp, (u8 __user *)dst,
384 ret, "b", "b", "iq", 1);
385 return ret;
386 }
387 case 2: {
388 u16 tmp;
389 __get_user_asm(tmp, (u16 __user *)src,
390 ret, "w", "w", "=r", 2);
391 if (likely(!ret))
392 __put_user_asm(tmp, (u16 __user *)dst,
393 ret, "w", "w", "ir", 2);
394 return ret;
395 }
396
397 case 4: {
398 u32 tmp;
399 __get_user_asm(tmp, (u32 __user *)src,
400 ret, "l", "k", "=r", 4);
401 if (likely(!ret))
402 __put_user_asm(tmp, (u32 __user *)dst,
403 ret, "l", "k", "ir", 4);
404 return ret;
405 }
406 case 8: {
407 u64 tmp;
408 __get_user_asm(tmp, (u64 __user *)src,
409 ret, "q", "", "=r", 8);
410 if (likely(!ret))
411 __put_user_asm(tmp, (u64 __user *)dst,
412 ret, "q", "", "ir", 8);
413 return ret;
414 }
415 default:
416 return copy_user_generic((__force void *)dst,
417 (__force void *)src, size);
418 }
419 }
420
421 __must_check long
422 strncpy_from_user(char *dst, const char __user *src, long count);
423 __must_check long
424 __strncpy_from_user(char *dst, const char __user *src, long count);
425 __must_check long strnlen_user(const char __user *str, long n);
426 __must_check long __strnlen_user(const char __user *str, long n);
427 __must_check long strlen_user(const char __user *str);
428 __must_check unsigned long clear_user(void __user *mem, unsigned long len);
429 __must_check unsigned long __clear_user(void __user *mem, unsigned long len);
430
431 __must_check long __copy_from_user_inatomic(void *dst, const void __user *src,
432 unsigned size);
433
434 static __must_check __always_inline int
435 __copy_to_user_inatomic(void __user *dst, const void *src, unsigned size)
436 {
437 return copy_user_generic((__force void *)dst, src, size);
438 }
439
440 #define ARCH_HAS_NOCACHE_UACCESS 1
441 extern long __copy_user_nocache(void *dst, const void __user *src,
442 unsigned size, int zerorest);
443
444 static inline int __copy_from_user_nocache(void *dst, const void __user *src,
445 unsigned size)
446 {
447 might_sleep();
448 return __copy_user_nocache(dst, src, size, 1);
449 }
450
451 static inline int __copy_from_user_inatomic_nocache(void *dst,
452 const void __user *src,
453 unsigned size)
454 {
455 return __copy_user_nocache(dst, src, size, 0);
456 }
457
458 #endif /* __X86_64_UACCESS_H */
Mirror for ia64_pv_ops.git