?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 4.13.16
[linux/fpc-iii.git] / arch / s390 / lib / uaccess.c
blobb3bd3f23b8e851c7f67e46cb527e20603512e7f5
1 /*
2 * Standard user space access functions based on mvcp/mvcs and doing
3 * interesting things in the secondary space mode.
4 *
5 * Copyright IBM Corp. 2006,2014
6 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
7 * Gerald Schaefer (gerald.schaefer@de.ibm.com)
8 */
9
10 #include <linux/jump_label.h>
11 #include <linux/uaccess.h>
12 #include <linux/export.h>
13 #include <linux/errno.h>
14 #include <linux/mm.h>
15 #include <asm/mmu_context.h>
16 #include <asm/facility.h>
17
18 static DEFINE_STATIC_KEY_FALSE(have_mvcos);
19
20 static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr,
21 unsigned long size)
22 {
23 register unsigned long reg0 asm("0") = 0x81UL;
24 unsigned long tmp1, tmp2;
25
26 tmp1 = -4096UL;
27 asm volatile(
28 "0: .insn ss,0xc80000000000,0(%0,%2),0(%1),0\n"
29 "6: jz 4f\n"
30 "1: algr %0,%3\n"
31 " slgr %1,%3\n"
32 " slgr %2,%3\n"
33 " j 0b\n"
34 "2: la %4,4095(%1)\n"/* %4 = ptr + 4095 */
35 " nr %4,%3\n" /* %4 = (ptr + 4095) & -4096 */
36 " slgr %4,%1\n"
37 " clgr %0,%4\n" /* copy crosses next page boundary? */
38 " jnh 5f\n"
39 "3: .insn ss,0xc80000000000,0(%4,%2),0(%1),0\n"
40 "7: slgr %0,%4\n"
41 " j 5f\n"
42 "4: slgr %0,%0\n"
43 "5:\n"
44 EX_TABLE(0b,2b) EX_TABLE(3b,5b) EX_TABLE(6b,2b) EX_TABLE(7b,5b)
45 : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
46 : "d" (reg0) : "cc", "memory");
47 return size;
48 }
49
50 static inline unsigned long copy_from_user_mvcp(void *x, const void __user *ptr,
51 unsigned long size)
52 {
53 unsigned long tmp1, tmp2;
54
55 load_kernel_asce();
56 tmp1 = -256UL;
57 asm volatile(
58 " sacf 0\n"
59 "0: mvcp 0(%0,%2),0(%1),%3\n"
60 "7: jz 5f\n"
61 "1: algr %0,%3\n"
62 " la %1,256(%1)\n"
63 " la %2,256(%2)\n"
64 "2: mvcp 0(%0,%2),0(%1),%3\n"
65 "8: jnz 1b\n"
66 " j 5f\n"
67 "3: la %4,255(%1)\n" /* %4 = ptr + 255 */
68 " lghi %3,-4096\n"
69 " nr %4,%3\n" /* %4 = (ptr + 255) & -4096 */
70 " slgr %4,%1\n"
71 " clgr %0,%4\n" /* copy crosses next page boundary? */
72 " jnh 6f\n"
73 "4: mvcp 0(%4,%2),0(%1),%3\n"
74 "9: slgr %0,%4\n"
75 " j 6f\n"
76 "5: slgr %0,%0\n"
77 "6: sacf 768\n"
78 EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,6b)
79 EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b)
80 : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
81 : : "cc", "memory");
82 return size;
83 }
84
85 unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n)
86 {
87 if (static_branch_likely(&have_mvcos))
88 return copy_from_user_mvcos(to, from, n);
89 return copy_from_user_mvcp(to, from, n);
90 }
91 EXPORT_SYMBOL(raw_copy_from_user);
92
93 static inline unsigned long copy_to_user_mvcos(void __user *ptr, const void *x,
94 unsigned long size)
95 {
96 register unsigned long reg0 asm("0") = 0x810000UL;
97 unsigned long tmp1, tmp2;
98
99 tmp1 = -4096UL;
100 asm volatile(
101 "0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n"
102 "6: jz 4f\n"
103 "1: algr %0,%3\n"
104 " slgr %1,%3\n"
105 " slgr %2,%3\n"
106 " j 0b\n"
107 "2: la %4,4095(%1)\n"/* %4 = ptr + 4095 */
108 " nr %4,%3\n" /* %4 = (ptr + 4095) & -4096 */
109 " slgr %4,%1\n"
110 " clgr %0,%4\n" /* copy crosses next page boundary? */
111 " jnh 5f\n"
112 "3: .insn ss,0xc80000000000,0(%4,%1),0(%2),0\n"
113 "7: slgr %0,%4\n"
114 " j 5f\n"
115 "4: slgr %0,%0\n"
116 "5:\n"
117 EX_TABLE(0b,2b) EX_TABLE(3b,5b) EX_TABLE(6b,2b) EX_TABLE(7b,5b)
118 : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
119 : "d" (reg0) : "cc", "memory");
120 return size;
121 }
122
123 static inline unsigned long copy_to_user_mvcs(void __user *ptr, const void *x,
124 unsigned long size)
125 {
126 unsigned long tmp1, tmp2;
127
128 load_kernel_asce();
129 tmp1 = -256UL;
130 asm volatile(
131 " sacf 0\n"
132 "0: mvcs 0(%0,%1),0(%2),%3\n"
133 "7: jz 5f\n"
134 "1: algr %0,%3\n"
135 " la %1,256(%1)\n"
136 " la %2,256(%2)\n"
137 "2: mvcs 0(%0,%1),0(%2),%3\n"
138 "8: jnz 1b\n"
139 " j 5f\n"
140 "3: la %4,255(%1)\n" /* %4 = ptr + 255 */
141 " lghi %3,-4096\n"
142 " nr %4,%3\n" /* %4 = (ptr + 255) & -4096 */
143 " slgr %4,%1\n"
144 " clgr %0,%4\n" /* copy crosses next page boundary? */
145 " jnh 6f\n"
146 "4: mvcs 0(%4,%1),0(%2),%3\n"
147 "9: slgr %0,%4\n"
148 " j 6f\n"
149 "5: slgr %0,%0\n"
150 "6: sacf 768\n"
151 EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,6b)
152 EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b)
153 : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
154 : : "cc", "memory");
155 return size;
156 }
157
158 unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n)
159 {
160 if (static_branch_likely(&have_mvcos))
161 return copy_to_user_mvcos(to, from, n);
162 return copy_to_user_mvcs(to, from, n);
163 }
164 EXPORT_SYMBOL(raw_copy_to_user);
165
166 static inline unsigned long copy_in_user_mvcos(void __user *to, const void __user *from,
167 unsigned long size)
168 {
169 register unsigned long reg0 asm("0") = 0x810081UL;
170 unsigned long tmp1, tmp2;
171
172 tmp1 = -4096UL;
173 /* FIXME: copy with reduced length. */
174 asm volatile(
175 "0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n"
176 " jz 2f\n"
177 "1: algr %0,%3\n"
178 " slgr %1,%3\n"
179 " slgr %2,%3\n"
180 " j 0b\n"
181 "2:slgr %0,%0\n"
182 "3: \n"
183 EX_TABLE(0b,3b)
184 : "+a" (size), "+a" (to), "+a" (from), "+a" (tmp1), "=a" (tmp2)
185 : "d" (reg0) : "cc", "memory");
186 return size;
187 }
188
189 static inline unsigned long copy_in_user_mvc(void __user *to, const void __user *from,
190 unsigned long size)
191 {
192 unsigned long tmp1;
193
194 load_kernel_asce();
195 asm volatile(
196 " sacf 256\n"
197 " aghi %0,-1\n"
198 " jo 5f\n"
199 " bras %3,3f\n"
200 "0: aghi %0,257\n"
201 "1: mvc 0(1,%1),0(%2)\n"
202 " la %1,1(%1)\n"
203 " la %2,1(%2)\n"
204 " aghi %0,-1\n"
205 " jnz 1b\n"
206 " j 5f\n"
207 "2: mvc 0(256,%1),0(%2)\n"
208 " la %1,256(%1)\n"
209 " la %2,256(%2)\n"
210 "3: aghi %0,-256\n"
211 " jnm 2b\n"
212 "4: ex %0,1b-0b(%3)\n"
213 "5: slgr %0,%0\n"
214 "6: sacf 768\n"
215 EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b)
216 : "+a" (size), "+a" (to), "+a" (from), "=a" (tmp1)
217 : : "cc", "memory");
218 return size;
219 }
220
221 unsigned long raw_copy_in_user(void __user *to, const void __user *from, unsigned long n)
222 {
223 if (static_branch_likely(&have_mvcos))
224 return copy_in_user_mvcos(to, from, n);
225 return copy_in_user_mvc(to, from, n);
226 }
227 EXPORT_SYMBOL(raw_copy_in_user);
228
229 static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size)
230 {
231 register unsigned long reg0 asm("0") = 0x810000UL;
232 unsigned long tmp1, tmp2;
233
234 tmp1 = -4096UL;
235 asm volatile(
236 "0: .insn ss,0xc80000000000,0(%0,%1),0(%4),0\n"
237 " jz 4f\n"
238 "1: algr %0,%2\n"
239 " slgr %1,%2\n"
240 " j 0b\n"
241 "2: la %3,4095(%1)\n"/* %4 = to + 4095 */
242 " nr %3,%2\n" /* %4 = (to + 4095) & -4096 */
243 " slgr %3,%1\n"
244 " clgr %0,%3\n" /* copy crosses next page boundary? */
245 " jnh 5f\n"
246 "3: .insn ss,0xc80000000000,0(%3,%1),0(%4),0\n"
247 " slgr %0,%3\n"
248 " j 5f\n"
249 "4: slgr %0,%0\n"
250 "5:\n"
251 EX_TABLE(0b,2b) EX_TABLE(3b,5b)
252 : "+a" (size), "+a" (to), "+a" (tmp1), "=a" (tmp2)
253 : "a" (empty_zero_page), "d" (reg0) : "cc", "memory");
254 return size;
255 }
256
257 static inline unsigned long clear_user_xc(void __user *to, unsigned long size)
258 {
259 unsigned long tmp1, tmp2;
260
261 load_kernel_asce();
262 asm volatile(
263 " sacf 256\n"
264 " aghi %0,-1\n"
265 " jo 5f\n"
266 " bras %3,3f\n"
267 " xc 0(1,%1),0(%1)\n"
268 "0: aghi %0,257\n"
269 " la %2,255(%1)\n" /* %2 = ptr + 255 */
270 " srl %2,12\n"
271 " sll %2,12\n" /* %2 = (ptr + 255) & -4096 */
272 " slgr %2,%1\n"
273 " clgr %0,%2\n" /* clear crosses next page boundary? */
274 " jnh 5f\n"
275 " aghi %2,-1\n"
276 "1: ex %2,0(%3)\n"
277 " aghi %2,1\n"
278 " slgr %0,%2\n"
279 " j 5f\n"
280 "2: xc 0(256,%1),0(%1)\n"
281 " la %1,256(%1)\n"
282 "3: aghi %0,-256\n"
283 " jnm 2b\n"
284 "4: ex %0,0(%3)\n"
285 "5: slgr %0,%0\n"
286 "6: sacf 768\n"
287 EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b)
288 : "+a" (size), "+a" (to), "=a" (tmp1), "=a" (tmp2)
289 : : "cc", "memory");
290 return size;
291 }
292
293 unsigned long __clear_user(void __user *to, unsigned long size)
294 {
295 if (static_branch_likely(&have_mvcos))
296 return clear_user_mvcos(to, size);
297 return clear_user_xc(to, size);
298 }
299 EXPORT_SYMBOL(__clear_user);
300
301 static inline unsigned long strnlen_user_srst(const char __user *src,
302 unsigned long size)
303 {
304 register unsigned long reg0 asm("0") = 0;
305 unsigned long tmp1, tmp2;
306
307 asm volatile(
308 " la %2,0(%1)\n"
309 " la %3,0(%0,%1)\n"
310 " slgr %0,%0\n"
311 " sacf 256\n"
312 "0: srst %3,%2\n"
313 " jo 0b\n"
314 " la %0,1(%3)\n" /* strnlen_user results includes \0 */
315 " slgr %0,%1\n"
316 "1: sacf 768\n"
317 EX_TABLE(0b,1b)
318 : "+a" (size), "+a" (src), "=a" (tmp1), "=a" (tmp2)
319 : "d" (reg0) : "cc", "memory");
320 return size;
321 }
322
323 unsigned long __strnlen_user(const char __user *src, unsigned long size)
324 {
325 if (unlikely(!size))
326 return 0;
327 load_kernel_asce();
328 return strnlen_user_srst(src, size);
329 }
330 EXPORT_SYMBOL(__strnlen_user);
331
332 long __strncpy_from_user(char *dst, const char __user *src, long size)
333 {
334 size_t done, len, offset, len_str;
335
336 if (unlikely(size <= 0))
337 return 0;
338 done = 0;
339 do {
340 offset = (size_t)src & (L1_CACHE_BYTES - 1);
341 len = min(size - done, L1_CACHE_BYTES - offset);
342 if (copy_from_user(dst, src, len))
343 return -EFAULT;
344 len_str = strnlen(dst, len);
345 done += len_str;
346 src += len_str;
347 dst += len_str;
348 } while ((len_str == len) && (done < size));
349 return done;
350 }
351 EXPORT_SYMBOL(__strncpy_from_user);
352
353 static int __init uaccess_init(void)
354 {
355 if (test_facility(27))
356 static_branch_enable(&have_mvcos);
357 return 0;
358 }
359 early_initcall(uaccess_init);
Linux with added FPC-III support