arch/x86/kernel/i387.c

   1 /*
   2  *  Copyright (C) 1994 Linus Torvalds
   3  *
   4  *  Pentium III FXSR, SSE support
   5  *  General FPU state handling cleanups
   6  *      Gareth Hughes <gareth@valinux.com>, May 2000
   7  */
   8
   9 #include <linux/sched.h>
  10 #include <linux/module.h>
  11 #include <linux/regset.h>
  12 #include <asm/processor.h>
  13 #include <asm/i387.h>
  14 #include <asm/math_emu.h>
  15 #include <asm/sigcontext.h>
  16 #include <asm/user.h>
  17 #include <asm/ptrace.h>
  18 #include <asm/uaccess.h>
  19
  20 #ifdef CONFIG_X86_64
  21
  22 #include <asm/sigcontext32.h>
  23 #include <asm/user32.h>
  24
  25 #else
  26
  27 #define save_i387_ia32          save_i387
  28 #define restore_i387_ia32       restore_i387
  29
  30 #define _fpstate_ia32           _fpstate
  31 #define user_i387_ia32_struct   user_i387_struct
  32 #define user32_fxsr_struct      user_fxsr_struct
  33
  34 #endif
  35
  36 #ifdef CONFIG_MATH_EMULATION
  37 #define HAVE_HWFP (boot_cpu_data.hard_math)
  38 #else
  39 #define HAVE_HWFP 1
  40 #endif
  41
  42 <<<<<<< HEAD:arch/x86/kernel/i387.c
  43 unsigned int mxcsr_feature_mask __read_mostly = 0xffffffffu;
  44 =======
  45 static unsigned int mxcsr_feature_mask __read_mostly = 0xffffffffu;
  46 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:arch/x86/kernel/i387.c
  47
  48 void mxcsr_feature_mask_init(void)
  49 {
  50         unsigned long mask = 0;
  51         clts();
  52         if (cpu_has_fxsr) {
  53                 memset(&current->thread.i387.fxsave, 0,
  54                        sizeof(struct i387_fxsave_struct));
  55                 asm volatile("fxsave %0" : : "m" (current->thread.i387.fxsave));
  56                 mask = current->thread.i387.fxsave.mxcsr_mask;
  57                 if (mask == 0)
  58                         mask = 0x0000ffbf;
  59         }
  60         mxcsr_feature_mask &= mask;
  61         stts();
  62 }
  63
  64 #ifdef CONFIG_X86_64
  65 /*
  66  * Called at bootup to set up the initial FPU state that is later cloned
  67  * into all processes.
  68  */
  69 void __cpuinit fpu_init(void)
  70 {
  71         unsigned long oldcr0 = read_cr0();
  72         extern void __bad_fxsave_alignment(void);
  73
  74         if (offsetof(struct task_struct, thread.i387.fxsave) & 15)
  75                 __bad_fxsave_alignment();
  76         set_in_cr4(X86_CR4_OSFXSR);
  77         set_in_cr4(X86_CR4_OSXMMEXCPT);
  78
  79         write_cr0(oldcr0 & ~((1UL<<3)|(1UL<<2))); /* clear TS and EM */
  80
  81         mxcsr_feature_mask_init();
  82         /* clean state in init */
  83         current_thread_info()->status = 0;
  84         clear_used_math();
  85 }
  86 #endif  /* CONFIG_X86_64 */
  87
  88 /*
  89  * The _current_ task is using the FPU for the first time
  90  * so initialize it and set the mxcsr to its default
  91  * value at reset if we support XMM instructions and then
  92  * remeber the current task has used the FPU.
  93  */
  94 void init_fpu(struct task_struct *tsk)
  95 {
  96         if (tsk_used_math(tsk)) {
  97                 if (tsk == current)
  98                         unlazy_fpu(tsk);
  99                 return;
 100         }
 101
 102         if (cpu_has_fxsr) {
 103                 memset(&tsk->thread.i387.fxsave, 0,
 104                        sizeof(struct i387_fxsave_struct));
 105                 tsk->thread.i387.fxsave.cwd = 0x37f;
 106                 if (cpu_has_xmm)
 107                         tsk->thread.i387.fxsave.mxcsr = MXCSR_DEFAULT;
 108         } else {
 109                 memset(&tsk->thread.i387.fsave, 0,
 110                        sizeof(struct i387_fsave_struct));
 111                 tsk->thread.i387.fsave.cwd = 0xffff037fu;
 112                 tsk->thread.i387.fsave.swd = 0xffff0000u;
 113                 tsk->thread.i387.fsave.twd = 0xffffffffu;
 114                 tsk->thread.i387.fsave.fos = 0xffff0000u;
 115         }
 116         /*
 117          * Only the device not available exception or ptrace can call init_fpu.
 118          */
 119         set_stopped_child_used_math(tsk);
 120 }
 121
 122 int fpregs_active(struct task_struct *target, const struct user_regset *regset)
 123 {
 124         return tsk_used_math(target) ? regset->n : 0;
 125 }
 126
 127 int xfpregs_active(struct task_struct *target, const struct user_regset *regset)
 128 {
 129         return (cpu_has_fxsr && tsk_used_math(target)) ? regset->n : 0;
 130 }
 131
 132 int xfpregs_get(struct task_struct *target, const struct user_regset *regset,
 133                 unsigned int pos, unsigned int count,
 134                 void *kbuf, void __user *ubuf)
 135 {
 136         if (!cpu_has_fxsr)
 137                 return -ENODEV;
 138
 139 <<<<<<< HEAD:arch/x86/kernel/i387.c
 140         unlazy_fpu(target);
 141 =======
 142         init_fpu(target);
 143 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:arch/x86/kernel/i387.c
 144
 145         return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
 146                                    &target->thread.i387.fxsave, 0, -1);
 147 }
 148
 149 int xfpregs_set(struct task_struct *target, const struct user_regset *regset,
 150                 unsigned int pos, unsigned int count,
 151                 const void *kbuf, const void __user *ubuf)
 152 {
 153         int ret;
 154
 155         if (!cpu_has_fxsr)
 156                 return -ENODEV;
 157
 158 <<<<<<< HEAD:arch/x86/kernel/i387.c
 159         unlazy_fpu(target);
 160 =======
 161         init_fpu(target);
 162 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:arch/x86/kernel/i387.c
 163         set_stopped_child_used_math(target);
 164
 165         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
 166                                  &target->thread.i387.fxsave, 0, -1);
 167
 168         /*
 169          * mxcsr reserved bits must be masked to zero for security reasons.
 170          */
 171         target->thread.i387.fxsave.mxcsr &= mxcsr_feature_mask;
 172
 173         return ret;
 174 }
 175
 176 #if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION
 177
 178 /*
 179  * FPU tag word conversions.
 180  */
 181
 182 static inline unsigned short twd_i387_to_fxsr(unsigned short twd)
 183 {
 184         unsigned int tmp; /* to avoid 16 bit prefixes in the code */
 185
 186         /* Transform each pair of bits into 01 (valid) or 00 (empty) */
 187         tmp = ~twd;
 188         tmp = (tmp | (tmp>>1)) & 0x5555; /* 0V0V0V0V0V0V0V0V */
 189         /* and move the valid bits to the lower byte. */
 190         tmp = (tmp | (tmp >> 1)) & 0x3333; /* 00VV00VV00VV00VV */
 191         tmp = (tmp | (tmp >> 2)) & 0x0f0f; /* 0000VVVV0000VVVV */
 192         tmp = (tmp | (tmp >> 4)) & 0x00ff; /* 00000000VVVVVVVV */
 193         return tmp;
 194 }
 195
 196 #define FPREG_ADDR(f, n)        ((void *)&(f)->st_space + (n) * 16);
 197 #define FP_EXP_TAG_VALID        0
 198 #define FP_EXP_TAG_ZERO         1
 199 #define FP_EXP_TAG_SPECIAL      2
 200 #define FP_EXP_TAG_EMPTY        3
 201
 202 static inline u32 twd_fxsr_to_i387(struct i387_fxsave_struct *fxsave)
 203 {
 204         struct _fpxreg *st;
 205         u32 tos = (fxsave->swd >> 11) & 7;
 206         u32 twd = (unsigned long) fxsave->twd;
 207         u32 tag;
 208         u32 ret = 0xffff0000u;
 209         int i;
 210
 211         for (i = 0; i < 8; i++, twd >>= 1) {
 212                 if (twd & 0x1) {
 213                         st = FPREG_ADDR(fxsave, (i - tos) & 7);
 214
 215                         switch (st->exponent & 0x7fff) {
 216                         case 0x7fff:
 217                                 tag = FP_EXP_TAG_SPECIAL;
 218                                 break;
 219                         case 0x0000:
 220                                 if (!st->significand[0] &&
 221                                     !st->significand[1] &&
 222                                     !st->significand[2] &&
 223                                     !st->significand[3])
 224                                         tag = FP_EXP_TAG_ZERO;
 225                                 else
 226                                         tag = FP_EXP_TAG_SPECIAL;
 227                                 break;
 228                         default:
 229                                 if (st->significand[3] & 0x8000)
 230                                         tag = FP_EXP_TAG_VALID;
 231                                 else
 232                                         tag = FP_EXP_TAG_SPECIAL;
 233                                 break;
 234                         }
 235                 } else {
 236                         tag = FP_EXP_TAG_EMPTY;
 237                 }
 238                 ret |= tag << (2 * i);
 239         }
 240         return ret;
 241 }
 242
 243 /*
 244  * FXSR floating point environment conversions.
 245  */
 246
 247 static void convert_from_fxsr(struct user_i387_ia32_struct *env,
 248                               struct task_struct *tsk)
 249 {
 250         struct i387_fxsave_struct *fxsave = &tsk->thread.i387.fxsave;
 251         struct _fpreg *to = (struct _fpreg *) &env->st_space[0];
 252         struct _fpxreg *from = (struct _fpxreg *) &fxsave->st_space[0];
 253         int i;
 254
 255         env->cwd = fxsave->cwd | 0xffff0000u;
 256         env->swd = fxsave->swd | 0xffff0000u;
 257         env->twd = twd_fxsr_to_i387(fxsave);
 258
 259 #ifdef CONFIG_X86_64
 260         env->fip = fxsave->rip;
 261         env->foo = fxsave->rdp;
 262         if (tsk == current) {
 263                 /*
 264                  * should be actually ds/cs at fpu exception time, but
 265                  * that information is not available in 64bit mode.
 266                  */
 267                 asm("mov %%ds,%0" : "=r" (env->fos));
 268                 asm("mov %%cs,%0" : "=r" (env->fcs));
 269         } else {
 270                 struct pt_regs *regs = task_pt_regs(tsk);
 271                 env->fos = 0xffff0000 | tsk->thread.ds;
 272                 env->fcs = regs->cs;
 273         }
 274 #else
 275         env->fip = fxsave->fip;
 276 <<<<<<< HEAD:arch/x86/kernel/i387.c
 277         env->fcs = fxsave->fcs;
 278 =======
 279         env->fcs = (u16) fxsave->fcs | ((u32) fxsave->fop << 16);
 280 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:arch/x86/kernel/i387.c
 281         env->foo = fxsave->foo;
 282         env->fos = fxsave->fos;
 283 #endif
 284
 285         for (i = 0; i < 8; ++i)
 286                 memcpy(&to[i], &from[i], sizeof(to[0]));
 287 }
 288
 289 static void convert_to_fxsr(struct task_struct *tsk,
 290                             const struct user_i387_ia32_struct *env)
 291
 292 {
 293         struct i387_fxsave_struct *fxsave = &tsk->thread.i387.fxsave;
 294         struct _fpreg *from = (struct _fpreg *) &env->st_space[0];
 295         struct _fpxreg *to = (struct _fpxreg *) &fxsave->st_space[0];
 296         int i;
 297
 298         fxsave->cwd = env->cwd;
 299         fxsave->swd = env->swd;
 300         fxsave->twd = twd_i387_to_fxsr(env->twd);
 301         fxsave->fop = (u16) ((u32) env->fcs >> 16);
 302 #ifdef CONFIG_X86_64
 303         fxsave->rip = env->fip;
 304         fxsave->rdp = env->foo;
 305         /* cs and ds ignored */
 306 #else
 307         fxsave->fip = env->fip;
 308         fxsave->fcs = (env->fcs & 0xffff);
 309         fxsave->foo = env->foo;
 310         fxsave->fos = env->fos;
 311 #endif
 312
 313         for (i = 0; i < 8; ++i)
 314                 memcpy(&to[i], &from[i], sizeof(from[0]));
 315 }
 316
 317 int fpregs_get(struct task_struct *target, const struct user_regset *regset,
 318                unsigned int pos, unsigned int count,
 319                void *kbuf, void __user *ubuf)
 320 {
 321         struct user_i387_ia32_struct env;
 322
 323         if (!HAVE_HWFP)
 324                 return fpregs_soft_get(target, regset, pos, count, kbuf, ubuf);
 325
 326 <<<<<<< HEAD:arch/x86/kernel/i387.c
 327         unlazy_fpu(target);
 328 =======
 329         init_fpu(target);
 330 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:arch/x86/kernel/i387.c
 331
 332         if (!cpu_has_fxsr)
 333                 return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
 334                                            &target->thread.i387.fsave, 0, -1);
 335
 336         if (kbuf && pos == 0 && count == sizeof(env)) {
 337                 convert_from_fxsr(kbuf, target);
 338                 return 0;
 339         }
 340
 341         convert_from_fxsr(&env, target);
 342         return user_regset_copyout(&pos, &count, &kbuf, &ubuf, &env, 0, -1);
 343 }
 344
 345 int fpregs_set(struct task_struct *target, const struct user_regset *regset,
 346                unsigned int pos, unsigned int count,
 347                const void *kbuf, const void __user *ubuf)
 348 {
 349         struct user_i387_ia32_struct env;
 350         int ret;
 351
 352         if (!HAVE_HWFP)
 353                 return fpregs_soft_set(target, regset, pos, count, kbuf, ubuf);
 354
 355 <<<<<<< HEAD:arch/x86/kernel/i387.c
 356         unlazy_fpu(target);
 357 =======
 358         init_fpu(target);
 359 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:arch/x86/kernel/i387.c
 360         set_stopped_child_used_math(target);
 361
 362         if (!cpu_has_fxsr)
 363                 return user_regset_copyin(&pos, &count, &kbuf, &ubuf,
 364                                           &target->thread.i387.fsave, 0, -1);
 365
 366         if (pos > 0 || count < sizeof(env))
 367                 convert_from_fxsr(&env, target);
 368
 369         ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &env, 0, -1);
 370         if (!ret)
 371                 convert_to_fxsr(target, &env);
 372
 373         return ret;
 374 }
 375
 376 /*
 377  * Signal frame handlers.
 378  */
 379
 380 static inline int save_i387_fsave(struct _fpstate_ia32 __user *buf)
 381 {
 382         struct task_struct *tsk = current;
 383
 384         unlazy_fpu(tsk);
 385         tsk->thread.i387.fsave.status = tsk->thread.i387.fsave.swd;
 386         if (__copy_to_user(buf, &tsk->thread.i387.fsave,
 387                            sizeof(struct i387_fsave_struct)))
 388                 return -1;
 389         return 1;
 390 }
 391
 392 static int save_i387_fxsave(struct _fpstate_ia32 __user *buf)
 393 {
 394         struct task_struct *tsk = current;
 395         struct user_i387_ia32_struct env;
 396         int err = 0;
 397
 398         unlazy_fpu(tsk);
 399
 400         convert_from_fxsr(&env, tsk);
 401         if (__copy_to_user(buf, &env, sizeof(env)))
 402                 return -1;
 403
 404         err |= __put_user(tsk->thread.i387.fxsave.swd, &buf->status);
 405         err |= __put_user(X86_FXSR_MAGIC, &buf->magic);
 406         if (err)
 407                 return -1;
 408
 409         if (__copy_to_user(&buf->_fxsr_env[0], &tsk->thread.i387.fxsave,
 410                            sizeof(struct i387_fxsave_struct)))
 411                 return -1;
 412         return 1;
 413 }
 414
 415 int save_i387_ia32(struct _fpstate_ia32 __user *buf)
 416 {
 417         if (!used_math())
 418                 return 0;
 419
 420         /* This will cause a "finit" to be triggered by the next
 421          * attempted FPU operation by the 'current' process.
 422          */
 423         clear_used_math();
 424
 425         if (HAVE_HWFP) {
 426                 if (cpu_has_fxsr) {
 427                         return save_i387_fxsave(buf);
 428                 } else {
 429                         return save_i387_fsave(buf);
 430                 }
 431         } else {
 432                 return fpregs_soft_get(current, NULL,
 433                                        0, sizeof(struct user_i387_ia32_struct),
 434                                        NULL, buf) ? -1 : 1;
 435         }
 436 }
 437
 438 static inline int restore_i387_fsave(struct _fpstate_ia32 __user *buf)
 439 {
 440         struct task_struct *tsk = current;
 441         clear_fpu(tsk);
 442         return __copy_from_user(&tsk->thread.i387.fsave, buf,
 443                                 sizeof(struct i387_fsave_struct));
 444 }
 445
 446 static int restore_i387_fxsave(struct _fpstate_ia32 __user *buf)
 447 {
 448         int err;
 449         struct task_struct *tsk = current;
 450         struct user_i387_ia32_struct env;
 451         clear_fpu(tsk);
 452         err = __copy_from_user(&tsk->thread.i387.fxsave, &buf->_fxsr_env[0],
 453                                sizeof(struct i387_fxsave_struct));
 454         /* mxcsr reserved bits must be masked to zero for security reasons */
 455         tsk->thread.i387.fxsave.mxcsr &= mxcsr_feature_mask;
 456         if (err || __copy_from_user(&env, buf, sizeof(env)))
 457                 return 1;
 458         convert_to_fxsr(tsk, &env);
 459         return 0;
 460 }
 461
 462 int restore_i387_ia32(struct _fpstate_ia32 __user *buf)
 463 {
 464         int err;
 465
 466         if (HAVE_HWFP) {
 467                 if (cpu_has_fxsr) {
 468                         err = restore_i387_fxsave(buf);
 469                 } else {
 470                         err = restore_i387_fsave(buf);
 471                 }
 472         } else {
 473                 err = fpregs_soft_set(current, NULL,
 474                                       0, sizeof(struct user_i387_ia32_struct),
 475                                       NULL, buf) != 0;
 476         }
 477         set_used_math();
 478         return err;
 479 }
 480
 481 /*
 482  * FPU state for core dumps.
 483  * This is only used for a.out dumps now.
 484  * It is declared generically using elf_fpregset_t (which is
 485  * struct user_i387_struct) but is in fact only used for 32-bit
 486  * dumps, so on 64-bit it is really struct user_i387_ia32_struct.
 487  */
 488 int dump_fpu(struct pt_regs *regs, struct user_i387_struct *fpu)
 489 {
 490         int fpvalid;
 491         struct task_struct *tsk = current;
 492
 493         fpvalid = !!used_math();
 494         if (fpvalid)
 495                 fpvalid = !fpregs_get(tsk, NULL,
 496                                       0, sizeof(struct user_i387_ia32_struct),
 497                                       fpu, NULL);
 498
 499         return fpvalid;
 500 }
 501 EXPORT_SYMBOL(dump_fpu);
 502
 503 #endif  /* CONFIG_X86_32 || CONFIG_IA32_EMULATION */