1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <asm/kvm_emulate.h>
26 #include <linux/stringify.h>
27 #include <asm/debugreg.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstMem16 (OpMem16 << DstShift)
90 #define DstImmUByte (OpImmUByte << DstShift)
91 #define DstDX (OpDX << DstShift)
92 #define DstAccLo (OpAccLo << DstShift)
93 #define DstMask (OpMask << DstShift)
94 /* Source operand type. */
96 #define SrcNone (OpNone << SrcShift)
97 #define SrcReg (OpReg << SrcShift)
98 #define SrcMem (OpMem << SrcShift)
99 #define SrcMem16 (OpMem16 << SrcShift)
100 #define SrcMem32 (OpMem32 << SrcShift)
101 #define SrcImm (OpImm << SrcShift)
102 #define SrcImmByte (OpImmByte << SrcShift)
103 #define SrcOne (OpOne << SrcShift)
104 #define SrcImmUByte (OpImmUByte << SrcShift)
105 #define SrcImmU (OpImmU << SrcShift)
106 #define SrcSI (OpSI << SrcShift)
107 #define SrcXLat (OpXLat << SrcShift)
108 #define SrcImmFAddr (OpImmFAddr << SrcShift)
109 #define SrcMemFAddr (OpMemFAddr << SrcShift)
110 #define SrcAcc (OpAcc << SrcShift)
111 #define SrcImmU16 (OpImmU16 << SrcShift)
112 #define SrcImm64 (OpImm64 << SrcShift)
113 #define SrcDX (OpDX << SrcShift)
114 #define SrcMem8 (OpMem8 << SrcShift)
115 #define SrcAccHi (OpAccHi << SrcShift)
116 #define SrcMask (OpMask << SrcShift)
117 #define BitOp (1<<11)
118 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
119 #define String (1<<13) /* String instruction (rep capable) */
120 #define Stack (1<<14) /* Stack instruction (push/pop) */
121 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
122 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
123 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
124 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
125 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
126 #define Escape (5<<15) /* Escape to coprocessor instruction */
127 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
128 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
129 #define Sse (1<<18) /* SSE Vector instruction */
130 /* Generic ModRM decode. */
131 #define ModRM (1<<19)
132 /* Destination is only written; never read. */
135 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
136 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
137 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
138 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
139 #define Undefined (1<<25) /* No Such Instruction */
140 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
141 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
143 #define PageTable (1 << 29) /* instruction used to write page table */
144 #define NotImpl (1 << 30) /* instruction is not implemented */
145 /* Source 2 operand type */
146 #define Src2Shift (31)
147 #define Src2None (OpNone << Src2Shift)
148 #define Src2Mem (OpMem << Src2Shift)
149 #define Src2CL (OpCL << Src2Shift)
150 #define Src2ImmByte (OpImmByte << Src2Shift)
151 #define Src2One (OpOne << Src2Shift)
152 #define Src2Imm (OpImm << Src2Shift)
153 #define Src2ES (OpES << Src2Shift)
154 #define Src2CS (OpCS << Src2Shift)
155 #define Src2SS (OpSS << Src2Shift)
156 #define Src2DS (OpDS << Src2Shift)
157 #define Src2FS (OpFS << Src2Shift)
158 #define Src2GS (OpGS << Src2Shift)
159 #define Src2Mask (OpMask << Src2Shift)
160 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
161 #define AlignMask ((u64)7 << 41)
162 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
163 #define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) */
164 #define Avx ((u64)3 << 41) /* Advanced Vector Extensions */
165 #define Aligned16 ((u64)4 << 41) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
166 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
167 #define NoWrite ((u64)1 << 45) /* No writeback */
168 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
169 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
170 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
171 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
172 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
173 #define NearBranch ((u64)1 << 52) /* Near branches */
174 #define No16 ((u64)1 << 53) /* No 16 bit operand */
175 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
176 #define TwoMemOp ((u64)1 << 55) /* Instruction has two memory operand */
178 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
180 #define X2(x...) x, x
181 #define X3(x...) X2(x), x
182 #define X4(x...) X2(x), X2(x)
183 #define X5(x...) X4(x), x
184 #define X6(x...) X4(x), X2(x)
185 #define X7(x...) X4(x), X3(x)
186 #define X8(x...) X4(x), X4(x)
187 #define X16(x...) X8(x), X8(x)
189 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
190 #define FASTOP_SIZE 8
193 * fastop functions have a special calling convention:
198 * flags: rflags (in/out)
199 * ex: rsi (in:fastop pointer, out:zero if exception)
201 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
202 * different operand sizes can be reached by calculation, rather than a jump
203 * table (which would be bigger than the code).
205 * fastop functions are declared as taking a never-defined fastop parameter,
206 * so they can't be called from C directly.
215 int (*execute
)(struct x86_emulate_ctxt
*ctxt
);
216 const struct opcode
*group
;
217 const struct group_dual
*gdual
;
218 const struct gprefix
*gprefix
;
219 const struct escape
*esc
;
220 const struct instr_dual
*idual
;
221 const struct mode_dual
*mdual
;
222 void (*fastop
)(struct fastop
*fake
);
224 int (*check_perm
)(struct x86_emulate_ctxt
*ctxt
);
228 struct opcode mod012
[8];
229 struct opcode mod3
[8];
233 struct opcode pfx_no
;
234 struct opcode pfx_66
;
235 struct opcode pfx_f2
;
236 struct opcode pfx_f3
;
241 struct opcode high
[64];
245 struct opcode mod012
;
250 struct opcode mode32
;
251 struct opcode mode64
;
254 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
256 enum x86_transfer_type
{
258 X86_TRANSFER_CALL_JMP
,
260 X86_TRANSFER_TASK_SWITCH
,
263 static ulong
reg_read(struct x86_emulate_ctxt
*ctxt
, unsigned nr
)
265 if (!(ctxt
->regs_valid
& (1 << nr
))) {
266 ctxt
->regs_valid
|= 1 << nr
;
267 ctxt
->_regs
[nr
] = ctxt
->ops
->read_gpr(ctxt
, nr
);
269 return ctxt
->_regs
[nr
];
272 static ulong
*reg_write(struct x86_emulate_ctxt
*ctxt
, unsigned nr
)
274 ctxt
->regs_valid
|= 1 << nr
;
275 ctxt
->regs_dirty
|= 1 << nr
;
276 return &ctxt
->_regs
[nr
];
279 static ulong
*reg_rmw(struct x86_emulate_ctxt
*ctxt
, unsigned nr
)
282 return reg_write(ctxt
, nr
);
285 static void writeback_registers(struct x86_emulate_ctxt
*ctxt
)
289 for_each_set_bit(reg
, (ulong
*)&ctxt
->regs_dirty
, 16)
290 ctxt
->ops
->write_gpr(ctxt
, reg
, ctxt
->_regs
[reg
]);
293 static void invalidate_registers(struct x86_emulate_ctxt
*ctxt
)
295 ctxt
->regs_dirty
= 0;
296 ctxt
->regs_valid
= 0;
300 * These EFLAGS bits are restored from saved value during emulation, and
301 * any changes are written back to the saved value after emulation.
303 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
304 X86_EFLAGS_PF|X86_EFLAGS_CF)
312 static int fastop(struct x86_emulate_ctxt
*ctxt
, void (*fop
)(struct fastop
*));
314 #define FOP_FUNC(name) \
315 ".align " __stringify(FASTOP_SIZE) " \n\t" \
316 ".type " name ", @function \n\t" \
319 #define FOP_RET "ret \n\t"
321 #define FOP_START(op) \
322 extern void em_##op(struct fastop *fake); \
323 asm(".pushsection .text, \"ax\" \n\t" \
324 ".global em_" #op " \n\t" \
331 FOP_FUNC(__stringify(__UNIQUE_ID(nop))) \
334 #define FOP1E(op, dst) \
335 FOP_FUNC(#op "_" #dst) \
336 "10: " #op " %" #dst " \n\t" FOP_RET
338 #define FOP1EEX(op, dst) \
339 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
341 #define FASTOP1(op) \
346 ON64(FOP1E(op##q, rax)) \
349 /* 1-operand, using src2 (for MUL/DIV r/m) */
350 #define FASTOP1SRC2(op, name) \
355 ON64(FOP1E(op, rcx)) \
358 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
359 #define FASTOP1SRC2EX(op, name) \
364 ON64(FOP1EEX(op, rcx)) \
367 #define FOP2E(op, dst, src) \
368 FOP_FUNC(#op "_" #dst "_" #src) \
369 #op " %" #src ", %" #dst " \n\t" FOP_RET
371 #define FASTOP2(op) \
373 FOP2E(op##b, al, dl) \
374 FOP2E(op##w, ax, dx) \
375 FOP2E(op##l, eax, edx) \
376 ON64(FOP2E(op##q, rax, rdx)) \
379 /* 2 operand, word only */
380 #define FASTOP2W(op) \
383 FOP2E(op##w, ax, dx) \
384 FOP2E(op##l, eax, edx) \
385 ON64(FOP2E(op##q, rax, rdx)) \
388 /* 2 operand, src is CL */
389 #define FASTOP2CL(op) \
391 FOP2E(op##b, al, cl) \
392 FOP2E(op##w, ax, cl) \
393 FOP2E(op##l, eax, cl) \
394 ON64(FOP2E(op##q, rax, cl)) \
397 /* 2 operand, src and dest are reversed */
398 #define FASTOP2R(op, name) \
400 FOP2E(op##b, dl, al) \
401 FOP2E(op##w, dx, ax) \
402 FOP2E(op##l, edx, eax) \
403 ON64(FOP2E(op##q, rdx, rax)) \
406 #define FOP3E(op, dst, src, src2) \
407 FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
408 #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
410 /* 3-operand, word-only, src2=cl */
411 #define FASTOP3WCL(op) \
414 FOP3E(op##w, ax, dx, cl) \
415 FOP3E(op##l, eax, edx, cl) \
416 ON64(FOP3E(op##q, rax, rdx, cl)) \
419 /* Special case for SETcc - 1 instruction per cc */
420 #define FOP_SETCC(op) \
422 ".type " #op ", @function \n\t" \
427 asm(".global kvm_fastop_exception \n"
428 "kvm_fastop_exception: xor %esi, %esi; ret");
449 FOP_START(salc
) "pushf; sbb %al, %al; popf \n\t" FOP_RET
453 * XXX: inoutclob user must know where the argument is being expanded.
454 * Relying on CC_HAVE_ASM_GOTO would allow us to remove _fault.
456 #define asm_safe(insn, inoutclob...) \
460 asm volatile("1:" insn "\n" \
462 ".pushsection .fixup, \"ax\"\n" \
463 "3: movl $1, %[_fault]\n" \
466 _ASM_EXTABLE(1b, 3b) \
467 : [_fault] "+qm"(_fault) inoutclob ); \
469 _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
472 static int emulator_check_intercept(struct x86_emulate_ctxt
*ctxt
,
473 enum x86_intercept intercept
,
474 enum x86_intercept_stage stage
)
476 struct x86_instruction_info info
= {
477 .intercept
= intercept
,
478 .rep_prefix
= ctxt
->rep_prefix
,
479 .modrm_mod
= ctxt
->modrm_mod
,
480 .modrm_reg
= ctxt
->modrm_reg
,
481 .modrm_rm
= ctxt
->modrm_rm
,
482 .src_val
= ctxt
->src
.val64
,
483 .dst_val
= ctxt
->dst
.val64
,
484 .src_bytes
= ctxt
->src
.bytes
,
485 .dst_bytes
= ctxt
->dst
.bytes
,
486 .ad_bytes
= ctxt
->ad_bytes
,
487 .next_rip
= ctxt
->eip
,
490 return ctxt
->ops
->intercept(ctxt
, &info
, stage
);
493 static void assign_masked(ulong
*dest
, ulong src
, ulong mask
)
495 *dest
= (*dest
& ~mask
) | (src
& mask
);
498 static void assign_register(unsigned long *reg
, u64 val
, int bytes
)
500 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
503 *(u8
*)reg
= (u8
)val
;
506 *(u16
*)reg
= (u16
)val
;
510 break; /* 64b: zero-extend */
517 static inline unsigned long ad_mask(struct x86_emulate_ctxt
*ctxt
)
519 return (1UL << (ctxt
->ad_bytes
<< 3)) - 1;
522 static ulong
stack_mask(struct x86_emulate_ctxt
*ctxt
)
525 struct desc_struct ss
;
527 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
529 ctxt
->ops
->get_segment(ctxt
, &sel
, &ss
, NULL
, VCPU_SREG_SS
);
530 return ~0U >> ((ss
.d
^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
533 static int stack_size(struct x86_emulate_ctxt
*ctxt
)
535 return (__fls(stack_mask(ctxt
)) + 1) >> 3;
538 /* Access/update address held in a register, based on addressing mode. */
539 static inline unsigned long
540 address_mask(struct x86_emulate_ctxt
*ctxt
, unsigned long reg
)
542 if (ctxt
->ad_bytes
== sizeof(unsigned long))
545 return reg
& ad_mask(ctxt
);
548 static inline unsigned long
549 register_address(struct x86_emulate_ctxt
*ctxt
, int reg
)
551 return address_mask(ctxt
, reg_read(ctxt
, reg
));
554 static void masked_increment(ulong
*reg
, ulong mask
, int inc
)
556 assign_masked(reg
, *reg
+ inc
, mask
);
560 register_address_increment(struct x86_emulate_ctxt
*ctxt
, int reg
, int inc
)
562 ulong
*preg
= reg_rmw(ctxt
, reg
);
564 assign_register(preg
, *preg
+ inc
, ctxt
->ad_bytes
);
567 static void rsp_increment(struct x86_emulate_ctxt
*ctxt
, int inc
)
569 masked_increment(reg_rmw(ctxt
, VCPU_REGS_RSP
), stack_mask(ctxt
), inc
);
572 static u32
desc_limit_scaled(struct desc_struct
*desc
)
574 u32 limit
= get_desc_limit(desc
);
576 return desc
->g
? (limit
<< 12) | 0xfff : limit
;
579 static unsigned long seg_base(struct x86_emulate_ctxt
*ctxt
, int seg
)
581 if (ctxt
->mode
== X86EMUL_MODE_PROT64
&& seg
< VCPU_SREG_FS
)
584 return ctxt
->ops
->get_cached_segment_base(ctxt
, seg
);
587 static int emulate_exception(struct x86_emulate_ctxt
*ctxt
, int vec
,
588 u32 error
, bool valid
)
591 ctxt
->exception
.vector
= vec
;
592 ctxt
->exception
.error_code
= error
;
593 ctxt
->exception
.error_code_valid
= valid
;
594 return X86EMUL_PROPAGATE_FAULT
;
597 static int emulate_db(struct x86_emulate_ctxt
*ctxt
)
599 return emulate_exception(ctxt
, DB_VECTOR
, 0, false);
602 static int emulate_gp(struct x86_emulate_ctxt
*ctxt
, int err
)
604 return emulate_exception(ctxt
, GP_VECTOR
, err
, true);
607 static int emulate_ss(struct x86_emulate_ctxt
*ctxt
, int err
)
609 return emulate_exception(ctxt
, SS_VECTOR
, err
, true);
612 static int emulate_ud(struct x86_emulate_ctxt
*ctxt
)
614 return emulate_exception(ctxt
, UD_VECTOR
, 0, false);
617 static int emulate_ts(struct x86_emulate_ctxt
*ctxt
, int err
)
619 return emulate_exception(ctxt
, TS_VECTOR
, err
, true);
622 static int emulate_de(struct x86_emulate_ctxt
*ctxt
)
624 return emulate_exception(ctxt
, DE_VECTOR
, 0, false);
627 static int emulate_nm(struct x86_emulate_ctxt
*ctxt
)
629 return emulate_exception(ctxt
, NM_VECTOR
, 0, false);
632 static u16
get_segment_selector(struct x86_emulate_ctxt
*ctxt
, unsigned seg
)
635 struct desc_struct desc
;
637 ctxt
->ops
->get_segment(ctxt
, &selector
, &desc
, NULL
, seg
);
641 static void set_segment_selector(struct x86_emulate_ctxt
*ctxt
, u16 selector
,
646 struct desc_struct desc
;
648 ctxt
->ops
->get_segment(ctxt
, &dummy
, &desc
, &base3
, seg
);
649 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, base3
, seg
);
653 * x86 defines three classes of vector instructions: explicitly
654 * aligned, explicitly unaligned, and the rest, which change behaviour
655 * depending on whether they're AVX encoded or not.
657 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
658 * subject to the same check. FXSAVE and FXRSTOR are checked here too as their
659 * 512 bytes of data must be aligned to a 16 byte boundary.
661 static unsigned insn_alignment(struct x86_emulate_ctxt
*ctxt
, unsigned size
)
663 u64 alignment
= ctxt
->d
& AlignMask
;
665 if (likely(size
< 16))
680 static __always_inline
int __linearize(struct x86_emulate_ctxt
*ctxt
,
681 struct segmented_address addr
,
682 unsigned *max_size
, unsigned size
,
683 bool write
, bool fetch
,
684 enum x86emul_mode mode
, ulong
*linear
)
686 struct desc_struct desc
;
692 la
= seg_base(ctxt
, addr
.seg
) + addr
.ea
;
695 case X86EMUL_MODE_PROT64
:
697 if (is_noncanonical_address(la
))
700 *max_size
= min_t(u64
, ~0u, (1ull << 48) - la
);
701 if (size
> *max_size
)
705 *linear
= la
= (u32
)la
;
706 usable
= ctxt
->ops
->get_segment(ctxt
, &sel
, &desc
, NULL
,
710 /* code segment in protected mode or read-only data segment */
711 if ((((ctxt
->mode
!= X86EMUL_MODE_REAL
) && (desc
.type
& 8))
712 || !(desc
.type
& 2)) && write
)
714 /* unreadable code segment */
715 if (!fetch
&& (desc
.type
& 8) && !(desc
.type
& 2))
717 lim
= desc_limit_scaled(&desc
);
718 if (!(desc
.type
& 8) && (desc
.type
& 4)) {
719 /* expand-down segment */
722 lim
= desc
.d
? 0xffffffff : 0xffff;
726 if (lim
== 0xffffffff)
729 *max_size
= (u64
)lim
+ 1 - addr
.ea
;
730 if (size
> *max_size
)
735 if (la
& (insn_alignment(ctxt
, size
) - 1))
736 return emulate_gp(ctxt
, 0);
737 return X86EMUL_CONTINUE
;
739 if (addr
.seg
== VCPU_SREG_SS
)
740 return emulate_ss(ctxt
, 0);
742 return emulate_gp(ctxt
, 0);
745 static int linearize(struct x86_emulate_ctxt
*ctxt
,
746 struct segmented_address addr
,
747 unsigned size
, bool write
,
751 return __linearize(ctxt
, addr
, &max_size
, size
, write
, false,
755 static inline int assign_eip(struct x86_emulate_ctxt
*ctxt
, ulong dst
,
756 enum x86emul_mode mode
)
761 struct segmented_address addr
= { .seg
= VCPU_SREG_CS
,
764 if (ctxt
->op_bytes
!= sizeof(unsigned long))
765 addr
.ea
= dst
& ((1UL << (ctxt
->op_bytes
<< 3)) - 1);
766 rc
= __linearize(ctxt
, addr
, &max_size
, 1, false, true, mode
, &linear
);
767 if (rc
== X86EMUL_CONTINUE
)
768 ctxt
->_eip
= addr
.ea
;
772 static inline int assign_eip_near(struct x86_emulate_ctxt
*ctxt
, ulong dst
)
774 return assign_eip(ctxt
, dst
, ctxt
->mode
);
777 static int assign_eip_far(struct x86_emulate_ctxt
*ctxt
, ulong dst
,
778 const struct desc_struct
*cs_desc
)
780 enum x86emul_mode mode
= ctxt
->mode
;
784 if (ctxt
->mode
>= X86EMUL_MODE_PROT16
) {
788 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
790 mode
= X86EMUL_MODE_PROT64
;
792 mode
= X86EMUL_MODE_PROT32
; /* temporary value */
795 if (mode
== X86EMUL_MODE_PROT16
|| mode
== X86EMUL_MODE_PROT32
)
796 mode
= cs_desc
->d
? X86EMUL_MODE_PROT32
: X86EMUL_MODE_PROT16
;
797 rc
= assign_eip(ctxt
, dst
, mode
);
798 if (rc
== X86EMUL_CONTINUE
)
803 static inline int jmp_rel(struct x86_emulate_ctxt
*ctxt
, int rel
)
805 return assign_eip_near(ctxt
, ctxt
->_eip
+ rel
);
808 static int segmented_read_std(struct x86_emulate_ctxt
*ctxt
,
809 struct segmented_address addr
,
816 rc
= linearize(ctxt
, addr
, size
, false, &linear
);
817 if (rc
!= X86EMUL_CONTINUE
)
819 return ctxt
->ops
->read_std(ctxt
, linear
, data
, size
, &ctxt
->exception
);
822 static int segmented_write_std(struct x86_emulate_ctxt
*ctxt
,
823 struct segmented_address addr
,
830 rc
= linearize(ctxt
, addr
, size
, true, &linear
);
831 if (rc
!= X86EMUL_CONTINUE
)
833 return ctxt
->ops
->write_std(ctxt
, linear
, data
, size
, &ctxt
->exception
);
837 * Prefetch the remaining bytes of the instruction without crossing page
838 * boundary if they are not in fetch_cache yet.
840 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt
*ctxt
, int op_size
)
843 unsigned size
, max_size
;
844 unsigned long linear
;
845 int cur_size
= ctxt
->fetch
.end
- ctxt
->fetch
.data
;
846 struct segmented_address addr
= { .seg
= VCPU_SREG_CS
,
847 .ea
= ctxt
->eip
+ cur_size
};
850 * We do not know exactly how many bytes will be needed, and
851 * __linearize is expensive, so fetch as much as possible. We
852 * just have to avoid going beyond the 15 byte limit, the end
853 * of the segment, or the end of the page.
855 * __linearize is called with size 0 so that it does not do any
856 * boundary check itself. Instead, we use max_size to check
859 rc
= __linearize(ctxt
, addr
, &max_size
, 0, false, true, ctxt
->mode
,
861 if (unlikely(rc
!= X86EMUL_CONTINUE
))
864 size
= min_t(unsigned, 15UL ^ cur_size
, max_size
);
865 size
= min_t(unsigned, size
, PAGE_SIZE
- offset_in_page(linear
));
868 * One instruction can only straddle two pages,
869 * and one has been loaded at the beginning of
870 * x86_decode_insn. So, if not enough bytes
871 * still, we must have hit the 15-byte boundary.
873 if (unlikely(size
< op_size
))
874 return emulate_gp(ctxt
, 0);
876 rc
= ctxt
->ops
->fetch(ctxt
, linear
, ctxt
->fetch
.end
,
877 size
, &ctxt
->exception
);
878 if (unlikely(rc
!= X86EMUL_CONTINUE
))
880 ctxt
->fetch
.end
+= size
;
881 return X86EMUL_CONTINUE
;
884 static __always_inline
int do_insn_fetch_bytes(struct x86_emulate_ctxt
*ctxt
,
887 unsigned done_size
= ctxt
->fetch
.end
- ctxt
->fetch
.ptr
;
889 if (unlikely(done_size
< size
))
890 return __do_insn_fetch_bytes(ctxt
, size
- done_size
);
892 return X86EMUL_CONTINUE
;
895 /* Fetch next part of the instruction being emulated. */
896 #define insn_fetch(_type, _ctxt) \
899 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
900 if (rc != X86EMUL_CONTINUE) \
902 ctxt->_eip += sizeof(_type); \
903 memcpy(&_x, ctxt->fetch.ptr, sizeof(_type)); \
904 ctxt->fetch.ptr += sizeof(_type); \
908 #define insn_fetch_arr(_arr, _size, _ctxt) \
910 rc = do_insn_fetch_bytes(_ctxt, _size); \
911 if (rc != X86EMUL_CONTINUE) \
913 ctxt->_eip += (_size); \
914 memcpy(_arr, ctxt->fetch.ptr, _size); \
915 ctxt->fetch.ptr += (_size); \
919 * Given the 'reg' portion of a ModRM byte, and a register block, return a
920 * pointer into the block that addresses the relevant register.
921 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
923 static void *decode_register(struct x86_emulate_ctxt
*ctxt
, u8 modrm_reg
,
927 int highbyte_regs
= (ctxt
->rex_prefix
== 0) && byteop
;
929 if (highbyte_regs
&& modrm_reg
>= 4 && modrm_reg
< 8)
930 p
= (unsigned char *)reg_rmw(ctxt
, modrm_reg
& 3) + 1;
932 p
= reg_rmw(ctxt
, modrm_reg
);
936 static int read_descriptor(struct x86_emulate_ctxt
*ctxt
,
937 struct segmented_address addr
,
938 u16
*size
, unsigned long *address
, int op_bytes
)
945 rc
= segmented_read_std(ctxt
, addr
, size
, 2);
946 if (rc
!= X86EMUL_CONTINUE
)
949 rc
= segmented_read_std(ctxt
, addr
, address
, op_bytes
);
963 FASTOP1SRC2(mul
, mul_ex
);
964 FASTOP1SRC2(imul
, imul_ex
);
965 FASTOP1SRC2EX(div
, div_ex
);
966 FASTOP1SRC2EX(idiv
, idiv_ex
);
995 FASTOP2R(cmp
, cmp_r
);
997 static int em_bsf_c(struct x86_emulate_ctxt
*ctxt
)
999 /* If src is zero, do not writeback, but update flags */
1000 if (ctxt
->src
.val
== 0)
1001 ctxt
->dst
.type
= OP_NONE
;
1002 return fastop(ctxt
, em_bsf
);
1005 static int em_bsr_c(struct x86_emulate_ctxt
*ctxt
)
1007 /* If src is zero, do not writeback, but update flags */
1008 if (ctxt
->src
.val
== 0)
1009 ctxt
->dst
.type
= OP_NONE
;
1010 return fastop(ctxt
, em_bsr
);
1013 static __always_inline u8
test_cc(unsigned int condition
, unsigned long flags
)
1016 void (*fop
)(void) = (void *)em_setcc
+ 4 * (condition
& 0xf);
1018 flags
= (flags
& EFLAGS_MASK
) | X86_EFLAGS_IF
;
1019 asm("push %[flags]; popf; call *%[fastop]"
1020 : "=a"(rc
) : [fastop
]"r"(fop
), [flags
]"r"(flags
));
1024 static void fetch_register_operand(struct operand
*op
)
1026 switch (op
->bytes
) {
1028 op
->val
= *(u8
*)op
->addr
.reg
;
1031 op
->val
= *(u16
*)op
->addr
.reg
;
1034 op
->val
= *(u32
*)op
->addr
.reg
;
1037 op
->val
= *(u64
*)op
->addr
.reg
;
1042 static void read_sse_reg(struct x86_emulate_ctxt
*ctxt
, sse128_t
*data
, int reg
)
1044 ctxt
->ops
->get_fpu(ctxt
);
1046 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data
)); break;
1047 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data
)); break;
1048 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data
)); break;
1049 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data
)); break;
1050 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data
)); break;
1051 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data
)); break;
1052 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data
)); break;
1053 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data
)); break;
1054 #ifdef CONFIG_X86_64
1055 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data
)); break;
1056 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data
)); break;
1057 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data
)); break;
1058 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data
)); break;
1059 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data
)); break;
1060 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data
)); break;
1061 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data
)); break;
1062 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data
)); break;
1066 ctxt
->ops
->put_fpu(ctxt
);
1069 static void write_sse_reg(struct x86_emulate_ctxt
*ctxt
, sse128_t
*data
,
1072 ctxt
->ops
->get_fpu(ctxt
);
1074 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data
)); break;
1075 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data
)); break;
1076 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data
)); break;
1077 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data
)); break;
1078 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data
)); break;
1079 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data
)); break;
1080 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data
)); break;
1081 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data
)); break;
1082 #ifdef CONFIG_X86_64
1083 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data
)); break;
1084 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data
)); break;
1085 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data
)); break;
1086 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data
)); break;
1087 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data
)); break;
1088 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data
)); break;
1089 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data
)); break;
1090 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data
)); break;
1094 ctxt
->ops
->put_fpu(ctxt
);
1097 static void read_mmx_reg(struct x86_emulate_ctxt
*ctxt
, u64
*data
, int reg
)
1099 ctxt
->ops
->get_fpu(ctxt
);
1101 case 0: asm("movq %%mm0, %0" : "=m"(*data
)); break;
1102 case 1: asm("movq %%mm1, %0" : "=m"(*data
)); break;
1103 case 2: asm("movq %%mm2, %0" : "=m"(*data
)); break;
1104 case 3: asm("movq %%mm3, %0" : "=m"(*data
)); break;
1105 case 4: asm("movq %%mm4, %0" : "=m"(*data
)); break;
1106 case 5: asm("movq %%mm5, %0" : "=m"(*data
)); break;
1107 case 6: asm("movq %%mm6, %0" : "=m"(*data
)); break;
1108 case 7: asm("movq %%mm7, %0" : "=m"(*data
)); break;
1111 ctxt
->ops
->put_fpu(ctxt
);
1114 static void write_mmx_reg(struct x86_emulate_ctxt
*ctxt
, u64
*data
, int reg
)
1116 ctxt
->ops
->get_fpu(ctxt
);
1118 case 0: asm("movq %0, %%mm0" : : "m"(*data
)); break;
1119 case 1: asm("movq %0, %%mm1" : : "m"(*data
)); break;
1120 case 2: asm("movq %0, %%mm2" : : "m"(*data
)); break;
1121 case 3: asm("movq %0, %%mm3" : : "m"(*data
)); break;
1122 case 4: asm("movq %0, %%mm4" : : "m"(*data
)); break;
1123 case 5: asm("movq %0, %%mm5" : : "m"(*data
)); break;
1124 case 6: asm("movq %0, %%mm6" : : "m"(*data
)); break;
1125 case 7: asm("movq %0, %%mm7" : : "m"(*data
)); break;
1128 ctxt
->ops
->put_fpu(ctxt
);
1131 static int em_fninit(struct x86_emulate_ctxt
*ctxt
)
1133 if (ctxt
->ops
->get_cr(ctxt
, 0) & (X86_CR0_TS
| X86_CR0_EM
))
1134 return emulate_nm(ctxt
);
1136 ctxt
->ops
->get_fpu(ctxt
);
1137 asm volatile("fninit");
1138 ctxt
->ops
->put_fpu(ctxt
);
1139 return X86EMUL_CONTINUE
;
1142 static int em_fnstcw(struct x86_emulate_ctxt
*ctxt
)
1146 if (ctxt
->ops
->get_cr(ctxt
, 0) & (X86_CR0_TS
| X86_CR0_EM
))
1147 return emulate_nm(ctxt
);
1149 ctxt
->ops
->get_fpu(ctxt
);
1150 asm volatile("fnstcw %0": "+m"(fcw
));
1151 ctxt
->ops
->put_fpu(ctxt
);
1153 ctxt
->dst
.val
= fcw
;
1155 return X86EMUL_CONTINUE
;
1158 static int em_fnstsw(struct x86_emulate_ctxt
*ctxt
)
1162 if (ctxt
->ops
->get_cr(ctxt
, 0) & (X86_CR0_TS
| X86_CR0_EM
))
1163 return emulate_nm(ctxt
);
1165 ctxt
->ops
->get_fpu(ctxt
);
1166 asm volatile("fnstsw %0": "+m"(fsw
));
1167 ctxt
->ops
->put_fpu(ctxt
);
1169 ctxt
->dst
.val
= fsw
;
1171 return X86EMUL_CONTINUE
;
1174 static void decode_register_operand(struct x86_emulate_ctxt
*ctxt
,
1177 unsigned reg
= ctxt
->modrm_reg
;
1179 if (!(ctxt
->d
& ModRM
))
1180 reg
= (ctxt
->b
& 7) | ((ctxt
->rex_prefix
& 1) << 3);
1182 if (ctxt
->d
& Sse
) {
1186 read_sse_reg(ctxt
, &op
->vec_val
, reg
);
1189 if (ctxt
->d
& Mmx
) {
1198 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
1199 op
->addr
.reg
= decode_register(ctxt
, reg
, ctxt
->d
& ByteOp
);
1201 fetch_register_operand(op
);
1202 op
->orig_val
= op
->val
;
1205 static void adjust_modrm_seg(struct x86_emulate_ctxt
*ctxt
, int base_reg
)
1207 if (base_reg
== VCPU_REGS_RSP
|| base_reg
== VCPU_REGS_RBP
)
1208 ctxt
->modrm_seg
= VCPU_SREG_SS
;
1211 static int decode_modrm(struct x86_emulate_ctxt
*ctxt
,
1215 int index_reg
, base_reg
, scale
;
1216 int rc
= X86EMUL_CONTINUE
;
1219 ctxt
->modrm_reg
= ((ctxt
->rex_prefix
<< 1) & 8); /* REX.R */
1220 index_reg
= (ctxt
->rex_prefix
<< 2) & 8; /* REX.X */
1221 base_reg
= (ctxt
->rex_prefix
<< 3) & 8; /* REX.B */
1223 ctxt
->modrm_mod
= (ctxt
->modrm
& 0xc0) >> 6;
1224 ctxt
->modrm_reg
|= (ctxt
->modrm
& 0x38) >> 3;
1225 ctxt
->modrm_rm
= base_reg
| (ctxt
->modrm
& 0x07);
1226 ctxt
->modrm_seg
= VCPU_SREG_DS
;
1228 if (ctxt
->modrm_mod
== 3 || (ctxt
->d
& NoMod
)) {
1230 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
1231 op
->addr
.reg
= decode_register(ctxt
, ctxt
->modrm_rm
,
1233 if (ctxt
->d
& Sse
) {
1236 op
->addr
.xmm
= ctxt
->modrm_rm
;
1237 read_sse_reg(ctxt
, &op
->vec_val
, ctxt
->modrm_rm
);
1240 if (ctxt
->d
& Mmx
) {
1243 op
->addr
.mm
= ctxt
->modrm_rm
& 7;
1246 fetch_register_operand(op
);
1252 if (ctxt
->ad_bytes
== 2) {
1253 unsigned bx
= reg_read(ctxt
, VCPU_REGS_RBX
);
1254 unsigned bp
= reg_read(ctxt
, VCPU_REGS_RBP
);
1255 unsigned si
= reg_read(ctxt
, VCPU_REGS_RSI
);
1256 unsigned di
= reg_read(ctxt
, VCPU_REGS_RDI
);
1258 /* 16-bit ModR/M decode. */
1259 switch (ctxt
->modrm_mod
) {
1261 if (ctxt
->modrm_rm
== 6)
1262 modrm_ea
+= insn_fetch(u16
, ctxt
);
1265 modrm_ea
+= insn_fetch(s8
, ctxt
);
1268 modrm_ea
+= insn_fetch(u16
, ctxt
);
1271 switch (ctxt
->modrm_rm
) {
1273 modrm_ea
+= bx
+ si
;
1276 modrm_ea
+= bx
+ di
;
1279 modrm_ea
+= bp
+ si
;
1282 modrm_ea
+= bp
+ di
;
1291 if (ctxt
->modrm_mod
!= 0)
1298 if (ctxt
->modrm_rm
== 2 || ctxt
->modrm_rm
== 3 ||
1299 (ctxt
->modrm_rm
== 6 && ctxt
->modrm_mod
!= 0))
1300 ctxt
->modrm_seg
= VCPU_SREG_SS
;
1301 modrm_ea
= (u16
)modrm_ea
;
1303 /* 32/64-bit ModR/M decode. */
1304 if ((ctxt
->modrm_rm
& 7) == 4) {
1305 sib
= insn_fetch(u8
, ctxt
);
1306 index_reg
|= (sib
>> 3) & 7;
1307 base_reg
|= sib
& 7;
1310 if ((base_reg
& 7) == 5 && ctxt
->modrm_mod
== 0)
1311 modrm_ea
+= insn_fetch(s32
, ctxt
);
1313 modrm_ea
+= reg_read(ctxt
, base_reg
);
1314 adjust_modrm_seg(ctxt
, base_reg
);
1315 /* Increment ESP on POP [ESP] */
1316 if ((ctxt
->d
& IncSP
) &&
1317 base_reg
== VCPU_REGS_RSP
)
1318 modrm_ea
+= ctxt
->op_bytes
;
1321 modrm_ea
+= reg_read(ctxt
, index_reg
) << scale
;
1322 } else if ((ctxt
->modrm_rm
& 7) == 5 && ctxt
->modrm_mod
== 0) {
1323 modrm_ea
+= insn_fetch(s32
, ctxt
);
1324 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
1325 ctxt
->rip_relative
= 1;
1327 base_reg
= ctxt
->modrm_rm
;
1328 modrm_ea
+= reg_read(ctxt
, base_reg
);
1329 adjust_modrm_seg(ctxt
, base_reg
);
1331 switch (ctxt
->modrm_mod
) {
1333 modrm_ea
+= insn_fetch(s8
, ctxt
);
1336 modrm_ea
+= insn_fetch(s32
, ctxt
);
1340 op
->addr
.mem
.ea
= modrm_ea
;
1341 if (ctxt
->ad_bytes
!= 8)
1342 ctxt
->memop
.addr
.mem
.ea
= (u32
)ctxt
->memop
.addr
.mem
.ea
;
1348 static int decode_abs(struct x86_emulate_ctxt
*ctxt
,
1351 int rc
= X86EMUL_CONTINUE
;
1354 switch (ctxt
->ad_bytes
) {
1356 op
->addr
.mem
.ea
= insn_fetch(u16
, ctxt
);
1359 op
->addr
.mem
.ea
= insn_fetch(u32
, ctxt
);
1362 op
->addr
.mem
.ea
= insn_fetch(u64
, ctxt
);
1369 static void fetch_bit_operand(struct x86_emulate_ctxt
*ctxt
)
1373 if (ctxt
->dst
.type
== OP_MEM
&& ctxt
->src
.type
== OP_REG
) {
1374 mask
= ~((long)ctxt
->dst
.bytes
* 8 - 1);
1376 if (ctxt
->src
.bytes
== 2)
1377 sv
= (s16
)ctxt
->src
.val
& (s16
)mask
;
1378 else if (ctxt
->src
.bytes
== 4)
1379 sv
= (s32
)ctxt
->src
.val
& (s32
)mask
;
1381 sv
= (s64
)ctxt
->src
.val
& (s64
)mask
;
1383 ctxt
->dst
.addr
.mem
.ea
= address_mask(ctxt
,
1384 ctxt
->dst
.addr
.mem
.ea
+ (sv
>> 3));
1387 /* only subword offset */
1388 ctxt
->src
.val
&= (ctxt
->dst
.bytes
<< 3) - 1;
1391 static int read_emulated(struct x86_emulate_ctxt
*ctxt
,
1392 unsigned long addr
, void *dest
, unsigned size
)
1395 struct read_cache
*mc
= &ctxt
->mem_read
;
1397 if (mc
->pos
< mc
->end
)
1400 WARN_ON((mc
->end
+ size
) >= sizeof(mc
->data
));
1402 rc
= ctxt
->ops
->read_emulated(ctxt
, addr
, mc
->data
+ mc
->end
, size
,
1404 if (rc
!= X86EMUL_CONTINUE
)
1410 memcpy(dest
, mc
->data
+ mc
->pos
, size
);
1412 return X86EMUL_CONTINUE
;
1415 static int segmented_read(struct x86_emulate_ctxt
*ctxt
,
1416 struct segmented_address addr
,
1423 rc
= linearize(ctxt
, addr
, size
, false, &linear
);
1424 if (rc
!= X86EMUL_CONTINUE
)
1426 return read_emulated(ctxt
, linear
, data
, size
);
1429 static int segmented_write(struct x86_emulate_ctxt
*ctxt
,
1430 struct segmented_address addr
,
1437 rc
= linearize(ctxt
, addr
, size
, true, &linear
);
1438 if (rc
!= X86EMUL_CONTINUE
)
1440 return ctxt
->ops
->write_emulated(ctxt
, linear
, data
, size
,
1444 static int segmented_cmpxchg(struct x86_emulate_ctxt
*ctxt
,
1445 struct segmented_address addr
,
1446 const void *orig_data
, const void *data
,
1452 rc
= linearize(ctxt
, addr
, size
, true, &linear
);
1453 if (rc
!= X86EMUL_CONTINUE
)
1455 return ctxt
->ops
->cmpxchg_emulated(ctxt
, linear
, orig_data
, data
,
1456 size
, &ctxt
->exception
);
1459 static int pio_in_emulated(struct x86_emulate_ctxt
*ctxt
,
1460 unsigned int size
, unsigned short port
,
1463 struct read_cache
*rc
= &ctxt
->io_read
;
1465 if (rc
->pos
== rc
->end
) { /* refill pio read ahead */
1466 unsigned int in_page
, n
;
1467 unsigned int count
= ctxt
->rep_prefix
?
1468 address_mask(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
)) : 1;
1469 in_page
= (ctxt
->eflags
& X86_EFLAGS_DF
) ?
1470 offset_in_page(reg_read(ctxt
, VCPU_REGS_RDI
)) :
1471 PAGE_SIZE
- offset_in_page(reg_read(ctxt
, VCPU_REGS_RDI
));
1472 n
= min3(in_page
, (unsigned int)sizeof(rc
->data
) / size
, count
);
1475 rc
->pos
= rc
->end
= 0;
1476 if (!ctxt
->ops
->pio_in_emulated(ctxt
, size
, port
, rc
->data
, n
))
1481 if (ctxt
->rep_prefix
&& (ctxt
->d
& String
) &&
1482 !(ctxt
->eflags
& X86_EFLAGS_DF
)) {
1483 ctxt
->dst
.data
= rc
->data
+ rc
->pos
;
1484 ctxt
->dst
.type
= OP_MEM_STR
;
1485 ctxt
->dst
.count
= (rc
->end
- rc
->pos
) / size
;
1488 memcpy(dest
, rc
->data
+ rc
->pos
, size
);
1494 static int read_interrupt_descriptor(struct x86_emulate_ctxt
*ctxt
,
1495 u16 index
, struct desc_struct
*desc
)
1500 ctxt
->ops
->get_idt(ctxt
, &dt
);
1502 if (dt
.size
< index
* 8 + 7)
1503 return emulate_gp(ctxt
, index
<< 3 | 0x2);
1505 addr
= dt
.address
+ index
* 8;
1506 return ctxt
->ops
->read_std(ctxt
, addr
, desc
, sizeof *desc
,
1510 static void get_descriptor_table_ptr(struct x86_emulate_ctxt
*ctxt
,
1511 u16 selector
, struct desc_ptr
*dt
)
1513 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
1516 if (selector
& 1 << 2) {
1517 struct desc_struct desc
;
1520 memset (dt
, 0, sizeof *dt
);
1521 if (!ops
->get_segment(ctxt
, &sel
, &desc
, &base3
,
1525 dt
->size
= desc_limit_scaled(&desc
); /* what if limit > 65535? */
1526 dt
->address
= get_desc_base(&desc
) | ((u64
)base3
<< 32);
1528 ops
->get_gdt(ctxt
, dt
);
1531 static int get_descriptor_ptr(struct x86_emulate_ctxt
*ctxt
,
1532 u16 selector
, ulong
*desc_addr_p
)
1535 u16 index
= selector
>> 3;
1538 get_descriptor_table_ptr(ctxt
, selector
, &dt
);
1540 if (dt
.size
< index
* 8 + 7)
1541 return emulate_gp(ctxt
, selector
& 0xfffc);
1543 addr
= dt
.address
+ index
* 8;
1545 #ifdef CONFIG_X86_64
1546 if (addr
>> 32 != 0) {
1549 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
1550 if (!(efer
& EFER_LMA
))
1555 *desc_addr_p
= addr
;
1556 return X86EMUL_CONTINUE
;
1559 /* allowed just for 8 bytes segments */
1560 static int read_segment_descriptor(struct x86_emulate_ctxt
*ctxt
,
1561 u16 selector
, struct desc_struct
*desc
,
1566 rc
= get_descriptor_ptr(ctxt
, selector
, desc_addr_p
);
1567 if (rc
!= X86EMUL_CONTINUE
)
1570 return ctxt
->ops
->read_std(ctxt
, *desc_addr_p
, desc
, sizeof(*desc
),
1574 /* allowed just for 8 bytes segments */
1575 static int write_segment_descriptor(struct x86_emulate_ctxt
*ctxt
,
1576 u16 selector
, struct desc_struct
*desc
)
1581 rc
= get_descriptor_ptr(ctxt
, selector
, &addr
);
1582 if (rc
!= X86EMUL_CONTINUE
)
1585 return ctxt
->ops
->write_std(ctxt
, addr
, desc
, sizeof *desc
,
1589 static int __load_segment_descriptor(struct x86_emulate_ctxt
*ctxt
,
1590 u16 selector
, int seg
, u8 cpl
,
1591 enum x86_transfer_type transfer
,
1592 struct desc_struct
*desc
)
1594 struct desc_struct seg_desc
, old_desc
;
1596 unsigned err_vec
= GP_VECTOR
;
1598 bool null_selector
= !(selector
& ~0x3); /* 0000-0003 are null */
1604 memset(&seg_desc
, 0, sizeof seg_desc
);
1606 if (ctxt
->mode
== X86EMUL_MODE_REAL
) {
1607 /* set real mode segment descriptor (keep limit etc. for
1609 ctxt
->ops
->get_segment(ctxt
, &dummy
, &seg_desc
, NULL
, seg
);
1610 set_desc_base(&seg_desc
, selector
<< 4);
1612 } else if (seg
<= VCPU_SREG_GS
&& ctxt
->mode
== X86EMUL_MODE_VM86
) {
1613 /* VM86 needs a clean new segment descriptor */
1614 set_desc_base(&seg_desc
, selector
<< 4);
1615 set_desc_limit(&seg_desc
, 0xffff);
1625 /* TR should be in GDT only */
1626 if (seg
== VCPU_SREG_TR
&& (selector
& (1 << 2)))
1629 /* NULL selector is not valid for TR, CS and (except for long mode) SS */
1630 if (null_selector
) {
1631 if (seg
== VCPU_SREG_CS
|| seg
== VCPU_SREG_TR
)
1634 if (seg
== VCPU_SREG_SS
) {
1635 if (ctxt
->mode
!= X86EMUL_MODE_PROT64
|| rpl
!= cpl
)
1639 * ctxt->ops->set_segment expects the CPL to be in
1640 * SS.DPL, so fake an expand-up 32-bit data segment.
1650 /* Skip all following checks */
1654 ret
= read_segment_descriptor(ctxt
, selector
, &seg_desc
, &desc_addr
);
1655 if (ret
!= X86EMUL_CONTINUE
)
1658 err_code
= selector
& 0xfffc;
1659 err_vec
= (transfer
== X86_TRANSFER_TASK_SWITCH
) ? TS_VECTOR
:
1662 /* can't load system descriptor into segment selector */
1663 if (seg
<= VCPU_SREG_GS
&& !seg_desc
.s
) {
1664 if (transfer
== X86_TRANSFER_CALL_JMP
)
1665 return X86EMUL_UNHANDLEABLE
;
1670 err_vec
= (seg
== VCPU_SREG_SS
) ? SS_VECTOR
: NP_VECTOR
;
1679 * segment is not a writable data segment or segment
1680 * selector's RPL != CPL or segment selector's RPL != CPL
1682 if (rpl
!= cpl
|| (seg_desc
.type
& 0xa) != 0x2 || dpl
!= cpl
)
1686 if (!(seg_desc
.type
& 8))
1689 if (seg_desc
.type
& 4) {
1695 if (rpl
> cpl
|| dpl
!= cpl
)
1698 /* in long-mode d/b must be clear if l is set */
1699 if (seg_desc
.d
&& seg_desc
.l
) {
1702 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
1703 if (efer
& EFER_LMA
)
1707 /* CS(RPL) <- CPL */
1708 selector
= (selector
& 0xfffc) | cpl
;
1711 if (seg_desc
.s
|| (seg_desc
.type
!= 1 && seg_desc
.type
!= 9))
1713 old_desc
= seg_desc
;
1714 seg_desc
.type
|= 2; /* busy */
1715 ret
= ctxt
->ops
->cmpxchg_emulated(ctxt
, desc_addr
, &old_desc
, &seg_desc
,
1716 sizeof(seg_desc
), &ctxt
->exception
);
1717 if (ret
!= X86EMUL_CONTINUE
)
1720 case VCPU_SREG_LDTR
:
1721 if (seg_desc
.s
|| seg_desc
.type
!= 2)
1724 default: /* DS, ES, FS, or GS */
1726 * segment is not a data or readable code segment or
1727 * ((segment is a data or nonconforming code segment)
1728 * and (both RPL and CPL > DPL))
1730 if ((seg_desc
.type
& 0xa) == 0x8 ||
1731 (((seg_desc
.type
& 0xc) != 0xc) &&
1732 (rpl
> dpl
&& cpl
> dpl
)))
1738 /* mark segment as accessed */
1739 if (!(seg_desc
.type
& 1)) {
1741 ret
= write_segment_descriptor(ctxt
, selector
,
1743 if (ret
!= X86EMUL_CONTINUE
)
1746 } else if (ctxt
->mode
== X86EMUL_MODE_PROT64
) {
1747 ret
= ctxt
->ops
->read_std(ctxt
, desc_addr
+8, &base3
,
1748 sizeof(base3
), &ctxt
->exception
);
1749 if (ret
!= X86EMUL_CONTINUE
)
1751 if (is_noncanonical_address(get_desc_base(&seg_desc
) |
1752 ((u64
)base3
<< 32)))
1753 return emulate_gp(ctxt
, 0);
1756 ctxt
->ops
->set_segment(ctxt
, selector
, &seg_desc
, base3
, seg
);
1759 return X86EMUL_CONTINUE
;
1761 return emulate_exception(ctxt
, err_vec
, err_code
, true);
1764 static int load_segment_descriptor(struct x86_emulate_ctxt
*ctxt
,
1765 u16 selector
, int seg
)
1767 u8 cpl
= ctxt
->ops
->cpl(ctxt
);
1770 * None of MOV, POP and LSS can load a NULL selector in CPL=3, but
1771 * they can load it at CPL<3 (Intel's manual says only LSS can,
1774 * However, the Intel manual says that putting IST=1/DPL=3 in
1775 * an interrupt gate will result in SS=3 (the AMD manual instead
1776 * says it doesn't), so allow SS=3 in __load_segment_descriptor
1777 * and only forbid it here.
1779 if (seg
== VCPU_SREG_SS
&& selector
== 3 &&
1780 ctxt
->mode
== X86EMUL_MODE_PROT64
)
1781 return emulate_exception(ctxt
, GP_VECTOR
, 0, true);
1783 return __load_segment_descriptor(ctxt
, selector
, seg
, cpl
,
1784 X86_TRANSFER_NONE
, NULL
);
1787 static void write_register_operand(struct operand
*op
)
1789 return assign_register(op
->addr
.reg
, op
->val
, op
->bytes
);
1792 static int writeback(struct x86_emulate_ctxt
*ctxt
, struct operand
*op
)
1796 write_register_operand(op
);
1799 if (ctxt
->lock_prefix
)
1800 return segmented_cmpxchg(ctxt
,
1806 return segmented_write(ctxt
,
1812 return segmented_write(ctxt
,
1815 op
->bytes
* op
->count
);
1818 write_sse_reg(ctxt
, &op
->vec_val
, op
->addr
.xmm
);
1821 write_mmx_reg(ctxt
, &op
->mm_val
, op
->addr
.mm
);
1829 return X86EMUL_CONTINUE
;
1832 static int push(struct x86_emulate_ctxt
*ctxt
, void *data
, int bytes
)
1834 struct segmented_address addr
;
1836 rsp_increment(ctxt
, -bytes
);
1837 addr
.ea
= reg_read(ctxt
, VCPU_REGS_RSP
) & stack_mask(ctxt
);
1838 addr
.seg
= VCPU_SREG_SS
;
1840 return segmented_write(ctxt
, addr
, data
, bytes
);
1843 static int em_push(struct x86_emulate_ctxt
*ctxt
)
1845 /* Disable writeback. */
1846 ctxt
->dst
.type
= OP_NONE
;
1847 return push(ctxt
, &ctxt
->src
.val
, ctxt
->op_bytes
);
1850 static int emulate_pop(struct x86_emulate_ctxt
*ctxt
,
1851 void *dest
, int len
)
1854 struct segmented_address addr
;
1856 addr
.ea
= reg_read(ctxt
, VCPU_REGS_RSP
) & stack_mask(ctxt
);
1857 addr
.seg
= VCPU_SREG_SS
;
1858 rc
= segmented_read(ctxt
, addr
, dest
, len
);
1859 if (rc
!= X86EMUL_CONTINUE
)
1862 rsp_increment(ctxt
, len
);
1866 static int em_pop(struct x86_emulate_ctxt
*ctxt
)
1868 return emulate_pop(ctxt
, &ctxt
->dst
.val
, ctxt
->op_bytes
);
1871 static int emulate_popf(struct x86_emulate_ctxt
*ctxt
,
1872 void *dest
, int len
)
1875 unsigned long val
, change_mask
;
1876 int iopl
= (ctxt
->eflags
& X86_EFLAGS_IOPL
) >> X86_EFLAGS_IOPL_BIT
;
1877 int cpl
= ctxt
->ops
->cpl(ctxt
);
1879 rc
= emulate_pop(ctxt
, &val
, len
);
1880 if (rc
!= X86EMUL_CONTINUE
)
1883 change_mask
= X86_EFLAGS_CF
| X86_EFLAGS_PF
| X86_EFLAGS_AF
|
1884 X86_EFLAGS_ZF
| X86_EFLAGS_SF
| X86_EFLAGS_OF
|
1885 X86_EFLAGS_TF
| X86_EFLAGS_DF
| X86_EFLAGS_NT
|
1886 X86_EFLAGS_AC
| X86_EFLAGS_ID
;
1888 switch(ctxt
->mode
) {
1889 case X86EMUL_MODE_PROT64
:
1890 case X86EMUL_MODE_PROT32
:
1891 case X86EMUL_MODE_PROT16
:
1893 change_mask
|= X86_EFLAGS_IOPL
;
1895 change_mask
|= X86_EFLAGS_IF
;
1897 case X86EMUL_MODE_VM86
:
1899 return emulate_gp(ctxt
, 0);
1900 change_mask
|= X86_EFLAGS_IF
;
1902 default: /* real mode */
1903 change_mask
|= (X86_EFLAGS_IOPL
| X86_EFLAGS_IF
);
1907 *(unsigned long *)dest
=
1908 (ctxt
->eflags
& ~change_mask
) | (val
& change_mask
);
1913 static int em_popf(struct x86_emulate_ctxt
*ctxt
)
1915 ctxt
->dst
.type
= OP_REG
;
1916 ctxt
->dst
.addr
.reg
= &ctxt
->eflags
;
1917 ctxt
->dst
.bytes
= ctxt
->op_bytes
;
1918 return emulate_popf(ctxt
, &ctxt
->dst
.val
, ctxt
->op_bytes
);
1921 static int em_enter(struct x86_emulate_ctxt
*ctxt
)
1924 unsigned frame_size
= ctxt
->src
.val
;
1925 unsigned nesting_level
= ctxt
->src2
.val
& 31;
1929 return X86EMUL_UNHANDLEABLE
;
1931 rbp
= reg_read(ctxt
, VCPU_REGS_RBP
);
1932 rc
= push(ctxt
, &rbp
, stack_size(ctxt
));
1933 if (rc
!= X86EMUL_CONTINUE
)
1935 assign_masked(reg_rmw(ctxt
, VCPU_REGS_RBP
), reg_read(ctxt
, VCPU_REGS_RSP
),
1937 assign_masked(reg_rmw(ctxt
, VCPU_REGS_RSP
),
1938 reg_read(ctxt
, VCPU_REGS_RSP
) - frame_size
,
1940 return X86EMUL_CONTINUE
;
1943 static int em_leave(struct x86_emulate_ctxt
*ctxt
)
1945 assign_masked(reg_rmw(ctxt
, VCPU_REGS_RSP
), reg_read(ctxt
, VCPU_REGS_RBP
),
1947 return emulate_pop(ctxt
, reg_rmw(ctxt
, VCPU_REGS_RBP
), ctxt
->op_bytes
);
1950 static int em_push_sreg(struct x86_emulate_ctxt
*ctxt
)
1952 int seg
= ctxt
->src2
.val
;
1954 ctxt
->src
.val
= get_segment_selector(ctxt
, seg
);
1955 if (ctxt
->op_bytes
== 4) {
1956 rsp_increment(ctxt
, -2);
1960 return em_push(ctxt
);
1963 static int em_pop_sreg(struct x86_emulate_ctxt
*ctxt
)
1965 int seg
= ctxt
->src2
.val
;
1966 unsigned long selector
;
1969 rc
= emulate_pop(ctxt
, &selector
, 2);
1970 if (rc
!= X86EMUL_CONTINUE
)
1973 if (ctxt
->modrm_reg
== VCPU_SREG_SS
)
1974 ctxt
->interruptibility
= KVM_X86_SHADOW_INT_MOV_SS
;
1975 if (ctxt
->op_bytes
> 2)
1976 rsp_increment(ctxt
, ctxt
->op_bytes
- 2);
1978 rc
= load_segment_descriptor(ctxt
, (u16
)selector
, seg
);
1982 static int em_pusha(struct x86_emulate_ctxt
*ctxt
)
1984 unsigned long old_esp
= reg_read(ctxt
, VCPU_REGS_RSP
);
1985 int rc
= X86EMUL_CONTINUE
;
1986 int reg
= VCPU_REGS_RAX
;
1988 while (reg
<= VCPU_REGS_RDI
) {
1989 (reg
== VCPU_REGS_RSP
) ?
1990 (ctxt
->src
.val
= old_esp
) : (ctxt
->src
.val
= reg_read(ctxt
, reg
));
1993 if (rc
!= X86EMUL_CONTINUE
)
2002 static int em_pushf(struct x86_emulate_ctxt
*ctxt
)
2004 ctxt
->src
.val
= (unsigned long)ctxt
->eflags
& ~X86_EFLAGS_VM
;
2005 return em_push(ctxt
);
2008 static int em_popa(struct x86_emulate_ctxt
*ctxt
)
2010 int rc
= X86EMUL_CONTINUE
;
2011 int reg
= VCPU_REGS_RDI
;
2014 while (reg
>= VCPU_REGS_RAX
) {
2015 if (reg
== VCPU_REGS_RSP
) {
2016 rsp_increment(ctxt
, ctxt
->op_bytes
);
2020 rc
= emulate_pop(ctxt
, &val
, ctxt
->op_bytes
);
2021 if (rc
!= X86EMUL_CONTINUE
)
2023 assign_register(reg_rmw(ctxt
, reg
), val
, ctxt
->op_bytes
);
2029 static int __emulate_int_real(struct x86_emulate_ctxt
*ctxt
, int irq
)
2031 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2038 /* TODO: Add limit checks */
2039 ctxt
->src
.val
= ctxt
->eflags
;
2041 if (rc
!= X86EMUL_CONTINUE
)
2044 ctxt
->eflags
&= ~(X86_EFLAGS_IF
| X86_EFLAGS_TF
| X86_EFLAGS_AC
);
2046 ctxt
->src
.val
= get_segment_selector(ctxt
, VCPU_SREG_CS
);
2048 if (rc
!= X86EMUL_CONTINUE
)
2051 ctxt
->src
.val
= ctxt
->_eip
;
2053 if (rc
!= X86EMUL_CONTINUE
)
2056 ops
->get_idt(ctxt
, &dt
);
2058 eip_addr
= dt
.address
+ (irq
<< 2);
2059 cs_addr
= dt
.address
+ (irq
<< 2) + 2;
2061 rc
= ops
->read_std(ctxt
, cs_addr
, &cs
, 2, &ctxt
->exception
);
2062 if (rc
!= X86EMUL_CONTINUE
)
2065 rc
= ops
->read_std(ctxt
, eip_addr
, &eip
, 2, &ctxt
->exception
);
2066 if (rc
!= X86EMUL_CONTINUE
)
2069 rc
= load_segment_descriptor(ctxt
, cs
, VCPU_SREG_CS
);
2070 if (rc
!= X86EMUL_CONTINUE
)
2078 int emulate_int_real(struct x86_emulate_ctxt
*ctxt
, int irq
)
2082 invalidate_registers(ctxt
);
2083 rc
= __emulate_int_real(ctxt
, irq
);
2084 if (rc
== X86EMUL_CONTINUE
)
2085 writeback_registers(ctxt
);
2089 static int emulate_int(struct x86_emulate_ctxt
*ctxt
, int irq
)
2091 switch(ctxt
->mode
) {
2092 case X86EMUL_MODE_REAL
:
2093 return __emulate_int_real(ctxt
, irq
);
2094 case X86EMUL_MODE_VM86
:
2095 case X86EMUL_MODE_PROT16
:
2096 case X86EMUL_MODE_PROT32
:
2097 case X86EMUL_MODE_PROT64
:
2099 /* Protected mode interrupts unimplemented yet */
2100 return X86EMUL_UNHANDLEABLE
;
2104 static int emulate_iret_real(struct x86_emulate_ctxt
*ctxt
)
2106 int rc
= X86EMUL_CONTINUE
;
2107 unsigned long temp_eip
= 0;
2108 unsigned long temp_eflags
= 0;
2109 unsigned long cs
= 0;
2110 unsigned long mask
= X86_EFLAGS_CF
| X86_EFLAGS_PF
| X86_EFLAGS_AF
|
2111 X86_EFLAGS_ZF
| X86_EFLAGS_SF
| X86_EFLAGS_TF
|
2112 X86_EFLAGS_IF
| X86_EFLAGS_DF
| X86_EFLAGS_OF
|
2113 X86_EFLAGS_IOPL
| X86_EFLAGS_NT
| X86_EFLAGS_RF
|
2114 X86_EFLAGS_AC
| X86_EFLAGS_ID
|
2116 unsigned long vm86_mask
= X86_EFLAGS_VM
| X86_EFLAGS_VIF
|
2119 /* TODO: Add stack limit check */
2121 rc
= emulate_pop(ctxt
, &temp_eip
, ctxt
->op_bytes
);
2123 if (rc
!= X86EMUL_CONTINUE
)
2126 if (temp_eip
& ~0xffff)
2127 return emulate_gp(ctxt
, 0);
2129 rc
= emulate_pop(ctxt
, &cs
, ctxt
->op_bytes
);
2131 if (rc
!= X86EMUL_CONTINUE
)
2134 rc
= emulate_pop(ctxt
, &temp_eflags
, ctxt
->op_bytes
);
2136 if (rc
!= X86EMUL_CONTINUE
)
2139 rc
= load_segment_descriptor(ctxt
, (u16
)cs
, VCPU_SREG_CS
);
2141 if (rc
!= X86EMUL_CONTINUE
)
2144 ctxt
->_eip
= temp_eip
;
2146 if (ctxt
->op_bytes
== 4)
2147 ctxt
->eflags
= ((temp_eflags
& mask
) | (ctxt
->eflags
& vm86_mask
));
2148 else if (ctxt
->op_bytes
== 2) {
2149 ctxt
->eflags
&= ~0xffff;
2150 ctxt
->eflags
|= temp_eflags
;
2153 ctxt
->eflags
&= ~EFLG_RESERVED_ZEROS_MASK
; /* Clear reserved zeros */
2154 ctxt
->eflags
|= X86_EFLAGS_FIXED
;
2155 ctxt
->ops
->set_nmi_mask(ctxt
, false);
2160 static int em_iret(struct x86_emulate_ctxt
*ctxt
)
2162 switch(ctxt
->mode
) {
2163 case X86EMUL_MODE_REAL
:
2164 return emulate_iret_real(ctxt
);
2165 case X86EMUL_MODE_VM86
:
2166 case X86EMUL_MODE_PROT16
:
2167 case X86EMUL_MODE_PROT32
:
2168 case X86EMUL_MODE_PROT64
:
2170 /* iret from protected mode unimplemented yet */
2171 return X86EMUL_UNHANDLEABLE
;
2175 static int em_jmp_far(struct x86_emulate_ctxt
*ctxt
)
2179 struct desc_struct new_desc
;
2180 u8 cpl
= ctxt
->ops
->cpl(ctxt
);
2182 memcpy(&sel
, ctxt
->src
.valptr
+ ctxt
->op_bytes
, 2);
2184 rc
= __load_segment_descriptor(ctxt
, sel
, VCPU_SREG_CS
, cpl
,
2185 X86_TRANSFER_CALL_JMP
,
2187 if (rc
!= X86EMUL_CONTINUE
)
2190 rc
= assign_eip_far(ctxt
, ctxt
->src
.val
, &new_desc
);
2191 /* Error handling is not implemented. */
2192 if (rc
!= X86EMUL_CONTINUE
)
2193 return X86EMUL_UNHANDLEABLE
;
2198 static int em_jmp_abs(struct x86_emulate_ctxt
*ctxt
)
2200 return assign_eip_near(ctxt
, ctxt
->src
.val
);
2203 static int em_call_near_abs(struct x86_emulate_ctxt
*ctxt
)
2208 old_eip
= ctxt
->_eip
;
2209 rc
= assign_eip_near(ctxt
, ctxt
->src
.val
);
2210 if (rc
!= X86EMUL_CONTINUE
)
2212 ctxt
->src
.val
= old_eip
;
2217 static int em_cmpxchg8b(struct x86_emulate_ctxt
*ctxt
)
2219 u64 old
= ctxt
->dst
.orig_val64
;
2221 if (ctxt
->dst
.bytes
== 16)
2222 return X86EMUL_UNHANDLEABLE
;
2224 if (((u32
) (old
>> 0) != (u32
) reg_read(ctxt
, VCPU_REGS_RAX
)) ||
2225 ((u32
) (old
>> 32) != (u32
) reg_read(ctxt
, VCPU_REGS_RDX
))) {
2226 *reg_write(ctxt
, VCPU_REGS_RAX
) = (u32
) (old
>> 0);
2227 *reg_write(ctxt
, VCPU_REGS_RDX
) = (u32
) (old
>> 32);
2228 ctxt
->eflags
&= ~X86_EFLAGS_ZF
;
2230 ctxt
->dst
.val64
= ((u64
)reg_read(ctxt
, VCPU_REGS_RCX
) << 32) |
2231 (u32
) reg_read(ctxt
, VCPU_REGS_RBX
);
2233 ctxt
->eflags
|= X86_EFLAGS_ZF
;
2235 return X86EMUL_CONTINUE
;
2238 static int em_ret(struct x86_emulate_ctxt
*ctxt
)
2243 rc
= emulate_pop(ctxt
, &eip
, ctxt
->op_bytes
);
2244 if (rc
!= X86EMUL_CONTINUE
)
2247 return assign_eip_near(ctxt
, eip
);
2250 static int em_ret_far(struct x86_emulate_ctxt
*ctxt
)
2253 unsigned long eip
, cs
;
2254 int cpl
= ctxt
->ops
->cpl(ctxt
);
2255 struct desc_struct new_desc
;
2257 rc
= emulate_pop(ctxt
, &eip
, ctxt
->op_bytes
);
2258 if (rc
!= X86EMUL_CONTINUE
)
2260 rc
= emulate_pop(ctxt
, &cs
, ctxt
->op_bytes
);
2261 if (rc
!= X86EMUL_CONTINUE
)
2263 /* Outer-privilege level return is not implemented */
2264 if (ctxt
->mode
>= X86EMUL_MODE_PROT16
&& (cs
& 3) > cpl
)
2265 return X86EMUL_UNHANDLEABLE
;
2266 rc
= __load_segment_descriptor(ctxt
, (u16
)cs
, VCPU_SREG_CS
, cpl
,
2269 if (rc
!= X86EMUL_CONTINUE
)
2271 rc
= assign_eip_far(ctxt
, eip
, &new_desc
);
2272 /* Error handling is not implemented. */
2273 if (rc
!= X86EMUL_CONTINUE
)
2274 return X86EMUL_UNHANDLEABLE
;
2279 static int em_ret_far_imm(struct x86_emulate_ctxt
*ctxt
)
2283 rc
= em_ret_far(ctxt
);
2284 if (rc
!= X86EMUL_CONTINUE
)
2286 rsp_increment(ctxt
, ctxt
->src
.val
);
2287 return X86EMUL_CONTINUE
;
2290 static int em_cmpxchg(struct x86_emulate_ctxt
*ctxt
)
2292 /* Save real source value, then compare EAX against destination. */
2293 ctxt
->dst
.orig_val
= ctxt
->dst
.val
;
2294 ctxt
->dst
.val
= reg_read(ctxt
, VCPU_REGS_RAX
);
2295 ctxt
->src
.orig_val
= ctxt
->src
.val
;
2296 ctxt
->src
.val
= ctxt
->dst
.orig_val
;
2297 fastop(ctxt
, em_cmp
);
2299 if (ctxt
->eflags
& X86_EFLAGS_ZF
) {
2300 /* Success: write back to memory; no update of EAX */
2301 ctxt
->src
.type
= OP_NONE
;
2302 ctxt
->dst
.val
= ctxt
->src
.orig_val
;
2304 /* Failure: write the value we saw to EAX. */
2305 ctxt
->src
.type
= OP_REG
;
2306 ctxt
->src
.addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RAX
);
2307 ctxt
->src
.val
= ctxt
->dst
.orig_val
;
2308 /* Create write-cycle to dest by writing the same value */
2309 ctxt
->dst
.val
= ctxt
->dst
.orig_val
;
2311 return X86EMUL_CONTINUE
;
2314 static int em_lseg(struct x86_emulate_ctxt
*ctxt
)
2316 int seg
= ctxt
->src2
.val
;
2320 memcpy(&sel
, ctxt
->src
.valptr
+ ctxt
->op_bytes
, 2);
2322 rc
= load_segment_descriptor(ctxt
, sel
, seg
);
2323 if (rc
!= X86EMUL_CONTINUE
)
2326 ctxt
->dst
.val
= ctxt
->src
.val
;
2330 static int emulator_has_longmode(struct x86_emulate_ctxt
*ctxt
)
2332 u32 eax
, ebx
, ecx
, edx
;
2336 ctxt
->ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
, &edx
);
2337 return edx
& bit(X86_FEATURE_LM
);
2340 #define GET_SMSTATE(type, smbase, offset) \
2343 int r = ctxt->ops->read_phys(ctxt, smbase + offset, &__val, \
2345 if (r != X86EMUL_CONTINUE) \
2346 return X86EMUL_UNHANDLEABLE; \
2350 static void rsm_set_desc_flags(struct desc_struct
*desc
, u32 flags
)
2352 desc
->g
= (flags
>> 23) & 1;
2353 desc
->d
= (flags
>> 22) & 1;
2354 desc
->l
= (flags
>> 21) & 1;
2355 desc
->avl
= (flags
>> 20) & 1;
2356 desc
->p
= (flags
>> 15) & 1;
2357 desc
->dpl
= (flags
>> 13) & 3;
2358 desc
->s
= (flags
>> 12) & 1;
2359 desc
->type
= (flags
>> 8) & 15;
2362 static int rsm_load_seg_32(struct x86_emulate_ctxt
*ctxt
, u64 smbase
, int n
)
2364 struct desc_struct desc
;
2368 selector
= GET_SMSTATE(u32
, smbase
, 0x7fa8 + n
* 4);
2371 offset
= 0x7f84 + n
* 12;
2373 offset
= 0x7f2c + (n
- 3) * 12;
2375 set_desc_base(&desc
, GET_SMSTATE(u32
, smbase
, offset
+ 8));
2376 set_desc_limit(&desc
, GET_SMSTATE(u32
, smbase
, offset
+ 4));
2377 rsm_set_desc_flags(&desc
, GET_SMSTATE(u32
, smbase
, offset
));
2378 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, 0, n
);
2379 return X86EMUL_CONTINUE
;
2382 static int rsm_load_seg_64(struct x86_emulate_ctxt
*ctxt
, u64 smbase
, int n
)
2384 struct desc_struct desc
;
2389 offset
= 0x7e00 + n
* 16;
2391 selector
= GET_SMSTATE(u16
, smbase
, offset
);
2392 rsm_set_desc_flags(&desc
, GET_SMSTATE(u16
, smbase
, offset
+ 2) << 8);
2393 set_desc_limit(&desc
, GET_SMSTATE(u32
, smbase
, offset
+ 4));
2394 set_desc_base(&desc
, GET_SMSTATE(u32
, smbase
, offset
+ 8));
2395 base3
= GET_SMSTATE(u32
, smbase
, offset
+ 12);
2397 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, base3
, n
);
2398 return X86EMUL_CONTINUE
;
2401 static int rsm_enter_protected_mode(struct x86_emulate_ctxt
*ctxt
,
2407 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
2408 * Then enable protected mode. However, PCID cannot be enabled
2409 * if EFER.LMA=0, so set it separately.
2411 bad
= ctxt
->ops
->set_cr(ctxt
, 4, cr4
& ~X86_CR4_PCIDE
);
2413 return X86EMUL_UNHANDLEABLE
;
2415 bad
= ctxt
->ops
->set_cr(ctxt
, 0, cr0
);
2417 return X86EMUL_UNHANDLEABLE
;
2419 if (cr4
& X86_CR4_PCIDE
) {
2420 bad
= ctxt
->ops
->set_cr(ctxt
, 4, cr4
);
2422 return X86EMUL_UNHANDLEABLE
;
2425 return X86EMUL_CONTINUE
;
2428 static int rsm_load_state_32(struct x86_emulate_ctxt
*ctxt
, u64 smbase
)
2430 struct desc_struct desc
;
2436 cr0
= GET_SMSTATE(u32
, smbase
, 0x7ffc);
2437 ctxt
->ops
->set_cr(ctxt
, 3, GET_SMSTATE(u32
, smbase
, 0x7ff8));
2438 ctxt
->eflags
= GET_SMSTATE(u32
, smbase
, 0x7ff4) | X86_EFLAGS_FIXED
;
2439 ctxt
->_eip
= GET_SMSTATE(u32
, smbase
, 0x7ff0);
2441 for (i
= 0; i
< 8; i
++)
2442 *reg_write(ctxt
, i
) = GET_SMSTATE(u32
, smbase
, 0x7fd0 + i
* 4);
2444 val
= GET_SMSTATE(u32
, smbase
, 0x7fcc);
2445 ctxt
->ops
->set_dr(ctxt
, 6, (val
& DR6_VOLATILE
) | DR6_FIXED_1
);
2446 val
= GET_SMSTATE(u32
, smbase
, 0x7fc8);
2447 ctxt
->ops
->set_dr(ctxt
, 7, (val
& DR7_VOLATILE
) | DR7_FIXED_1
);
2449 selector
= GET_SMSTATE(u32
, smbase
, 0x7fc4);
2450 set_desc_base(&desc
, GET_SMSTATE(u32
, smbase
, 0x7f64));
2451 set_desc_limit(&desc
, GET_SMSTATE(u32
, smbase
, 0x7f60));
2452 rsm_set_desc_flags(&desc
, GET_SMSTATE(u32
, smbase
, 0x7f5c));
2453 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, 0, VCPU_SREG_TR
);
2455 selector
= GET_SMSTATE(u32
, smbase
, 0x7fc0);
2456 set_desc_base(&desc
, GET_SMSTATE(u32
, smbase
, 0x7f80));
2457 set_desc_limit(&desc
, GET_SMSTATE(u32
, smbase
, 0x7f7c));
2458 rsm_set_desc_flags(&desc
, GET_SMSTATE(u32
, smbase
, 0x7f78));
2459 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, 0, VCPU_SREG_LDTR
);
2461 dt
.address
= GET_SMSTATE(u32
, smbase
, 0x7f74);
2462 dt
.size
= GET_SMSTATE(u32
, smbase
, 0x7f70);
2463 ctxt
->ops
->set_gdt(ctxt
, &dt
);
2465 dt
.address
= GET_SMSTATE(u32
, smbase
, 0x7f58);
2466 dt
.size
= GET_SMSTATE(u32
, smbase
, 0x7f54);
2467 ctxt
->ops
->set_idt(ctxt
, &dt
);
2469 for (i
= 0; i
< 6; i
++) {
2470 int r
= rsm_load_seg_32(ctxt
, smbase
, i
);
2471 if (r
!= X86EMUL_CONTINUE
)
2475 cr4
= GET_SMSTATE(u32
, smbase
, 0x7f14);
2477 ctxt
->ops
->set_smbase(ctxt
, GET_SMSTATE(u32
, smbase
, 0x7ef8));
2479 return rsm_enter_protected_mode(ctxt
, cr0
, cr4
);
2482 static int rsm_load_state_64(struct x86_emulate_ctxt
*ctxt
, u64 smbase
)
2484 struct desc_struct desc
;
2491 for (i
= 0; i
< 16; i
++)
2492 *reg_write(ctxt
, i
) = GET_SMSTATE(u64
, smbase
, 0x7ff8 - i
* 8);
2494 ctxt
->_eip
= GET_SMSTATE(u64
, smbase
, 0x7f78);
2495 ctxt
->eflags
= GET_SMSTATE(u32
, smbase
, 0x7f70) | X86_EFLAGS_FIXED
;
2497 val
= GET_SMSTATE(u32
, smbase
, 0x7f68);
2498 ctxt
->ops
->set_dr(ctxt
, 6, (val
& DR6_VOLATILE
) | DR6_FIXED_1
);
2499 val
= GET_SMSTATE(u32
, smbase
, 0x7f60);
2500 ctxt
->ops
->set_dr(ctxt
, 7, (val
& DR7_VOLATILE
) | DR7_FIXED_1
);
2502 cr0
= GET_SMSTATE(u64
, smbase
, 0x7f58);
2503 ctxt
->ops
->set_cr(ctxt
, 3, GET_SMSTATE(u64
, smbase
, 0x7f50));
2504 cr4
= GET_SMSTATE(u64
, smbase
, 0x7f48);
2505 ctxt
->ops
->set_smbase(ctxt
, GET_SMSTATE(u32
, smbase
, 0x7f00));
2506 val
= GET_SMSTATE(u64
, smbase
, 0x7ed0);
2507 ctxt
->ops
->set_msr(ctxt
, MSR_EFER
, val
& ~EFER_LMA
);
2509 selector
= GET_SMSTATE(u32
, smbase
, 0x7e90);
2510 rsm_set_desc_flags(&desc
, GET_SMSTATE(u32
, smbase
, 0x7e92) << 8);
2511 set_desc_limit(&desc
, GET_SMSTATE(u32
, smbase
, 0x7e94));
2512 set_desc_base(&desc
, GET_SMSTATE(u32
, smbase
, 0x7e98));
2513 base3
= GET_SMSTATE(u32
, smbase
, 0x7e9c);
2514 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, base3
, VCPU_SREG_TR
);
2516 dt
.size
= GET_SMSTATE(u32
, smbase
, 0x7e84);
2517 dt
.address
= GET_SMSTATE(u64
, smbase
, 0x7e88);
2518 ctxt
->ops
->set_idt(ctxt
, &dt
);
2520 selector
= GET_SMSTATE(u32
, smbase
, 0x7e70);
2521 rsm_set_desc_flags(&desc
, GET_SMSTATE(u32
, smbase
, 0x7e72) << 8);
2522 set_desc_limit(&desc
, GET_SMSTATE(u32
, smbase
, 0x7e74));
2523 set_desc_base(&desc
, GET_SMSTATE(u32
, smbase
, 0x7e78));
2524 base3
= GET_SMSTATE(u32
, smbase
, 0x7e7c);
2525 ctxt
->ops
->set_segment(ctxt
, selector
, &desc
, base3
, VCPU_SREG_LDTR
);
2527 dt
.size
= GET_SMSTATE(u32
, smbase
, 0x7e64);
2528 dt
.address
= GET_SMSTATE(u64
, smbase
, 0x7e68);
2529 ctxt
->ops
->set_gdt(ctxt
, &dt
);
2531 r
= rsm_enter_protected_mode(ctxt
, cr0
, cr4
);
2532 if (r
!= X86EMUL_CONTINUE
)
2535 for (i
= 0; i
< 6; i
++) {
2536 r
= rsm_load_seg_64(ctxt
, smbase
, i
);
2537 if (r
!= X86EMUL_CONTINUE
)
2541 return X86EMUL_CONTINUE
;
2544 static int em_rsm(struct x86_emulate_ctxt
*ctxt
)
2546 unsigned long cr0
, cr4
, efer
;
2550 if ((ctxt
->ops
->get_hflags(ctxt
) & X86EMUL_SMM_MASK
) == 0)
2551 return emulate_ud(ctxt
);
2554 * Get back to real mode, to prepare a safe state in which to load
2555 * CR0/CR3/CR4/EFER. It's all a bit more complicated if the vCPU
2556 * supports long mode.
2558 cr4
= ctxt
->ops
->get_cr(ctxt
, 4);
2559 if (emulator_has_longmode(ctxt
)) {
2560 struct desc_struct cs_desc
;
2562 /* Zero CR4.PCIDE before CR0.PG. */
2563 if (cr4
& X86_CR4_PCIDE
) {
2564 ctxt
->ops
->set_cr(ctxt
, 4, cr4
& ~X86_CR4_PCIDE
);
2565 cr4
&= ~X86_CR4_PCIDE
;
2568 /* A 32-bit code segment is required to clear EFER.LMA. */
2569 memset(&cs_desc
, 0, sizeof(cs_desc
));
2571 cs_desc
.s
= cs_desc
.g
= cs_desc
.p
= 1;
2572 ctxt
->ops
->set_segment(ctxt
, 0, &cs_desc
, 0, VCPU_SREG_CS
);
2575 /* For the 64-bit case, this will clear EFER.LMA. */
2576 cr0
= ctxt
->ops
->get_cr(ctxt
, 0);
2577 if (cr0
& X86_CR0_PE
)
2578 ctxt
->ops
->set_cr(ctxt
, 0, cr0
& ~(X86_CR0_PG
| X86_CR0_PE
));
2580 /* Now clear CR4.PAE (which must be done before clearing EFER.LME). */
2581 if (cr4
& X86_CR4_PAE
)
2582 ctxt
->ops
->set_cr(ctxt
, 4, cr4
& ~X86_CR4_PAE
);
2584 /* And finally go back to 32-bit mode. */
2586 ctxt
->ops
->set_msr(ctxt
, MSR_EFER
, efer
);
2588 smbase
= ctxt
->ops
->get_smbase(ctxt
);
2589 if (emulator_has_longmode(ctxt
))
2590 ret
= rsm_load_state_64(ctxt
, smbase
+ 0x8000);
2592 ret
= rsm_load_state_32(ctxt
, smbase
+ 0x8000);
2594 if (ret
!= X86EMUL_CONTINUE
) {
2595 /* FIXME: should triple fault */
2596 return X86EMUL_UNHANDLEABLE
;
2599 if ((ctxt
->ops
->get_hflags(ctxt
) & X86EMUL_SMM_INSIDE_NMI_MASK
) == 0)
2600 ctxt
->ops
->set_nmi_mask(ctxt
, false);
2602 ctxt
->ops
->set_hflags(ctxt
, ctxt
->ops
->get_hflags(ctxt
) &
2603 ~(X86EMUL_SMM_INSIDE_NMI_MASK
| X86EMUL_SMM_MASK
));
2604 return X86EMUL_CONTINUE
;
2608 setup_syscalls_segments(struct x86_emulate_ctxt
*ctxt
,
2609 struct desc_struct
*cs
, struct desc_struct
*ss
)
2611 cs
->l
= 0; /* will be adjusted later */
2612 set_desc_base(cs
, 0); /* flat segment */
2613 cs
->g
= 1; /* 4kb granularity */
2614 set_desc_limit(cs
, 0xfffff); /* 4GB limit */
2615 cs
->type
= 0x0b; /* Read, Execute, Accessed */
2617 cs
->dpl
= 0; /* will be adjusted later */
2622 set_desc_base(ss
, 0); /* flat segment */
2623 set_desc_limit(ss
, 0xfffff); /* 4GB limit */
2624 ss
->g
= 1; /* 4kb granularity */
2626 ss
->type
= 0x03; /* Read/Write, Accessed */
2627 ss
->d
= 1; /* 32bit stack segment */
2634 static bool vendor_intel(struct x86_emulate_ctxt
*ctxt
)
2636 u32 eax
, ebx
, ecx
, edx
;
2639 ctxt
->ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
, &edx
);
2640 return ebx
== X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2641 && ecx
== X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2642 && edx
== X86EMUL_CPUID_VENDOR_GenuineIntel_edx
;
2645 static bool em_syscall_is_enabled(struct x86_emulate_ctxt
*ctxt
)
2647 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2648 u32 eax
, ebx
, ecx
, edx
;
2651 * syscall should always be enabled in longmode - so only become
2652 * vendor specific (cpuid) if other modes are active...
2654 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
2659 ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
, &edx
);
2661 * Intel ("GenuineIntel")
2662 * remark: Intel CPUs only support "syscall" in 64bit
2663 * longmode. Also an 64bit guest with a
2664 * 32bit compat-app running will #UD !! While this
2665 * behaviour can be fixed (by emulating) into AMD
2666 * response - CPUs of AMD can't behave like Intel.
2668 if (ebx
== X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
&&
2669 ecx
== X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
&&
2670 edx
== X86EMUL_CPUID_VENDOR_GenuineIntel_edx
)
2673 /* AMD ("AuthenticAMD") */
2674 if (ebx
== X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx
&&
2675 ecx
== X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx
&&
2676 edx
== X86EMUL_CPUID_VENDOR_AuthenticAMD_edx
)
2679 /* AMD ("AMDisbetter!") */
2680 if (ebx
== X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx
&&
2681 ecx
== X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx
&&
2682 edx
== X86EMUL_CPUID_VENDOR_AMDisbetterI_edx
)
2685 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2689 static int em_syscall(struct x86_emulate_ctxt
*ctxt
)
2691 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2692 struct desc_struct cs
, ss
;
2697 /* syscall is not available in real mode */
2698 if (ctxt
->mode
== X86EMUL_MODE_REAL
||
2699 ctxt
->mode
== X86EMUL_MODE_VM86
)
2700 return emulate_ud(ctxt
);
2702 if (!(em_syscall_is_enabled(ctxt
)))
2703 return emulate_ud(ctxt
);
2705 ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
2706 setup_syscalls_segments(ctxt
, &cs
, &ss
);
2708 if (!(efer
& EFER_SCE
))
2709 return emulate_ud(ctxt
);
2711 ops
->get_msr(ctxt
, MSR_STAR
, &msr_data
);
2713 cs_sel
= (u16
)(msr_data
& 0xfffc);
2714 ss_sel
= (u16
)(msr_data
+ 8);
2716 if (efer
& EFER_LMA
) {
2720 ops
->set_segment(ctxt
, cs_sel
, &cs
, 0, VCPU_SREG_CS
);
2721 ops
->set_segment(ctxt
, ss_sel
, &ss
, 0, VCPU_SREG_SS
);
2723 *reg_write(ctxt
, VCPU_REGS_RCX
) = ctxt
->_eip
;
2724 if (efer
& EFER_LMA
) {
2725 #ifdef CONFIG_X86_64
2726 *reg_write(ctxt
, VCPU_REGS_R11
) = ctxt
->eflags
;
2729 ctxt
->mode
== X86EMUL_MODE_PROT64
?
2730 MSR_LSTAR
: MSR_CSTAR
, &msr_data
);
2731 ctxt
->_eip
= msr_data
;
2733 ops
->get_msr(ctxt
, MSR_SYSCALL_MASK
, &msr_data
);
2734 ctxt
->eflags
&= ~msr_data
;
2735 ctxt
->eflags
|= X86_EFLAGS_FIXED
;
2739 ops
->get_msr(ctxt
, MSR_STAR
, &msr_data
);
2740 ctxt
->_eip
= (u32
)msr_data
;
2742 ctxt
->eflags
&= ~(X86_EFLAGS_VM
| X86_EFLAGS_IF
);
2745 ctxt
->tf
= (ctxt
->eflags
& X86_EFLAGS_TF
) != 0;
2746 return X86EMUL_CONTINUE
;
2749 static int em_sysenter(struct x86_emulate_ctxt
*ctxt
)
2751 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2752 struct desc_struct cs
, ss
;
2757 ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
2758 /* inject #GP if in real mode */
2759 if (ctxt
->mode
== X86EMUL_MODE_REAL
)
2760 return emulate_gp(ctxt
, 0);
2763 * Not recognized on AMD in compat mode (but is recognized in legacy
2766 if ((ctxt
->mode
!= X86EMUL_MODE_PROT64
) && (efer
& EFER_LMA
)
2767 && !vendor_intel(ctxt
))
2768 return emulate_ud(ctxt
);
2770 /* sysenter/sysexit have not been tested in 64bit mode. */
2771 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
2772 return X86EMUL_UNHANDLEABLE
;
2774 setup_syscalls_segments(ctxt
, &cs
, &ss
);
2776 ops
->get_msr(ctxt
, MSR_IA32_SYSENTER_CS
, &msr_data
);
2777 if ((msr_data
& 0xfffc) == 0x0)
2778 return emulate_gp(ctxt
, 0);
2780 ctxt
->eflags
&= ~(X86_EFLAGS_VM
| X86_EFLAGS_IF
);
2781 cs_sel
= (u16
)msr_data
& ~SEGMENT_RPL_MASK
;
2782 ss_sel
= cs_sel
+ 8;
2783 if (efer
& EFER_LMA
) {
2788 ops
->set_segment(ctxt
, cs_sel
, &cs
, 0, VCPU_SREG_CS
);
2789 ops
->set_segment(ctxt
, ss_sel
, &ss
, 0, VCPU_SREG_SS
);
2791 ops
->get_msr(ctxt
, MSR_IA32_SYSENTER_EIP
, &msr_data
);
2792 ctxt
->_eip
= (efer
& EFER_LMA
) ? msr_data
: (u32
)msr_data
;
2794 ops
->get_msr(ctxt
, MSR_IA32_SYSENTER_ESP
, &msr_data
);
2795 *reg_write(ctxt
, VCPU_REGS_RSP
) = (efer
& EFER_LMA
) ? msr_data
:
2798 return X86EMUL_CONTINUE
;
2801 static int em_sysexit(struct x86_emulate_ctxt
*ctxt
)
2803 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2804 struct desc_struct cs
, ss
;
2805 u64 msr_data
, rcx
, rdx
;
2807 u16 cs_sel
= 0, ss_sel
= 0;
2809 /* inject #GP if in real mode or Virtual 8086 mode */
2810 if (ctxt
->mode
== X86EMUL_MODE_REAL
||
2811 ctxt
->mode
== X86EMUL_MODE_VM86
)
2812 return emulate_gp(ctxt
, 0);
2814 setup_syscalls_segments(ctxt
, &cs
, &ss
);
2816 if ((ctxt
->rex_prefix
& 0x8) != 0x0)
2817 usermode
= X86EMUL_MODE_PROT64
;
2819 usermode
= X86EMUL_MODE_PROT32
;
2821 rcx
= reg_read(ctxt
, VCPU_REGS_RCX
);
2822 rdx
= reg_read(ctxt
, VCPU_REGS_RDX
);
2826 ops
->get_msr(ctxt
, MSR_IA32_SYSENTER_CS
, &msr_data
);
2828 case X86EMUL_MODE_PROT32
:
2829 cs_sel
= (u16
)(msr_data
+ 16);
2830 if ((msr_data
& 0xfffc) == 0x0)
2831 return emulate_gp(ctxt
, 0);
2832 ss_sel
= (u16
)(msr_data
+ 24);
2836 case X86EMUL_MODE_PROT64
:
2837 cs_sel
= (u16
)(msr_data
+ 32);
2838 if (msr_data
== 0x0)
2839 return emulate_gp(ctxt
, 0);
2840 ss_sel
= cs_sel
+ 8;
2843 if (is_noncanonical_address(rcx
) ||
2844 is_noncanonical_address(rdx
))
2845 return emulate_gp(ctxt
, 0);
2848 cs_sel
|= SEGMENT_RPL_MASK
;
2849 ss_sel
|= SEGMENT_RPL_MASK
;
2851 ops
->set_segment(ctxt
, cs_sel
, &cs
, 0, VCPU_SREG_CS
);
2852 ops
->set_segment(ctxt
, ss_sel
, &ss
, 0, VCPU_SREG_SS
);
2855 *reg_write(ctxt
, VCPU_REGS_RSP
) = rcx
;
2857 return X86EMUL_CONTINUE
;
2860 static bool emulator_bad_iopl(struct x86_emulate_ctxt
*ctxt
)
2863 if (ctxt
->mode
== X86EMUL_MODE_REAL
)
2865 if (ctxt
->mode
== X86EMUL_MODE_VM86
)
2867 iopl
= (ctxt
->eflags
& X86_EFLAGS_IOPL
) >> X86_EFLAGS_IOPL_BIT
;
2868 return ctxt
->ops
->cpl(ctxt
) > iopl
;
2871 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt
*ctxt
,
2874 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
2875 struct desc_struct tr_seg
;
2878 u16 tr
, io_bitmap_ptr
, perm
, bit_idx
= port
& 0x7;
2879 unsigned mask
= (1 << len
) - 1;
2882 ops
->get_segment(ctxt
, &tr
, &tr_seg
, &base3
, VCPU_SREG_TR
);
2885 if (desc_limit_scaled(&tr_seg
) < 103)
2887 base
= get_desc_base(&tr_seg
);
2888 #ifdef CONFIG_X86_64
2889 base
|= ((u64
)base3
) << 32;
2891 r
= ops
->read_std(ctxt
, base
+ 102, &io_bitmap_ptr
, 2, NULL
);
2892 if (r
!= X86EMUL_CONTINUE
)
2894 if (io_bitmap_ptr
+ port
/8 > desc_limit_scaled(&tr_seg
))
2896 r
= ops
->read_std(ctxt
, base
+ io_bitmap_ptr
+ port
/8, &perm
, 2, NULL
);
2897 if (r
!= X86EMUL_CONTINUE
)
2899 if ((perm
>> bit_idx
) & mask
)
2904 static bool emulator_io_permited(struct x86_emulate_ctxt
*ctxt
,
2910 if (emulator_bad_iopl(ctxt
))
2911 if (!emulator_io_port_access_allowed(ctxt
, port
, len
))
2914 ctxt
->perm_ok
= true;
2919 static void string_registers_quirk(struct x86_emulate_ctxt
*ctxt
)
2922 * Intel CPUs mask the counter and pointers in quite strange
2923 * manner when ECX is zero due to REP-string optimizations.
2925 #ifdef CONFIG_X86_64
2926 if (ctxt
->ad_bytes
!= 4 || !vendor_intel(ctxt
))
2929 *reg_write(ctxt
, VCPU_REGS_RCX
) = 0;
2932 case 0xa4: /* movsb */
2933 case 0xa5: /* movsd/w */
2934 *reg_rmw(ctxt
, VCPU_REGS_RSI
) &= (u32
)-1;
2936 case 0xaa: /* stosb */
2937 case 0xab: /* stosd/w */
2938 *reg_rmw(ctxt
, VCPU_REGS_RDI
) &= (u32
)-1;
2943 static void save_state_to_tss16(struct x86_emulate_ctxt
*ctxt
,
2944 struct tss_segment_16
*tss
)
2946 tss
->ip
= ctxt
->_eip
;
2947 tss
->flag
= ctxt
->eflags
;
2948 tss
->ax
= reg_read(ctxt
, VCPU_REGS_RAX
);
2949 tss
->cx
= reg_read(ctxt
, VCPU_REGS_RCX
);
2950 tss
->dx
= reg_read(ctxt
, VCPU_REGS_RDX
);
2951 tss
->bx
= reg_read(ctxt
, VCPU_REGS_RBX
);
2952 tss
->sp
= reg_read(ctxt
, VCPU_REGS_RSP
);
2953 tss
->bp
= reg_read(ctxt
, VCPU_REGS_RBP
);
2954 tss
->si
= reg_read(ctxt
, VCPU_REGS_RSI
);
2955 tss
->di
= reg_read(ctxt
, VCPU_REGS_RDI
);
2957 tss
->es
= get_segment_selector(ctxt
, VCPU_SREG_ES
);
2958 tss
->cs
= get_segment_selector(ctxt
, VCPU_SREG_CS
);
2959 tss
->ss
= get_segment_selector(ctxt
, VCPU_SREG_SS
);
2960 tss
->ds
= get_segment_selector(ctxt
, VCPU_SREG_DS
);
2961 tss
->ldt
= get_segment_selector(ctxt
, VCPU_SREG_LDTR
);
2964 static int load_state_from_tss16(struct x86_emulate_ctxt
*ctxt
,
2965 struct tss_segment_16
*tss
)
2970 ctxt
->_eip
= tss
->ip
;
2971 ctxt
->eflags
= tss
->flag
| 2;
2972 *reg_write(ctxt
, VCPU_REGS_RAX
) = tss
->ax
;
2973 *reg_write(ctxt
, VCPU_REGS_RCX
) = tss
->cx
;
2974 *reg_write(ctxt
, VCPU_REGS_RDX
) = tss
->dx
;
2975 *reg_write(ctxt
, VCPU_REGS_RBX
) = tss
->bx
;
2976 *reg_write(ctxt
, VCPU_REGS_RSP
) = tss
->sp
;
2977 *reg_write(ctxt
, VCPU_REGS_RBP
) = tss
->bp
;
2978 *reg_write(ctxt
, VCPU_REGS_RSI
) = tss
->si
;
2979 *reg_write(ctxt
, VCPU_REGS_RDI
) = tss
->di
;
2982 * SDM says that segment selectors are loaded before segment
2985 set_segment_selector(ctxt
, tss
->ldt
, VCPU_SREG_LDTR
);
2986 set_segment_selector(ctxt
, tss
->es
, VCPU_SREG_ES
);
2987 set_segment_selector(ctxt
, tss
->cs
, VCPU_SREG_CS
);
2988 set_segment_selector(ctxt
, tss
->ss
, VCPU_SREG_SS
);
2989 set_segment_selector(ctxt
, tss
->ds
, VCPU_SREG_DS
);
2994 * Now load segment descriptors. If fault happens at this stage
2995 * it is handled in a context of new task
2997 ret
= __load_segment_descriptor(ctxt
, tss
->ldt
, VCPU_SREG_LDTR
, cpl
,
2998 X86_TRANSFER_TASK_SWITCH
, NULL
);
2999 if (ret
!= X86EMUL_CONTINUE
)
3001 ret
= __load_segment_descriptor(ctxt
, tss
->es
, VCPU_SREG_ES
, cpl
,
3002 X86_TRANSFER_TASK_SWITCH
, NULL
);
3003 if (ret
!= X86EMUL_CONTINUE
)
3005 ret
= __load_segment_descriptor(ctxt
, tss
->cs
, VCPU_SREG_CS
, cpl
,
3006 X86_TRANSFER_TASK_SWITCH
, NULL
);
3007 if (ret
!= X86EMUL_CONTINUE
)
3009 ret
= __load_segment_descriptor(ctxt
, tss
->ss
, VCPU_SREG_SS
, cpl
,
3010 X86_TRANSFER_TASK_SWITCH
, NULL
);
3011 if (ret
!= X86EMUL_CONTINUE
)
3013 ret
= __load_segment_descriptor(ctxt
, tss
->ds
, VCPU_SREG_DS
, cpl
,
3014 X86_TRANSFER_TASK_SWITCH
, NULL
);
3015 if (ret
!= X86EMUL_CONTINUE
)
3018 return X86EMUL_CONTINUE
;
3021 static int task_switch_16(struct x86_emulate_ctxt
*ctxt
,
3022 u16 tss_selector
, u16 old_tss_sel
,
3023 ulong old_tss_base
, struct desc_struct
*new_desc
)
3025 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
3026 struct tss_segment_16 tss_seg
;
3028 u32 new_tss_base
= get_desc_base(new_desc
);
3030 ret
= ops
->read_std(ctxt
, old_tss_base
, &tss_seg
, sizeof tss_seg
,
3032 if (ret
!= X86EMUL_CONTINUE
)
3035 save_state_to_tss16(ctxt
, &tss_seg
);
3037 ret
= ops
->write_std(ctxt
, old_tss_base
, &tss_seg
, sizeof tss_seg
,
3039 if (ret
!= X86EMUL_CONTINUE
)
3042 ret
= ops
->read_std(ctxt
, new_tss_base
, &tss_seg
, sizeof tss_seg
,
3044 if (ret
!= X86EMUL_CONTINUE
)
3047 if (old_tss_sel
!= 0xffff) {
3048 tss_seg
.prev_task_link
= old_tss_sel
;
3050 ret
= ops
->write_std(ctxt
, new_tss_base
,
3051 &tss_seg
.prev_task_link
,
3052 sizeof tss_seg
.prev_task_link
,
3054 if (ret
!= X86EMUL_CONTINUE
)
3058 return load_state_from_tss16(ctxt
, &tss_seg
);
3061 static void save_state_to_tss32(struct x86_emulate_ctxt
*ctxt
,
3062 struct tss_segment_32
*tss
)
3064 /* CR3 and ldt selector are not saved intentionally */
3065 tss
->eip
= ctxt
->_eip
;
3066 tss
->eflags
= ctxt
->eflags
;
3067 tss
->eax
= reg_read(ctxt
, VCPU_REGS_RAX
);
3068 tss
->ecx
= reg_read(ctxt
, VCPU_REGS_RCX
);
3069 tss
->edx
= reg_read(ctxt
, VCPU_REGS_RDX
);
3070 tss
->ebx
= reg_read(ctxt
, VCPU_REGS_RBX
);
3071 tss
->esp
= reg_read(ctxt
, VCPU_REGS_RSP
);
3072 tss
->ebp
= reg_read(ctxt
, VCPU_REGS_RBP
);
3073 tss
->esi
= reg_read(ctxt
, VCPU_REGS_RSI
);
3074 tss
->edi
= reg_read(ctxt
, VCPU_REGS_RDI
);
3076 tss
->es
= get_segment_selector(ctxt
, VCPU_SREG_ES
);
3077 tss
->cs
= get_segment_selector(ctxt
, VCPU_SREG_CS
);
3078 tss
->ss
= get_segment_selector(ctxt
, VCPU_SREG_SS
);
3079 tss
->ds
= get_segment_selector(ctxt
, VCPU_SREG_DS
);
3080 tss
->fs
= get_segment_selector(ctxt
, VCPU_SREG_FS
);
3081 tss
->gs
= get_segment_selector(ctxt
, VCPU_SREG_GS
);
3084 static int load_state_from_tss32(struct x86_emulate_ctxt
*ctxt
,
3085 struct tss_segment_32
*tss
)
3090 if (ctxt
->ops
->set_cr(ctxt
, 3, tss
->cr3
))
3091 return emulate_gp(ctxt
, 0);
3092 ctxt
->_eip
= tss
->eip
;
3093 ctxt
->eflags
= tss
->eflags
| 2;
3095 /* General purpose registers */
3096 *reg_write(ctxt
, VCPU_REGS_RAX
) = tss
->eax
;
3097 *reg_write(ctxt
, VCPU_REGS_RCX
) = tss
->ecx
;
3098 *reg_write(ctxt
, VCPU_REGS_RDX
) = tss
->edx
;
3099 *reg_write(ctxt
, VCPU_REGS_RBX
) = tss
->ebx
;
3100 *reg_write(ctxt
, VCPU_REGS_RSP
) = tss
->esp
;
3101 *reg_write(ctxt
, VCPU_REGS_RBP
) = tss
->ebp
;
3102 *reg_write(ctxt
, VCPU_REGS_RSI
) = tss
->esi
;
3103 *reg_write(ctxt
, VCPU_REGS_RDI
) = tss
->edi
;
3106 * SDM says that segment selectors are loaded before segment
3107 * descriptors. This is important because CPL checks will
3110 set_segment_selector(ctxt
, tss
->ldt_selector
, VCPU_SREG_LDTR
);
3111 set_segment_selector(ctxt
, tss
->es
, VCPU_SREG_ES
);
3112 set_segment_selector(ctxt
, tss
->cs
, VCPU_SREG_CS
);
3113 set_segment_selector(ctxt
, tss
->ss
, VCPU_SREG_SS
);
3114 set_segment_selector(ctxt
, tss
->ds
, VCPU_SREG_DS
);
3115 set_segment_selector(ctxt
, tss
->fs
, VCPU_SREG_FS
);
3116 set_segment_selector(ctxt
, tss
->gs
, VCPU_SREG_GS
);
3119 * If we're switching between Protected Mode and VM86, we need to make
3120 * sure to update the mode before loading the segment descriptors so
3121 * that the selectors are interpreted correctly.
3123 if (ctxt
->eflags
& X86_EFLAGS_VM
) {
3124 ctxt
->mode
= X86EMUL_MODE_VM86
;
3127 ctxt
->mode
= X86EMUL_MODE_PROT32
;
3132 * Now load segment descriptors. If fault happenes at this stage
3133 * it is handled in a context of new task
3135 ret
= __load_segment_descriptor(ctxt
, tss
->ldt_selector
, VCPU_SREG_LDTR
,
3136 cpl
, X86_TRANSFER_TASK_SWITCH
, NULL
);
3137 if (ret
!= X86EMUL_CONTINUE
)
3139 ret
= __load_segment_descriptor(ctxt
, tss
->es
, VCPU_SREG_ES
, cpl
,
3140 X86_TRANSFER_TASK_SWITCH
, NULL
);
3141 if (ret
!= X86EMUL_CONTINUE
)
3143 ret
= __load_segment_descriptor(ctxt
, tss
->cs
, VCPU_SREG_CS
, cpl
,
3144 X86_TRANSFER_TASK_SWITCH
, NULL
);
3145 if (ret
!= X86EMUL_CONTINUE
)
3147 ret
= __load_segment_descriptor(ctxt
, tss
->ss
, VCPU_SREG_SS
, cpl
,
3148 X86_TRANSFER_TASK_SWITCH
, NULL
);
3149 if (ret
!= X86EMUL_CONTINUE
)
3151 ret
= __load_segment_descriptor(ctxt
, tss
->ds
, VCPU_SREG_DS
, cpl
,
3152 X86_TRANSFER_TASK_SWITCH
, NULL
);
3153 if (ret
!= X86EMUL_CONTINUE
)
3155 ret
= __load_segment_descriptor(ctxt
, tss
->fs
, VCPU_SREG_FS
, cpl
,
3156 X86_TRANSFER_TASK_SWITCH
, NULL
);
3157 if (ret
!= X86EMUL_CONTINUE
)
3159 ret
= __load_segment_descriptor(ctxt
, tss
->gs
, VCPU_SREG_GS
, cpl
,
3160 X86_TRANSFER_TASK_SWITCH
, NULL
);
3165 static int task_switch_32(struct x86_emulate_ctxt
*ctxt
,
3166 u16 tss_selector
, u16 old_tss_sel
,
3167 ulong old_tss_base
, struct desc_struct
*new_desc
)
3169 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
3170 struct tss_segment_32 tss_seg
;
3172 u32 new_tss_base
= get_desc_base(new_desc
);
3173 u32 eip_offset
= offsetof(struct tss_segment_32
, eip
);
3174 u32 ldt_sel_offset
= offsetof(struct tss_segment_32
, ldt_selector
);
3176 ret
= ops
->read_std(ctxt
, old_tss_base
, &tss_seg
, sizeof tss_seg
,
3178 if (ret
!= X86EMUL_CONTINUE
)
3181 save_state_to_tss32(ctxt
, &tss_seg
);
3183 /* Only GP registers and segment selectors are saved */
3184 ret
= ops
->write_std(ctxt
, old_tss_base
+ eip_offset
, &tss_seg
.eip
,
3185 ldt_sel_offset
- eip_offset
, &ctxt
->exception
);
3186 if (ret
!= X86EMUL_CONTINUE
)
3189 ret
= ops
->read_std(ctxt
, new_tss_base
, &tss_seg
, sizeof tss_seg
,
3191 if (ret
!= X86EMUL_CONTINUE
)
3194 if (old_tss_sel
!= 0xffff) {
3195 tss_seg
.prev_task_link
= old_tss_sel
;
3197 ret
= ops
->write_std(ctxt
, new_tss_base
,
3198 &tss_seg
.prev_task_link
,
3199 sizeof tss_seg
.prev_task_link
,
3201 if (ret
!= X86EMUL_CONTINUE
)
3205 return load_state_from_tss32(ctxt
, &tss_seg
);
3208 static int emulator_do_task_switch(struct x86_emulate_ctxt
*ctxt
,
3209 u16 tss_selector
, int idt_index
, int reason
,
3210 bool has_error_code
, u32 error_code
)
3212 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
3213 struct desc_struct curr_tss_desc
, next_tss_desc
;
3215 u16 old_tss_sel
= get_segment_selector(ctxt
, VCPU_SREG_TR
);
3216 ulong old_tss_base
=
3217 ops
->get_cached_segment_base(ctxt
, VCPU_SREG_TR
);
3219 ulong desc_addr
, dr7
;
3221 /* FIXME: old_tss_base == ~0 ? */
3223 ret
= read_segment_descriptor(ctxt
, tss_selector
, &next_tss_desc
, &desc_addr
);
3224 if (ret
!= X86EMUL_CONTINUE
)
3226 ret
= read_segment_descriptor(ctxt
, old_tss_sel
, &curr_tss_desc
, &desc_addr
);
3227 if (ret
!= X86EMUL_CONTINUE
)
3230 /* FIXME: check that next_tss_desc is tss */
3233 * Check privileges. The three cases are task switch caused by...
3235 * 1. jmp/call/int to task gate: Check against DPL of the task gate
3236 * 2. Exception/IRQ/iret: No check is performed
3237 * 3. jmp/call to TSS/task-gate: No check is performed since the
3238 * hardware checks it before exiting.
3240 if (reason
== TASK_SWITCH_GATE
) {
3241 if (idt_index
!= -1) {
3242 /* Software interrupts */
3243 struct desc_struct task_gate_desc
;
3246 ret
= read_interrupt_descriptor(ctxt
, idt_index
,
3248 if (ret
!= X86EMUL_CONTINUE
)
3251 dpl
= task_gate_desc
.dpl
;
3252 if ((tss_selector
& 3) > dpl
|| ops
->cpl(ctxt
) > dpl
)
3253 return emulate_gp(ctxt
, (idt_index
<< 3) | 0x2);
3257 desc_limit
= desc_limit_scaled(&next_tss_desc
);
3258 if (!next_tss_desc
.p
||
3259 ((desc_limit
< 0x67 && (next_tss_desc
.type
& 8)) ||
3260 desc_limit
< 0x2b)) {
3261 return emulate_ts(ctxt
, tss_selector
& 0xfffc);
3264 if (reason
== TASK_SWITCH_IRET
|| reason
== TASK_SWITCH_JMP
) {
3265 curr_tss_desc
.type
&= ~(1 << 1); /* clear busy flag */
3266 write_segment_descriptor(ctxt
, old_tss_sel
, &curr_tss_desc
);
3269 if (reason
== TASK_SWITCH_IRET
)
3270 ctxt
->eflags
= ctxt
->eflags
& ~X86_EFLAGS_NT
;
3272 /* set back link to prev task only if NT bit is set in eflags
3273 note that old_tss_sel is not used after this point */
3274 if (reason
!= TASK_SWITCH_CALL
&& reason
!= TASK_SWITCH_GATE
)
3275 old_tss_sel
= 0xffff;
3277 if (next_tss_desc
.type
& 8)
3278 ret
= task_switch_32(ctxt
, tss_selector
, old_tss_sel
,
3279 old_tss_base
, &next_tss_desc
);
3281 ret
= task_switch_16(ctxt
, tss_selector
, old_tss_sel
,
3282 old_tss_base
, &next_tss_desc
);
3283 if (ret
!= X86EMUL_CONTINUE
)
3286 if (reason
== TASK_SWITCH_CALL
|| reason
== TASK_SWITCH_GATE
)
3287 ctxt
->eflags
= ctxt
->eflags
| X86_EFLAGS_NT
;
3289 if (reason
!= TASK_SWITCH_IRET
) {
3290 next_tss_desc
.type
|= (1 << 1); /* set busy flag */
3291 write_segment_descriptor(ctxt
, tss_selector
, &next_tss_desc
);
3294 ops
->set_cr(ctxt
, 0, ops
->get_cr(ctxt
, 0) | X86_CR0_TS
);
3295 ops
->set_segment(ctxt
, tss_selector
, &next_tss_desc
, 0, VCPU_SREG_TR
);
3297 if (has_error_code
) {
3298 ctxt
->op_bytes
= ctxt
->ad_bytes
= (next_tss_desc
.type
& 8) ? 4 : 2;
3299 ctxt
->lock_prefix
= 0;
3300 ctxt
->src
.val
= (unsigned long) error_code
;
3301 ret
= em_push(ctxt
);
3304 ops
->get_dr(ctxt
, 7, &dr7
);
3305 ops
->set_dr(ctxt
, 7, dr7
& ~(DR_LOCAL_ENABLE_MASK
| DR_LOCAL_SLOWDOWN
));
3310 int emulator_task_switch(struct x86_emulate_ctxt
*ctxt
,
3311 u16 tss_selector
, int idt_index
, int reason
,
3312 bool has_error_code
, u32 error_code
)
3316 invalidate_registers(ctxt
);
3317 ctxt
->_eip
= ctxt
->eip
;
3318 ctxt
->dst
.type
= OP_NONE
;
3320 rc
= emulator_do_task_switch(ctxt
, tss_selector
, idt_index
, reason
,
3321 has_error_code
, error_code
);
3323 if (rc
== X86EMUL_CONTINUE
) {
3324 ctxt
->eip
= ctxt
->_eip
;
3325 writeback_registers(ctxt
);
3328 return (rc
== X86EMUL_UNHANDLEABLE
) ? EMULATION_FAILED
: EMULATION_OK
;
3331 static void string_addr_inc(struct x86_emulate_ctxt
*ctxt
, int reg
,
3334 int df
= (ctxt
->eflags
& X86_EFLAGS_DF
) ? -op
->count
: op
->count
;
3336 register_address_increment(ctxt
, reg
, df
* op
->bytes
);
3337 op
->addr
.mem
.ea
= register_address(ctxt
, reg
);
3340 static int em_das(struct x86_emulate_ctxt
*ctxt
)
3343 bool af
, cf
, old_cf
;
3345 cf
= ctxt
->eflags
& X86_EFLAGS_CF
;
3351 af
= ctxt
->eflags
& X86_EFLAGS_AF
;
3352 if ((al
& 0x0f) > 9 || af
) {
3354 cf
= old_cf
| (al
>= 250);
3359 if (old_al
> 0x99 || old_cf
) {
3365 /* Set PF, ZF, SF */
3366 ctxt
->src
.type
= OP_IMM
;
3368 ctxt
->src
.bytes
= 1;
3369 fastop(ctxt
, em_or
);
3370 ctxt
->eflags
&= ~(X86_EFLAGS_AF
| X86_EFLAGS_CF
);
3372 ctxt
->eflags
|= X86_EFLAGS_CF
;
3374 ctxt
->eflags
|= X86_EFLAGS_AF
;
3375 return X86EMUL_CONTINUE
;
3378 static int em_aam(struct x86_emulate_ctxt
*ctxt
)
3382 if (ctxt
->src
.val
== 0)
3383 return emulate_de(ctxt
);
3385 al
= ctxt
->dst
.val
& 0xff;
3386 ah
= al
/ ctxt
->src
.val
;
3387 al
%= ctxt
->src
.val
;
3389 ctxt
->dst
.val
= (ctxt
->dst
.val
& 0xffff0000) | al
| (ah
<< 8);
3391 /* Set PF, ZF, SF */
3392 ctxt
->src
.type
= OP_IMM
;
3394 ctxt
->src
.bytes
= 1;
3395 fastop(ctxt
, em_or
);
3397 return X86EMUL_CONTINUE
;
3400 static int em_aad(struct x86_emulate_ctxt
*ctxt
)
3402 u8 al
= ctxt
->dst
.val
& 0xff;
3403 u8 ah
= (ctxt
->dst
.val
>> 8) & 0xff;
3405 al
= (al
+ (ah
* ctxt
->src
.val
)) & 0xff;
3407 ctxt
->dst
.val
= (ctxt
->dst
.val
& 0xffff0000) | al
;
3409 /* Set PF, ZF, SF */
3410 ctxt
->src
.type
= OP_IMM
;
3412 ctxt
->src
.bytes
= 1;
3413 fastop(ctxt
, em_or
);
3415 return X86EMUL_CONTINUE
;
3418 static int em_call(struct x86_emulate_ctxt
*ctxt
)
3421 long rel
= ctxt
->src
.val
;
3423 ctxt
->src
.val
= (unsigned long)ctxt
->_eip
;
3424 rc
= jmp_rel(ctxt
, rel
);
3425 if (rc
!= X86EMUL_CONTINUE
)
3427 return em_push(ctxt
);
3430 static int em_call_far(struct x86_emulate_ctxt
*ctxt
)
3435 struct desc_struct old_desc
, new_desc
;
3436 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
3437 int cpl
= ctxt
->ops
->cpl(ctxt
);
3438 enum x86emul_mode prev_mode
= ctxt
->mode
;
3440 old_eip
= ctxt
->_eip
;
3441 ops
->get_segment(ctxt
, &old_cs
, &old_desc
, NULL
, VCPU_SREG_CS
);
3443 memcpy(&sel
, ctxt
->src
.valptr
+ ctxt
->op_bytes
, 2);
3444 rc
= __load_segment_descriptor(ctxt
, sel
, VCPU_SREG_CS
, cpl
,
3445 X86_TRANSFER_CALL_JMP
, &new_desc
);
3446 if (rc
!= X86EMUL_CONTINUE
)
3449 rc
= assign_eip_far(ctxt
, ctxt
->src
.val
, &new_desc
);
3450 if (rc
!= X86EMUL_CONTINUE
)
3453 ctxt
->src
.val
= old_cs
;
3455 if (rc
!= X86EMUL_CONTINUE
)
3458 ctxt
->src
.val
= old_eip
;
3460 /* If we failed, we tainted the memory, but the very least we should
3462 if (rc
!= X86EMUL_CONTINUE
) {
3463 pr_warn_once("faulting far call emulation tainted memory\n");
3468 ops
->set_segment(ctxt
, old_cs
, &old_desc
, 0, VCPU_SREG_CS
);
3469 ctxt
->mode
= prev_mode
;
3474 static int em_ret_near_imm(struct x86_emulate_ctxt
*ctxt
)
3479 rc
= emulate_pop(ctxt
, &eip
, ctxt
->op_bytes
);
3480 if (rc
!= X86EMUL_CONTINUE
)
3482 rc
= assign_eip_near(ctxt
, eip
);
3483 if (rc
!= X86EMUL_CONTINUE
)
3485 rsp_increment(ctxt
, ctxt
->src
.val
);
3486 return X86EMUL_CONTINUE
;
3489 static int em_xchg(struct x86_emulate_ctxt
*ctxt
)
3491 /* Write back the register source. */
3492 ctxt
->src
.val
= ctxt
->dst
.val
;
3493 write_register_operand(&ctxt
->src
);
3495 /* Write back the memory destination with implicit LOCK prefix. */
3496 ctxt
->dst
.val
= ctxt
->src
.orig_val
;
3497 ctxt
->lock_prefix
= 1;
3498 return X86EMUL_CONTINUE
;
3501 static int em_imul_3op(struct x86_emulate_ctxt
*ctxt
)
3503 ctxt
->dst
.val
= ctxt
->src2
.val
;
3504 return fastop(ctxt
, em_imul
);
3507 static int em_cwd(struct x86_emulate_ctxt
*ctxt
)
3509 ctxt
->dst
.type
= OP_REG
;
3510 ctxt
->dst
.bytes
= ctxt
->src
.bytes
;
3511 ctxt
->dst
.addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RDX
);
3512 ctxt
->dst
.val
= ~((ctxt
->src
.val
>> (ctxt
->src
.bytes
* 8 - 1)) - 1);
3514 return X86EMUL_CONTINUE
;
3517 static int em_rdtsc(struct x86_emulate_ctxt
*ctxt
)
3521 ctxt
->ops
->get_msr(ctxt
, MSR_IA32_TSC
, &tsc
);
3522 *reg_write(ctxt
, VCPU_REGS_RAX
) = (u32
)tsc
;
3523 *reg_write(ctxt
, VCPU_REGS_RDX
) = tsc
>> 32;
3524 return X86EMUL_CONTINUE
;
3527 static int em_rdpmc(struct x86_emulate_ctxt
*ctxt
)
3531 if (ctxt
->ops
->read_pmc(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
), &pmc
))
3532 return emulate_gp(ctxt
, 0);
3533 *reg_write(ctxt
, VCPU_REGS_RAX
) = (u32
)pmc
;
3534 *reg_write(ctxt
, VCPU_REGS_RDX
) = pmc
>> 32;
3535 return X86EMUL_CONTINUE
;
3538 static int em_mov(struct x86_emulate_ctxt
*ctxt
)
3540 memcpy(ctxt
->dst
.valptr
, ctxt
->src
.valptr
, sizeof(ctxt
->src
.valptr
));
3541 return X86EMUL_CONTINUE
;
3544 #define FFL(x) bit(X86_FEATURE_##x)
3546 static int em_movbe(struct x86_emulate_ctxt
*ctxt
)
3548 u32 ebx
, ecx
, edx
, eax
= 1;
3552 * Check MOVBE is set in the guest-visible CPUID leaf.
3554 ctxt
->ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
, &edx
);
3555 if (!(ecx
& FFL(MOVBE
)))
3556 return emulate_ud(ctxt
);
3558 switch (ctxt
->op_bytes
) {
3561 * From MOVBE definition: "...When the operand size is 16 bits,
3562 * the upper word of the destination register remains unchanged
3565 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3566 * rules so we have to do the operation almost per hand.
3568 tmp
= (u16
)ctxt
->src
.val
;
3569 ctxt
->dst
.val
&= ~0xffffUL
;
3570 ctxt
->dst
.val
|= (unsigned long)swab16(tmp
);
3573 ctxt
->dst
.val
= swab32((u32
)ctxt
->src
.val
);
3576 ctxt
->dst
.val
= swab64(ctxt
->src
.val
);
3581 return X86EMUL_CONTINUE
;
3584 static int em_cr_write(struct x86_emulate_ctxt
*ctxt
)
3586 if (ctxt
->ops
->set_cr(ctxt
, ctxt
->modrm_reg
, ctxt
->src
.val
))
3587 return emulate_gp(ctxt
, 0);
3589 /* Disable writeback. */
3590 ctxt
->dst
.type
= OP_NONE
;
3591 return X86EMUL_CONTINUE
;
3594 static int em_dr_write(struct x86_emulate_ctxt
*ctxt
)
3598 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
3599 val
= ctxt
->src
.val
& ~0ULL;
3601 val
= ctxt
->src
.val
& ~0U;
3603 /* #UD condition is already handled. */
3604 if (ctxt
->ops
->set_dr(ctxt
, ctxt
->modrm_reg
, val
) < 0)
3605 return emulate_gp(ctxt
, 0);
3607 /* Disable writeback. */
3608 ctxt
->dst
.type
= OP_NONE
;
3609 return X86EMUL_CONTINUE
;
3612 static int em_wrmsr(struct x86_emulate_ctxt
*ctxt
)
3616 msr_data
= (u32
)reg_read(ctxt
, VCPU_REGS_RAX
)
3617 | ((u64
)reg_read(ctxt
, VCPU_REGS_RDX
) << 32);
3618 if (ctxt
->ops
->set_msr(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
), msr_data
))
3619 return emulate_gp(ctxt
, 0);
3621 return X86EMUL_CONTINUE
;
3624 static int em_rdmsr(struct x86_emulate_ctxt
*ctxt
)
3628 if (ctxt
->ops
->get_msr(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
), &msr_data
))
3629 return emulate_gp(ctxt
, 0);
3631 *reg_write(ctxt
, VCPU_REGS_RAX
) = (u32
)msr_data
;
3632 *reg_write(ctxt
, VCPU_REGS_RDX
) = msr_data
>> 32;
3633 return X86EMUL_CONTINUE
;
3636 static int em_mov_rm_sreg(struct x86_emulate_ctxt
*ctxt
)
3638 if (ctxt
->modrm_reg
> VCPU_SREG_GS
)
3639 return emulate_ud(ctxt
);
3641 ctxt
->dst
.val
= get_segment_selector(ctxt
, ctxt
->modrm_reg
);
3642 if (ctxt
->dst
.bytes
== 4 && ctxt
->dst
.type
== OP_MEM
)
3643 ctxt
->dst
.bytes
= 2;
3644 return X86EMUL_CONTINUE
;
3647 static int em_mov_sreg_rm(struct x86_emulate_ctxt
*ctxt
)
3649 u16 sel
= ctxt
->src
.val
;
3651 if (ctxt
->modrm_reg
== VCPU_SREG_CS
|| ctxt
->modrm_reg
> VCPU_SREG_GS
)
3652 return emulate_ud(ctxt
);
3654 if (ctxt
->modrm_reg
== VCPU_SREG_SS
)
3655 ctxt
->interruptibility
= KVM_X86_SHADOW_INT_MOV_SS
;
3657 /* Disable writeback. */
3658 ctxt
->dst
.type
= OP_NONE
;
3659 return load_segment_descriptor(ctxt
, sel
, ctxt
->modrm_reg
);
3662 static int em_lldt(struct x86_emulate_ctxt
*ctxt
)
3664 u16 sel
= ctxt
->src
.val
;
3666 /* Disable writeback. */
3667 ctxt
->dst
.type
= OP_NONE
;
3668 return load_segment_descriptor(ctxt
, sel
, VCPU_SREG_LDTR
);
3671 static int em_ltr(struct x86_emulate_ctxt
*ctxt
)
3673 u16 sel
= ctxt
->src
.val
;
3675 /* Disable writeback. */
3676 ctxt
->dst
.type
= OP_NONE
;
3677 return load_segment_descriptor(ctxt
, sel
, VCPU_SREG_TR
);
3680 static int em_invlpg(struct x86_emulate_ctxt
*ctxt
)
3685 rc
= linearize(ctxt
, ctxt
->src
.addr
.mem
, 1, false, &linear
);
3686 if (rc
== X86EMUL_CONTINUE
)
3687 ctxt
->ops
->invlpg(ctxt
, linear
);
3688 /* Disable writeback. */
3689 ctxt
->dst
.type
= OP_NONE
;
3690 return X86EMUL_CONTINUE
;
3693 static int em_clts(struct x86_emulate_ctxt
*ctxt
)
3697 cr0
= ctxt
->ops
->get_cr(ctxt
, 0);
3699 ctxt
->ops
->set_cr(ctxt
, 0, cr0
);
3700 return X86EMUL_CONTINUE
;
3703 static int em_hypercall(struct x86_emulate_ctxt
*ctxt
)
3705 int rc
= ctxt
->ops
->fix_hypercall(ctxt
);
3707 if (rc
!= X86EMUL_CONTINUE
)
3710 /* Let the processor re-execute the fixed hypercall */
3711 ctxt
->_eip
= ctxt
->eip
;
3712 /* Disable writeback. */
3713 ctxt
->dst
.type
= OP_NONE
;
3714 return X86EMUL_CONTINUE
;
3717 static int emulate_store_desc_ptr(struct x86_emulate_ctxt
*ctxt
,
3718 void (*get
)(struct x86_emulate_ctxt
*ctxt
,
3719 struct desc_ptr
*ptr
))
3721 struct desc_ptr desc_ptr
;
3723 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
3725 get(ctxt
, &desc_ptr
);
3726 if (ctxt
->op_bytes
== 2) {
3728 desc_ptr
.address
&= 0x00ffffff;
3730 /* Disable writeback. */
3731 ctxt
->dst
.type
= OP_NONE
;
3732 return segmented_write_std(ctxt
, ctxt
->dst
.addr
.mem
,
3733 &desc_ptr
, 2 + ctxt
->op_bytes
);
3736 static int em_sgdt(struct x86_emulate_ctxt
*ctxt
)
3738 return emulate_store_desc_ptr(ctxt
, ctxt
->ops
->get_gdt
);
3741 static int em_sidt(struct x86_emulate_ctxt
*ctxt
)
3743 return emulate_store_desc_ptr(ctxt
, ctxt
->ops
->get_idt
);
3746 static int em_lgdt_lidt(struct x86_emulate_ctxt
*ctxt
, bool lgdt
)
3748 struct desc_ptr desc_ptr
;
3751 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
3753 rc
= read_descriptor(ctxt
, ctxt
->src
.addr
.mem
,
3754 &desc_ptr
.size
, &desc_ptr
.address
,
3756 if (rc
!= X86EMUL_CONTINUE
)
3758 if (ctxt
->mode
== X86EMUL_MODE_PROT64
&&
3759 is_noncanonical_address(desc_ptr
.address
))
3760 return emulate_gp(ctxt
, 0);
3762 ctxt
->ops
->set_gdt(ctxt
, &desc_ptr
);
3764 ctxt
->ops
->set_idt(ctxt
, &desc_ptr
);
3765 /* Disable writeback. */
3766 ctxt
->dst
.type
= OP_NONE
;
3767 return X86EMUL_CONTINUE
;
3770 static int em_lgdt(struct x86_emulate_ctxt
*ctxt
)
3772 return em_lgdt_lidt(ctxt
, true);
3775 static int em_lidt(struct x86_emulate_ctxt
*ctxt
)
3777 return em_lgdt_lidt(ctxt
, false);
3780 static int em_smsw(struct x86_emulate_ctxt
*ctxt
)
3782 if (ctxt
->dst
.type
== OP_MEM
)
3783 ctxt
->dst
.bytes
= 2;
3784 ctxt
->dst
.val
= ctxt
->ops
->get_cr(ctxt
, 0);
3785 return X86EMUL_CONTINUE
;
3788 static int em_lmsw(struct x86_emulate_ctxt
*ctxt
)
3790 ctxt
->ops
->set_cr(ctxt
, 0, (ctxt
->ops
->get_cr(ctxt
, 0) & ~0x0eul
)
3791 | (ctxt
->src
.val
& 0x0f));
3792 ctxt
->dst
.type
= OP_NONE
;
3793 return X86EMUL_CONTINUE
;
3796 static int em_loop(struct x86_emulate_ctxt
*ctxt
)
3798 int rc
= X86EMUL_CONTINUE
;
3800 register_address_increment(ctxt
, VCPU_REGS_RCX
, -1);
3801 if ((address_mask(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
)) != 0) &&
3802 (ctxt
->b
== 0xe2 || test_cc(ctxt
->b
^ 0x5, ctxt
->eflags
)))
3803 rc
= jmp_rel(ctxt
, ctxt
->src
.val
);
3808 static int em_jcxz(struct x86_emulate_ctxt
*ctxt
)
3810 int rc
= X86EMUL_CONTINUE
;
3812 if (address_mask(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
)) == 0)
3813 rc
= jmp_rel(ctxt
, ctxt
->src
.val
);
3818 static int em_in(struct x86_emulate_ctxt
*ctxt
)
3820 if (!pio_in_emulated(ctxt
, ctxt
->dst
.bytes
, ctxt
->src
.val
,
3822 return X86EMUL_IO_NEEDED
;
3824 return X86EMUL_CONTINUE
;
3827 static int em_out(struct x86_emulate_ctxt
*ctxt
)
3829 ctxt
->ops
->pio_out_emulated(ctxt
, ctxt
->src
.bytes
, ctxt
->dst
.val
,
3831 /* Disable writeback. */
3832 ctxt
->dst
.type
= OP_NONE
;
3833 return X86EMUL_CONTINUE
;
3836 static int em_cli(struct x86_emulate_ctxt
*ctxt
)
3838 if (emulator_bad_iopl(ctxt
))
3839 return emulate_gp(ctxt
, 0);
3841 ctxt
->eflags
&= ~X86_EFLAGS_IF
;
3842 return X86EMUL_CONTINUE
;
3845 static int em_sti(struct x86_emulate_ctxt
*ctxt
)
3847 if (emulator_bad_iopl(ctxt
))
3848 return emulate_gp(ctxt
, 0);
3850 ctxt
->interruptibility
= KVM_X86_SHADOW_INT_STI
;
3851 ctxt
->eflags
|= X86_EFLAGS_IF
;
3852 return X86EMUL_CONTINUE
;
3855 static int em_cpuid(struct x86_emulate_ctxt
*ctxt
)
3857 u32 eax
, ebx
, ecx
, edx
;
3860 ctxt
->ops
->get_msr(ctxt
, MSR_MISC_FEATURES_ENABLES
, &msr
);
3861 if (msr
& MSR_MISC_FEATURES_ENABLES_CPUID_FAULT
&&
3862 ctxt
->ops
->cpl(ctxt
)) {
3863 return emulate_gp(ctxt
, 0);
3866 eax
= reg_read(ctxt
, VCPU_REGS_RAX
);
3867 ecx
= reg_read(ctxt
, VCPU_REGS_RCX
);
3868 ctxt
->ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
, &edx
);
3869 *reg_write(ctxt
, VCPU_REGS_RAX
) = eax
;
3870 *reg_write(ctxt
, VCPU_REGS_RBX
) = ebx
;
3871 *reg_write(ctxt
, VCPU_REGS_RCX
) = ecx
;
3872 *reg_write(ctxt
, VCPU_REGS_RDX
) = edx
;
3873 return X86EMUL_CONTINUE
;
3876 static int em_sahf(struct x86_emulate_ctxt
*ctxt
)
3880 flags
= X86_EFLAGS_CF
| X86_EFLAGS_PF
| X86_EFLAGS_AF
| X86_EFLAGS_ZF
|
3882 flags
&= *reg_rmw(ctxt
, VCPU_REGS_RAX
) >> 8;
3884 ctxt
->eflags
&= ~0xffUL
;
3885 ctxt
->eflags
|= flags
| X86_EFLAGS_FIXED
;
3886 return X86EMUL_CONTINUE
;
3889 static int em_lahf(struct x86_emulate_ctxt
*ctxt
)
3891 *reg_rmw(ctxt
, VCPU_REGS_RAX
) &= ~0xff00UL
;
3892 *reg_rmw(ctxt
, VCPU_REGS_RAX
) |= (ctxt
->eflags
& 0xff) << 8;
3893 return X86EMUL_CONTINUE
;
3896 static int em_bswap(struct x86_emulate_ctxt
*ctxt
)
3898 switch (ctxt
->op_bytes
) {
3899 #ifdef CONFIG_X86_64
3901 asm("bswap %0" : "+r"(ctxt
->dst
.val
));
3905 asm("bswap %0" : "+r"(*(u32
*)&ctxt
->dst
.val
));
3908 return X86EMUL_CONTINUE
;
3911 static int em_clflush(struct x86_emulate_ctxt
*ctxt
)
3913 /* emulating clflush regardless of cpuid */
3914 return X86EMUL_CONTINUE
;
3917 static int em_movsxd(struct x86_emulate_ctxt
*ctxt
)
3919 ctxt
->dst
.val
= (s32
) ctxt
->src
.val
;
3920 return X86EMUL_CONTINUE
;
3923 static int check_fxsr(struct x86_emulate_ctxt
*ctxt
)
3925 u32 eax
= 1, ebx
, ecx
= 0, edx
;
3927 ctxt
->ops
->get_cpuid(ctxt
, &eax
, &ebx
, &ecx
, &edx
);
3928 if (!(edx
& FFL(FXSR
)))
3929 return emulate_ud(ctxt
);
3931 if (ctxt
->ops
->get_cr(ctxt
, 0) & (X86_CR0_TS
| X86_CR0_EM
))
3932 return emulate_nm(ctxt
);
3935 * Don't emulate a case that should never be hit, instead of working
3936 * around a lack of fxsave64/fxrstor64 on old compilers.
3938 if (ctxt
->mode
>= X86EMUL_MODE_PROT64
)
3939 return X86EMUL_UNHANDLEABLE
;
3941 return X86EMUL_CONTINUE
;
3945 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but does save
3946 * and restore MXCSR.
3948 static size_t __fxstate_size(int nregs
)
3950 return offsetof(struct fxregs_state
, xmm_space
[0]) + nregs
* 16;
3953 static inline size_t fxstate_size(struct x86_emulate_ctxt
*ctxt
)
3956 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
3957 return __fxstate_size(16);
3959 cr4_osfxsr
= ctxt
->ops
->get_cr(ctxt
, 4) & X86_CR4_OSFXSR
;
3960 return __fxstate_size(cr4_osfxsr
? 8 : 0);
3964 * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
3967 * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs
3968 * preserve whole 32 bit values, though, so (1) and (2) are the same wrt.
3970 * 3) 64-bit mode with REX.W prefix
3971 * - like (2), but XMM 8-15 are being saved and restored
3972 * 4) 64-bit mode without REX.W prefix
3973 * - like (3), but FIP and FDP are 64 bit
3975 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
3976 * desired result. (4) is not emulated.
3978 * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS
3979 * and FPU DS) should match.
3981 static int em_fxsave(struct x86_emulate_ctxt
*ctxt
)
3983 struct fxregs_state fx_state
;
3986 rc
= check_fxsr(ctxt
);
3987 if (rc
!= X86EMUL_CONTINUE
)
3990 ctxt
->ops
->get_fpu(ctxt
);
3992 rc
= asm_safe("fxsave %[fx]", , [fx
] "+m"(fx_state
));
3994 ctxt
->ops
->put_fpu(ctxt
);
3996 if (rc
!= X86EMUL_CONTINUE
)
3999 return segmented_write_std(ctxt
, ctxt
->memop
.addr
.mem
, &fx_state
,
4000 fxstate_size(ctxt
));
4003 static int em_fxrstor(struct x86_emulate_ctxt
*ctxt
)
4005 struct fxregs_state fx_state
;
4009 rc
= check_fxsr(ctxt
);
4010 if (rc
!= X86EMUL_CONTINUE
)
4013 ctxt
->ops
->get_fpu(ctxt
);
4015 size
= fxstate_size(ctxt
);
4016 if (size
< __fxstate_size(16)) {
4017 rc
= asm_safe("fxsave %[fx]", , [fx
] "+m"(fx_state
));
4018 if (rc
!= X86EMUL_CONTINUE
)
4022 rc
= segmented_read_std(ctxt
, ctxt
->memop
.addr
.mem
, &fx_state
, size
);
4023 if (rc
!= X86EMUL_CONTINUE
)
4026 if (fx_state
.mxcsr
>> 16) {
4027 rc
= emulate_gp(ctxt
, 0);
4031 if (rc
== X86EMUL_CONTINUE
)
4032 rc
= asm_safe("fxrstor %[fx]", : [fx
] "m"(fx_state
));
4035 ctxt
->ops
->put_fpu(ctxt
);
4040 static bool valid_cr(int nr
)
4052 static int check_cr_read(struct x86_emulate_ctxt
*ctxt
)
4054 if (!valid_cr(ctxt
->modrm_reg
))
4055 return emulate_ud(ctxt
);
4057 return X86EMUL_CONTINUE
;
4060 static int check_cr_write(struct x86_emulate_ctxt
*ctxt
)
4062 u64 new_val
= ctxt
->src
.val64
;
4063 int cr
= ctxt
->modrm_reg
;
4066 static u64 cr_reserved_bits
[] = {
4067 0xffffffff00000000ULL
,
4068 0, 0, 0, /* CR3 checked later */
4075 return emulate_ud(ctxt
);
4077 if (new_val
& cr_reserved_bits
[cr
])
4078 return emulate_gp(ctxt
, 0);
4083 if (((new_val
& X86_CR0_PG
) && !(new_val
& X86_CR0_PE
)) ||
4084 ((new_val
& X86_CR0_NW
) && !(new_val
& X86_CR0_CD
)))
4085 return emulate_gp(ctxt
, 0);
4087 cr4
= ctxt
->ops
->get_cr(ctxt
, 4);
4088 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
4090 if ((new_val
& X86_CR0_PG
) && (efer
& EFER_LME
) &&
4091 !(cr4
& X86_CR4_PAE
))
4092 return emulate_gp(ctxt
, 0);
4099 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
4100 if (efer
& EFER_LMA
)
4101 rsvd
= CR3_L_MODE_RESERVED_BITS
& ~CR3_PCID_INVD
;
4104 return emulate_gp(ctxt
, 0);
4109 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
4111 if ((efer
& EFER_LMA
) && !(new_val
& X86_CR4_PAE
))
4112 return emulate_gp(ctxt
, 0);
4118 return X86EMUL_CONTINUE
;
4121 static int check_dr7_gd(struct x86_emulate_ctxt
*ctxt
)
4125 ctxt
->ops
->get_dr(ctxt
, 7, &dr7
);
4127 /* Check if DR7.Global_Enable is set */
4128 return dr7
& (1 << 13);
4131 static int check_dr_read(struct x86_emulate_ctxt
*ctxt
)
4133 int dr
= ctxt
->modrm_reg
;
4137 return emulate_ud(ctxt
);
4139 cr4
= ctxt
->ops
->get_cr(ctxt
, 4);
4140 if ((cr4
& X86_CR4_DE
) && (dr
== 4 || dr
== 5))
4141 return emulate_ud(ctxt
);
4143 if (check_dr7_gd(ctxt
)) {
4146 ctxt
->ops
->get_dr(ctxt
, 6, &dr6
);
4148 dr6
|= DR6_BD
| DR6_RTM
;
4149 ctxt
->ops
->set_dr(ctxt
, 6, dr6
);
4150 return emulate_db(ctxt
);
4153 return X86EMUL_CONTINUE
;
4156 static int check_dr_write(struct x86_emulate_ctxt
*ctxt
)
4158 u64 new_val
= ctxt
->src
.val64
;
4159 int dr
= ctxt
->modrm_reg
;
4161 if ((dr
== 6 || dr
== 7) && (new_val
& 0xffffffff00000000ULL
))
4162 return emulate_gp(ctxt
, 0);
4164 return check_dr_read(ctxt
);
4167 static int check_svme(struct x86_emulate_ctxt
*ctxt
)
4171 ctxt
->ops
->get_msr(ctxt
, MSR_EFER
, &efer
);
4173 if (!(efer
& EFER_SVME
))
4174 return emulate_ud(ctxt
);
4176 return X86EMUL_CONTINUE
;
4179 static int check_svme_pa(struct x86_emulate_ctxt
*ctxt
)
4181 u64 rax
= reg_read(ctxt
, VCPU_REGS_RAX
);
4183 /* Valid physical address? */
4184 if (rax
& 0xffff000000000000ULL
)
4185 return emulate_gp(ctxt
, 0);
4187 return check_svme(ctxt
);
4190 static int check_rdtsc(struct x86_emulate_ctxt
*ctxt
)
4192 u64 cr4
= ctxt
->ops
->get_cr(ctxt
, 4);
4194 if (cr4
& X86_CR4_TSD
&& ctxt
->ops
->cpl(ctxt
))
4195 return emulate_ud(ctxt
);
4197 return X86EMUL_CONTINUE
;
4200 static int check_rdpmc(struct x86_emulate_ctxt
*ctxt
)
4202 u64 cr4
= ctxt
->ops
->get_cr(ctxt
, 4);
4203 u64 rcx
= reg_read(ctxt
, VCPU_REGS_RCX
);
4205 if ((!(cr4
& X86_CR4_PCE
) && ctxt
->ops
->cpl(ctxt
)) ||
4206 ctxt
->ops
->check_pmc(ctxt
, rcx
))
4207 return emulate_gp(ctxt
, 0);
4209 return X86EMUL_CONTINUE
;
4212 static int check_perm_in(struct x86_emulate_ctxt
*ctxt
)
4214 ctxt
->dst
.bytes
= min(ctxt
->dst
.bytes
, 4u);
4215 if (!emulator_io_permited(ctxt
, ctxt
->src
.val
, ctxt
->dst
.bytes
))
4216 return emulate_gp(ctxt
, 0);
4218 return X86EMUL_CONTINUE
;
4221 static int check_perm_out(struct x86_emulate_ctxt
*ctxt
)
4223 ctxt
->src
.bytes
= min(ctxt
->src
.bytes
, 4u);
4224 if (!emulator_io_permited(ctxt
, ctxt
->dst
.val
, ctxt
->src
.bytes
))
4225 return emulate_gp(ctxt
, 0);
4227 return X86EMUL_CONTINUE
;
4230 #define D(_y) { .flags = (_y) }
4231 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4232 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4233 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4234 #define N D(NotImpl)
4235 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4236 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4237 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4238 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4239 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4240 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4241 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4242 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4243 #define II(_f, _e, _i) \
4244 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4245 #define IIP(_f, _e, _i, _p) \
4246 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4247 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4248 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4250 #define D2bv(_f) D((_f) | ByteOp), D(_f)
4251 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4252 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
4253 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
4254 #define I2bvIP(_f, _e, _i, _p) \
4255 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4257 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
4258 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
4259 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4261 static const struct opcode group7_rm0
[] = {
4263 I(SrcNone
| Priv
| EmulateOnUD
, em_hypercall
),
4267 static const struct opcode group7_rm1
[] = {
4268 DI(SrcNone
| Priv
, monitor
),
4269 DI(SrcNone
| Priv
, mwait
),
4273 static const struct opcode group7_rm3
[] = {
4274 DIP(SrcNone
| Prot
| Priv
, vmrun
, check_svme_pa
),
4275 II(SrcNone
| Prot
| EmulateOnUD
, em_hypercall
, vmmcall
),
4276 DIP(SrcNone
| Prot
| Priv
, vmload
, check_svme_pa
),
4277 DIP(SrcNone
| Prot
| Priv
, vmsave
, check_svme_pa
),
4278 DIP(SrcNone
| Prot
| Priv
, stgi
, check_svme
),
4279 DIP(SrcNone
| Prot
| Priv
, clgi
, check_svme
),
4280 DIP(SrcNone
| Prot
| Priv
, skinit
, check_svme
),
4281 DIP(SrcNone
| Prot
| Priv
, invlpga
, check_svme
),
4284 static const struct opcode group7_rm7
[] = {
4286 DIP(SrcNone
, rdtscp
, check_rdtsc
),
4290 static const struct opcode group1
[] = {
4292 F(Lock
| PageTable
, em_or
),
4295 F(Lock
| PageTable
, em_and
),
4301 static const struct opcode group1A
[] = {
4302 I(DstMem
| SrcNone
| Mov
| Stack
| IncSP
| TwoMemOp
, em_pop
), N
, N
, N
, N
, N
, N
, N
,
4305 static const struct opcode group2
[] = {
4306 F(DstMem
| ModRM
, em_rol
),
4307 F(DstMem
| ModRM
, em_ror
),
4308 F(DstMem
| ModRM
, em_rcl
),
4309 F(DstMem
| ModRM
, em_rcr
),
4310 F(DstMem
| ModRM
, em_shl
),
4311 F(DstMem
| ModRM
, em_shr
),
4312 F(DstMem
| ModRM
, em_shl
),
4313 F(DstMem
| ModRM
, em_sar
),
4316 static const struct opcode group3
[] = {
4317 F(DstMem
| SrcImm
| NoWrite
, em_test
),
4318 F(DstMem
| SrcImm
| NoWrite
, em_test
),
4319 F(DstMem
| SrcNone
| Lock
, em_not
),
4320 F(DstMem
| SrcNone
| Lock
, em_neg
),
4321 F(DstXacc
| Src2Mem
, em_mul_ex
),
4322 F(DstXacc
| Src2Mem
, em_imul_ex
),
4323 F(DstXacc
| Src2Mem
, em_div_ex
),
4324 F(DstXacc
| Src2Mem
, em_idiv_ex
),
4327 static const struct opcode group4
[] = {
4328 F(ByteOp
| DstMem
| SrcNone
| Lock
, em_inc
),
4329 F(ByteOp
| DstMem
| SrcNone
| Lock
, em_dec
),
4333 static const struct opcode group5
[] = {
4334 F(DstMem
| SrcNone
| Lock
, em_inc
),
4335 F(DstMem
| SrcNone
| Lock
, em_dec
),
4336 I(SrcMem
| NearBranch
, em_call_near_abs
),
4337 I(SrcMemFAddr
| ImplicitOps
, em_call_far
),
4338 I(SrcMem
| NearBranch
, em_jmp_abs
),
4339 I(SrcMemFAddr
| ImplicitOps
, em_jmp_far
),
4340 I(SrcMem
| Stack
| TwoMemOp
, em_push
), D(Undefined
),
4343 static const struct opcode group6
[] = {
4344 DI(Prot
| DstMem
, sldt
),
4345 DI(Prot
| DstMem
, str
),
4346 II(Prot
| Priv
| SrcMem16
, em_lldt
, lldt
),
4347 II(Prot
| Priv
| SrcMem16
, em_ltr
, ltr
),
4351 static const struct group_dual group7
= { {
4352 II(Mov
| DstMem
, em_sgdt
, sgdt
),
4353 II(Mov
| DstMem
, em_sidt
, sidt
),
4354 II(SrcMem
| Priv
, em_lgdt
, lgdt
),
4355 II(SrcMem
| Priv
, em_lidt
, lidt
),
4356 II(SrcNone
| DstMem
| Mov
, em_smsw
, smsw
), N
,
4357 II(SrcMem16
| Mov
| Priv
, em_lmsw
, lmsw
),
4358 II(SrcMem
| ByteOp
| Priv
| NoAccess
, em_invlpg
, invlpg
),
4362 N
, EXT(0, group7_rm3
),
4363 II(SrcNone
| DstMem
| Mov
, em_smsw
, smsw
), N
,
4364 II(SrcMem16
| Mov
| Priv
, em_lmsw
, lmsw
),
4368 static const struct opcode group8
[] = {
4370 F(DstMem
| SrcImmByte
| NoWrite
, em_bt
),
4371 F(DstMem
| SrcImmByte
| Lock
| PageTable
, em_bts
),
4372 F(DstMem
| SrcImmByte
| Lock
, em_btr
),
4373 F(DstMem
| SrcImmByte
| Lock
| PageTable
, em_btc
),
4376 static const struct group_dual group9
= { {
4377 N
, I(DstMem64
| Lock
| PageTable
, em_cmpxchg8b
), N
, N
, N
, N
, N
, N
,
4379 N
, N
, N
, N
, N
, N
, N
, N
,
4382 static const struct opcode group11
[] = {
4383 I(DstMem
| SrcImm
| Mov
| PageTable
, em_mov
),
4387 static const struct gprefix pfx_0f_ae_7
= {
4388 I(SrcMem
| ByteOp
, em_clflush
), N
, N
, N
,
4391 static const struct group_dual group15
= { {
4392 I(ModRM
| Aligned16
, em_fxsave
),
4393 I(ModRM
| Aligned16
, em_fxrstor
),
4394 N
, N
, N
, N
, N
, GP(0, &pfx_0f_ae_7
),
4396 N
, N
, N
, N
, N
, N
, N
, N
,
4399 static const struct gprefix pfx_0f_6f_0f_7f
= {
4400 I(Mmx
, em_mov
), I(Sse
| Aligned
, em_mov
), N
, I(Sse
| Unaligned
, em_mov
),
4403 static const struct instr_dual instr_dual_0f_2b
= {
4407 static const struct gprefix pfx_0f_2b
= {
4408 ID(0, &instr_dual_0f_2b
), ID(0, &instr_dual_0f_2b
), N
, N
,
4411 static const struct gprefix pfx_0f_28_0f_29
= {
4412 I(Aligned
, em_mov
), I(Aligned
, em_mov
), N
, N
,
4415 static const struct gprefix pfx_0f_e7
= {
4416 N
, I(Sse
, em_mov
), N
, N
,
4419 static const struct escape escape_d9
= { {
4420 N
, N
, N
, N
, N
, N
, N
, I(DstMem16
| Mov
, em_fnstcw
),
4423 N
, N
, N
, N
, N
, N
, N
, N
,
4425 N
, N
, N
, N
, N
, N
, N
, N
,
4427 N
, N
, N
, N
, N
, N
, N
, N
,
4429 N
, N
, N
, N
, N
, N
, N
, N
,
4431 N
, N
, N
, N
, N
, N
, N
, N
,
4433 N
, N
, N
, N
, N
, N
, N
, N
,
4435 N
, N
, N
, N
, N
, N
, N
, N
,
4437 N
, N
, N
, N
, N
, N
, N
, N
,
4440 static const struct escape escape_db
= { {
4441 N
, N
, N
, N
, N
, N
, N
, N
,
4444 N
, N
, N
, N
, N
, N
, N
, N
,
4446 N
, N
, N
, N
, N
, N
, N
, N
,
4448 N
, N
, N
, N
, N
, N
, N
, N
,
4450 N
, N
, N
, N
, N
, N
, N
, N
,
4452 N
, N
, N
, I(ImplicitOps
, em_fninit
), N
, N
, N
, N
,
4454 N
, N
, N
, N
, N
, N
, N
, N
,
4456 N
, N
, N
, N
, N
, N
, N
, N
,
4458 N
, N
, N
, N
, N
, N
, N
, N
,
4461 static const struct escape escape_dd
= { {
4462 N
, N
, N
, N
, N
, N
, N
, I(DstMem16
| Mov
, em_fnstsw
),
4465 N
, N
, N
, N
, N
, N
, N
, N
,
4467 N
, N
, N
, N
, N
, N
, N
, N
,
4469 N
, N
, N
, N
, N
, N
, N
, N
,
4471 N
, N
, N
, N
, N
, N
, N
, N
,
4473 N
, N
, N
, N
, N
, N
, N
, N
,
4475 N
, N
, N
, N
, N
, N
, N
, N
,
4477 N
, N
, N
, N
, N
, N
, N
, N
,
4479 N
, N
, N
, N
, N
, N
, N
, N
,
4482 static const struct instr_dual instr_dual_0f_c3
= {
4483 I(DstMem
| SrcReg
| ModRM
| No16
| Mov
, em_mov
), N
4486 static const struct mode_dual mode_dual_63
= {
4487 N
, I(DstReg
| SrcMem32
| ModRM
| Mov
, em_movsxd
)
4490 static const struct opcode opcode_table
[256] = {
4492 F6ALU(Lock
, em_add
),
4493 I(ImplicitOps
| Stack
| No64
| Src2ES
, em_push_sreg
),
4494 I(ImplicitOps
| Stack
| No64
| Src2ES
, em_pop_sreg
),
4496 F6ALU(Lock
| PageTable
, em_or
),
4497 I(ImplicitOps
| Stack
| No64
| Src2CS
, em_push_sreg
),
4500 F6ALU(Lock
, em_adc
),
4501 I(ImplicitOps
| Stack
| No64
| Src2SS
, em_push_sreg
),
4502 I(ImplicitOps
| Stack
| No64
| Src2SS
, em_pop_sreg
),
4504 F6ALU(Lock
, em_sbb
),
4505 I(ImplicitOps
| Stack
| No64
| Src2DS
, em_push_sreg
),
4506 I(ImplicitOps
| Stack
| No64
| Src2DS
, em_pop_sreg
),
4508 F6ALU(Lock
| PageTable
, em_and
), N
, N
,
4510 F6ALU(Lock
, em_sub
), N
, I(ByteOp
| DstAcc
| No64
, em_das
),
4512 F6ALU(Lock
, em_xor
), N
, N
,
4514 F6ALU(NoWrite
, em_cmp
), N
, N
,
4516 X8(F(DstReg
, em_inc
)), X8(F(DstReg
, em_dec
)),
4518 X8(I(SrcReg
| Stack
, em_push
)),
4520 X8(I(DstReg
| Stack
, em_pop
)),
4522 I(ImplicitOps
| Stack
| No64
, em_pusha
),
4523 I(ImplicitOps
| Stack
| No64
, em_popa
),
4524 N
, MD(ModRM
, &mode_dual_63
),
4527 I(SrcImm
| Mov
| Stack
, em_push
),
4528 I(DstReg
| SrcMem
| ModRM
| Src2Imm
, em_imul_3op
),
4529 I(SrcImmByte
| Mov
| Stack
, em_push
),
4530 I(DstReg
| SrcMem
| ModRM
| Src2ImmByte
, em_imul_3op
),
4531 I2bvIP(DstDI
| SrcDX
| Mov
| String
| Unaligned
, em_in
, ins
, check_perm_in
), /* insb, insw/insd */
4532 I2bvIP(SrcSI
| DstDX
| String
, em_out
, outs
, check_perm_out
), /* outsb, outsw/outsd */
4534 X16(D(SrcImmByte
| NearBranch
)),
4536 G(ByteOp
| DstMem
| SrcImm
, group1
),
4537 G(DstMem
| SrcImm
, group1
),
4538 G(ByteOp
| DstMem
| SrcImm
| No64
, group1
),
4539 G(DstMem
| SrcImmByte
, group1
),
4540 F2bv(DstMem
| SrcReg
| ModRM
| NoWrite
, em_test
),
4541 I2bv(DstMem
| SrcReg
| ModRM
| Lock
| PageTable
, em_xchg
),
4543 I2bv(DstMem
| SrcReg
| ModRM
| Mov
| PageTable
, em_mov
),
4544 I2bv(DstReg
| SrcMem
| ModRM
| Mov
, em_mov
),
4545 I(DstMem
| SrcNone
| ModRM
| Mov
| PageTable
, em_mov_rm_sreg
),
4546 D(ModRM
| SrcMem
| NoAccess
| DstReg
),
4547 I(ImplicitOps
| SrcMem16
| ModRM
, em_mov_sreg_rm
),
4550 DI(SrcAcc
| DstReg
, pause
), X7(D(SrcAcc
| DstReg
)),
4552 D(DstAcc
| SrcNone
), I(ImplicitOps
| SrcAcc
, em_cwd
),
4553 I(SrcImmFAddr
| No64
, em_call_far
), N
,
4554 II(ImplicitOps
| Stack
, em_pushf
, pushf
),
4555 II(ImplicitOps
| Stack
, em_popf
, popf
),
4556 I(ImplicitOps
, em_sahf
), I(ImplicitOps
, em_lahf
),
4558 I2bv(DstAcc
| SrcMem
| Mov
| MemAbs
, em_mov
),
4559 I2bv(DstMem
| SrcAcc
| Mov
| MemAbs
| PageTable
, em_mov
),
4560 I2bv(SrcSI
| DstDI
| Mov
| String
| TwoMemOp
, em_mov
),
4561 F2bv(SrcSI
| DstDI
| String
| NoWrite
| TwoMemOp
, em_cmp_r
),
4563 F2bv(DstAcc
| SrcImm
| NoWrite
, em_test
),
4564 I2bv(SrcAcc
| DstDI
| Mov
| String
, em_mov
),
4565 I2bv(SrcSI
| DstAcc
| Mov
| String
, em_mov
),
4566 F2bv(SrcAcc
| DstDI
| String
| NoWrite
, em_cmp_r
),
4568 X8(I(ByteOp
| DstReg
| SrcImm
| Mov
, em_mov
)),
4570 X8(I(DstReg
| SrcImm64
| Mov
, em_mov
)),
4572 G(ByteOp
| Src2ImmByte
, group2
), G(Src2ImmByte
, group2
),
4573 I(ImplicitOps
| NearBranch
| SrcImmU16
, em_ret_near_imm
),
4574 I(ImplicitOps
| NearBranch
, em_ret
),
4575 I(DstReg
| SrcMemFAddr
| ModRM
| No64
| Src2ES
, em_lseg
),
4576 I(DstReg
| SrcMemFAddr
| ModRM
| No64
| Src2DS
, em_lseg
),
4577 G(ByteOp
, group11
), G(0, group11
),
4579 I(Stack
| SrcImmU16
| Src2ImmByte
, em_enter
), I(Stack
, em_leave
),
4580 I(ImplicitOps
| SrcImmU16
, em_ret_far_imm
),
4581 I(ImplicitOps
, em_ret_far
),
4582 D(ImplicitOps
), DI(SrcImmByte
, intn
),
4583 D(ImplicitOps
| No64
), II(ImplicitOps
, em_iret
, iret
),
4585 G(Src2One
| ByteOp
, group2
), G(Src2One
, group2
),
4586 G(Src2CL
| ByteOp
, group2
), G(Src2CL
, group2
),
4587 I(DstAcc
| SrcImmUByte
| No64
, em_aam
),
4588 I(DstAcc
| SrcImmUByte
| No64
, em_aad
),
4589 F(DstAcc
| ByteOp
| No64
, em_salc
),
4590 I(DstAcc
| SrcXLat
| ByteOp
, em_mov
),
4592 N
, E(0, &escape_d9
), N
, E(0, &escape_db
), N
, E(0, &escape_dd
), N
, N
,
4594 X3(I(SrcImmByte
| NearBranch
, em_loop
)),
4595 I(SrcImmByte
| NearBranch
, em_jcxz
),
4596 I2bvIP(SrcImmUByte
| DstAcc
, em_in
, in
, check_perm_in
),
4597 I2bvIP(SrcAcc
| DstImmUByte
, em_out
, out
, check_perm_out
),
4599 I(SrcImm
| NearBranch
, em_call
), D(SrcImm
| ImplicitOps
| NearBranch
),
4600 I(SrcImmFAddr
| No64
, em_jmp_far
),
4601 D(SrcImmByte
| ImplicitOps
| NearBranch
),
4602 I2bvIP(SrcDX
| DstAcc
, em_in
, in
, check_perm_in
),
4603 I2bvIP(SrcAcc
| DstDX
, em_out
, out
, check_perm_out
),
4605 N
, DI(ImplicitOps
, icebp
), N
, N
,
4606 DI(ImplicitOps
| Priv
, hlt
), D(ImplicitOps
),
4607 G(ByteOp
, group3
), G(0, group3
),
4609 D(ImplicitOps
), D(ImplicitOps
),
4610 I(ImplicitOps
, em_cli
), I(ImplicitOps
, em_sti
),
4611 D(ImplicitOps
), D(ImplicitOps
), G(0, group4
), G(0, group5
),
4614 static const struct opcode twobyte_table
[256] = {
4616 G(0, group6
), GD(0, &group7
), N
, N
,
4617 N
, I(ImplicitOps
| EmulateOnUD
, em_syscall
),
4618 II(ImplicitOps
| Priv
, em_clts
, clts
), N
,
4619 DI(ImplicitOps
| Priv
, invd
), DI(ImplicitOps
| Priv
, wbinvd
), N
, N
,
4620 N
, D(ImplicitOps
| ModRM
| SrcMem
| NoAccess
), N
, N
,
4622 N
, N
, N
, N
, N
, N
, N
, N
,
4623 D(ImplicitOps
| ModRM
| SrcMem
| NoAccess
),
4624 N
, N
, N
, N
, N
, N
, D(ImplicitOps
| ModRM
| SrcMem
| NoAccess
),
4626 DIP(ModRM
| DstMem
| Priv
| Op3264
| NoMod
, cr_read
, check_cr_read
),
4627 DIP(ModRM
| DstMem
| Priv
| Op3264
| NoMod
, dr_read
, check_dr_read
),
4628 IIP(ModRM
| SrcMem
| Priv
| Op3264
| NoMod
, em_cr_write
, cr_write
,
4630 IIP(ModRM
| SrcMem
| Priv
| Op3264
| NoMod
, em_dr_write
, dr_write
,
4633 GP(ModRM
| DstReg
| SrcMem
| Mov
| Sse
, &pfx_0f_28_0f_29
),
4634 GP(ModRM
| DstMem
| SrcReg
| Mov
| Sse
, &pfx_0f_28_0f_29
),
4635 N
, GP(ModRM
| DstMem
| SrcReg
| Mov
| Sse
, &pfx_0f_2b
),
4638 II(ImplicitOps
| Priv
, em_wrmsr
, wrmsr
),
4639 IIP(ImplicitOps
, em_rdtsc
, rdtsc
, check_rdtsc
),
4640 II(ImplicitOps
| Priv
, em_rdmsr
, rdmsr
),
4641 IIP(ImplicitOps
, em_rdpmc
, rdpmc
, check_rdpmc
),
4642 I(ImplicitOps
| EmulateOnUD
, em_sysenter
),
4643 I(ImplicitOps
| Priv
| EmulateOnUD
, em_sysexit
),
4645 N
, N
, N
, N
, N
, N
, N
, N
,
4647 X16(D(DstReg
| SrcMem
| ModRM
)),
4649 N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
,
4654 N
, N
, N
, GP(SrcMem
| DstReg
| ModRM
| Mov
, &pfx_0f_6f_0f_7f
),
4659 N
, N
, N
, GP(SrcReg
| DstMem
| ModRM
| Mov
, &pfx_0f_6f_0f_7f
),
4661 X16(D(SrcImm
| NearBranch
)),
4663 X16(D(ByteOp
| DstMem
| SrcNone
| ModRM
| Mov
)),
4665 I(Stack
| Src2FS
, em_push_sreg
), I(Stack
| Src2FS
, em_pop_sreg
),
4666 II(ImplicitOps
, em_cpuid
, cpuid
),
4667 F(DstMem
| SrcReg
| ModRM
| BitOp
| NoWrite
, em_bt
),
4668 F(DstMem
| SrcReg
| Src2ImmByte
| ModRM
, em_shld
),
4669 F(DstMem
| SrcReg
| Src2CL
| ModRM
, em_shld
), N
, N
,
4671 I(Stack
| Src2GS
, em_push_sreg
), I(Stack
| Src2GS
, em_pop_sreg
),
4672 II(EmulateOnUD
| ImplicitOps
, em_rsm
, rsm
),
4673 F(DstMem
| SrcReg
| ModRM
| BitOp
| Lock
| PageTable
, em_bts
),
4674 F(DstMem
| SrcReg
| Src2ImmByte
| ModRM
, em_shrd
),
4675 F(DstMem
| SrcReg
| Src2CL
| ModRM
, em_shrd
),
4676 GD(0, &group15
), F(DstReg
| SrcMem
| ModRM
, em_imul
),
4678 I2bv(DstMem
| SrcReg
| ModRM
| Lock
| PageTable
| SrcWrite
, em_cmpxchg
),
4679 I(DstReg
| SrcMemFAddr
| ModRM
| Src2SS
, em_lseg
),
4680 F(DstMem
| SrcReg
| ModRM
| BitOp
| Lock
, em_btr
),
4681 I(DstReg
| SrcMemFAddr
| ModRM
| Src2FS
, em_lseg
),
4682 I(DstReg
| SrcMemFAddr
| ModRM
| Src2GS
, em_lseg
),
4683 D(DstReg
| SrcMem8
| ModRM
| Mov
), D(DstReg
| SrcMem16
| ModRM
| Mov
),
4687 F(DstMem
| SrcReg
| ModRM
| BitOp
| Lock
| PageTable
, em_btc
),
4688 I(DstReg
| SrcMem
| ModRM
, em_bsf_c
),
4689 I(DstReg
| SrcMem
| ModRM
, em_bsr_c
),
4690 D(DstReg
| SrcMem8
| ModRM
| Mov
), D(DstReg
| SrcMem16
| ModRM
| Mov
),
4692 F2bv(DstMem
| SrcReg
| ModRM
| SrcWrite
| Lock
, em_xadd
),
4693 N
, ID(0, &instr_dual_0f_c3
),
4694 N
, N
, N
, GD(0, &group9
),
4696 X8(I(DstReg
, em_bswap
)),
4698 N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
,
4700 N
, N
, N
, N
, N
, N
, N
, GP(SrcReg
| DstMem
| ModRM
| Mov
, &pfx_0f_e7
),
4701 N
, N
, N
, N
, N
, N
, N
, N
,
4703 N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
, N
4706 static const struct instr_dual instr_dual_0f_38_f0
= {
4707 I(DstReg
| SrcMem
| Mov
, em_movbe
), N
4710 static const struct instr_dual instr_dual_0f_38_f1
= {
4711 I(DstMem
| SrcReg
| Mov
, em_movbe
), N
4714 static const struct gprefix three_byte_0f_38_f0
= {
4715 ID(0, &instr_dual_0f_38_f0
), N
, N
, N
4718 static const struct gprefix three_byte_0f_38_f1
= {
4719 ID(0, &instr_dual_0f_38_f1
), N
, N
, N
4723 * Insns below are selected by the prefix which indexed by the third opcode
4726 static const struct opcode opcode_map_0f_38
[256] = {
4728 X16(N
), X16(N
), X16(N
), X16(N
), X16(N
), X16(N
), X16(N
), X16(N
),
4730 X16(N
), X16(N
), X16(N
), X16(N
), X16(N
), X16(N
), X16(N
),
4732 GP(EmulateOnUD
| ModRM
, &three_byte_0f_38_f0
),
4733 GP(EmulateOnUD
| ModRM
, &three_byte_0f_38_f1
),
4754 static unsigned imm_size(struct x86_emulate_ctxt
*ctxt
)
4758 size
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4764 static int decode_imm(struct x86_emulate_ctxt
*ctxt
, struct operand
*op
,
4765 unsigned size
, bool sign_extension
)
4767 int rc
= X86EMUL_CONTINUE
;
4771 op
->addr
.mem
.ea
= ctxt
->_eip
;
4772 /* NB. Immediates are sign-extended as necessary. */
4773 switch (op
->bytes
) {
4775 op
->val
= insn_fetch(s8
, ctxt
);
4778 op
->val
= insn_fetch(s16
, ctxt
);
4781 op
->val
= insn_fetch(s32
, ctxt
);
4784 op
->val
= insn_fetch(s64
, ctxt
);
4787 if (!sign_extension
) {
4788 switch (op
->bytes
) {
4796 op
->val
&= 0xffffffff;
4804 static int decode_operand(struct x86_emulate_ctxt
*ctxt
, struct operand
*op
,
4807 int rc
= X86EMUL_CONTINUE
;
4811 decode_register_operand(ctxt
, op
);
4814 rc
= decode_imm(ctxt
, op
, 1, false);
4817 ctxt
->memop
.bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4821 if (ctxt
->d
& BitOp
)
4822 fetch_bit_operand(ctxt
);
4823 op
->orig_val
= op
->val
;
4826 ctxt
->memop
.bytes
= (ctxt
->op_bytes
== 8) ? 16 : 8;
4830 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4831 op
->addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RAX
);
4832 fetch_register_operand(op
);
4833 op
->orig_val
= op
->val
;
4837 op
->bytes
= (ctxt
->d
& ByteOp
) ? 2 : ctxt
->op_bytes
;
4838 op
->addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RAX
);
4839 fetch_register_operand(op
);
4840 op
->orig_val
= op
->val
;
4843 if (ctxt
->d
& ByteOp
) {
4848 op
->bytes
= ctxt
->op_bytes
;
4849 op
->addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RDX
);
4850 fetch_register_operand(op
);
4851 op
->orig_val
= op
->val
;
4855 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4857 register_address(ctxt
, VCPU_REGS_RDI
);
4858 op
->addr
.mem
.seg
= VCPU_SREG_ES
;
4865 op
->addr
.reg
= reg_rmw(ctxt
, VCPU_REGS_RDX
);
4866 fetch_register_operand(op
);
4871 op
->val
= reg_read(ctxt
, VCPU_REGS_RCX
) & 0xff;
4874 rc
= decode_imm(ctxt
, op
, 1, true);
4882 rc
= decode_imm(ctxt
, op
, imm_size(ctxt
), true);
4885 rc
= decode_imm(ctxt
, op
, ctxt
->op_bytes
, true);
4888 ctxt
->memop
.bytes
= 1;
4889 if (ctxt
->memop
.type
== OP_REG
) {
4890 ctxt
->memop
.addr
.reg
= decode_register(ctxt
,
4891 ctxt
->modrm_rm
, true);
4892 fetch_register_operand(&ctxt
->memop
);
4896 ctxt
->memop
.bytes
= 2;
4899 ctxt
->memop
.bytes
= 4;
4902 rc
= decode_imm(ctxt
, op
, 2, false);
4905 rc
= decode_imm(ctxt
, op
, imm_size(ctxt
), false);
4909 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4911 register_address(ctxt
, VCPU_REGS_RSI
);
4912 op
->addr
.mem
.seg
= ctxt
->seg_override
;
4918 op
->bytes
= (ctxt
->d
& ByteOp
) ? 1 : ctxt
->op_bytes
;
4921 reg_read(ctxt
, VCPU_REGS_RBX
) +
4922 (reg_read(ctxt
, VCPU_REGS_RAX
) & 0xff));
4923 op
->addr
.mem
.seg
= ctxt
->seg_override
;
4928 op
->addr
.mem
.ea
= ctxt
->_eip
;
4929 op
->bytes
= ctxt
->op_bytes
+ 2;
4930 insn_fetch_arr(op
->valptr
, op
->bytes
, ctxt
);
4933 ctxt
->memop
.bytes
= ctxt
->op_bytes
+ 2;
4937 op
->val
= VCPU_SREG_ES
;
4941 op
->val
= VCPU_SREG_CS
;
4945 op
->val
= VCPU_SREG_SS
;
4949 op
->val
= VCPU_SREG_DS
;
4953 op
->val
= VCPU_SREG_FS
;
4957 op
->val
= VCPU_SREG_GS
;
4960 /* Special instructions do their own operand decoding. */
4962 op
->type
= OP_NONE
; /* Disable writeback. */
4970 int x86_decode_insn(struct x86_emulate_ctxt
*ctxt
, void *insn
, int insn_len
)
4972 int rc
= X86EMUL_CONTINUE
;
4973 int mode
= ctxt
->mode
;
4974 int def_op_bytes
, def_ad_bytes
, goffset
, simd_prefix
;
4975 bool op_prefix
= false;
4976 bool has_seg_override
= false;
4977 struct opcode opcode
;
4979 ctxt
->memop
.type
= OP_NONE
;
4980 ctxt
->memopp
= NULL
;
4981 ctxt
->_eip
= ctxt
->eip
;
4982 ctxt
->fetch
.ptr
= ctxt
->fetch
.data
;
4983 ctxt
->fetch
.end
= ctxt
->fetch
.data
+ insn_len
;
4984 ctxt
->opcode_len
= 1;
4986 memcpy(ctxt
->fetch
.data
, insn
, insn_len
);
4988 rc
= __do_insn_fetch_bytes(ctxt
, 1);
4989 if (rc
!= X86EMUL_CONTINUE
)
4994 case X86EMUL_MODE_REAL
:
4995 case X86EMUL_MODE_VM86
:
4996 case X86EMUL_MODE_PROT16
:
4997 def_op_bytes
= def_ad_bytes
= 2;
4999 case X86EMUL_MODE_PROT32
:
5000 def_op_bytes
= def_ad_bytes
= 4;
5002 #ifdef CONFIG_X86_64
5003 case X86EMUL_MODE_PROT64
:
5009 return EMULATION_FAILED
;
5012 ctxt
->op_bytes
= def_op_bytes
;
5013 ctxt
->ad_bytes
= def_ad_bytes
;
5015 /* Legacy prefixes. */
5017 switch (ctxt
->b
= insn_fetch(u8
, ctxt
)) {
5018 case 0x66: /* operand-size override */
5020 /* switch between 2/4 bytes */
5021 ctxt
->op_bytes
= def_op_bytes
^ 6;
5023 case 0x67: /* address-size override */
5024 if (mode
== X86EMUL_MODE_PROT64
)
5025 /* switch between 4/8 bytes */
5026 ctxt
->ad_bytes
= def_ad_bytes
^ 12;
5028 /* switch between 2/4 bytes */
5029 ctxt
->ad_bytes
= def_ad_bytes
^ 6;
5031 case 0x26: /* ES override */
5032 case 0x2e: /* CS override */
5033 case 0x36: /* SS override */
5034 case 0x3e: /* DS override */
5035 has_seg_override
= true;
5036 ctxt
->seg_override
= (ctxt
->b
>> 3) & 3;
5038 case 0x64: /* FS override */
5039 case 0x65: /* GS override */
5040 has_seg_override
= true;
5041 ctxt
->seg_override
= ctxt
->b
& 7;
5043 case 0x40 ... 0x4f: /* REX */
5044 if (mode
!= X86EMUL_MODE_PROT64
)
5046 ctxt
->rex_prefix
= ctxt
->b
;
5048 case 0xf0: /* LOCK */
5049 ctxt
->lock_prefix
= 1;
5051 case 0xf2: /* REPNE/REPNZ */
5052 case 0xf3: /* REP/REPE/REPZ */
5053 ctxt
->rep_prefix
= ctxt
->b
;
5059 /* Any legacy prefix after a REX prefix nullifies its effect. */
5061 ctxt
->rex_prefix
= 0;
5067 if (ctxt
->rex_prefix
& 8)
5068 ctxt
->op_bytes
= 8; /* REX.W */
5070 /* Opcode byte(s). */
5071 opcode
= opcode_table
[ctxt
->b
];
5072 /* Two-byte opcode? */
5073 if (ctxt
->b
== 0x0f) {
5074 ctxt
->opcode_len
= 2;
5075 ctxt
->b
= insn_fetch(u8
, ctxt
);
5076 opcode
= twobyte_table
[ctxt
->b
];
5078 /* 0F_38 opcode map */
5079 if (ctxt
->b
== 0x38) {
5080 ctxt
->opcode_len
= 3;
5081 ctxt
->b
= insn_fetch(u8
, ctxt
);
5082 opcode
= opcode_map_0f_38
[ctxt
->b
];
5085 ctxt
->d
= opcode
.flags
;
5087 if (ctxt
->d
& ModRM
)
5088 ctxt
->modrm
= insn_fetch(u8
, ctxt
);
5090 /* vex-prefix instructions are not implemented */
5091 if (ctxt
->opcode_len
== 1 && (ctxt
->b
== 0xc5 || ctxt
->b
== 0xc4) &&
5092 (mode
== X86EMUL_MODE_PROT64
|| (ctxt
->modrm
& 0xc0) == 0xc0)) {
5096 while (ctxt
->d
& GroupMask
) {
5097 switch (ctxt
->d
& GroupMask
) {
5099 goffset
= (ctxt
->modrm
>> 3) & 7;
5100 opcode
= opcode
.u
.group
[goffset
];
5103 goffset
= (ctxt
->modrm
>> 3) & 7;
5104 if ((ctxt
->modrm
>> 6) == 3)
5105 opcode
= opcode
.u
.gdual
->mod3
[goffset
];
5107 opcode
= opcode
.u
.gdual
->mod012
[goffset
];
5110 goffset
= ctxt
->modrm
& 7;
5111 opcode
= opcode
.u
.group
[goffset
];
5114 if (ctxt
->rep_prefix
&& op_prefix
)
5115 return EMULATION_FAILED
;
5116 simd_prefix
= op_prefix
? 0x66 : ctxt
->rep_prefix
;
5117 switch (simd_prefix
) {
5118 case 0x00: opcode
= opcode
.u
.gprefix
->pfx_no
; break;
5119 case 0x66: opcode
= opcode
.u
.gprefix
->pfx_66
; break;
5120 case 0xf2: opcode
= opcode
.u
.gprefix
->pfx_f2
; break;
5121 case 0xf3: opcode
= opcode
.u
.gprefix
->pfx_f3
; break;
5125 if (ctxt
->modrm
> 0xbf)
5126 opcode
= opcode
.u
.esc
->high
[ctxt
->modrm
- 0xc0];
5128 opcode
= opcode
.u
.esc
->op
[(ctxt
->modrm
>> 3) & 7];
5131 if ((ctxt
->modrm
>> 6) == 3)
5132 opcode
= opcode
.u
.idual
->mod3
;
5134 opcode
= opcode
.u
.idual
->mod012
;
5137 if (ctxt
->mode
== X86EMUL_MODE_PROT64
)
5138 opcode
= opcode
.u
.mdual
->mode64
;
5140 opcode
= opcode
.u
.mdual
->mode32
;
5143 return EMULATION_FAILED
;
5146 ctxt
->d
&= ~(u64
)GroupMask
;
5147 ctxt
->d
|= opcode
.flags
;
5152 return EMULATION_FAILED
;
5154 ctxt
->execute
= opcode
.u
.execute
;
5156 if (unlikely(ctxt
->ud
) && likely(!(ctxt
->d
& EmulateOnUD
)))
5157 return EMULATION_FAILED
;
5159 if (unlikely(ctxt
->d
&
5160 (NotImpl
|Stack
|Op3264
|Sse
|Mmx
|Intercept
|CheckPerm
|NearBranch
|
5163 * These are copied unconditionally here, and checked unconditionally
5164 * in x86_emulate_insn.
5166 ctxt
->check_perm
= opcode
.check_perm
;
5167 ctxt
->intercept
= opcode
.intercept
;
5169 if (ctxt
->d
& NotImpl
)
5170 return EMULATION_FAILED
;
5172 if (mode
== X86EMUL_MODE_PROT64
) {
5173 if (ctxt
->op_bytes
== 4 && (ctxt
->d
& Stack
))
5175 else if (ctxt
->d
& NearBranch
)
5179 if (ctxt
->d
& Op3264
) {
5180 if (mode
== X86EMUL_MODE_PROT64
)
5186 if ((ctxt
->d
& No16
) && ctxt
->op_bytes
== 2)
5190 ctxt
->op_bytes
= 16;
5191 else if (ctxt
->d
& Mmx
)
5195 /* ModRM and SIB bytes. */
5196 if (ctxt
->d
& ModRM
) {
5197 rc
= decode_modrm(ctxt
, &ctxt
->memop
);
5198 if (!has_seg_override
) {
5199 has_seg_override
= true;
5200 ctxt
->seg_override
= ctxt
->modrm_seg
;
5202 } else if (ctxt
->d
& MemAbs
)
5203 rc
= decode_abs(ctxt
, &ctxt
->memop
);
5204 if (rc
!= X86EMUL_CONTINUE
)
5207 if (!has_seg_override
)
5208 ctxt
->seg_override
= VCPU_SREG_DS
;
5210 ctxt
->memop
.addr
.mem
.seg
= ctxt
->seg_override
;
5213 * Decode and fetch the source operand: register, memory
5216 rc
= decode_operand(ctxt
, &ctxt
->src
, (ctxt
->d
>> SrcShift
) & OpMask
);
5217 if (rc
!= X86EMUL_CONTINUE
)
5221 * Decode and fetch the second source operand: register, memory
5224 rc
= decode_operand(ctxt
, &ctxt
->src2
, (ctxt
->d
>> Src2Shift
) & OpMask
);
5225 if (rc
!= X86EMUL_CONTINUE
)
5228 /* Decode and fetch the destination operand: register or memory. */
5229 rc
= decode_operand(ctxt
, &ctxt
->dst
, (ctxt
->d
>> DstShift
) & OpMask
);
5231 if (ctxt
->rip_relative
&& likely(ctxt
->memopp
))
5232 ctxt
->memopp
->addr
.mem
.ea
= address_mask(ctxt
,
5233 ctxt
->memopp
->addr
.mem
.ea
+ ctxt
->_eip
);
5236 return (rc
!= X86EMUL_CONTINUE
) ? EMULATION_FAILED
: EMULATION_OK
;
5239 bool x86_page_table_writing_insn(struct x86_emulate_ctxt
*ctxt
)
5241 return ctxt
->d
& PageTable
;
5244 static bool string_insn_completed(struct x86_emulate_ctxt
*ctxt
)
5246 /* The second termination condition only applies for REPE
5247 * and REPNE. Test if the repeat string operation prefix is
5248 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
5249 * corresponding termination condition according to:
5250 * - if REPE/REPZ and ZF = 0 then done
5251 * - if REPNE/REPNZ and ZF = 1 then done
5253 if (((ctxt
->b
== 0xa6) || (ctxt
->b
== 0xa7) ||
5254 (ctxt
->b
== 0xae) || (ctxt
->b
== 0xaf))
5255 && (((ctxt
->rep_prefix
== REPE_PREFIX
) &&
5256 ((ctxt
->eflags
& X86_EFLAGS_ZF
) == 0))
5257 || ((ctxt
->rep_prefix
== REPNE_PREFIX
) &&
5258 ((ctxt
->eflags
& X86_EFLAGS_ZF
) == X86_EFLAGS_ZF
))))
5264 static int flush_pending_x87_faults(struct x86_emulate_ctxt
*ctxt
)
5268 ctxt
->ops
->get_fpu(ctxt
);
5269 rc
= asm_safe("fwait");
5270 ctxt
->ops
->put_fpu(ctxt
);
5272 if (unlikely(rc
!= X86EMUL_CONTINUE
))
5273 return emulate_exception(ctxt
, MF_VECTOR
, 0, false);
5275 return X86EMUL_CONTINUE
;
5278 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt
*ctxt
,
5281 if (op
->type
== OP_MM
)
5282 read_mmx_reg(ctxt
, &op
->mm_val
, op
->addr
.mm
);
5285 static int fastop(struct x86_emulate_ctxt
*ctxt
, void (*fop
)(struct fastop
*))
5287 register void *__sp
asm(_ASM_SP
);
5288 ulong flags
= (ctxt
->eflags
& EFLAGS_MASK
) | X86_EFLAGS_IF
;
5290 if (!(ctxt
->d
& ByteOp
))
5291 fop
+= __ffs(ctxt
->dst
.bytes
) * FASTOP_SIZE
;
5293 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
5294 : "+a"(ctxt
->dst
.val
), "+d"(ctxt
->src
.val
), [flags
]"+D"(flags
),
5295 [fastop
]"+S"(fop
), "+r"(__sp
)
5296 : "c"(ctxt
->src2
.val
));
5298 ctxt
->eflags
= (ctxt
->eflags
& ~EFLAGS_MASK
) | (flags
& EFLAGS_MASK
);
5299 if (!fop
) /* exception is returned in fop variable */
5300 return emulate_de(ctxt
);
5301 return X86EMUL_CONTINUE
;
5304 void init_decode_cache(struct x86_emulate_ctxt
*ctxt
)
5306 memset(&ctxt
->rip_relative
, 0,
5307 (void *)&ctxt
->modrm
- (void *)&ctxt
->rip_relative
);
5309 ctxt
->io_read
.pos
= 0;
5310 ctxt
->io_read
.end
= 0;
5311 ctxt
->mem_read
.end
= 0;
5314 int x86_emulate_insn(struct x86_emulate_ctxt
*ctxt
)
5316 const struct x86_emulate_ops
*ops
= ctxt
->ops
;
5317 int rc
= X86EMUL_CONTINUE
;
5318 int saved_dst_type
= ctxt
->dst
.type
;
5319 unsigned emul_flags
;
5321 ctxt
->mem_read
.pos
= 0;
5323 /* LOCK prefix is allowed only with some instructions */
5324 if (ctxt
->lock_prefix
&& (!(ctxt
->d
& Lock
) || ctxt
->dst
.type
!= OP_MEM
)) {
5325 rc
= emulate_ud(ctxt
);
5329 if ((ctxt
->d
& SrcMask
) == SrcMemFAddr
&& ctxt
->src
.type
!= OP_MEM
) {
5330 rc
= emulate_ud(ctxt
);
5334 emul_flags
= ctxt
->ops
->get_hflags(ctxt
);
5335 if (unlikely(ctxt
->d
&
5336 (No64
|Undefined
|Sse
|Mmx
|Intercept
|CheckPerm
|Priv
|Prot
|String
))) {
5337 if ((ctxt
->mode
== X86EMUL_MODE_PROT64
&& (ctxt
->d
& No64
)) ||
5338 (ctxt
->d
& Undefined
)) {
5339 rc
= emulate_ud(ctxt
);
5343 if (((ctxt
->d
& (Sse
|Mmx
)) && ((ops
->get_cr(ctxt
, 0) & X86_CR0_EM
)))
5344 || ((ctxt
->d
& Sse
) && !(ops
->get_cr(ctxt
, 4) & X86_CR4_OSFXSR
))) {
5345 rc
= emulate_ud(ctxt
);
5349 if ((ctxt
->d
& (Sse
|Mmx
)) && (ops
->get_cr(ctxt
, 0) & X86_CR0_TS
)) {
5350 rc
= emulate_nm(ctxt
);
5354 if (ctxt
->d
& Mmx
) {
5355 rc
= flush_pending_x87_faults(ctxt
);
5356 if (rc
!= X86EMUL_CONTINUE
)
5359 * Now that we know the fpu is exception safe, we can fetch
5362 fetch_possible_mmx_operand(ctxt
, &ctxt
->src
);
5363 fetch_possible_mmx_operand(ctxt
, &ctxt
->src2
);
5364 if (!(ctxt
->d
& Mov
))
5365 fetch_possible_mmx_operand(ctxt
, &ctxt
->dst
);
5368 if (unlikely(emul_flags
& X86EMUL_GUEST_MASK
) && ctxt
->intercept
) {
5369 rc
= emulator_check_intercept(ctxt
, ctxt
->intercept
,
5370 X86_ICPT_PRE_EXCEPT
);
5371 if (rc
!= X86EMUL_CONTINUE
)
5375 /* Instruction can only be executed in protected mode */
5376 if ((ctxt
->d
& Prot
) && ctxt
->mode
< X86EMUL_MODE_PROT16
) {
5377 rc
= emulate_ud(ctxt
);
5381 /* Privileged instruction can be executed only in CPL=0 */
5382 if ((ctxt
->d
& Priv
) && ops
->cpl(ctxt
)) {
5383 if (ctxt
->d
& PrivUD
)
5384 rc
= emulate_ud(ctxt
);
5386 rc
= emulate_gp(ctxt
, 0);
5390 /* Do instruction specific permission checks */
5391 if (ctxt
->d
& CheckPerm
) {
5392 rc
= ctxt
->check_perm(ctxt
);
5393 if (rc
!= X86EMUL_CONTINUE
)
5397 if (unlikely(emul_flags
& X86EMUL_GUEST_MASK
) && (ctxt
->d
& Intercept
)) {
5398 rc
= emulator_check_intercept(ctxt
, ctxt
->intercept
,
5399 X86_ICPT_POST_EXCEPT
);
5400 if (rc
!= X86EMUL_CONTINUE
)
5404 if (ctxt
->rep_prefix
&& (ctxt
->d
& String
)) {
5405 /* All REP prefixes have the same first termination condition */
5406 if (address_mask(ctxt
, reg_read(ctxt
, VCPU_REGS_RCX
)) == 0) {
5407 string_registers_quirk(ctxt
);
5408 ctxt
->eip
= ctxt
->_eip
;
5409 ctxt
->eflags
&= ~X86_EFLAGS_RF
;
5415 if ((ctxt
->src
.type
== OP_MEM
) && !(ctxt
->d
& NoAccess
)) {
5416 rc
= segmented_read(ctxt
, ctxt
->src
.addr
.mem
,
5417 ctxt
->src
.valptr
, ctxt
->src
.bytes
);
5418 if (rc
!= X86EMUL_CONTINUE
)
5420 ctxt
->src
.orig_val64
= ctxt
->src
.val64
;
5423 if (ctxt
->src2
.type
== OP_MEM
) {
5424 rc
= segmented_read(ctxt
, ctxt
->src2
.addr
.mem
,
5425 &ctxt
->src2
.val
, ctxt
->src2
.bytes
);
5426 if (rc
!= X86EMUL_CONTINUE
)
5430 if ((ctxt
->d
& DstMask
) == ImplicitOps
)
5434 if ((ctxt
->dst
.type
== OP_MEM
) && !(ctxt
->d
& Mov
)) {
5435 /* optimisation - avoid slow emulated read if Mov */
5436 rc
= segmented_read(ctxt
, ctxt
->dst
.addr
.mem
,
5437 &ctxt
->dst
.val
, ctxt
->dst
.bytes
);
5438 if (rc
!= X86EMUL_CONTINUE
) {
5439 if (!(ctxt
->d
& NoWrite
) &&
5440 rc
== X86EMUL_PROPAGATE_FAULT
&&
5441 ctxt
->exception
.vector
== PF_VECTOR
)
5442 ctxt
->exception
.error_code
|= PFERR_WRITE_MASK
;
5446 /* Copy full 64-bit value for CMPXCHG8B. */
5447 ctxt
->dst
.orig_val64
= ctxt
->dst
.val64
;
5451 if (unlikely(emul_flags
& X86EMUL_GUEST_MASK
) && (ctxt
->d
& Intercept
)) {
5452 rc
= emulator_check_intercept(ctxt
, ctxt
->intercept
,
5453 X86_ICPT_POST_MEMACCESS
);
5454 if (rc
!= X86EMUL_CONTINUE
)
5458 if (ctxt
->rep_prefix
&& (ctxt
->d
& String
))
5459 ctxt
->eflags
|= X86_EFLAGS_RF
;
5461 ctxt
->eflags
&= ~X86_EFLAGS_RF
;
5463 if (ctxt
->execute
) {
5464 if (ctxt
->d
& Fastop
) {
5465 void (*fop
)(struct fastop
*) = (void *)ctxt
->execute
;
5466 rc
= fastop(ctxt
, fop
);
5467 if (rc
!= X86EMUL_CONTINUE
)
5471 rc
= ctxt
->execute(ctxt
);
5472 if (rc
!= X86EMUL_CONTINUE
)
5477 if (ctxt
->opcode_len
== 2)
5479 else if (ctxt
->opcode_len
== 3)
5480 goto threebyte_insn
;
5483 case 0x70 ... 0x7f: /* jcc (short) */
5484 if (test_cc(ctxt
->b
, ctxt
->eflags
))
5485 rc
= jmp_rel(ctxt
, ctxt
->src
.val
);
5487 case 0x8d: /* lea r16/r32, m */
5488 ctxt
->dst
.val
= ctxt
->src
.addr
.mem
.ea
;
5490 case 0x90 ... 0x97: /* nop / xchg reg, rax */
5491 if (ctxt
->dst
.addr
.reg
== reg_rmw(ctxt
, VCPU_REGS_RAX
))
5492 ctxt
->dst
.type
= OP_NONE
;
5496 case 0x98: /* cbw/cwde/cdqe */
5497 switch (ctxt
->op_bytes
) {
5498 case 2: ctxt
->dst
.val
= (s8
)ctxt
->dst
.val
; break;
5499 case 4: ctxt
->dst
.val
= (s16
)ctxt
->dst
.val
; break;
5500 case 8: ctxt
->dst
.val
= (s32
)ctxt
->dst
.val
; break;
5503 case 0xcc: /* int3 */
5504 rc
= emulate_int(ctxt
, 3);
5506 case 0xcd: /* int n */
5507 rc
= emulate_int(ctxt
, ctxt
->src
.val
);
5509 case 0xce: /* into */
5510 if (ctxt
->eflags
& X86_EFLAGS_OF
)
5511 rc
= emulate_int(ctxt
, 4);
5513 case 0xe9: /* jmp rel */
5514 case 0xeb: /* jmp rel short */
5515 rc
= jmp_rel(ctxt
, ctxt
->src
.val
);
5516 ctxt
->dst
.type
= OP_NONE
; /* Disable writeback. */
5518 case 0xf4: /* hlt */
5519 ctxt
->ops
->halt(ctxt
);
5521 case 0xf5: /* cmc */
5522 /* complement carry flag from eflags reg */
5523 ctxt
->eflags
^= X86_EFLAGS_CF
;
5525 case 0xf8: /* clc */
5526 ctxt
->eflags
&= ~X86_EFLAGS_CF
;
5528 case 0xf9: /* stc */
5529 ctxt
->eflags
|= X86_EFLAGS_CF
;
5531 case 0xfc: /* cld */
5532 ctxt
->eflags
&= ~X86_EFLAGS_DF
;
5534 case 0xfd: /* std */
5535 ctxt
->eflags
|= X86_EFLAGS_DF
;
5538 goto cannot_emulate
;
5541 if (rc
!= X86EMUL_CONTINUE
)
5545 if (ctxt
->d
& SrcWrite
) {
5546 BUG_ON(ctxt
->src
.type
== OP_MEM
|| ctxt
->src
.type
== OP_MEM_STR
);
5547 rc
= writeback(ctxt
, &ctxt
->src
);
5548 if (rc
!= X86EMUL_CONTINUE
)
5551 if (!(ctxt
->d
& NoWrite
)) {
5552 rc
= writeback(ctxt
, &ctxt
->dst
);
5553 if (rc
!= X86EMUL_CONTINUE
)
5558 * restore dst type in case the decoding will be reused
5559 * (happens for string instruction )
5561 ctxt
->dst
.type
= saved_dst_type
;
5563 if ((ctxt
->d
& SrcMask
) == SrcSI
)
5564 string_addr_inc(ctxt
, VCPU_REGS_RSI
, &ctxt
->src
);
5566 if ((ctxt
->d
& DstMask
) == DstDI
)
5567 string_addr_inc(ctxt
, VCPU_REGS_RDI
, &ctxt
->dst
);
5569 if (ctxt
->rep_prefix
&& (ctxt
->d
& String
)) {
5571 struct read_cache
*r
= &ctxt
->io_read
;
5572 if ((ctxt
->d
& SrcMask
) == SrcSI
)
5573 count
= ctxt
->src
.count
;
5575 count
= ctxt
->dst
.count
;
5576 register_address_increment(ctxt
, VCPU_REGS_RCX
, -count
);
5578 if (!string_insn_completed(ctxt
)) {
5580 * Re-enter guest when pio read ahead buffer is empty
5581 * or, if it is not used, after each 1024 iteration.
5583 if ((r
->end
!= 0 || reg_read(ctxt
, VCPU_REGS_RCX
) & 0x3ff) &&
5584 (r
->end
== 0 || r
->end
!= r
->pos
)) {
5586 * Reset read cache. Usually happens before
5587 * decode, but since instruction is restarted
5588 * we have to do it here.
5590 ctxt
->mem_read
.end
= 0;
5591 writeback_registers(ctxt
);
5592 return EMULATION_RESTART
;
5594 goto done
; /* skip rip writeback */
5596 ctxt
->eflags
&= ~X86_EFLAGS_RF
;
5599 ctxt
->eip
= ctxt
->_eip
;
5602 if (rc
== X86EMUL_PROPAGATE_FAULT
) {
5603 WARN_ON(ctxt
->exception
.vector
> 0x1f);
5604 ctxt
->have_exception
= true;
5606 if (rc
== X86EMUL_INTERCEPTED
)
5607 return EMULATION_INTERCEPTED
;
5609 if (rc
== X86EMUL_CONTINUE
)
5610 writeback_registers(ctxt
);
5612 return (rc
== X86EMUL_UNHANDLEABLE
) ? EMULATION_FAILED
: EMULATION_OK
;
5616 case 0x09: /* wbinvd */
5617 (ctxt
->ops
->wbinvd
)(ctxt
);
5619 case 0x08: /* invd */
5620 case 0x0d: /* GrpP (prefetch) */
5621 case 0x18: /* Grp16 (prefetch/nop) */
5622 case 0x1f: /* nop */
5624 case 0x20: /* mov cr, reg */
5625 ctxt
->dst
.val
= ops
->get_cr(ctxt
, ctxt
->modrm_reg
);
5627 case 0x21: /* mov from dr to reg */
5628 ops
->get_dr(ctxt
, ctxt
->modrm_reg
, &ctxt
->dst
.val
);
5630 case 0x40 ... 0x4f: /* cmov */
5631 if (test_cc(ctxt
->b
, ctxt
->eflags
))
5632 ctxt
->dst
.val
= ctxt
->src
.val
;
5633 else if (ctxt
->op_bytes
!= 4)
5634 ctxt
->dst
.type
= OP_NONE
; /* no writeback */
5636 case 0x80 ... 0x8f: /* jnz rel, etc*/
5637 if (test_cc(ctxt
->b
, ctxt
->eflags
))
5638 rc
= jmp_rel(ctxt
, ctxt
->src
.val
);
5640 case 0x90 ... 0x9f: /* setcc r/m8 */
5641 ctxt
->dst
.val
= test_cc(ctxt
->b
, ctxt
->eflags
);
5643 case 0xb6 ... 0xb7: /* movzx */
5644 ctxt
->dst
.bytes
= ctxt
->op_bytes
;
5645 ctxt
->dst
.val
= (ctxt
->src
.bytes
== 1) ? (u8
) ctxt
->src
.val
5646 : (u16
) ctxt
->src
.val
;
5648 case 0xbe ... 0xbf: /* movsx */
5649 ctxt
->dst
.bytes
= ctxt
->op_bytes
;
5650 ctxt
->dst
.val
= (ctxt
->src
.bytes
== 1) ? (s8
) ctxt
->src
.val
:
5651 (s16
) ctxt
->src
.val
;
5654 goto cannot_emulate
;
5659 if (rc
!= X86EMUL_CONTINUE
)
5665 return EMULATION_FAILED
;
5668 void emulator_invalidate_register_cache(struct x86_emulate_ctxt
*ctxt
)
5670 invalidate_registers(ctxt
);
5673 void emulator_writeback_register_cache(struct x86_emulate_ctxt
*ctxt
)
5675 writeback_registers(ctxt
);
5678 bool emulator_can_use_gpa(struct x86_emulate_ctxt
*ctxt
)
5680 if (ctxt
->rep_prefix
&& (ctxt
->d
& String
))
5683 if (ctxt
->d
& TwoMemOp
)