1 /* bpf_jit_comp.c : BPF JIT compiler
3 * Copyright (C) 2011-2013 Eric Dumazet (eric.dumazet@gmail.com)
4 * Internal BPF Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; version 2
11 #include <linux/netdevice.h>
12 #include <linux/filter.h>
13 #include <linux/if_vlan.h>
14 #include <asm/cacheflush.h>
15 #include <linux/bpf.h>
17 int bpf_jit_enable __read_mostly
;
20 * assembly code in arch/x86/net/bpf_jit.S
22 extern u8 sk_load_word
[], sk_load_half
[], sk_load_byte
[];
23 extern u8 sk_load_word_positive_offset
[], sk_load_half_positive_offset
[];
24 extern u8 sk_load_byte_positive_offset
[];
25 extern u8 sk_load_word_negative_offset
[], sk_load_half_negative_offset
[];
26 extern u8 sk_load_byte_negative_offset
[];
28 static u8
*emit_code(u8
*ptr
, u32 bytes
, unsigned int len
)
41 #define EMIT(bytes, len) \
42 do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)
44 #define EMIT1(b1) EMIT(b1, 1)
45 #define EMIT2(b1, b2) EMIT((b1) + ((b2) << 8), 2)
46 #define EMIT3(b1, b2, b3) EMIT((b1) + ((b2) << 8) + ((b3) << 16), 3)
47 #define EMIT4(b1, b2, b3, b4) EMIT((b1) + ((b2) << 8) + ((b3) << 16) + ((b4) << 24), 4)
48 #define EMIT1_off32(b1, off) \
49 do {EMIT1(b1); EMIT(off, 4); } while (0)
50 #define EMIT2_off32(b1, b2, off) \
51 do {EMIT2(b1, b2); EMIT(off, 4); } while (0)
52 #define EMIT3_off32(b1, b2, b3, off) \
53 do {EMIT3(b1, b2, b3); EMIT(off, 4); } while (0)
54 #define EMIT4_off32(b1, b2, b3, b4, off) \
55 do {EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0)
57 static bool is_imm8(int value
)
59 return value
<= 127 && value
>= -128;
62 static bool is_simm32(s64 value
)
64 return value
== (s64
) (s32
) value
;
68 #define EMIT_mov(DST, SRC) \
70 EMIT3(add_2mod(0x48, DST, SRC), 0x89, add_2reg(0xC0, DST, SRC)); \
73 static int bpf_size_to_x86_bytes(int bpf_size
)
75 if (bpf_size
== BPF_W
)
77 else if (bpf_size
== BPF_H
)
79 else if (bpf_size
== BPF_B
)
81 else if (bpf_size
== BPF_DW
)
87 /* list of x86 cond jumps opcodes (. + s8)
88 * Add 0x10 (and an extra 0x0f) to generate far jumps (. + s32)
99 static void bpf_flush_icache(void *start
, void *end
)
101 mm_segment_t old_fs
= get_fs();
105 flush_icache_range((unsigned long)start
, (unsigned long)end
);
109 #define CHOOSE_LOAD_FUNC(K, func) \
110 ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset)
112 /* pick a register outside of BPF range for JIT internal work */
113 #define AUX_REG (MAX_BPF_JIT_REG + 1)
115 /* The following table maps BPF registers to x64 registers.
117 * x64 register r12 is unused, since if used as base address
118 * register in load/store instructions, it always needs an
119 * extra byte of encoding and is callee saved.
121 * r9 caches skb->len - skb->data_len
122 * r10 caches skb->data, and used for blinding (if enabled)
124 static const int reg2hex
[] = {
125 [BPF_REG_0
] = 0, /* rax */
126 [BPF_REG_1
] = 7, /* rdi */
127 [BPF_REG_2
] = 6, /* rsi */
128 [BPF_REG_3
] = 2, /* rdx */
129 [BPF_REG_4
] = 1, /* rcx */
130 [BPF_REG_5
] = 0, /* r8 */
131 [BPF_REG_6
] = 3, /* rbx callee saved */
132 [BPF_REG_7
] = 5, /* r13 callee saved */
133 [BPF_REG_8
] = 6, /* r14 callee saved */
134 [BPF_REG_9
] = 7, /* r15 callee saved */
135 [BPF_REG_FP
] = 5, /* rbp readonly */
136 [BPF_REG_AX
] = 2, /* r10 temp register */
137 [AUX_REG
] = 3, /* r11 temp register */
140 /* is_ereg() == true if BPF register 'reg' maps to x64 r8..r15
141 * which need extra byte of encoding.
142 * rax,rcx,...,rbp have simpler encoding
144 static bool is_ereg(u32 reg
)
146 return (1 << reg
) & (BIT(BPF_REG_5
) |
154 /* add modifiers if 'reg' maps to x64 registers r8..r15 */
155 static u8
add_1mod(u8 byte
, u32 reg
)
162 static u8
add_2mod(u8 byte
, u32 r1
, u32 r2
)
171 /* encode 'dst_reg' register into x64 opcode 'byte' */
172 static u8
add_1reg(u8 byte
, u32 dst_reg
)
174 return byte
+ reg2hex
[dst_reg
];
177 /* encode 'dst_reg' and 'src_reg' registers into x64 opcode 'byte' */
178 static u8
add_2reg(u8 byte
, u32 dst_reg
, u32 src_reg
)
180 return byte
+ reg2hex
[dst_reg
] + (reg2hex
[src_reg
] << 3);
183 static void jit_fill_hole(void *area
, unsigned int size
)
185 /* fill whole space with int3 instructions */
186 memset(area
, 0xcc, size
);
190 int cleanup_addr
; /* epilogue code offset */
195 /* maximum number of bytes emitted while JITing one eBPF insn */
196 #define BPF_MAX_INSN_SIZE 128
197 #define BPF_INSN_SAFETY 64
201 32 /* space for rbx, r13, r14, r15 */ + \
202 8 /* space for skb_copy_bits() buffer */)
204 #define PROLOGUE_SIZE 48
206 /* emit x64 prologue code for BPF program and check it's size.
207 * bpf_tail_call helper will skip it while jumping into another program
209 static void emit_prologue(u8
**pprog
)
214 EMIT1(0x55); /* push rbp */
215 EMIT3(0x48, 0x89, 0xE5); /* mov rbp,rsp */
217 /* sub rsp, STACKSIZE */
218 EMIT3_off32(0x48, 0x81, 0xEC, STACKSIZE
);
220 /* all classic BPF filters use R6(rbx) save it */
222 /* mov qword ptr [rbp-X],rbx */
223 EMIT3_off32(0x48, 0x89, 0x9D, -STACKSIZE
);
225 /* bpf_convert_filter() maps classic BPF register X to R7 and uses R8
226 * as temporary, so all tcpdump filters need to spill/fill R7(r13) and
227 * R8(r14). R9(r15) spill could be made conditional, but there is only
228 * one 'bpf_error' return path out of helper functions inside bpf_jit.S
229 * The overhead of extra spill is negligible for any filter other
230 * than synthetic ones. Therefore not worth adding complexity.
233 /* mov qword ptr [rbp-X],r13 */
234 EMIT3_off32(0x4C, 0x89, 0xAD, -STACKSIZE
+ 8);
235 /* mov qword ptr [rbp-X],r14 */
236 EMIT3_off32(0x4C, 0x89, 0xB5, -STACKSIZE
+ 16);
237 /* mov qword ptr [rbp-X],r15 */
238 EMIT3_off32(0x4C, 0x89, 0xBD, -STACKSIZE
+ 24);
240 /* Clear the tail call counter (tail_call_cnt): for eBPF tail calls
241 * we need to reset the counter to 0. It's done in two instructions,
242 * resetting rax register to 0 (xor on eax gets 0 extended), and
243 * moving it to the counter location.
248 /* mov qword ptr [rbp-X], rax */
249 EMIT3_off32(0x48, 0x89, 0x85, -STACKSIZE
+ 32);
251 BUILD_BUG_ON(cnt
!= PROLOGUE_SIZE
);
255 /* generate the following code:
256 * ... bpf_tail_call(void *ctx, struct bpf_array *array, u64 index) ...
257 * if (index >= array->map.max_entries)
259 * if (++tail_call_cnt > MAX_TAIL_CALL_CNT)
261 * prog = array->ptrs[index];
264 * goto *(prog->bpf_func + prologue_size);
267 static void emit_bpf_tail_call(u8
**pprog
)
270 int label1
, label2
, label3
;
273 /* rdi - pointer to ctx
274 * rsi - pointer to bpf_array
275 * rdx - index in bpf_array
278 /* if (index >= array->map.max_entries)
281 EMIT4(0x48, 0x8B, 0x46, /* mov rax, qword ptr [rsi + 16] */
282 offsetof(struct bpf_array
, map
.max_entries
));
283 EMIT3(0x48, 0x39, 0xD0); /* cmp rax, rdx */
284 #define OFFSET1 47 /* number of bytes to jump */
285 EMIT2(X86_JBE
, OFFSET1
); /* jbe out */
288 /* if (tail_call_cnt > MAX_TAIL_CALL_CNT)
291 EMIT2_off32(0x8B, 0x85, -STACKSIZE
+ 36); /* mov eax, dword ptr [rbp - 516] */
292 EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT
); /* cmp eax, MAX_TAIL_CALL_CNT */
294 EMIT2(X86_JA
, OFFSET2
); /* ja out */
296 EMIT3(0x83, 0xC0, 0x01); /* add eax, 1 */
297 EMIT2_off32(0x89, 0x85, -STACKSIZE
+ 36); /* mov dword ptr [rbp - 516], eax */
299 /* prog = array->ptrs[index]; */
300 EMIT4_off32(0x48, 0x8D, 0x84, 0xD6, /* lea rax, [rsi + rdx * 8 + offsetof(...)] */
301 offsetof(struct bpf_array
, ptrs
));
302 EMIT3(0x48, 0x8B, 0x00); /* mov rax, qword ptr [rax] */
307 EMIT4(0x48, 0x83, 0xF8, 0x00); /* cmp rax, 0 */
309 EMIT2(X86_JE
, OFFSET3
); /* je out */
312 /* goto *(prog->bpf_func + prologue_size); */
313 EMIT4(0x48, 0x8B, 0x40, /* mov rax, qword ptr [rax + 32] */
314 offsetof(struct bpf_prog
, bpf_func
));
315 EMIT4(0x48, 0x83, 0xC0, PROLOGUE_SIZE
); /* add rax, prologue_size */
317 /* now we're ready to jump into next BPF program
318 * rdi == ctx (1st arg)
319 * rax == prog->bpf_func + prologue_size
321 EMIT2(0xFF, 0xE0); /* jmp rax */
324 BUILD_BUG_ON(cnt
- label1
!= OFFSET1
);
325 BUILD_BUG_ON(cnt
- label2
!= OFFSET2
);
326 BUILD_BUG_ON(cnt
- label3
!= OFFSET3
);
331 static void emit_load_skb_data_hlen(u8
**pprog
)
336 /* r9d = skb->len - skb->data_len (headlen)
339 /* mov %r9d, off32(%rdi) */
340 EMIT3_off32(0x44, 0x8b, 0x8f, offsetof(struct sk_buff
, len
));
342 /* sub %r9d, off32(%rdi) */
343 EMIT3_off32(0x44, 0x2b, 0x8f, offsetof(struct sk_buff
, data_len
));
345 /* mov %r10, off32(%rdi) */
346 EMIT3_off32(0x4c, 0x8b, 0x97, offsetof(struct sk_buff
, data
));
350 static int do_jit(struct bpf_prog
*bpf_prog
, int *addrs
, u8
*image
,
351 int oldproglen
, struct jit_context
*ctx
)
353 struct bpf_insn
*insn
= bpf_prog
->insnsi
;
354 int insn_cnt
= bpf_prog
->len
;
355 bool seen_ld_abs
= ctx
->seen_ld_abs
| (oldproglen
== 0);
356 bool seen_ax_reg
= ctx
->seen_ax_reg
| (oldproglen
== 0);
357 bool seen_exit
= false;
358 u8 temp
[BPF_MAX_INSN_SIZE
+ BPF_INSN_SAFETY
];
363 emit_prologue(&prog
);
366 emit_load_skb_data_hlen(&prog
);
368 for (i
= 0; i
< insn_cnt
; i
++, insn
++) {
369 const s32 imm32
= insn
->imm
;
370 u32 dst_reg
= insn
->dst_reg
;
371 u32 src_reg
= insn
->src_reg
;
372 u8 b1
= 0, b2
= 0, b3
= 0;
375 bool reload_skb_data
;
379 if (dst_reg
== BPF_REG_AX
|| src_reg
== BPF_REG_AX
)
380 ctx
->seen_ax_reg
= seen_ax_reg
= true;
382 switch (insn
->code
) {
384 case BPF_ALU
| BPF_ADD
| BPF_X
:
385 case BPF_ALU
| BPF_SUB
| BPF_X
:
386 case BPF_ALU
| BPF_AND
| BPF_X
:
387 case BPF_ALU
| BPF_OR
| BPF_X
:
388 case BPF_ALU
| BPF_XOR
| BPF_X
:
389 case BPF_ALU64
| BPF_ADD
| BPF_X
:
390 case BPF_ALU64
| BPF_SUB
| BPF_X
:
391 case BPF_ALU64
| BPF_AND
| BPF_X
:
392 case BPF_ALU64
| BPF_OR
| BPF_X
:
393 case BPF_ALU64
| BPF_XOR
| BPF_X
:
394 switch (BPF_OP(insn
->code
)) {
395 case BPF_ADD
: b2
= 0x01; break;
396 case BPF_SUB
: b2
= 0x29; break;
397 case BPF_AND
: b2
= 0x21; break;
398 case BPF_OR
: b2
= 0x09; break;
399 case BPF_XOR
: b2
= 0x31; break;
401 if (BPF_CLASS(insn
->code
) == BPF_ALU64
)
402 EMIT1(add_2mod(0x48, dst_reg
, src_reg
));
403 else if (is_ereg(dst_reg
) || is_ereg(src_reg
))
404 EMIT1(add_2mod(0x40, dst_reg
, src_reg
));
405 EMIT2(b2
, add_2reg(0xC0, dst_reg
, src_reg
));
409 case BPF_ALU64
| BPF_MOV
| BPF_X
:
410 EMIT_mov(dst_reg
, src_reg
);
414 case BPF_ALU
| BPF_MOV
| BPF_X
:
415 if (is_ereg(dst_reg
) || is_ereg(src_reg
))
416 EMIT1(add_2mod(0x40, dst_reg
, src_reg
));
417 EMIT2(0x89, add_2reg(0xC0, dst_reg
, src_reg
));
421 case BPF_ALU
| BPF_NEG
:
422 case BPF_ALU64
| BPF_NEG
:
423 if (BPF_CLASS(insn
->code
) == BPF_ALU64
)
424 EMIT1(add_1mod(0x48, dst_reg
));
425 else if (is_ereg(dst_reg
))
426 EMIT1(add_1mod(0x40, dst_reg
));
427 EMIT2(0xF7, add_1reg(0xD8, dst_reg
));
430 case BPF_ALU
| BPF_ADD
| BPF_K
:
431 case BPF_ALU
| BPF_SUB
| BPF_K
:
432 case BPF_ALU
| BPF_AND
| BPF_K
:
433 case BPF_ALU
| BPF_OR
| BPF_K
:
434 case BPF_ALU
| BPF_XOR
| BPF_K
:
435 case BPF_ALU64
| BPF_ADD
| BPF_K
:
436 case BPF_ALU64
| BPF_SUB
| BPF_K
:
437 case BPF_ALU64
| BPF_AND
| BPF_K
:
438 case BPF_ALU64
| BPF_OR
| BPF_K
:
439 case BPF_ALU64
| BPF_XOR
| BPF_K
:
440 if (BPF_CLASS(insn
->code
) == BPF_ALU64
)
441 EMIT1(add_1mod(0x48, dst_reg
));
442 else if (is_ereg(dst_reg
))
443 EMIT1(add_1mod(0x40, dst_reg
));
445 switch (BPF_OP(insn
->code
)) {
446 case BPF_ADD
: b3
= 0xC0; break;
447 case BPF_SUB
: b3
= 0xE8; break;
448 case BPF_AND
: b3
= 0xE0; break;
449 case BPF_OR
: b3
= 0xC8; break;
450 case BPF_XOR
: b3
= 0xF0; break;
454 EMIT3(0x83, add_1reg(b3
, dst_reg
), imm32
);
456 EMIT2_off32(0x81, add_1reg(b3
, dst_reg
), imm32
);
459 case BPF_ALU64
| BPF_MOV
| BPF_K
:
460 /* optimization: if imm32 is positive,
461 * use 'mov eax, imm32' (which zero-extends imm32)
465 /* 'mov rax, imm32' sign extends imm32 */
466 b1
= add_1mod(0x48, dst_reg
);
469 EMIT3_off32(b1
, b2
, add_1reg(b3
, dst_reg
), imm32
);
473 case BPF_ALU
| BPF_MOV
| BPF_K
:
474 /* optimization: if imm32 is zero, use 'xor <dst>,<dst>'
478 if (is_ereg(dst_reg
))
479 EMIT1(add_2mod(0x40, dst_reg
, dst_reg
));
482 EMIT2(b2
, add_2reg(b3
, dst_reg
, dst_reg
));
486 /* mov %eax, imm32 */
487 if (is_ereg(dst_reg
))
488 EMIT1(add_1mod(0x40, dst_reg
));
489 EMIT1_off32(add_1reg(0xB8, dst_reg
), imm32
);
492 case BPF_LD
| BPF_IMM
| BPF_DW
:
493 if (insn
[1].code
!= 0 || insn
[1].src_reg
!= 0 ||
494 insn
[1].dst_reg
!= 0 || insn
[1].off
!= 0) {
495 /* verifier must catch invalid insns */
496 pr_err("invalid BPF_LD_IMM64 insn\n");
500 /* optimization: if imm64 is zero, use 'xor <dst>,<dst>'
503 if (insn
[0].imm
== 0 && insn
[1].imm
== 0) {
504 b1
= add_2mod(0x48, dst_reg
, dst_reg
);
507 EMIT3(b1
, b2
, add_2reg(b3
, dst_reg
, dst_reg
));
514 /* movabsq %rax, imm64 */
515 EMIT2(add_1mod(0x48, dst_reg
), add_1reg(0xB8, dst_reg
));
516 EMIT(insn
[0].imm
, 4);
517 EMIT(insn
[1].imm
, 4);
523 /* dst %= src, dst /= src, dst %= imm32, dst /= imm32 */
524 case BPF_ALU
| BPF_MOD
| BPF_X
:
525 case BPF_ALU
| BPF_DIV
| BPF_X
:
526 case BPF_ALU
| BPF_MOD
| BPF_K
:
527 case BPF_ALU
| BPF_DIV
| BPF_K
:
528 case BPF_ALU64
| BPF_MOD
| BPF_X
:
529 case BPF_ALU64
| BPF_DIV
| BPF_X
:
530 case BPF_ALU64
| BPF_MOD
| BPF_K
:
531 case BPF_ALU64
| BPF_DIV
| BPF_K
:
532 EMIT1(0x50); /* push rax */
533 EMIT1(0x52); /* push rdx */
535 if (BPF_SRC(insn
->code
) == BPF_X
)
536 /* mov r11, src_reg */
537 EMIT_mov(AUX_REG
, src_reg
);
540 EMIT3_off32(0x49, 0xC7, 0xC3, imm32
);
542 /* mov rax, dst_reg */
543 EMIT_mov(BPF_REG_0
, dst_reg
);
546 * equivalent to 'xor rdx, rdx', but one byte less
550 if (BPF_SRC(insn
->code
) == BPF_X
) {
551 /* if (src_reg == 0) return 0 */
554 EMIT4(0x49, 0x83, 0xFB, 0x00);
556 /* jne .+9 (skip over pop, pop, xor and jmp) */
557 EMIT2(X86_JNE
, 1 + 1 + 2 + 5);
558 EMIT1(0x5A); /* pop rdx */
559 EMIT1(0x58); /* pop rax */
560 EMIT2(0x31, 0xc0); /* xor eax, eax */
563 * addrs[i] - 11, because there are 11 bytes
564 * after this insn: div, mov, pop, pop, mov
566 jmp_offset
= ctx
->cleanup_addr
- (addrs
[i
] - 11);
567 EMIT1_off32(0xE9, jmp_offset
);
570 if (BPF_CLASS(insn
->code
) == BPF_ALU64
)
572 EMIT3(0x49, 0xF7, 0xF3);
575 EMIT3(0x41, 0xF7, 0xF3);
577 if (BPF_OP(insn
->code
) == BPF_MOD
)
579 EMIT3(0x49, 0x89, 0xD3);
582 EMIT3(0x49, 0x89, 0xC3);
584 EMIT1(0x5A); /* pop rdx */
585 EMIT1(0x58); /* pop rax */
587 /* mov dst_reg, r11 */
588 EMIT_mov(dst_reg
, AUX_REG
);
591 case BPF_ALU
| BPF_MUL
| BPF_K
:
592 case BPF_ALU
| BPF_MUL
| BPF_X
:
593 case BPF_ALU64
| BPF_MUL
| BPF_K
:
594 case BPF_ALU64
| BPF_MUL
| BPF_X
:
595 EMIT1(0x50); /* push rax */
596 EMIT1(0x52); /* push rdx */
598 /* mov r11, dst_reg */
599 EMIT_mov(AUX_REG
, dst_reg
);
601 if (BPF_SRC(insn
->code
) == BPF_X
)
602 /* mov rax, src_reg */
603 EMIT_mov(BPF_REG_0
, src_reg
);
606 EMIT3_off32(0x48, 0xC7, 0xC0, imm32
);
608 if (BPF_CLASS(insn
->code
) == BPF_ALU64
)
609 EMIT1(add_1mod(0x48, AUX_REG
));
610 else if (is_ereg(AUX_REG
))
611 EMIT1(add_1mod(0x40, AUX_REG
));
613 EMIT2(0xF7, add_1reg(0xE0, AUX_REG
));
616 EMIT_mov(AUX_REG
, BPF_REG_0
);
618 EMIT1(0x5A); /* pop rdx */
619 EMIT1(0x58); /* pop rax */
621 /* mov dst_reg, r11 */
622 EMIT_mov(dst_reg
, AUX_REG
);
626 case BPF_ALU
| BPF_LSH
| BPF_K
:
627 case BPF_ALU
| BPF_RSH
| BPF_K
:
628 case BPF_ALU
| BPF_ARSH
| BPF_K
:
629 case BPF_ALU64
| BPF_LSH
| BPF_K
:
630 case BPF_ALU64
| BPF_RSH
| BPF_K
:
631 case BPF_ALU64
| BPF_ARSH
| BPF_K
:
632 if (BPF_CLASS(insn
->code
) == BPF_ALU64
)
633 EMIT1(add_1mod(0x48, dst_reg
));
634 else if (is_ereg(dst_reg
))
635 EMIT1(add_1mod(0x40, dst_reg
));
637 switch (BPF_OP(insn
->code
)) {
638 case BPF_LSH
: b3
= 0xE0; break;
639 case BPF_RSH
: b3
= 0xE8; break;
640 case BPF_ARSH
: b3
= 0xF8; break;
642 EMIT3(0xC1, add_1reg(b3
, dst_reg
), imm32
);
645 case BPF_ALU
| BPF_LSH
| BPF_X
:
646 case BPF_ALU
| BPF_RSH
| BPF_X
:
647 case BPF_ALU
| BPF_ARSH
| BPF_X
:
648 case BPF_ALU64
| BPF_LSH
| BPF_X
:
649 case BPF_ALU64
| BPF_RSH
| BPF_X
:
650 case BPF_ALU64
| BPF_ARSH
| BPF_X
:
652 /* check for bad case when dst_reg == rcx */
653 if (dst_reg
== BPF_REG_4
) {
654 /* mov r11, dst_reg */
655 EMIT_mov(AUX_REG
, dst_reg
);
659 if (src_reg
!= BPF_REG_4
) { /* common case */
660 EMIT1(0x51); /* push rcx */
662 /* mov rcx, src_reg */
663 EMIT_mov(BPF_REG_4
, src_reg
);
666 /* shl %rax, %cl | shr %rax, %cl | sar %rax, %cl */
667 if (BPF_CLASS(insn
->code
) == BPF_ALU64
)
668 EMIT1(add_1mod(0x48, dst_reg
));
669 else if (is_ereg(dst_reg
))
670 EMIT1(add_1mod(0x40, dst_reg
));
672 switch (BPF_OP(insn
->code
)) {
673 case BPF_LSH
: b3
= 0xE0; break;
674 case BPF_RSH
: b3
= 0xE8; break;
675 case BPF_ARSH
: b3
= 0xF8; break;
677 EMIT2(0xD3, add_1reg(b3
, dst_reg
));
679 if (src_reg
!= BPF_REG_4
)
680 EMIT1(0x59); /* pop rcx */
682 if (insn
->dst_reg
== BPF_REG_4
)
683 /* mov dst_reg, r11 */
684 EMIT_mov(insn
->dst_reg
, AUX_REG
);
687 case BPF_ALU
| BPF_END
| BPF_FROM_BE
:
690 /* emit 'ror %ax, 8' to swap lower 2 bytes */
692 if (is_ereg(dst_reg
))
694 EMIT3(0xC1, add_1reg(0xC8, dst_reg
), 8);
696 /* emit 'movzwl eax, ax' */
697 if (is_ereg(dst_reg
))
698 EMIT3(0x45, 0x0F, 0xB7);
701 EMIT1(add_2reg(0xC0, dst_reg
, dst_reg
));
704 /* emit 'bswap eax' to swap lower 4 bytes */
705 if (is_ereg(dst_reg
))
709 EMIT1(add_1reg(0xC8, dst_reg
));
712 /* emit 'bswap rax' to swap 8 bytes */
713 EMIT3(add_1mod(0x48, dst_reg
), 0x0F,
714 add_1reg(0xC8, dst_reg
));
719 case BPF_ALU
| BPF_END
| BPF_FROM_LE
:
722 /* emit 'movzwl eax, ax' to zero extend 16-bit
725 if (is_ereg(dst_reg
))
726 EMIT3(0x45, 0x0F, 0xB7);
729 EMIT1(add_2reg(0xC0, dst_reg
, dst_reg
));
732 /* emit 'mov eax, eax' to clear upper 32-bits */
733 if (is_ereg(dst_reg
))
735 EMIT2(0x89, add_2reg(0xC0, dst_reg
, dst_reg
));
743 /* ST: *(u8*)(dst_reg + off) = imm */
744 case BPF_ST
| BPF_MEM
| BPF_B
:
745 if (is_ereg(dst_reg
))
750 case BPF_ST
| BPF_MEM
| BPF_H
:
751 if (is_ereg(dst_reg
))
752 EMIT3(0x66, 0x41, 0xC7);
756 case BPF_ST
| BPF_MEM
| BPF_W
:
757 if (is_ereg(dst_reg
))
762 case BPF_ST
| BPF_MEM
| BPF_DW
:
763 EMIT2(add_1mod(0x48, dst_reg
), 0xC7);
765 st
: if (is_imm8(insn
->off
))
766 EMIT2(add_1reg(0x40, dst_reg
), insn
->off
);
768 EMIT1_off32(add_1reg(0x80, dst_reg
), insn
->off
);
770 EMIT(imm32
, bpf_size_to_x86_bytes(BPF_SIZE(insn
->code
)));
773 /* STX: *(u8*)(dst_reg + off) = src_reg */
774 case BPF_STX
| BPF_MEM
| BPF_B
:
775 /* emit 'mov byte ptr [rax + off], al' */
776 if (is_ereg(dst_reg
) || is_ereg(src_reg
) ||
777 /* have to add extra byte for x86 SIL, DIL regs */
778 src_reg
== BPF_REG_1
|| src_reg
== BPF_REG_2
)
779 EMIT2(add_2mod(0x40, dst_reg
, src_reg
), 0x88);
783 case BPF_STX
| BPF_MEM
| BPF_H
:
784 if (is_ereg(dst_reg
) || is_ereg(src_reg
))
785 EMIT3(0x66, add_2mod(0x40, dst_reg
, src_reg
), 0x89);
789 case BPF_STX
| BPF_MEM
| BPF_W
:
790 if (is_ereg(dst_reg
) || is_ereg(src_reg
))
791 EMIT2(add_2mod(0x40, dst_reg
, src_reg
), 0x89);
795 case BPF_STX
| BPF_MEM
| BPF_DW
:
796 EMIT2(add_2mod(0x48, dst_reg
, src_reg
), 0x89);
797 stx
: if (is_imm8(insn
->off
))
798 EMIT2(add_2reg(0x40, dst_reg
, src_reg
), insn
->off
);
800 EMIT1_off32(add_2reg(0x80, dst_reg
, src_reg
),
804 /* LDX: dst_reg = *(u8*)(src_reg + off) */
805 case BPF_LDX
| BPF_MEM
| BPF_B
:
806 /* emit 'movzx rax, byte ptr [rax + off]' */
807 EMIT3(add_2mod(0x48, src_reg
, dst_reg
), 0x0F, 0xB6);
809 case BPF_LDX
| BPF_MEM
| BPF_H
:
810 /* emit 'movzx rax, word ptr [rax + off]' */
811 EMIT3(add_2mod(0x48, src_reg
, dst_reg
), 0x0F, 0xB7);
813 case BPF_LDX
| BPF_MEM
| BPF_W
:
814 /* emit 'mov eax, dword ptr [rax+0x14]' */
815 if (is_ereg(dst_reg
) || is_ereg(src_reg
))
816 EMIT2(add_2mod(0x40, src_reg
, dst_reg
), 0x8B);
820 case BPF_LDX
| BPF_MEM
| BPF_DW
:
821 /* emit 'mov rax, qword ptr [rax+0x14]' */
822 EMIT2(add_2mod(0x48, src_reg
, dst_reg
), 0x8B);
823 ldx
: /* if insn->off == 0 we can save one extra byte, but
824 * special case of x86 r13 which always needs an offset
825 * is not worth the hassle
827 if (is_imm8(insn
->off
))
828 EMIT2(add_2reg(0x40, src_reg
, dst_reg
), insn
->off
);
830 EMIT1_off32(add_2reg(0x80, src_reg
, dst_reg
),
834 /* STX XADD: lock *(u32*)(dst_reg + off) += src_reg */
835 case BPF_STX
| BPF_XADD
| BPF_W
:
836 /* emit 'lock add dword ptr [rax + off], eax' */
837 if (is_ereg(dst_reg
) || is_ereg(src_reg
))
838 EMIT3(0xF0, add_2mod(0x40, dst_reg
, src_reg
), 0x01);
842 case BPF_STX
| BPF_XADD
| BPF_DW
:
843 EMIT3(0xF0, add_2mod(0x48, dst_reg
, src_reg
), 0x01);
844 xadd
: if (is_imm8(insn
->off
))
845 EMIT2(add_2reg(0x40, dst_reg
, src_reg
), insn
->off
);
847 EMIT1_off32(add_2reg(0x80, dst_reg
, src_reg
),
852 case BPF_JMP
| BPF_CALL
:
853 func
= (u8
*) __bpf_call_base
+ imm32
;
854 jmp_offset
= func
- (image
+ addrs
[i
]);
856 reload_skb_data
= bpf_helper_changes_pkt_data(func
);
857 if (reload_skb_data
) {
858 EMIT1(0x57); /* push %rdi */
859 jmp_offset
+= 22; /* pop, mov, sub, mov */
861 EMIT2(0x41, 0x52); /* push %r10 */
862 EMIT2(0x41, 0x51); /* push %r9 */
863 /* need to adjust jmp offset, since
864 * pop %r9, pop %r10 take 4 bytes after call insn
869 if (!imm32
|| !is_simm32(jmp_offset
)) {
870 pr_err("unsupported bpf func %d addr %p image %p\n",
874 EMIT1_off32(0xE8, jmp_offset
);
876 if (reload_skb_data
) {
877 EMIT1(0x5F); /* pop %rdi */
878 emit_load_skb_data_hlen(&prog
);
880 EMIT2(0x41, 0x59); /* pop %r9 */
881 EMIT2(0x41, 0x5A); /* pop %r10 */
886 case BPF_JMP
| BPF_CALL
| BPF_X
:
887 emit_bpf_tail_call(&prog
);
891 case BPF_JMP
| BPF_JEQ
| BPF_X
:
892 case BPF_JMP
| BPF_JNE
| BPF_X
:
893 case BPF_JMP
| BPF_JGT
| BPF_X
:
894 case BPF_JMP
| BPF_JGE
| BPF_X
:
895 case BPF_JMP
| BPF_JSGT
| BPF_X
:
896 case BPF_JMP
| BPF_JSGE
| BPF_X
:
897 /* cmp dst_reg, src_reg */
898 EMIT3(add_2mod(0x48, dst_reg
, src_reg
), 0x39,
899 add_2reg(0xC0, dst_reg
, src_reg
));
902 case BPF_JMP
| BPF_JSET
| BPF_X
:
903 /* test dst_reg, src_reg */
904 EMIT3(add_2mod(0x48, dst_reg
, src_reg
), 0x85,
905 add_2reg(0xC0, dst_reg
, src_reg
));
908 case BPF_JMP
| BPF_JSET
| BPF_K
:
909 /* test dst_reg, imm32 */
910 EMIT1(add_1mod(0x48, dst_reg
));
911 EMIT2_off32(0xF7, add_1reg(0xC0, dst_reg
), imm32
);
914 case BPF_JMP
| BPF_JEQ
| BPF_K
:
915 case BPF_JMP
| BPF_JNE
| BPF_K
:
916 case BPF_JMP
| BPF_JGT
| BPF_K
:
917 case BPF_JMP
| BPF_JGE
| BPF_K
:
918 case BPF_JMP
| BPF_JSGT
| BPF_K
:
919 case BPF_JMP
| BPF_JSGE
| BPF_K
:
920 /* cmp dst_reg, imm8/32 */
921 EMIT1(add_1mod(0x48, dst_reg
));
924 EMIT3(0x83, add_1reg(0xF8, dst_reg
), imm32
);
926 EMIT2_off32(0x81, add_1reg(0xF8, dst_reg
), imm32
);
928 emit_cond_jmp
: /* convert BPF opcode to x86 */
929 switch (BPF_OP(insn
->code
)) {
938 /* GT is unsigned '>', JA in x86 */
942 /* GE is unsigned '>=', JAE in x86 */
946 /* signed '>', GT in x86 */
950 /* signed '>=', GE in x86 */
953 default: /* to silence gcc warning */
956 jmp_offset
= addrs
[i
+ insn
->off
] - addrs
[i
];
957 if (is_imm8(jmp_offset
)) {
958 EMIT2(jmp_cond
, jmp_offset
);
959 } else if (is_simm32(jmp_offset
)) {
960 EMIT2_off32(0x0F, jmp_cond
+ 0x10, jmp_offset
);
962 pr_err("cond_jmp gen bug %llx\n", jmp_offset
);
968 case BPF_JMP
| BPF_JA
:
969 jmp_offset
= addrs
[i
+ insn
->off
] - addrs
[i
];
971 /* optimize out nop jumps */
974 if (is_imm8(jmp_offset
)) {
975 EMIT2(0xEB, jmp_offset
);
976 } else if (is_simm32(jmp_offset
)) {
977 EMIT1_off32(0xE9, jmp_offset
);
979 pr_err("jmp gen bug %llx\n", jmp_offset
);
984 case BPF_LD
| BPF_IND
| BPF_W
:
987 case BPF_LD
| BPF_ABS
| BPF_W
:
988 func
= CHOOSE_LOAD_FUNC(imm32
, sk_load_word
);
990 ctx
->seen_ld_abs
= seen_ld_abs
= true;
991 jmp_offset
= func
- (image
+ addrs
[i
]);
992 if (!func
|| !is_simm32(jmp_offset
)) {
993 pr_err("unsupported bpf func %d addr %p image %p\n",
997 if (BPF_MODE(insn
->code
) == BPF_ABS
) {
998 /* mov %esi, imm32 */
999 EMIT1_off32(0xBE, imm32
);
1001 /* mov %rsi, src_reg */
1002 EMIT_mov(BPF_REG_2
, src_reg
);
1005 /* add %esi, imm8 */
1006 EMIT3(0x83, 0xC6, imm32
);
1008 /* add %esi, imm32 */
1009 EMIT2_off32(0x81, 0xC6, imm32
);
1012 /* skb pointer is in R6 (%rbx), it will be copied into
1013 * %rdi if skb_copy_bits() call is necessary.
1014 * sk_load_* helpers also use %r10 and %r9d.
1018 /* r10 = skb->data, mov %r10, off32(%rbx) */
1019 EMIT3_off32(0x4c, 0x8b, 0x93,
1020 offsetof(struct sk_buff
, data
));
1021 EMIT1_off32(0xE8, jmp_offset
); /* call */
1024 case BPF_LD
| BPF_IND
| BPF_H
:
1025 func
= sk_load_half
;
1027 case BPF_LD
| BPF_ABS
| BPF_H
:
1028 func
= CHOOSE_LOAD_FUNC(imm32
, sk_load_half
);
1030 case BPF_LD
| BPF_IND
| BPF_B
:
1031 func
= sk_load_byte
;
1033 case BPF_LD
| BPF_ABS
| BPF_B
:
1034 func
= CHOOSE_LOAD_FUNC(imm32
, sk_load_byte
);
1037 case BPF_JMP
| BPF_EXIT
:
1039 jmp_offset
= ctx
->cleanup_addr
- addrs
[i
];
1043 /* update cleanup_addr */
1044 ctx
->cleanup_addr
= proglen
;
1045 /* mov rbx, qword ptr [rbp-X] */
1046 EMIT3_off32(0x48, 0x8B, 0x9D, -STACKSIZE
);
1047 /* mov r13, qword ptr [rbp-X] */
1048 EMIT3_off32(0x4C, 0x8B, 0xAD, -STACKSIZE
+ 8);
1049 /* mov r14, qword ptr [rbp-X] */
1050 EMIT3_off32(0x4C, 0x8B, 0xB5, -STACKSIZE
+ 16);
1051 /* mov r15, qword ptr [rbp-X] */
1052 EMIT3_off32(0x4C, 0x8B, 0xBD, -STACKSIZE
+ 24);
1054 EMIT1(0xC9); /* leave */
1055 EMIT1(0xC3); /* ret */
1059 /* By design x64 JIT should support all BPF instructions
1060 * This error will be seen if new instruction was added
1061 * to interpreter, but not to JIT
1062 * or if there is junk in bpf_prog
1064 pr_err("bpf_jit: unknown opcode %02x\n", insn
->code
);
1069 if (ilen
> BPF_MAX_INSN_SIZE
) {
1070 pr_err("bpf_jit: fatal insn size error\n");
1075 if (unlikely(proglen
+ ilen
> oldproglen
)) {
1076 pr_err("bpf_jit: fatal error\n");
1079 memcpy(image
+ proglen
, temp
, ilen
);
1088 struct bpf_prog
*bpf_int_jit_compile(struct bpf_prog
*prog
)
1090 struct bpf_binary_header
*header
= NULL
;
1091 struct bpf_prog
*tmp
, *orig_prog
= prog
;
1092 int proglen
, oldproglen
= 0;
1093 struct jit_context ctx
= {};
1094 bool tmp_blinded
= false;
1100 if (!bpf_jit_enable
)
1103 tmp
= bpf_jit_blind_constants(prog
);
1104 /* If blinding was requested and we failed during blinding,
1105 * we must fall back to the interpreter.
1114 addrs
= kmalloc(prog
->len
* sizeof(*addrs
), GFP_KERNEL
);
1120 /* Before first pass, make a rough estimation of addrs[]
1121 * each bpf instruction is translated to less than 64 bytes
1123 for (proglen
= 0, i
= 0; i
< prog
->len
; i
++) {
1127 ctx
.cleanup_addr
= proglen
;
1129 /* JITed image shrinks with every pass and the loop iterates
1130 * until the image stops shrinking. Very large bpf programs
1131 * may converge on the last pass. In such case do one more
1132 * pass to emit the final image
1134 for (pass
= 0; pass
< 10 || image
; pass
++) {
1135 proglen
= do_jit(prog
, addrs
, image
, oldproglen
, &ctx
);
1139 bpf_jit_binary_free(header
);
1144 if (proglen
!= oldproglen
) {
1145 pr_err("bpf_jit: proglen=%d != oldproglen=%d\n",
1146 proglen
, oldproglen
);
1152 if (proglen
== oldproglen
) {
1153 header
= bpf_jit_binary_alloc(proglen
, &image
,
1160 oldproglen
= proglen
;
1163 if (bpf_jit_enable
> 1)
1164 bpf_jit_dump(prog
->len
, proglen
, pass
+ 1, image
);
1167 bpf_flush_icache(header
, image
+ proglen
);
1168 bpf_jit_binary_lock_ro(header
);
1169 prog
->bpf_func
= (void *)image
;
1179 bpf_jit_prog_release_other(prog
, prog
== orig_prog
?