?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
irqchip: Fix dependencies for archs w/o HAS_IOMEM
[linux/fpc-iii.git] / arch / s390 / net / bpf_jit_comp.c
blob9a0c4c22e53670b1d813f3ddfb328b76c8e06c78
1 /*
2 * BPF Jit compiler for s390.
3 *
4 * Minimum build requirements:
5 *
6 * - HAVE_MARCH_Z196_FEATURES: laal, laalg
7 * - HAVE_MARCH_Z10_FEATURES: msfi, cgrj, clgrj
8 * - HAVE_MARCH_Z9_109_FEATURES: alfi, llilf, clfi, oilf, nilf
9 * - PACK_STACK
10 * - 64BIT
11 *
12 * Copyright IBM Corp. 2012,2015
13 *
14 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
15 * Michael Holzheu <holzheu@linux.vnet.ibm.com>
16 */
17
18 #define KMSG_COMPONENT "bpf_jit"
19 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
20
21 #include <linux/netdevice.h>
22 #include <linux/filter.h>
23 #include <linux/init.h>
24 #include <linux/bpf.h>
25 #include <asm/cacheflush.h>
26 #include <asm/dis.h>
27 #include "bpf_jit.h"
28
29 int bpf_jit_enable __read_mostly;
30
31 struct bpf_jit {
32 u32 seen; /* Flags to remember seen eBPF instructions */
33 u32 seen_reg[16]; /* Array to remember which registers are used */
34 u32 *addrs; /* Array with relative instruction addresses */
35 u8 *prg_buf; /* Start of program */
36 int size; /* Size of program and literal pool */
37 int size_prg; /* Size of program */
38 int prg; /* Current position in program */
39 int lit_start; /* Start of literal pool */
40 int lit; /* Current position in literal pool */
41 int base_ip; /* Base address for literal pool */
42 int ret0_ip; /* Address of return 0 */
43 int exit_ip; /* Address of exit */
44 int tail_call_start; /* Tail call start offset */
45 int labels[1]; /* Labels for local jumps */
46 };
47
48 #define BPF_SIZE_MAX 0x7ffff /* Max size for program (20 bit signed displ) */
49
50 #define SEEN_SKB 1 /* skb access */
51 #define SEEN_MEM 2 /* use mem[] for temporary storage */
52 #define SEEN_RET0 4 /* ret0_ip points to a valid return 0 */
53 #define SEEN_LITERAL 8 /* code uses literals */
54 #define SEEN_FUNC 16 /* calls C functions */
55 #define SEEN_TAIL_CALL 32 /* code uses tail calls */
56 #define SEEN_SKB_CHANGE 64 /* code changes skb data */
57 #define SEEN_STACK (SEEN_FUNC | SEEN_MEM | SEEN_SKB)
58
59 /*
60 * s390 registers
61 */
62 #define REG_W0 (__MAX_BPF_REG+0) /* Work register 1 (even) */
63 #define REG_W1 (__MAX_BPF_REG+1) /* Work register 2 (odd) */
64 #define REG_SKB_DATA (__MAX_BPF_REG+2) /* SKB data register */
65 #define REG_L (__MAX_BPF_REG+3) /* Literal pool register */
66 #define REG_15 (__MAX_BPF_REG+4) /* Register 15 */
67 #define REG_0 REG_W0 /* Register 0 */
68 #define REG_1 REG_W1 /* Register 1 */
69 #define REG_2 BPF_REG_1 /* Register 2 */
70 #define REG_14 BPF_REG_0 /* Register 14 */
71
72 /*
73 * Mapping of BPF registers to s390 registers
74 */
75 static const int reg2hex[] = {
76 /* Return code */
77 [BPF_REG_0] = 14,
78 /* Function parameters */
79 [BPF_REG_1] = 2,
80 [BPF_REG_2] = 3,
81 [BPF_REG_3] = 4,
82 [BPF_REG_4] = 5,
83 [BPF_REG_5] = 6,
84 /* Call saved registers */
85 [BPF_REG_6] = 7,
86 [BPF_REG_7] = 8,
87 [BPF_REG_8] = 9,
88 [BPF_REG_9] = 10,
89 /* BPF stack pointer */
90 [BPF_REG_FP] = 13,
91 /* SKB data pointer */
92 [REG_SKB_DATA] = 12,
93 /* Work registers for s390x backend */
94 [REG_W0] = 0,
95 [REG_W1] = 1,
96 [REG_L] = 11,
97 [REG_15] = 15,
98 };
99
100 static inline u32 reg(u32 dst_reg, u32 src_reg)
101 {
102 return reg2hex[dst_reg] << 4 | reg2hex[src_reg];
103 }
104
105 static inline u32 reg_high(u32 reg)
106 {
107 return reg2hex[reg] << 4;
108 }
109
110 static inline void reg_set_seen(struct bpf_jit *jit, u32 b1)
111 {
112 u32 r1 = reg2hex[b1];
113
114 if (!jit->seen_reg[r1] && r1 >= 6 && r1 <= 15)
115 jit->seen_reg[r1] = 1;
116 }
117
118 #define REG_SET_SEEN(b1) \
119 ({ \
120 reg_set_seen(jit, b1); \
121 })
122
123 #define REG_SEEN(b1) jit->seen_reg[reg2hex[(b1)]]
124
125 /*
126 * EMIT macros for code generation
127 */
128
129 #define _EMIT2(op) \
130 ({ \
131 if (jit->prg_buf) \
132 *(u16 *) (jit->prg_buf + jit->prg) = op; \
133 jit->prg += 2; \
134 })
135
136 #define EMIT2(op, b1, b2) \
137 ({ \
138 _EMIT2(op | reg(b1, b2)); \
139 REG_SET_SEEN(b1); \
140 REG_SET_SEEN(b2); \
141 })
142
143 #define _EMIT4(op) \
144 ({ \
145 if (jit->prg_buf) \
146 *(u32 *) (jit->prg_buf + jit->prg) = op; \
147 jit->prg += 4; \
148 })
149
150 #define EMIT4(op, b1, b2) \
151 ({ \
152 _EMIT4(op | reg(b1, b2)); \
153 REG_SET_SEEN(b1); \
154 REG_SET_SEEN(b2); \
155 })
156
157 #define EMIT4_RRF(op, b1, b2, b3) \
158 ({ \
159 _EMIT4(op | reg_high(b3) << 8 | reg(b1, b2)); \
160 REG_SET_SEEN(b1); \
161 REG_SET_SEEN(b2); \
162 REG_SET_SEEN(b3); \
163 })
164
165 #define _EMIT4_DISP(op, disp) \
166 ({ \
167 unsigned int __disp = (disp) & 0xfff; \
168 _EMIT4(op | __disp); \
169 })
170
171 #define EMIT4_DISP(op, b1, b2, disp) \
172 ({ \
173 _EMIT4_DISP(op | reg_high(b1) << 16 | \
174 reg_high(b2) << 8, disp); \
175 REG_SET_SEEN(b1); \
176 REG_SET_SEEN(b2); \
177 })
178
179 #define EMIT4_IMM(op, b1, imm) \
180 ({ \
181 unsigned int __imm = (imm) & 0xffff; \
182 _EMIT4(op | reg_high(b1) << 16 | __imm); \
183 REG_SET_SEEN(b1); \
184 })
185
186 #define EMIT4_PCREL(op, pcrel) \
187 ({ \
188 long __pcrel = ((pcrel) >> 1) & 0xffff; \
189 _EMIT4(op | __pcrel); \
190 })
191
192 #define _EMIT6(op1, op2) \
193 ({ \
194 if (jit->prg_buf) { \
195 *(u32 *) (jit->prg_buf + jit->prg) = op1; \
196 *(u16 *) (jit->prg_buf + jit->prg + 4) = op2; \
197 } \
198 jit->prg += 6; \
199 })
200
201 #define _EMIT6_DISP(op1, op2, disp) \
202 ({ \
203 unsigned int __disp = (disp) & 0xfff; \
204 _EMIT6(op1 | __disp, op2); \
205 })
206
207 #define _EMIT6_DISP_LH(op1, op2, disp) \
208 ({ \
209 u32 _disp = (u32) disp; \
210 unsigned int __disp_h = _disp & 0xff000; \
211 unsigned int __disp_l = _disp & 0x00fff; \
212 _EMIT6(op1 | __disp_l, op2 | __disp_h >> 4); \
213 })
214
215 #define EMIT6_DISP_LH(op1, op2, b1, b2, b3, disp) \
216 ({ \
217 _EMIT6_DISP_LH(op1 | reg(b1, b2) << 16 | \
218 reg_high(b3) << 8, op2, disp); \
219 REG_SET_SEEN(b1); \
220 REG_SET_SEEN(b2); \
221 REG_SET_SEEN(b3); \
222 })
223
224 #define EMIT6_PCREL_LABEL(op1, op2, b1, b2, label, mask) \
225 ({ \
226 int rel = (jit->labels[label] - jit->prg) >> 1; \
227 _EMIT6(op1 | reg(b1, b2) << 16 | (rel & 0xffff), \
228 op2 | mask << 12); \
229 REG_SET_SEEN(b1); \
230 REG_SET_SEEN(b2); \
231 })
232
233 #define EMIT6_PCREL_IMM_LABEL(op1, op2, b1, imm, label, mask) \
234 ({ \
235 int rel = (jit->labels[label] - jit->prg) >> 1; \
236 _EMIT6(op1 | (reg_high(b1) | mask) << 16 | \
237 (rel & 0xffff), op2 | (imm & 0xff) << 8); \
238 REG_SET_SEEN(b1); \
239 BUILD_BUG_ON(((unsigned long) imm) > 0xff); \
240 })
241
242 #define EMIT6_PCREL(op1, op2, b1, b2, i, off, mask) \
243 ({ \
244 /* Branch instruction needs 6 bytes */ \
245 int rel = (addrs[i + off + 1] - (addrs[i + 1] - 6)) / 2;\
246 _EMIT6(op1 | reg(b1, b2) << 16 | (rel & 0xffff), op2 | mask); \
247 REG_SET_SEEN(b1); \
248 REG_SET_SEEN(b2); \
249 })
250
251 #define _EMIT6_IMM(op, imm) \
252 ({ \
253 unsigned int __imm = (imm); \
254 _EMIT6(op | (__imm >> 16), __imm & 0xffff); \
255 })
256
257 #define EMIT6_IMM(op, b1, imm) \
258 ({ \
259 _EMIT6_IMM(op | reg_high(b1) << 16, imm); \
260 REG_SET_SEEN(b1); \
261 })
262
263 #define EMIT_CONST_U32(val) \
264 ({ \
265 unsigned int ret; \
266 ret = jit->lit - jit->base_ip; \
267 jit->seen |= SEEN_LITERAL; \
268 if (jit->prg_buf) \
269 *(u32 *) (jit->prg_buf + jit->lit) = (u32) val; \
270 jit->lit += 4; \
271 ret; \
272 })
273
274 #define EMIT_CONST_U64(val) \
275 ({ \
276 unsigned int ret; \
277 ret = jit->lit - jit->base_ip; \
278 jit->seen |= SEEN_LITERAL; \
279 if (jit->prg_buf) \
280 *(u64 *) (jit->prg_buf + jit->lit) = (u64) val; \
281 jit->lit += 8; \
282 ret; \
283 })
284
285 #define EMIT_ZERO(b1) \
286 ({ \
287 /* llgfr %dst,%dst (zero extend to 64 bit) */ \
288 EMIT4(0xb9160000, b1, b1); \
289 REG_SET_SEEN(b1); \
290 })
291
292 /*
293 * Fill whole space with illegal instructions
294 */
295 static void jit_fill_hole(void *area, unsigned int size)
296 {
297 memset(area, 0, size);
298 }
299
300 /*
301 * Save registers from "rs" (register start) to "re" (register end) on stack
302 */
303 static void save_regs(struct bpf_jit *jit, u32 rs, u32 re)
304 {
305 u32 off = STK_OFF_R6 + (rs - 6) * 8;
306
307 if (rs == re)
308 /* stg %rs,off(%r15) */
309 _EMIT6(0xe300f000 | rs << 20 | off, 0x0024);
310 else
311 /* stmg %rs,%re,off(%r15) */
312 _EMIT6_DISP(0xeb00f000 | rs << 20 | re << 16, 0x0024, off);
313 }
314
315 /*
316 * Restore registers from "rs" (register start) to "re" (register end) on stack
317 */
318 static void restore_regs(struct bpf_jit *jit, u32 rs, u32 re)
319 {
320 u32 off = STK_OFF_R6 + (rs - 6) * 8;
321
322 if (jit->seen & SEEN_STACK)
323 off += STK_OFF;
324
325 if (rs == re)
326 /* lg %rs,off(%r15) */
327 _EMIT6(0xe300f000 | rs << 20 | off, 0x0004);
328 else
329 /* lmg %rs,%re,off(%r15) */
330 _EMIT6_DISP(0xeb00f000 | rs << 20 | re << 16, 0x0004, off);
331 }
332
333 /*
334 * Return first seen register (from start)
335 */
336 static int get_start(struct bpf_jit *jit, int start)
337 {
338 int i;
339
340 for (i = start; i <= 15; i++) {
341 if (jit->seen_reg[i])
342 return i;
343 }
344 return 0;
345 }
346
347 /*
348 * Return last seen register (from start) (gap >= 2)
349 */
350 static int get_end(struct bpf_jit *jit, int start)
351 {
352 int i;
353
354 for (i = start; i < 15; i++) {
355 if (!jit->seen_reg[i] && !jit->seen_reg[i + 1])
356 return i - 1;
357 }
358 return jit->seen_reg[15] ? 15 : 14;
359 }
360
361 #define REGS_SAVE 1
362 #define REGS_RESTORE 0
363 /*
364 * Save and restore clobbered registers (6-15) on stack.
365 * We save/restore registers in chunks with gap >= 2 registers.
366 */
367 static void save_restore_regs(struct bpf_jit *jit, int op)
368 {
369
370 int re = 6, rs;
371
372 do {
373 rs = get_start(jit, re);
374 if (!rs)
375 break;
376 re = get_end(jit, rs + 1);
377 if (op == REGS_SAVE)
378 save_regs(jit, rs, re);
379 else
380 restore_regs(jit, rs, re);
381 re++;
382 } while (re <= 15);
383 }
384
385 /*
386 * For SKB access %b1 contains the SKB pointer. For "bpf_jit.S"
387 * we store the SKB header length on the stack and the SKB data
388 * pointer in REG_SKB_DATA.
389 */
390 static void emit_load_skb_data_hlen(struct bpf_jit *jit)
391 {
392 /* Header length: llgf %w1,<len>(%b1) */
393 EMIT6_DISP_LH(0xe3000000, 0x0016, REG_W1, REG_0, BPF_REG_1,
394 offsetof(struct sk_buff, len));
395 /* s %w1,<data_len>(%b1) */
396 EMIT4_DISP(0x5b000000, REG_W1, BPF_REG_1,
397 offsetof(struct sk_buff, data_len));
398 /* stg %w1,ST_OFF_HLEN(%r0,%r15) */
399 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, REG_15, STK_OFF_HLEN);
400 /* lg %skb_data,data_off(%b1) */
401 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_SKB_DATA, REG_0,
402 BPF_REG_1, offsetof(struct sk_buff, data));
403 }
404
405 /*
406 * Emit function prologue
407 *
408 * Save registers and create stack frame if necessary.
409 * See stack frame layout desription in "bpf_jit.h"!
410 */
411 static void bpf_jit_prologue(struct bpf_jit *jit, bool is_classic)
412 {
413 if (jit->seen & SEEN_TAIL_CALL) {
414 /* xc STK_OFF_TCCNT(4,%r15),STK_OFF_TCCNT(%r15) */
415 _EMIT6(0xd703f000 | STK_OFF_TCCNT, 0xf000 | STK_OFF_TCCNT);
416 } else {
417 /* j tail_call_start: NOP if no tail calls are used */
418 EMIT4_PCREL(0xa7f40000, 6);
419 _EMIT2(0);
420 }
421 /* Tail calls have to skip above initialization */
422 jit->tail_call_start = jit->prg;
423 /* Save registers */
424 save_restore_regs(jit, REGS_SAVE);
425 /* Setup literal pool */
426 if (jit->seen & SEEN_LITERAL) {
427 /* basr %r13,0 */
428 EMIT2(0x0d00, REG_L, REG_0);
429 jit->base_ip = jit->prg;
430 }
431 /* Setup stack and backchain */
432 if (jit->seen & SEEN_STACK) {
433 if (jit->seen & SEEN_FUNC)
434 /* lgr %w1,%r15 (backchain) */
435 EMIT4(0xb9040000, REG_W1, REG_15);
436 /* la %bfp,STK_160_UNUSED(%r15) (BPF frame pointer) */
437 EMIT4_DISP(0x41000000, BPF_REG_FP, REG_15, STK_160_UNUSED);
438 /* aghi %r15,-STK_OFF */
439 EMIT4_IMM(0xa70b0000, REG_15, -STK_OFF);
440 if (jit->seen & SEEN_FUNC)
441 /* stg %w1,152(%r15) (backchain) */
442 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0,
443 REG_15, 152);
444 }
445 if (jit->seen & SEEN_SKB)
446 emit_load_skb_data_hlen(jit);
447 if (jit->seen & SEEN_SKB_CHANGE)
448 /* stg %b1,ST_OFF_SKBP(%r0,%r15) */
449 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, REG_15,
450 STK_OFF_SKBP);
451 /* Clear A (%b0) and X (%b7) registers for converted BPF programs */
452 if (is_classic) {
453 if (REG_SEEN(BPF_REG_A))
454 /* lghi %ba,0 */
455 EMIT4_IMM(0xa7090000, BPF_REG_A, 0);
456 if (REG_SEEN(BPF_REG_X))
457 /* lghi %bx,0 */
458 EMIT4_IMM(0xa7090000, BPF_REG_X, 0);
459 }
460 }
461
462 /*
463 * Function epilogue
464 */
465 static void bpf_jit_epilogue(struct bpf_jit *jit)
466 {
467 /* Return 0 */
468 if (jit->seen & SEEN_RET0) {
469 jit->ret0_ip = jit->prg;
470 /* lghi %b0,0 */
471 EMIT4_IMM(0xa7090000, BPF_REG_0, 0);
472 }
473 jit->exit_ip = jit->prg;
474 /* Load exit code: lgr %r2,%b0 */
475 EMIT4(0xb9040000, REG_2, BPF_REG_0);
476 /* Restore registers */
477 save_restore_regs(jit, REGS_RESTORE);
478 /* br %r14 */
479 _EMIT2(0x07fe);
480 }
481
482 /*
483 * Compile one eBPF instruction into s390x code
484 *
485 * NOTE: Use noinline because for gcov (-fprofile-arcs) gcc allocates a lot of
486 * stack space for the large switch statement.
487 */
488 static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i)
489 {
490 struct bpf_insn *insn = &fp->insnsi[i];
491 int jmp_off, last, insn_count = 1;
492 unsigned int func_addr, mask;
493 u32 dst_reg = insn->dst_reg;
494 u32 src_reg = insn->src_reg;
495 u32 *addrs = jit->addrs;
496 s32 imm = insn->imm;
497 s16 off = insn->off;
498
499 switch (insn->code) {
500 /*
501 * BPF_MOV
502 */
503 case BPF_ALU | BPF_MOV | BPF_X: /* dst = (u32) src */
504 /* llgfr %dst,%src */
505 EMIT4(0xb9160000, dst_reg, src_reg);
506 break;
507 case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */
508 /* lgr %dst,%src */
509 EMIT4(0xb9040000, dst_reg, src_reg);
510 break;
511 case BPF_ALU | BPF_MOV | BPF_K: /* dst = (u32) imm */
512 /* llilf %dst,imm */
513 EMIT6_IMM(0xc00f0000, dst_reg, imm);
514 break;
515 case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = imm */
516 /* lgfi %dst,imm */
517 EMIT6_IMM(0xc0010000, dst_reg, imm);
518 break;
519 /*
520 * BPF_LD 64
521 */
522 case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */
523 {
524 /* 16 byte instruction that uses two 'struct bpf_insn' */
525 u64 imm64;
526
527 imm64 = (u64)(u32) insn[0].imm | ((u64)(u32) insn[1].imm) << 32;
528 /* lg %dst,<d(imm)>(%l) */
529 EMIT6_DISP_LH(0xe3000000, 0x0004, dst_reg, REG_0, REG_L,
530 EMIT_CONST_U64(imm64));
531 insn_count = 2;
532 break;
533 }
534 /*
535 * BPF_ADD
536 */
537 case BPF_ALU | BPF_ADD | BPF_X: /* dst = (u32) dst + (u32) src */
538 /* ar %dst,%src */
539 EMIT2(0x1a00, dst_reg, src_reg);
540 EMIT_ZERO(dst_reg);
541 break;
542 case BPF_ALU64 | BPF_ADD | BPF_X: /* dst = dst + src */
543 /* agr %dst,%src */
544 EMIT4(0xb9080000, dst_reg, src_reg);
545 break;
546 case BPF_ALU | BPF_ADD | BPF_K: /* dst = (u32) dst + (u32) imm */
547 if (!imm)
548 break;
549 /* alfi %dst,imm */
550 EMIT6_IMM(0xc20b0000, dst_reg, imm);
551 EMIT_ZERO(dst_reg);
552 break;
553 case BPF_ALU64 | BPF_ADD | BPF_K: /* dst = dst + imm */
554 if (!imm)
555 break;
556 /* agfi %dst,imm */
557 EMIT6_IMM(0xc2080000, dst_reg, imm);
558 break;
559 /*
560 * BPF_SUB
561 */
562 case BPF_ALU | BPF_SUB | BPF_X: /* dst = (u32) dst - (u32) src */
563 /* sr %dst,%src */
564 EMIT2(0x1b00, dst_reg, src_reg);
565 EMIT_ZERO(dst_reg);
566 break;
567 case BPF_ALU64 | BPF_SUB | BPF_X: /* dst = dst - src */
568 /* sgr %dst,%src */
569 EMIT4(0xb9090000, dst_reg, src_reg);
570 break;
571 case BPF_ALU | BPF_SUB | BPF_K: /* dst = (u32) dst - (u32) imm */
572 if (!imm)
573 break;
574 /* alfi %dst,-imm */
575 EMIT6_IMM(0xc20b0000, dst_reg, -imm);
576 EMIT_ZERO(dst_reg);
577 break;
578 case BPF_ALU64 | BPF_SUB | BPF_K: /* dst = dst - imm */
579 if (!imm)
580 break;
581 /* agfi %dst,-imm */
582 EMIT6_IMM(0xc2080000, dst_reg, -imm);
583 break;
584 /*
585 * BPF_MUL
586 */
587 case BPF_ALU | BPF_MUL | BPF_X: /* dst = (u32) dst * (u32) src */
588 /* msr %dst,%src */
589 EMIT4(0xb2520000, dst_reg, src_reg);
590 EMIT_ZERO(dst_reg);
591 break;
592 case BPF_ALU64 | BPF_MUL | BPF_X: /* dst = dst * src */
593 /* msgr %dst,%src */
594 EMIT4(0xb90c0000, dst_reg, src_reg);
595 break;
596 case BPF_ALU | BPF_MUL | BPF_K: /* dst = (u32) dst * (u32) imm */
597 if (imm == 1)
598 break;
599 /* msfi %r5,imm */
600 EMIT6_IMM(0xc2010000, dst_reg, imm);
601 EMIT_ZERO(dst_reg);
602 break;
603 case BPF_ALU64 | BPF_MUL | BPF_K: /* dst = dst * imm */
604 if (imm == 1)
605 break;
606 /* msgfi %dst,imm */
607 EMIT6_IMM(0xc2000000, dst_reg, imm);
608 break;
609 /*
610 * BPF_DIV / BPF_MOD
611 */
612 case BPF_ALU | BPF_DIV | BPF_X: /* dst = (u32) dst / (u32) src */
613 case BPF_ALU | BPF_MOD | BPF_X: /* dst = (u32) dst % (u32) src */
614 {
615 int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;
616
617 jit->seen |= SEEN_RET0;
618 /* ltr %src,%src (if src == 0 goto fail) */
619 EMIT2(0x1200, src_reg, src_reg);
620 /* jz <ret0> */
621 EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg);
622 /* lhi %w0,0 */
623 EMIT4_IMM(0xa7080000, REG_W0, 0);
624 /* lr %w1,%dst */
625 EMIT2(0x1800, REG_W1, dst_reg);
626 /* dlr %w0,%src */
627 EMIT4(0xb9970000, REG_W0, src_reg);
628 /* llgfr %dst,%rc */
629 EMIT4(0xb9160000, dst_reg, rc_reg);
630 break;
631 }
632 case BPF_ALU64 | BPF_DIV | BPF_X: /* dst = dst / src */
633 case BPF_ALU64 | BPF_MOD | BPF_X: /* dst = dst % src */
634 {
635 int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;
636
637 jit->seen |= SEEN_RET0;
638 /* ltgr %src,%src (if src == 0 goto fail) */
639 EMIT4(0xb9020000, src_reg, src_reg);
640 /* jz <ret0> */
641 EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg);
642 /* lghi %w0,0 */
643 EMIT4_IMM(0xa7090000, REG_W0, 0);
644 /* lgr %w1,%dst */
645 EMIT4(0xb9040000, REG_W1, dst_reg);
646 /* dlgr %w0,%dst */
647 EMIT4(0xb9870000, REG_W0, src_reg);
648 /* lgr %dst,%rc */
649 EMIT4(0xb9040000, dst_reg, rc_reg);
650 break;
651 }
652 case BPF_ALU | BPF_DIV | BPF_K: /* dst = (u32) dst / (u32) imm */
653 case BPF_ALU | BPF_MOD | BPF_K: /* dst = (u32) dst % (u32) imm */
654 {
655 int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;
656
657 if (imm == 1) {
658 if (BPF_OP(insn->code) == BPF_MOD)
659 /* lhgi %dst,0 */
660 EMIT4_IMM(0xa7090000, dst_reg, 0);
661 break;
662 }
663 /* lhi %w0,0 */
664 EMIT4_IMM(0xa7080000, REG_W0, 0);
665 /* lr %w1,%dst */
666 EMIT2(0x1800, REG_W1, dst_reg);
667 /* dl %w0,<d(imm)>(%l) */
668 EMIT6_DISP_LH(0xe3000000, 0x0097, REG_W0, REG_0, REG_L,
669 EMIT_CONST_U32(imm));
670 /* llgfr %dst,%rc */
671 EMIT4(0xb9160000, dst_reg, rc_reg);
672 break;
673 }
674 case BPF_ALU64 | BPF_DIV | BPF_K: /* dst = dst / imm */
675 case BPF_ALU64 | BPF_MOD | BPF_K: /* dst = dst % imm */
676 {
677 int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;
678
679 if (imm == 1) {
680 if (BPF_OP(insn->code) == BPF_MOD)
681 /* lhgi %dst,0 */
682 EMIT4_IMM(0xa7090000, dst_reg, 0);
683 break;
684 }
685 /* lghi %w0,0 */
686 EMIT4_IMM(0xa7090000, REG_W0, 0);
687 /* lgr %w1,%dst */
688 EMIT4(0xb9040000, REG_W1, dst_reg);
689 /* dlg %w0,<d(imm)>(%l) */
690 EMIT6_DISP_LH(0xe3000000, 0x0087, REG_W0, REG_0, REG_L,
691 EMIT_CONST_U64(imm));
692 /* lgr %dst,%rc */
693 EMIT4(0xb9040000, dst_reg, rc_reg);
694 break;
695 }
696 /*
697 * BPF_AND
698 */
699 case BPF_ALU | BPF_AND | BPF_X: /* dst = (u32) dst & (u32) src */
700 /* nr %dst,%src */
701 EMIT2(0x1400, dst_reg, src_reg);
702 EMIT_ZERO(dst_reg);
703 break;
704 case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */
705 /* ngr %dst,%src */
706 EMIT4(0xb9800000, dst_reg, src_reg);
707 break;
708 case BPF_ALU | BPF_AND | BPF_K: /* dst = (u32) dst & (u32) imm */
709 /* nilf %dst,imm */
710 EMIT6_IMM(0xc00b0000, dst_reg, imm);
711 EMIT_ZERO(dst_reg);
712 break;
713 case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */
714 /* ng %dst,<d(imm)>(%l) */
715 EMIT6_DISP_LH(0xe3000000, 0x0080, dst_reg, REG_0, REG_L,
716 EMIT_CONST_U64(imm));
717 break;
718 /*
719 * BPF_OR
720 */
721 case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */
722 /* or %dst,%src */
723 EMIT2(0x1600, dst_reg, src_reg);
724 EMIT_ZERO(dst_reg);
725 break;
726 case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */
727 /* ogr %dst,%src */
728 EMIT4(0xb9810000, dst_reg, src_reg);
729 break;
730 case BPF_ALU | BPF_OR | BPF_K: /* dst = (u32) dst | (u32) imm */
731 /* oilf %dst,imm */
732 EMIT6_IMM(0xc00d0000, dst_reg, imm);
733 EMIT_ZERO(dst_reg);
734 break;
735 case BPF_ALU64 | BPF_OR | BPF_K: /* dst = dst | imm */
736 /* og %dst,<d(imm)>(%l) */
737 EMIT6_DISP_LH(0xe3000000, 0x0081, dst_reg, REG_0, REG_L,
738 EMIT_CONST_U64(imm));
739 break;
740 /*
741 * BPF_XOR
742 */
743 case BPF_ALU | BPF_XOR | BPF_X: /* dst = (u32) dst ^ (u32) src */
744 /* xr %dst,%src */
745 EMIT2(0x1700, dst_reg, src_reg);
746 EMIT_ZERO(dst_reg);
747 break;
748 case BPF_ALU64 | BPF_XOR | BPF_X: /* dst = dst ^ src */
749 /* xgr %dst,%src */
750 EMIT4(0xb9820000, dst_reg, src_reg);
751 break;
752 case BPF_ALU | BPF_XOR | BPF_K: /* dst = (u32) dst ^ (u32) imm */
753 if (!imm)
754 break;
755 /* xilf %dst,imm */
756 EMIT6_IMM(0xc0070000, dst_reg, imm);
757 EMIT_ZERO(dst_reg);
758 break;
759 case BPF_ALU64 | BPF_XOR | BPF_K: /* dst = dst ^ imm */
760 /* xg %dst,<d(imm)>(%l) */
761 EMIT6_DISP_LH(0xe3000000, 0x0082, dst_reg, REG_0, REG_L,
762 EMIT_CONST_U64(imm));
763 break;
764 /*
765 * BPF_LSH
766 */
767 case BPF_ALU | BPF_LSH | BPF_X: /* dst = (u32) dst << (u32) src */
768 /* sll %dst,0(%src) */
769 EMIT4_DISP(0x89000000, dst_reg, src_reg, 0);
770 EMIT_ZERO(dst_reg);
771 break;
772 case BPF_ALU64 | BPF_LSH | BPF_X: /* dst = dst << src */
773 /* sllg %dst,%dst,0(%src) */
774 EMIT6_DISP_LH(0xeb000000, 0x000d, dst_reg, dst_reg, src_reg, 0);
775 break;
776 case BPF_ALU | BPF_LSH | BPF_K: /* dst = (u32) dst << (u32) imm */
777 if (imm == 0)
778 break;
779 /* sll %dst,imm(%r0) */
780 EMIT4_DISP(0x89000000, dst_reg, REG_0, imm);
781 EMIT_ZERO(dst_reg);
782 break;
783 case BPF_ALU64 | BPF_LSH | BPF_K: /* dst = dst << imm */
784 if (imm == 0)
785 break;
786 /* sllg %dst,%dst,imm(%r0) */
787 EMIT6_DISP_LH(0xeb000000, 0x000d, dst_reg, dst_reg, REG_0, imm);
788 break;
789 /*
790 * BPF_RSH
791 */
792 case BPF_ALU | BPF_RSH | BPF_X: /* dst = (u32) dst >> (u32) src */
793 /* srl %dst,0(%src) */
794 EMIT4_DISP(0x88000000, dst_reg, src_reg, 0);
795 EMIT_ZERO(dst_reg);
796 break;
797 case BPF_ALU64 | BPF_RSH | BPF_X: /* dst = dst >> src */
798 /* srlg %dst,%dst,0(%src) */
799 EMIT6_DISP_LH(0xeb000000, 0x000c, dst_reg, dst_reg, src_reg, 0);
800 break;
801 case BPF_ALU | BPF_RSH | BPF_K: /* dst = (u32) dst >> (u32) imm */
802 if (imm == 0)
803 break;
804 /* srl %dst,imm(%r0) */
805 EMIT4_DISP(0x88000000, dst_reg, REG_0, imm);
806 EMIT_ZERO(dst_reg);
807 break;
808 case BPF_ALU64 | BPF_RSH | BPF_K: /* dst = dst >> imm */
809 if (imm == 0)
810 break;
811 /* srlg %dst,%dst,imm(%r0) */
812 EMIT6_DISP_LH(0xeb000000, 0x000c, dst_reg, dst_reg, REG_0, imm);
813 break;
814 /*
815 * BPF_ARSH
816 */
817 case BPF_ALU64 | BPF_ARSH | BPF_X: /* ((s64) dst) >>= src */
818 /* srag %dst,%dst,0(%src) */
819 EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, src_reg, 0);
820 break;
821 case BPF_ALU64 | BPF_ARSH | BPF_K: /* ((s64) dst) >>= imm */
822 if (imm == 0)
823 break;
824 /* srag %dst,%dst,imm(%r0) */
825 EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, REG_0, imm);
826 break;
827 /*
828 * BPF_NEG
829 */
830 case BPF_ALU | BPF_NEG: /* dst = (u32) -dst */
831 /* lcr %dst,%dst */
832 EMIT2(0x1300, dst_reg, dst_reg);
833 EMIT_ZERO(dst_reg);
834 break;
835 case BPF_ALU64 | BPF_NEG: /* dst = -dst */
836 /* lcgr %dst,%dst */
837 EMIT4(0xb9130000, dst_reg, dst_reg);
838 break;
839 /*
840 * BPF_FROM_BE/LE
841 */
842 case BPF_ALU | BPF_END | BPF_FROM_BE:
843 /* s390 is big endian, therefore only clear high order bytes */
844 switch (imm) {
845 case 16: /* dst = (u16) cpu_to_be16(dst) */
846 /* llghr %dst,%dst */
847 EMIT4(0xb9850000, dst_reg, dst_reg);
848 break;
849 case 32: /* dst = (u32) cpu_to_be32(dst) */
850 /* llgfr %dst,%dst */
851 EMIT4(0xb9160000, dst_reg, dst_reg);
852 break;
853 case 64: /* dst = (u64) cpu_to_be64(dst) */
854 break;
855 }
856 break;
857 case BPF_ALU | BPF_END | BPF_FROM_LE:
858 switch (imm) {
859 case 16: /* dst = (u16) cpu_to_le16(dst) */
860 /* lrvr %dst,%dst */
861 EMIT4(0xb91f0000, dst_reg, dst_reg);
862 /* srl %dst,16(%r0) */
863 EMIT4_DISP(0x88000000, dst_reg, REG_0, 16);
864 /* llghr %dst,%dst */
865 EMIT4(0xb9850000, dst_reg, dst_reg);
866 break;
867 case 32: /* dst = (u32) cpu_to_le32(dst) */
868 /* lrvr %dst,%dst */
869 EMIT4(0xb91f0000, dst_reg, dst_reg);
870 /* llgfr %dst,%dst */
871 EMIT4(0xb9160000, dst_reg, dst_reg);
872 break;
873 case 64: /* dst = (u64) cpu_to_le64(dst) */
874 /* lrvgr %dst,%dst */
875 EMIT4(0xb90f0000, dst_reg, dst_reg);
876 break;
877 }
878 break;
879 /*
880 * BPF_ST(X)
881 */
882 case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src_reg */
883 /* stcy %src,off(%dst) */
884 EMIT6_DISP_LH(0xe3000000, 0x0072, src_reg, dst_reg, REG_0, off);
885 jit->seen |= SEEN_MEM;
886 break;
887 case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */
888 /* sthy %src,off(%dst) */
889 EMIT6_DISP_LH(0xe3000000, 0x0070, src_reg, dst_reg, REG_0, off);
890 jit->seen |= SEEN_MEM;
891 break;
892 case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */
893 /* sty %src,off(%dst) */
894 EMIT6_DISP_LH(0xe3000000, 0x0050, src_reg, dst_reg, REG_0, off);
895 jit->seen |= SEEN_MEM;
896 break;
897 case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */
898 /* stg %src,off(%dst) */
899 EMIT6_DISP_LH(0xe3000000, 0x0024, src_reg, dst_reg, REG_0, off);
900 jit->seen |= SEEN_MEM;
901 break;
902 case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */
903 /* lhi %w0,imm */
904 EMIT4_IMM(0xa7080000, REG_W0, (u8) imm);
905 /* stcy %w0,off(dst) */
906 EMIT6_DISP_LH(0xe3000000, 0x0072, REG_W0, dst_reg, REG_0, off);
907 jit->seen |= SEEN_MEM;
908 break;
909 case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */
910 /* lhi %w0,imm */
911 EMIT4_IMM(0xa7080000, REG_W0, (u16) imm);
912 /* sthy %w0,off(dst) */
913 EMIT6_DISP_LH(0xe3000000, 0x0070, REG_W0, dst_reg, REG_0, off);
914 jit->seen |= SEEN_MEM;
915 break;
916 case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */
917 /* llilf %w0,imm */
918 EMIT6_IMM(0xc00f0000, REG_W0, (u32) imm);
919 /* sty %w0,off(%dst) */
920 EMIT6_DISP_LH(0xe3000000, 0x0050, REG_W0, dst_reg, REG_0, off);
921 jit->seen |= SEEN_MEM;
922 break;
923 case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */
924 /* lgfi %w0,imm */
925 EMIT6_IMM(0xc0010000, REG_W0, imm);
926 /* stg %w0,off(%dst) */
927 EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W0, dst_reg, REG_0, off);
928 jit->seen |= SEEN_MEM;
929 break;
930 /*
931 * BPF_STX XADD (atomic_add)
932 */
933 case BPF_STX | BPF_XADD | BPF_W: /* *(u32 *)(dst + off) += src */
934 /* laal %w0,%src,off(%dst) */
935 EMIT6_DISP_LH(0xeb000000, 0x00fa, REG_W0, src_reg,
936 dst_reg, off);
937 jit->seen |= SEEN_MEM;
938 break;
939 case BPF_STX | BPF_XADD | BPF_DW: /* *(u64 *)(dst + off) += src */
940 /* laalg %w0,%src,off(%dst) */
941 EMIT6_DISP_LH(0xeb000000, 0x00ea, REG_W0, src_reg,
942 dst_reg, off);
943 jit->seen |= SEEN_MEM;
944 break;
945 /*
946 * BPF_LDX
947 */
948 case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */
949 /* llgc %dst,0(off,%src) */
950 EMIT6_DISP_LH(0xe3000000, 0x0090, dst_reg, src_reg, REG_0, off);
951 jit->seen |= SEEN_MEM;
952 break;
953 case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */
954 /* llgh %dst,0(off,%src) */
955 EMIT6_DISP_LH(0xe3000000, 0x0091, dst_reg, src_reg, REG_0, off);
956 jit->seen |= SEEN_MEM;
957 break;
958 case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */
959 /* llgf %dst,off(%src) */
960 jit->seen |= SEEN_MEM;
961 EMIT6_DISP_LH(0xe3000000, 0x0016, dst_reg, src_reg, REG_0, off);
962 break;
963 case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */
964 /* lg %dst,0(off,%src) */
965 jit->seen |= SEEN_MEM;
966 EMIT6_DISP_LH(0xe3000000, 0x0004, dst_reg, src_reg, REG_0, off);
967 break;
968 /*
969 * BPF_JMP / CALL
970 */
971 case BPF_JMP | BPF_CALL:
972 {
973 /*
974 * b0 = (__bpf_call_base + imm)(b1, b2, b3, b4, b5)
975 */
976 const u64 func = (u64)__bpf_call_base + imm;
977
978 REG_SET_SEEN(BPF_REG_5);
979 jit->seen |= SEEN_FUNC;
980 /* lg %w1,<d(imm)>(%l) */
981 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_W1, REG_0, REG_L,
982 EMIT_CONST_U64(func));
983 /* basr %r14,%w1 */
984 EMIT2(0x0d00, REG_14, REG_W1);
985 /* lgr %b0,%r2: load return value into %b0 */
986 EMIT4(0xb9040000, BPF_REG_0, REG_2);
987 if (bpf_helper_changes_skb_data((void *)func)) {
988 jit->seen |= SEEN_SKB_CHANGE;
989 /* lg %b1,ST_OFF_SKBP(%r15) */
990 EMIT6_DISP_LH(0xe3000000, 0x0004, BPF_REG_1, REG_0,
991 REG_15, STK_OFF_SKBP);
992 emit_load_skb_data_hlen(jit);
993 }
994 break;
995 }
996 case BPF_JMP | BPF_CALL | BPF_X:
997 /*
998 * Implicit input:
999 * B1: pointer to ctx
1000 * B2: pointer to bpf_array
1001 * B3: index in bpf_array
1002 */
1003 jit->seen |= SEEN_TAIL_CALL;
1004
1005 /*
1006 * if (index >= array->map.max_entries)
1007 * goto out;
1008 */
1009
1010 /* llgf %w1,map.max_entries(%b2) */
1011 EMIT6_DISP_LH(0xe3000000, 0x0016, REG_W1, REG_0, BPF_REG_2,
1012 offsetof(struct bpf_array, map.max_entries));
1013 /* clgrj %b3,%w1,0xa,label0: if %b3 >= %w1 goto out */
1014 EMIT6_PCREL_LABEL(0xec000000, 0x0065, BPF_REG_3,
1015 REG_W1, 0, 0xa);
1016
1017 /*
1018 * if (tail_call_cnt++ > MAX_TAIL_CALL_CNT)
1019 * goto out;
1020 */
1021
1022 if (jit->seen & SEEN_STACK)
1023 off = STK_OFF_TCCNT + STK_OFF;
1024 else
1025 off = STK_OFF_TCCNT;
1026 /* lhi %w0,1 */
1027 EMIT4_IMM(0xa7080000, REG_W0, 1);
1028 /* laal %w1,%w0,off(%r15) */
1029 EMIT6_DISP_LH(0xeb000000, 0x00fa, REG_W1, REG_W0, REG_15, off);
1030 /* clij %w1,MAX_TAIL_CALL_CNT,0x2,label0 */
1031 EMIT6_PCREL_IMM_LABEL(0xec000000, 0x007f, REG_W1,
1032 MAX_TAIL_CALL_CNT, 0, 0x2);
1033
1034 /*
1035 * prog = array->ptrs[index];
1036 * if (prog == NULL)
1037 * goto out;
1038 */
1039
1040 /* sllg %r1,%b3,3: %r1 = index * 8 */
1041 EMIT6_DISP_LH(0xeb000000, 0x000d, REG_1, BPF_REG_3, REG_0, 3);
1042 /* lg %r1,prog(%b2,%r1) */
1043 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_1, BPF_REG_2,
1044 REG_1, offsetof(struct bpf_array, ptrs));
1045 /* clgij %r1,0,0x8,label0 */
1046 EMIT6_PCREL_IMM_LABEL(0xec000000, 0x007d, REG_1, 0, 0, 0x8);
1047
1048 /*
1049 * Restore registers before calling function
1050 */
1051 save_restore_regs(jit, REGS_RESTORE);
1052
1053 /*
1054 * goto *(prog->bpf_func + tail_call_start);
1055 */
1056
1057 /* lg %r1,bpf_func(%r1) */
1058 EMIT6_DISP_LH(0xe3000000, 0x0004, REG_1, REG_1, REG_0,
1059 offsetof(struct bpf_prog, bpf_func));
1060 /* bc 0xf,tail_call_start(%r1) */
1061 _EMIT4(0x47f01000 + jit->tail_call_start);
1062 /* out: */
1063 jit->labels[0] = jit->prg;
1064 break;
1065 case BPF_JMP | BPF_EXIT: /* return b0 */
1066 last = (i == fp->len - 1) ? 1 : 0;
1067 if (last && !(jit->seen & SEEN_RET0))
1068 break;
1069 /* j <exit> */
1070 EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg);
1071 break;
1072 /*
1073 * Branch relative (number of skipped instructions) to offset on
1074 * condition.
1075 *
1076 * Condition code to mask mapping:
1077 *
1078 * CC | Description | Mask
1079 * ------------------------------
1080 * 0 | Operands equal | 8
1081 * 1 | First operand low | 4
1082 * 2 | First operand high | 2
1083 * 3 | Unused | 1
1084 *
1085 * For s390x relative branches: ip = ip + off_bytes
1086 * For BPF relative branches: insn = insn + off_insns + 1
1087 *
1088 * For example for s390x with offset 0 we jump to the branch
1089 * instruction itself (loop) and for BPF with offset 0 we
1090 * branch to the instruction behind the branch.
1091 */
1092 case BPF_JMP | BPF_JA: /* if (true) */
1093 mask = 0xf000; /* j */
1094 goto branch_oc;
1095 case BPF_JMP | BPF_JSGT | BPF_K: /* ((s64) dst > (s64) imm) */
1096 mask = 0x2000; /* jh */
1097 goto branch_ks;
1098 case BPF_JMP | BPF_JSGE | BPF_K: /* ((s64) dst >= (s64) imm) */
1099 mask = 0xa000; /* jhe */
1100 goto branch_ks;
1101 case BPF_JMP | BPF_JGT | BPF_K: /* (dst_reg > imm) */
1102 mask = 0x2000; /* jh */
1103 goto branch_ku;
1104 case BPF_JMP | BPF_JGE | BPF_K: /* (dst_reg >= imm) */
1105 mask = 0xa000; /* jhe */
1106 goto branch_ku;
1107 case BPF_JMP | BPF_JNE | BPF_K: /* (dst_reg != imm) */
1108 mask = 0x7000; /* jne */
1109 goto branch_ku;
1110 case BPF_JMP | BPF_JEQ | BPF_K: /* (dst_reg == imm) */
1111 mask = 0x8000; /* je */
1112 goto branch_ku;
1113 case BPF_JMP | BPF_JSET | BPF_K: /* (dst_reg & imm) */
1114 mask = 0x7000; /* jnz */
1115 /* lgfi %w1,imm (load sign extend imm) */
1116 EMIT6_IMM(0xc0010000, REG_W1, imm);
1117 /* ngr %w1,%dst */
1118 EMIT4(0xb9800000, REG_W1, dst_reg);
1119 goto branch_oc;
1120
1121 case BPF_JMP | BPF_JSGT | BPF_X: /* ((s64) dst > (s64) src) */
1122 mask = 0x2000; /* jh */
1123 goto branch_xs;
1124 case BPF_JMP | BPF_JSGE | BPF_X: /* ((s64) dst >= (s64) src) */
1125 mask = 0xa000; /* jhe */
1126 goto branch_xs;
1127 case BPF_JMP | BPF_JGT | BPF_X: /* (dst > src) */
1128 mask = 0x2000; /* jh */
1129 goto branch_xu;
1130 case BPF_JMP | BPF_JGE | BPF_X: /* (dst >= src) */
1131 mask = 0xa000; /* jhe */
1132 goto branch_xu;
1133 case BPF_JMP | BPF_JNE | BPF_X: /* (dst != src) */
1134 mask = 0x7000; /* jne */
1135 goto branch_xu;
1136 case BPF_JMP | BPF_JEQ | BPF_X: /* (dst == src) */
1137 mask = 0x8000; /* je */
1138 goto branch_xu;
1139 case BPF_JMP | BPF_JSET | BPF_X: /* (dst & src) */
1140 mask = 0x7000; /* jnz */
1141 /* ngrk %w1,%dst,%src */
1142 EMIT4_RRF(0xb9e40000, REG_W1, dst_reg, src_reg);
1143 goto branch_oc;
1144 branch_ks:
1145 /* lgfi %w1,imm (load sign extend imm) */
1146 EMIT6_IMM(0xc0010000, REG_W1, imm);
1147 /* cgrj %dst,%w1,mask,off */
1148 EMIT6_PCREL(0xec000000, 0x0064, dst_reg, REG_W1, i, off, mask);
1149 break;
1150 branch_ku:
1151 /* lgfi %w1,imm (load sign extend imm) */
1152 EMIT6_IMM(0xc0010000, REG_W1, imm);
1153 /* clgrj %dst,%w1,mask,off */
1154 EMIT6_PCREL(0xec000000, 0x0065, dst_reg, REG_W1, i, off, mask);
1155 break;
1156 branch_xs:
1157 /* cgrj %dst,%src,mask,off */
1158 EMIT6_PCREL(0xec000000, 0x0064, dst_reg, src_reg, i, off, mask);
1159 break;
1160 branch_xu:
1161 /* clgrj %dst,%src,mask,off */
1162 EMIT6_PCREL(0xec000000, 0x0065, dst_reg, src_reg, i, off, mask);
1163 break;
1164 branch_oc:
1165 /* brc mask,jmp_off (branch instruction needs 4 bytes) */
1166 jmp_off = addrs[i + off + 1] - (addrs[i + 1] - 4);
1167 EMIT4_PCREL(0xa7040000 | mask << 8, jmp_off);
1168 break;
1169 /*
1170 * BPF_LD
1171 */
1172 case BPF_LD | BPF_ABS | BPF_B: /* b0 = *(u8 *) (skb->data+imm) */
1173 case BPF_LD | BPF_IND | BPF_B: /* b0 = *(u8 *) (skb->data+imm+src) */
1174 if ((BPF_MODE(insn->code) == BPF_ABS) && (imm >= 0))
1175 func_addr = __pa(sk_load_byte_pos);
1176 else
1177 func_addr = __pa(sk_load_byte);
1178 goto call_fn;
1179 case BPF_LD | BPF_ABS | BPF_H: /* b0 = *(u16 *) (skb->data+imm) */
1180 case BPF_LD | BPF_IND | BPF_H: /* b0 = *(u16 *) (skb->data+imm+src) */
1181 if ((BPF_MODE(insn->code) == BPF_ABS) && (imm >= 0))
1182 func_addr = __pa(sk_load_half_pos);
1183 else
1184 func_addr = __pa(sk_load_half);
1185 goto call_fn;
1186 case BPF_LD | BPF_ABS | BPF_W: /* b0 = *(u32 *) (skb->data+imm) */
1187 case BPF_LD | BPF_IND | BPF_W: /* b0 = *(u32 *) (skb->data+imm+src) */
1188 if ((BPF_MODE(insn->code) == BPF_ABS) && (imm >= 0))
1189 func_addr = __pa(sk_load_word_pos);
1190 else
1191 func_addr = __pa(sk_load_word);
1192 goto call_fn;
1193 call_fn:
1194 jit->seen |= SEEN_SKB | SEEN_RET0 | SEEN_FUNC;
1195 REG_SET_SEEN(REG_14); /* Return address of possible func call */
1196
1197 /*
1198 * Implicit input:
1199 * BPF_REG_6 (R7) : skb pointer
1200 * REG_SKB_DATA (R12): skb data pointer
1201 *
1202 * Calculated input:
1203 * BPF_REG_2 (R3) : offset of byte(s) to fetch in skb
1204 * BPF_REG_5 (R6) : return address
1205 *
1206 * Output:
1207 * BPF_REG_0 (R14): data read from skb
1208 *
1209 * Scratch registers (BPF_REG_1-5)
1210 */
1211
1212 /* Call function: llilf %w1,func_addr */
1213 EMIT6_IMM(0xc00f0000, REG_W1, func_addr);
1214
1215 /* Offset: lgfi %b2,imm */
1216 EMIT6_IMM(0xc0010000, BPF_REG_2, imm);
1217 if (BPF_MODE(insn->code) == BPF_IND)
1218 /* agfr %b2,%src (%src is s32 here) */
1219 EMIT4(0xb9180000, BPF_REG_2, src_reg);
1220
1221 /* basr %b5,%w1 (%b5 is call saved) */
1222 EMIT2(0x0d00, BPF_REG_5, REG_W1);
1223
1224 /*
1225 * Note: For fast access we jump directly after the
1226 * jnz instruction from bpf_jit.S
1227 */
1228 /* jnz <ret0> */
1229 EMIT4_PCREL(0xa7740000, jit->ret0_ip - jit->prg);
1230 break;
1231 default: /* too complex, give up */
1232 pr_err("Unknown opcode %02x\n", insn->code);
1233 return -1;
1234 }
1235 return insn_count;
1236 }
1237
1238 /*
1239 * Compile eBPF program into s390x code
1240 */
1241 static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp)
1242 {
1243 int i, insn_count;
1244
1245 jit->lit = jit->lit_start;
1246 jit->prg = 0;
1247
1248 bpf_jit_prologue(jit, bpf_prog_was_classic(fp));
1249 for (i = 0; i < fp->len; i += insn_count) {
1250 insn_count = bpf_jit_insn(jit, fp, i);
1251 if (insn_count < 0)
1252 return -1;
1253 jit->addrs[i + 1] = jit->prg; /* Next instruction address */
1254 }
1255 bpf_jit_epilogue(jit);
1256
1257 jit->lit_start = jit->prg;
1258 jit->size = jit->lit;
1259 jit->size_prg = jit->prg;
1260 return 0;
1261 }
1262
1263 /*
1264 * Classic BPF function stub. BPF programs will be converted into
1265 * eBPF and then bpf_int_jit_compile() will be called.
1266 */
1267 void bpf_jit_compile(struct bpf_prog *fp)
1268 {
1269 }
1270
1271 /*
1272 * Compile eBPF program "fp"
1273 */
1274 void bpf_int_jit_compile(struct bpf_prog *fp)
1275 {
1276 struct bpf_binary_header *header;
1277 struct bpf_jit jit;
1278 int pass;
1279
1280 if (!bpf_jit_enable)
1281 return;
1282 memset(&jit, 0, sizeof(jit));
1283 jit.addrs = kcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL);
1284 if (jit.addrs == NULL)
1285 return;
1286 /*
1287 * Three initial passes:
1288 * - 1/2: Determine clobbered registers
1289 * - 3: Calculate program size and addrs arrray
1290 */
1291 for (pass = 1; pass <= 3; pass++) {
1292 if (bpf_jit_prog(&jit, fp))
1293 goto free_addrs;
1294 }
1295 /*
1296 * Final pass: Allocate and generate program
1297 */
1298 if (jit.size >= BPF_SIZE_MAX)
1299 goto free_addrs;
1300 header = bpf_jit_binary_alloc(jit.size, &jit.prg_buf, 2, jit_fill_hole);
1301 if (!header)
1302 goto free_addrs;
1303 if (bpf_jit_prog(&jit, fp))
1304 goto free_addrs;
1305 if (bpf_jit_enable > 1) {
1306 bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf);
1307 if (jit.prg_buf)
1308 print_fn_code(jit.prg_buf, jit.size_prg);
1309 }
1310 if (jit.prg_buf) {
1311 set_memory_ro((unsigned long)header, header->pages);
1312 fp->bpf_func = (void *) jit.prg_buf;
1313 fp->jited = 1;
1314 }
1315 free_addrs:
1316 kfree(jit.addrs);
1317 }
1318
1319 /*
1320 * Free eBPF program
1321 */
1322 void bpf_jit_free(struct bpf_prog *fp)
1323 {
1324 unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK;
1325 struct bpf_binary_header *header = (void *)addr;
1326
1327 if (!fp->jited)
1328 goto free_filter;
1329
1330 set_memory_rw(addr, header->pages);
1331 bpf_jit_binary_free(header);
1332
1333 free_filter:
1334 bpf_prog_unlock_free(fp);
1335 }
Linux with added FPC-III support