?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Revert "tty: hvc: Fix data abort due to race in hvc_open"
[linux/fpc-iii.git] / arch / riscv / net / bpf_jit_comp64.c
blobd208a9fd6c528e4599cfc6aecf12151f03372e87
1 // SPDX-License-Identifier: GPL-2.0
2 /* BPF JIT compiler for RV64G
3 *
4 * Copyright(c) 2019 Björn Töpel <bjorn.topel@gmail.com>
5 *
6 */
7
8 #include <linux/bpf.h>
9 #include <linux/filter.h>
10 #include "bpf_jit.h"
11
12 #define RV_REG_TCC RV_REG_A6
13 #define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */
14
15 static const int regmap[] = {
16 [BPF_REG_0] = RV_REG_A5,
17 [BPF_REG_1] = RV_REG_A0,
18 [BPF_REG_2] = RV_REG_A1,
19 [BPF_REG_3] = RV_REG_A2,
20 [BPF_REG_4] = RV_REG_A3,
21 [BPF_REG_5] = RV_REG_A4,
22 [BPF_REG_6] = RV_REG_S1,
23 [BPF_REG_7] = RV_REG_S2,
24 [BPF_REG_8] = RV_REG_S3,
25 [BPF_REG_9] = RV_REG_S4,
26 [BPF_REG_FP] = RV_REG_S5,
27 [BPF_REG_AX] = RV_REG_T0,
28 };
29
30 enum {
31 RV_CTX_F_SEEN_TAIL_CALL = 0,
32 RV_CTX_F_SEEN_CALL = RV_REG_RA,
33 RV_CTX_F_SEEN_S1 = RV_REG_S1,
34 RV_CTX_F_SEEN_S2 = RV_REG_S2,
35 RV_CTX_F_SEEN_S3 = RV_REG_S3,
36 RV_CTX_F_SEEN_S4 = RV_REG_S4,
37 RV_CTX_F_SEEN_S5 = RV_REG_S5,
38 RV_CTX_F_SEEN_S6 = RV_REG_S6,
39 };
40
41 static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx)
42 {
43 u8 reg = regmap[bpf_reg];
44
45 switch (reg) {
46 case RV_CTX_F_SEEN_S1:
47 case RV_CTX_F_SEEN_S2:
48 case RV_CTX_F_SEEN_S3:
49 case RV_CTX_F_SEEN_S4:
50 case RV_CTX_F_SEEN_S5:
51 case RV_CTX_F_SEEN_S6:
52 __set_bit(reg, &ctx->flags);
53 }
54 return reg;
55 };
56
57 static bool seen_reg(int reg, struct rv_jit_context *ctx)
58 {
59 switch (reg) {
60 case RV_CTX_F_SEEN_CALL:
61 case RV_CTX_F_SEEN_S1:
62 case RV_CTX_F_SEEN_S2:
63 case RV_CTX_F_SEEN_S3:
64 case RV_CTX_F_SEEN_S4:
65 case RV_CTX_F_SEEN_S5:
66 case RV_CTX_F_SEEN_S6:
67 return test_bit(reg, &ctx->flags);
68 }
69 return false;
70 }
71
72 static void mark_fp(struct rv_jit_context *ctx)
73 {
74 __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags);
75 }
76
77 static void mark_call(struct rv_jit_context *ctx)
78 {
79 __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags);
80 }
81
82 static bool seen_call(struct rv_jit_context *ctx)
83 {
84 return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags);
85 }
86
87 static void mark_tail_call(struct rv_jit_context *ctx)
88 {
89 __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags);
90 }
91
92 static bool seen_tail_call(struct rv_jit_context *ctx)
93 {
94 return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags);
95 }
96
97 static u8 rv_tail_call_reg(struct rv_jit_context *ctx)
98 {
99 mark_tail_call(ctx);
100
101 if (seen_call(ctx)) {
102 __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags);
103 return RV_REG_S6;
104 }
105 return RV_REG_A6;
106 }
107
108 static bool is_32b_int(s64 val)
109 {
110 return -(1L << 31) <= val && val < (1L << 31);
111 }
112
113 static bool in_auipc_jalr_range(s64 val)
114 {
115 /*
116 * auipc+jalr can reach any signed PC-relative offset in the range
117 * [-2^31 - 2^11, 2^31 - 2^11).
118 */
119 return (-(1L << 31) - (1L << 11)) <= val &&
120 val < ((1L << 31) - (1L << 11));
121 }
122
123 static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx)
124 {
125 /* Note that the immediate from the add is sign-extended,
126 * which means that we need to compensate this by adding 2^12,
127 * when the 12th bit is set. A simpler way of doing this, and
128 * getting rid of the check, is to just add 2**11 before the
129 * shift. The "Loading a 32-Bit constant" example from the
130 * "Computer Organization and Design, RISC-V edition" book by
131 * Patterson/Hennessy highlights this fact.
132 *
133 * This also means that we need to process LSB to MSB.
134 */
135 s64 upper = (val + (1 << 11)) >> 12, lower = val & 0xfff;
136 int shift;
137
138 if (is_32b_int(val)) {
139 if (upper)
140 emit(rv_lui(rd, upper), ctx);
141
142 if (!upper) {
143 emit(rv_addi(rd, RV_REG_ZERO, lower), ctx);
144 return;
145 }
146
147 emit(rv_addiw(rd, rd, lower), ctx);
148 return;
149 }
150
151 shift = __ffs(upper);
152 upper >>= shift;
153 shift += 12;
154
155 emit_imm(rd, upper, ctx);
156
157 emit(rv_slli(rd, rd, shift), ctx);
158 if (lower)
159 emit(rv_addi(rd, rd, lower), ctx);
160 }
161
162 static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx)
163 {
164 int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8;
165
166 if (seen_reg(RV_REG_RA, ctx)) {
167 emit(rv_ld(RV_REG_RA, store_offset, RV_REG_SP), ctx);
168 store_offset -= 8;
169 }
170 emit(rv_ld(RV_REG_FP, store_offset, RV_REG_SP), ctx);
171 store_offset -= 8;
172 if (seen_reg(RV_REG_S1, ctx)) {
173 emit(rv_ld(RV_REG_S1, store_offset, RV_REG_SP), ctx);
174 store_offset -= 8;
175 }
176 if (seen_reg(RV_REG_S2, ctx)) {
177 emit(rv_ld(RV_REG_S2, store_offset, RV_REG_SP), ctx);
178 store_offset -= 8;
179 }
180 if (seen_reg(RV_REG_S3, ctx)) {
181 emit(rv_ld(RV_REG_S3, store_offset, RV_REG_SP), ctx);
182 store_offset -= 8;
183 }
184 if (seen_reg(RV_REG_S4, ctx)) {
185 emit(rv_ld(RV_REG_S4, store_offset, RV_REG_SP), ctx);
186 store_offset -= 8;
187 }
188 if (seen_reg(RV_REG_S5, ctx)) {
189 emit(rv_ld(RV_REG_S5, store_offset, RV_REG_SP), ctx);
190 store_offset -= 8;
191 }
192 if (seen_reg(RV_REG_S6, ctx)) {
193 emit(rv_ld(RV_REG_S6, store_offset, RV_REG_SP), ctx);
194 store_offset -= 8;
195 }
196
197 emit(rv_addi(RV_REG_SP, RV_REG_SP, stack_adjust), ctx);
198 /* Set return value. */
199 if (!is_tail_call)
200 emit(rv_addi(RV_REG_A0, RV_REG_A5, 0), ctx);
201 emit(rv_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA,
202 is_tail_call ? 4 : 0), /* skip TCC init */
203 ctx);
204 }
205
206 static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff,
207 struct rv_jit_context *ctx)
208 {
209 switch (cond) {
210 case BPF_JEQ:
211 emit(rv_beq(rd, rs, rvoff >> 1), ctx);
212 return;
213 case BPF_JGT:
214 emit(rv_bltu(rs, rd, rvoff >> 1), ctx);
215 return;
216 case BPF_JLT:
217 emit(rv_bltu(rd, rs, rvoff >> 1), ctx);
218 return;
219 case BPF_JGE:
220 emit(rv_bgeu(rd, rs, rvoff >> 1), ctx);
221 return;
222 case BPF_JLE:
223 emit(rv_bgeu(rs, rd, rvoff >> 1), ctx);
224 return;
225 case BPF_JNE:
226 emit(rv_bne(rd, rs, rvoff >> 1), ctx);
227 return;
228 case BPF_JSGT:
229 emit(rv_blt(rs, rd, rvoff >> 1), ctx);
230 return;
231 case BPF_JSLT:
232 emit(rv_blt(rd, rs, rvoff >> 1), ctx);
233 return;
234 case BPF_JSGE:
235 emit(rv_bge(rd, rs, rvoff >> 1), ctx);
236 return;
237 case BPF_JSLE:
238 emit(rv_bge(rs, rd, rvoff >> 1), ctx);
239 }
240 }
241
242 static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff,
243 struct rv_jit_context *ctx)
244 {
245 s64 upper, lower;
246
247 if (is_13b_int(rvoff)) {
248 emit_bcc(cond, rd, rs, rvoff, ctx);
249 return;
250 }
251
252 /* Adjust for jal */
253 rvoff -= 4;
254
255 /* Transform, e.g.:
256 * bne rd,rs,foo
257 * to
258 * beq rd,rs,<.L1>
259 * (auipc foo)
260 * jal(r) foo
261 * .L1
262 */
263 cond = invert_bpf_cond(cond);
264 if (is_21b_int(rvoff)) {
265 emit_bcc(cond, rd, rs, 8, ctx);
266 emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx);
267 return;
268 }
269
270 /* 32b No need for an additional rvoff adjustment, since we
271 * get that from the auipc at PC', where PC = PC' + 4.
272 */
273 upper = (rvoff + (1 << 11)) >> 12;
274 lower = rvoff & 0xfff;
275
276 emit_bcc(cond, rd, rs, 12, ctx);
277 emit(rv_auipc(RV_REG_T1, upper), ctx);
278 emit(rv_jalr(RV_REG_ZERO, RV_REG_T1, lower), ctx);
279 }
280
281 static void emit_zext_32(u8 reg, struct rv_jit_context *ctx)
282 {
283 emit(rv_slli(reg, reg, 32), ctx);
284 emit(rv_srli(reg, reg, 32), ctx);
285 }
286
287 static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx)
288 {
289 int tc_ninsn, off, start_insn = ctx->ninsns;
290 u8 tcc = rv_tail_call_reg(ctx);
291
292 /* a0: &ctx
293 * a1: &array
294 * a2: index
295 *
296 * if (index >= array->map.max_entries)
297 * goto out;
298 */
299 tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] :
300 ctx->offset[0];
301 emit_zext_32(RV_REG_A2, ctx);
302
303 off = offsetof(struct bpf_array, map.max_entries);
304 if (is_12b_check(off, insn))
305 return -1;
306 emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx);
307 off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2;
308 emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx);
309
310 /* if (TCC-- < 0)
311 * goto out;
312 */
313 emit(rv_addi(RV_REG_T1, tcc, -1), ctx);
314 off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2;
315 emit_branch(BPF_JSLT, tcc, RV_REG_ZERO, off, ctx);
316
317 /* prog = array->ptrs[index];
318 * if (!prog)
319 * goto out;
320 */
321 emit(rv_slli(RV_REG_T2, RV_REG_A2, 3), ctx);
322 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_A1), ctx);
323 off = offsetof(struct bpf_array, ptrs);
324 if (is_12b_check(off, insn))
325 return -1;
326 emit(rv_ld(RV_REG_T2, off, RV_REG_T2), ctx);
327 off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2;
328 emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx);
329
330 /* goto *(prog->bpf_func + 4); */
331 off = offsetof(struct bpf_prog, bpf_func);
332 if (is_12b_check(off, insn))
333 return -1;
334 emit(rv_ld(RV_REG_T3, off, RV_REG_T2), ctx);
335 emit(rv_addi(RV_REG_TCC, RV_REG_T1, 0), ctx);
336 __build_epilogue(true, ctx);
337 return 0;
338 }
339
340 static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn,
341 struct rv_jit_context *ctx)
342 {
343 u8 code = insn->code;
344
345 switch (code) {
346 case BPF_JMP | BPF_JA:
347 case BPF_JMP | BPF_CALL:
348 case BPF_JMP | BPF_EXIT:
349 case BPF_JMP | BPF_TAIL_CALL:
350 break;
351 default:
352 *rd = bpf_to_rv_reg(insn->dst_reg, ctx);
353 }
354
355 if (code & (BPF_ALU | BPF_X) || code & (BPF_ALU64 | BPF_X) ||
356 code & (BPF_JMP | BPF_X) || code & (BPF_JMP32 | BPF_X) ||
357 code & BPF_LDX || code & BPF_STX)
358 *rs = bpf_to_rv_reg(insn->src_reg, ctx);
359 }
360
361 static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx)
362 {
363 emit(rv_addi(RV_REG_T2, *rd, 0), ctx);
364 emit_zext_32(RV_REG_T2, ctx);
365 emit(rv_addi(RV_REG_T1, *rs, 0), ctx);
366 emit_zext_32(RV_REG_T1, ctx);
367 *rd = RV_REG_T2;
368 *rs = RV_REG_T1;
369 }
370
371 static void emit_sext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx)
372 {
373 emit(rv_addiw(RV_REG_T2, *rd, 0), ctx);
374 emit(rv_addiw(RV_REG_T1, *rs, 0), ctx);
375 *rd = RV_REG_T2;
376 *rs = RV_REG_T1;
377 }
378
379 static void emit_zext_32_rd_t1(u8 *rd, struct rv_jit_context *ctx)
380 {
381 emit(rv_addi(RV_REG_T2, *rd, 0), ctx);
382 emit_zext_32(RV_REG_T2, ctx);
383 emit_zext_32(RV_REG_T1, ctx);
384 *rd = RV_REG_T2;
385 }
386
387 static void emit_sext_32_rd(u8 *rd, struct rv_jit_context *ctx)
388 {
389 emit(rv_addiw(RV_REG_T2, *rd, 0), ctx);
390 *rd = RV_REG_T2;
391 }
392
393 static int emit_jump_and_link(u8 rd, s64 rvoff, bool force_jalr,
394 struct rv_jit_context *ctx)
395 {
396 s64 upper, lower;
397
398 if (rvoff && is_21b_int(rvoff) && !force_jalr) {
399 emit(rv_jal(rd, rvoff >> 1), ctx);
400 return 0;
401 } else if (in_auipc_jalr_range(rvoff)) {
402 upper = (rvoff + (1 << 11)) >> 12;
403 lower = rvoff & 0xfff;
404 emit(rv_auipc(RV_REG_T1, upper), ctx);
405 emit(rv_jalr(rd, RV_REG_T1, lower), ctx);
406 return 0;
407 }
408
409 pr_err("bpf-jit: target offset 0x%llx is out of range\n", rvoff);
410 return -ERANGE;
411 }
412
413 static bool is_signed_bpf_cond(u8 cond)
414 {
415 return cond == BPF_JSGT || cond == BPF_JSLT ||
416 cond == BPF_JSGE || cond == BPF_JSLE;
417 }
418
419 static int emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx)
420 {
421 s64 off = 0;
422 u64 ip;
423 u8 rd;
424 int ret;
425
426 if (addr && ctx->insns) {
427 ip = (u64)(long)(ctx->insns + ctx->ninsns);
428 off = addr - ip;
429 }
430
431 ret = emit_jump_and_link(RV_REG_RA, off, !fixed, ctx);
432 if (ret)
433 return ret;
434 rd = bpf_to_rv_reg(BPF_REG_0, ctx);
435 emit(rv_addi(rd, RV_REG_A0, 0), ctx);
436 return 0;
437 }
438
439 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
440 bool extra_pass)
441 {
442 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 ||
443 BPF_CLASS(insn->code) == BPF_JMP;
444 int s, e, rvoff, ret, i = insn - ctx->prog->insnsi;
445 struct bpf_prog_aux *aux = ctx->prog->aux;
446 u8 rd = -1, rs = -1, code = insn->code;
447 s16 off = insn->off;
448 s32 imm = insn->imm;
449
450 init_regs(&rd, &rs, insn, ctx);
451
452 switch (code) {
453 /* dst = src */
454 case BPF_ALU | BPF_MOV | BPF_X:
455 case BPF_ALU64 | BPF_MOV | BPF_X:
456 if (imm == 1) {
457 /* Special mov32 for zext */
458 emit_zext_32(rd, ctx);
459 break;
460 }
461 emit(is64 ? rv_addi(rd, rs, 0) : rv_addiw(rd, rs, 0), ctx);
462 if (!is64 && !aux->verifier_zext)
463 emit_zext_32(rd, ctx);
464 break;
465
466 /* dst = dst OP src */
467 case BPF_ALU | BPF_ADD | BPF_X:
468 case BPF_ALU64 | BPF_ADD | BPF_X:
469 emit(is64 ? rv_add(rd, rd, rs) : rv_addw(rd, rd, rs), ctx);
470 if (!is64 && !aux->verifier_zext)
471 emit_zext_32(rd, ctx);
472 break;
473 case BPF_ALU | BPF_SUB | BPF_X:
474 case BPF_ALU64 | BPF_SUB | BPF_X:
475 emit(is64 ? rv_sub(rd, rd, rs) : rv_subw(rd, rd, rs), ctx);
476 if (!is64 && !aux->verifier_zext)
477 emit_zext_32(rd, ctx);
478 break;
479 case BPF_ALU | BPF_AND | BPF_X:
480 case BPF_ALU64 | BPF_AND | BPF_X:
481 emit(rv_and(rd, rd, rs), ctx);
482 if (!is64 && !aux->verifier_zext)
483 emit_zext_32(rd, ctx);
484 break;
485 case BPF_ALU | BPF_OR | BPF_X:
486 case BPF_ALU64 | BPF_OR | BPF_X:
487 emit(rv_or(rd, rd, rs), ctx);
488 if (!is64 && !aux->verifier_zext)
489 emit_zext_32(rd, ctx);
490 break;
491 case BPF_ALU | BPF_XOR | BPF_X:
492 case BPF_ALU64 | BPF_XOR | BPF_X:
493 emit(rv_xor(rd, rd, rs), ctx);
494 if (!is64 && !aux->verifier_zext)
495 emit_zext_32(rd, ctx);
496 break;
497 case BPF_ALU | BPF_MUL | BPF_X:
498 case BPF_ALU64 | BPF_MUL | BPF_X:
499 emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx);
500 if (!is64 && !aux->verifier_zext)
501 emit_zext_32(rd, ctx);
502 break;
503 case BPF_ALU | BPF_DIV | BPF_X:
504 case BPF_ALU64 | BPF_DIV | BPF_X:
505 emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx);
506 if (!is64 && !aux->verifier_zext)
507 emit_zext_32(rd, ctx);
508 break;
509 case BPF_ALU | BPF_MOD | BPF_X:
510 case BPF_ALU64 | BPF_MOD | BPF_X:
511 emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx);
512 if (!is64 && !aux->verifier_zext)
513 emit_zext_32(rd, ctx);
514 break;
515 case BPF_ALU | BPF_LSH | BPF_X:
516 case BPF_ALU64 | BPF_LSH | BPF_X:
517 emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx);
518 if (!is64)
519 emit_zext_32(rd, ctx);
520 break;
521 case BPF_ALU | BPF_RSH | BPF_X:
522 case BPF_ALU64 | BPF_RSH | BPF_X:
523 emit(is64 ? rv_srl(rd, rd, rs) : rv_srlw(rd, rd, rs), ctx);
524 if (!is64 && !aux->verifier_zext)
525 emit_zext_32(rd, ctx);
526 break;
527 case BPF_ALU | BPF_ARSH | BPF_X:
528 case BPF_ALU64 | BPF_ARSH | BPF_X:
529 emit(is64 ? rv_sra(rd, rd, rs) : rv_sraw(rd, rd, rs), ctx);
530 if (!is64 && !aux->verifier_zext)
531 emit_zext_32(rd, ctx);
532 break;
533
534 /* dst = -dst */
535 case BPF_ALU | BPF_NEG:
536 case BPF_ALU64 | BPF_NEG:
537 emit(is64 ? rv_sub(rd, RV_REG_ZERO, rd) :
538 rv_subw(rd, RV_REG_ZERO, rd), ctx);
539 if (!is64 && !aux->verifier_zext)
540 emit_zext_32(rd, ctx);
541 break;
542
543 /* dst = BSWAP##imm(dst) */
544 case BPF_ALU | BPF_END | BPF_FROM_LE:
545 {
546 int shift = 64 - imm;
547
548 emit(rv_slli(rd, rd, shift), ctx);
549 emit(rv_srli(rd, rd, shift), ctx);
550 break;
551 }
552 case BPF_ALU | BPF_END | BPF_FROM_BE:
553 emit(rv_addi(RV_REG_T2, RV_REG_ZERO, 0), ctx);
554
555 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
556 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
557 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
558 emit(rv_srli(rd, rd, 8), ctx);
559 if (imm == 16)
560 goto out_be;
561
562 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
563 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
564 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
565 emit(rv_srli(rd, rd, 8), ctx);
566
567 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
568 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
569 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
570 emit(rv_srli(rd, rd, 8), ctx);
571 if (imm == 32)
572 goto out_be;
573
574 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
575 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
576 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
577 emit(rv_srli(rd, rd, 8), ctx);
578
579 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
580 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
581 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
582 emit(rv_srli(rd, rd, 8), ctx);
583
584 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
585 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
586 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
587 emit(rv_srli(rd, rd, 8), ctx);
588
589 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
590 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
591 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
592 emit(rv_srli(rd, rd, 8), ctx);
593 out_be:
594 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
595 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
596
597 emit(rv_addi(rd, RV_REG_T2, 0), ctx);
598 break;
599
600 /* dst = imm */
601 case BPF_ALU | BPF_MOV | BPF_K:
602 case BPF_ALU64 | BPF_MOV | BPF_K:
603 emit_imm(rd, imm, ctx);
604 if (!is64 && !aux->verifier_zext)
605 emit_zext_32(rd, ctx);
606 break;
607
608 /* dst = dst OP imm */
609 case BPF_ALU | BPF_ADD | BPF_K:
610 case BPF_ALU64 | BPF_ADD | BPF_K:
611 if (is_12b_int(imm)) {
612 emit(is64 ? rv_addi(rd, rd, imm) :
613 rv_addiw(rd, rd, imm), ctx);
614 } else {
615 emit_imm(RV_REG_T1, imm, ctx);
616 emit(is64 ? rv_add(rd, rd, RV_REG_T1) :
617 rv_addw(rd, rd, RV_REG_T1), ctx);
618 }
619 if (!is64 && !aux->verifier_zext)
620 emit_zext_32(rd, ctx);
621 break;
622 case BPF_ALU | BPF_SUB | BPF_K:
623 case BPF_ALU64 | BPF_SUB | BPF_K:
624 if (is_12b_int(-imm)) {
625 emit(is64 ? rv_addi(rd, rd, -imm) :
626 rv_addiw(rd, rd, -imm), ctx);
627 } else {
628 emit_imm(RV_REG_T1, imm, ctx);
629 emit(is64 ? rv_sub(rd, rd, RV_REG_T1) :
630 rv_subw(rd, rd, RV_REG_T1), ctx);
631 }
632 if (!is64 && !aux->verifier_zext)
633 emit_zext_32(rd, ctx);
634 break;
635 case BPF_ALU | BPF_AND | BPF_K:
636 case BPF_ALU64 | BPF_AND | BPF_K:
637 if (is_12b_int(imm)) {
638 emit(rv_andi(rd, rd, imm), ctx);
639 } else {
640 emit_imm(RV_REG_T1, imm, ctx);
641 emit(rv_and(rd, rd, RV_REG_T1), ctx);
642 }
643 if (!is64 && !aux->verifier_zext)
644 emit_zext_32(rd, ctx);
645 break;
646 case BPF_ALU | BPF_OR | BPF_K:
647 case BPF_ALU64 | BPF_OR | BPF_K:
648 if (is_12b_int(imm)) {
649 emit(rv_ori(rd, rd, imm), ctx);
650 } else {
651 emit_imm(RV_REG_T1, imm, ctx);
652 emit(rv_or(rd, rd, RV_REG_T1), ctx);
653 }
654 if (!is64 && !aux->verifier_zext)
655 emit_zext_32(rd, ctx);
656 break;
657 case BPF_ALU | BPF_XOR | BPF_K:
658 case BPF_ALU64 | BPF_XOR | BPF_K:
659 if (is_12b_int(imm)) {
660 emit(rv_xori(rd, rd, imm), ctx);
661 } else {
662 emit_imm(RV_REG_T1, imm, ctx);
663 emit(rv_xor(rd, rd, RV_REG_T1), ctx);
664 }
665 if (!is64 && !aux->verifier_zext)
666 emit_zext_32(rd, ctx);
667 break;
668 case BPF_ALU | BPF_MUL | BPF_K:
669 case BPF_ALU64 | BPF_MUL | BPF_K:
670 emit_imm(RV_REG_T1, imm, ctx);
671 emit(is64 ? rv_mul(rd, rd, RV_REG_T1) :
672 rv_mulw(rd, rd, RV_REG_T1), ctx);
673 if (!is64 && !aux->verifier_zext)
674 emit_zext_32(rd, ctx);
675 break;
676 case BPF_ALU | BPF_DIV | BPF_K:
677 case BPF_ALU64 | BPF_DIV | BPF_K:
678 emit_imm(RV_REG_T1, imm, ctx);
679 emit(is64 ? rv_divu(rd, rd, RV_REG_T1) :
680 rv_divuw(rd, rd, RV_REG_T1), ctx);
681 if (!is64 && !aux->verifier_zext)
682 emit_zext_32(rd, ctx);
683 break;
684 case BPF_ALU | BPF_MOD | BPF_K:
685 case BPF_ALU64 | BPF_MOD | BPF_K:
686 emit_imm(RV_REG_T1, imm, ctx);
687 emit(is64 ? rv_remu(rd, rd, RV_REG_T1) :
688 rv_remuw(rd, rd, RV_REG_T1), ctx);
689 if (!is64 && !aux->verifier_zext)
690 emit_zext_32(rd, ctx);
691 break;
692 case BPF_ALU | BPF_LSH | BPF_K:
693 case BPF_ALU64 | BPF_LSH | BPF_K:
694 emit(is64 ? rv_slli(rd, rd, imm) : rv_slliw(rd, rd, imm), ctx);
695 if (!is64)
696 emit_zext_32(rd, ctx);
697 break;
698 case BPF_ALU | BPF_RSH | BPF_K:
699 case BPF_ALU64 | BPF_RSH | BPF_K:
700 emit(is64 ? rv_srli(rd, rd, imm) : rv_srliw(rd, rd, imm), ctx);
701 if (!is64)
702 emit_zext_32(rd, ctx);
703 break;
704 case BPF_ALU | BPF_ARSH | BPF_K:
705 case BPF_ALU64 | BPF_ARSH | BPF_K:
706 emit(is64 ? rv_srai(rd, rd, imm) : rv_sraiw(rd, rd, imm), ctx);
707 if (!is64)
708 emit_zext_32(rd, ctx);
709 break;
710
711 /* JUMP off */
712 case BPF_JMP | BPF_JA:
713 rvoff = rv_offset(i, off, ctx);
714 ret = emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx);
715 if (ret)
716 return ret;
717 break;
718
719 /* IF (dst COND src) JUMP off */
720 case BPF_JMP | BPF_JEQ | BPF_X:
721 case BPF_JMP32 | BPF_JEQ | BPF_X:
722 case BPF_JMP | BPF_JGT | BPF_X:
723 case BPF_JMP32 | BPF_JGT | BPF_X:
724 case BPF_JMP | BPF_JLT | BPF_X:
725 case BPF_JMP32 | BPF_JLT | BPF_X:
726 case BPF_JMP | BPF_JGE | BPF_X:
727 case BPF_JMP32 | BPF_JGE | BPF_X:
728 case BPF_JMP | BPF_JLE | BPF_X:
729 case BPF_JMP32 | BPF_JLE | BPF_X:
730 case BPF_JMP | BPF_JNE | BPF_X:
731 case BPF_JMP32 | BPF_JNE | BPF_X:
732 case BPF_JMP | BPF_JSGT | BPF_X:
733 case BPF_JMP32 | BPF_JSGT | BPF_X:
734 case BPF_JMP | BPF_JSLT | BPF_X:
735 case BPF_JMP32 | BPF_JSLT | BPF_X:
736 case BPF_JMP | BPF_JSGE | BPF_X:
737 case BPF_JMP32 | BPF_JSGE | BPF_X:
738 case BPF_JMP | BPF_JSLE | BPF_X:
739 case BPF_JMP32 | BPF_JSLE | BPF_X:
740 case BPF_JMP | BPF_JSET | BPF_X:
741 case BPF_JMP32 | BPF_JSET | BPF_X:
742 rvoff = rv_offset(i, off, ctx);
743 if (!is64) {
744 s = ctx->ninsns;
745 if (is_signed_bpf_cond(BPF_OP(code)))
746 emit_sext_32_rd_rs(&rd, &rs, ctx);
747 else
748 emit_zext_32_rd_rs(&rd, &rs, ctx);
749 e = ctx->ninsns;
750
751 /* Adjust for extra insns */
752 rvoff -= (e - s) << 2;
753 }
754
755 if (BPF_OP(code) == BPF_JSET) {
756 /* Adjust for and */
757 rvoff -= 4;
758 emit(rv_and(RV_REG_T1, rd, rs), ctx);
759 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff,
760 ctx);
761 } else {
762 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx);
763 }
764 break;
765
766 /* IF (dst COND imm) JUMP off */
767 case BPF_JMP | BPF_JEQ | BPF_K:
768 case BPF_JMP32 | BPF_JEQ | BPF_K:
769 case BPF_JMP | BPF_JGT | BPF_K:
770 case BPF_JMP32 | BPF_JGT | BPF_K:
771 case BPF_JMP | BPF_JLT | BPF_K:
772 case BPF_JMP32 | BPF_JLT | BPF_K:
773 case BPF_JMP | BPF_JGE | BPF_K:
774 case BPF_JMP32 | BPF_JGE | BPF_K:
775 case BPF_JMP | BPF_JLE | BPF_K:
776 case BPF_JMP32 | BPF_JLE | BPF_K:
777 case BPF_JMP | BPF_JNE | BPF_K:
778 case BPF_JMP32 | BPF_JNE | BPF_K:
779 case BPF_JMP | BPF_JSGT | BPF_K:
780 case BPF_JMP32 | BPF_JSGT | BPF_K:
781 case BPF_JMP | BPF_JSLT | BPF_K:
782 case BPF_JMP32 | BPF_JSLT | BPF_K:
783 case BPF_JMP | BPF_JSGE | BPF_K:
784 case BPF_JMP32 | BPF_JSGE | BPF_K:
785 case BPF_JMP | BPF_JSLE | BPF_K:
786 case BPF_JMP32 | BPF_JSLE | BPF_K:
787 case BPF_JMP | BPF_JSET | BPF_K:
788 case BPF_JMP32 | BPF_JSET | BPF_K:
789 rvoff = rv_offset(i, off, ctx);
790 s = ctx->ninsns;
791 emit_imm(RV_REG_T1, imm, ctx);
792 if (!is64) {
793 if (is_signed_bpf_cond(BPF_OP(code)))
794 emit_sext_32_rd(&rd, ctx);
795 else
796 emit_zext_32_rd_t1(&rd, ctx);
797 }
798 e = ctx->ninsns;
799
800 /* Adjust for extra insns */
801 rvoff -= (e - s) << 2;
802
803 if (BPF_OP(code) == BPF_JSET) {
804 /* Adjust for and */
805 rvoff -= 4;
806 emit(rv_and(RV_REG_T1, rd, RV_REG_T1), ctx);
807 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff,
808 ctx);
809 } else {
810 emit_branch(BPF_OP(code), rd, RV_REG_T1, rvoff, ctx);
811 }
812 break;
813
814 /* function call */
815 case BPF_JMP | BPF_CALL:
816 {
817 bool fixed;
818 u64 addr;
819
820 mark_call(ctx);
821 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, &addr,
822 &fixed);
823 if (ret < 0)
824 return ret;
825 ret = emit_call(fixed, addr, ctx);
826 if (ret)
827 return ret;
828 break;
829 }
830 /* tail call */
831 case BPF_JMP | BPF_TAIL_CALL:
832 if (emit_bpf_tail_call(i, ctx))
833 return -1;
834 break;
835
836 /* function return */
837 case BPF_JMP | BPF_EXIT:
838 if (i == ctx->prog->len - 1)
839 break;
840
841 rvoff = epilogue_offset(ctx);
842 ret = emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx);
843 if (ret)
844 return ret;
845 break;
846
847 /* dst = imm64 */
848 case BPF_LD | BPF_IMM | BPF_DW:
849 {
850 struct bpf_insn insn1 = insn[1];
851 u64 imm64;
852
853 imm64 = (u64)insn1.imm << 32 | (u32)imm;
854 emit_imm(rd, imm64, ctx);
855 return 1;
856 }
857
858 /* LDX: dst = *(size *)(src + off) */
859 case BPF_LDX | BPF_MEM | BPF_B:
860 if (is_12b_int(off)) {
861 emit(rv_lbu(rd, off, rs), ctx);
862 break;
863 }
864
865 emit_imm(RV_REG_T1, off, ctx);
866 emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx);
867 emit(rv_lbu(rd, 0, RV_REG_T1), ctx);
868 if (insn_is_zext(&insn[1]))
869 return 1;
870 break;
871 case BPF_LDX | BPF_MEM | BPF_H:
872 if (is_12b_int(off)) {
873 emit(rv_lhu(rd, off, rs), ctx);
874 break;
875 }
876
877 emit_imm(RV_REG_T1, off, ctx);
878 emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx);
879 emit(rv_lhu(rd, 0, RV_REG_T1), ctx);
880 if (insn_is_zext(&insn[1]))
881 return 1;
882 break;
883 case BPF_LDX | BPF_MEM | BPF_W:
884 if (is_12b_int(off)) {
885 emit(rv_lwu(rd, off, rs), ctx);
886 break;
887 }
888
889 emit_imm(RV_REG_T1, off, ctx);
890 emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx);
891 emit(rv_lwu(rd, 0, RV_REG_T1), ctx);
892 if (insn_is_zext(&insn[1]))
893 return 1;
894 break;
895 case BPF_LDX | BPF_MEM | BPF_DW:
896 if (is_12b_int(off)) {
897 emit(rv_ld(rd, off, rs), ctx);
898 break;
899 }
900
901 emit_imm(RV_REG_T1, off, ctx);
902 emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx);
903 emit(rv_ld(rd, 0, RV_REG_T1), ctx);
904 break;
905
906 /* ST: *(size *)(dst + off) = imm */
907 case BPF_ST | BPF_MEM | BPF_B:
908 emit_imm(RV_REG_T1, imm, ctx);
909 if (is_12b_int(off)) {
910 emit(rv_sb(rd, off, RV_REG_T1), ctx);
911 break;
912 }
913
914 emit_imm(RV_REG_T2, off, ctx);
915 emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx);
916 emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx);
917 break;
918
919 case BPF_ST | BPF_MEM | BPF_H:
920 emit_imm(RV_REG_T1, imm, ctx);
921 if (is_12b_int(off)) {
922 emit(rv_sh(rd, off, RV_REG_T1), ctx);
923 break;
924 }
925
926 emit_imm(RV_REG_T2, off, ctx);
927 emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx);
928 emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx);
929 break;
930 case BPF_ST | BPF_MEM | BPF_W:
931 emit_imm(RV_REG_T1, imm, ctx);
932 if (is_12b_int(off)) {
933 emit(rv_sw(rd, off, RV_REG_T1), ctx);
934 break;
935 }
936
937 emit_imm(RV_REG_T2, off, ctx);
938 emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx);
939 emit(rv_sw(RV_REG_T2, 0, RV_REG_T1), ctx);
940 break;
941 case BPF_ST | BPF_MEM | BPF_DW:
942 emit_imm(RV_REG_T1, imm, ctx);
943 if (is_12b_int(off)) {
944 emit(rv_sd(rd, off, RV_REG_T1), ctx);
945 break;
946 }
947
948 emit_imm(RV_REG_T2, off, ctx);
949 emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx);
950 emit(rv_sd(RV_REG_T2, 0, RV_REG_T1), ctx);
951 break;
952
953 /* STX: *(size *)(dst + off) = src */
954 case BPF_STX | BPF_MEM | BPF_B:
955 if (is_12b_int(off)) {
956 emit(rv_sb(rd, off, rs), ctx);
957 break;
958 }
959
960 emit_imm(RV_REG_T1, off, ctx);
961 emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx);
962 emit(rv_sb(RV_REG_T1, 0, rs), ctx);
963 break;
964 case BPF_STX | BPF_MEM | BPF_H:
965 if (is_12b_int(off)) {
966 emit(rv_sh(rd, off, rs), ctx);
967 break;
968 }
969
970 emit_imm(RV_REG_T1, off, ctx);
971 emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx);
972 emit(rv_sh(RV_REG_T1, 0, rs), ctx);
973 break;
974 case BPF_STX | BPF_MEM | BPF_W:
975 if (is_12b_int(off)) {
976 emit(rv_sw(rd, off, rs), ctx);
977 break;
978 }
979
980 emit_imm(RV_REG_T1, off, ctx);
981 emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx);
982 emit(rv_sw(RV_REG_T1, 0, rs), ctx);
983 break;
984 case BPF_STX | BPF_MEM | BPF_DW:
985 if (is_12b_int(off)) {
986 emit(rv_sd(rd, off, rs), ctx);
987 break;
988 }
989
990 emit_imm(RV_REG_T1, off, ctx);
991 emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx);
992 emit(rv_sd(RV_REG_T1, 0, rs), ctx);
993 break;
994 /* STX XADD: lock *(u32 *)(dst + off) += src */
995 case BPF_STX | BPF_XADD | BPF_W:
996 /* STX XADD: lock *(u64 *)(dst + off) += src */
997 case BPF_STX | BPF_XADD | BPF_DW:
998 if (off) {
999 if (is_12b_int(off)) {
1000 emit(rv_addi(RV_REG_T1, rd, off), ctx);
1001 } else {
1002 emit_imm(RV_REG_T1, off, ctx);
1003 emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx);
1004 }
1005
1006 rd = RV_REG_T1;
1007 }
1008
1009 emit(BPF_SIZE(code) == BPF_W ?
1010 rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0) :
1011 rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0), ctx);
1012 break;
1013 default:
1014 pr_err("bpf-jit: unknown opcode %02x\n", code);
1015 return -EINVAL;
1016 }
1017
1018 return 0;
1019 }
1020
1021 void bpf_jit_build_prologue(struct rv_jit_context *ctx)
1022 {
1023 int stack_adjust = 0, store_offset, bpf_stack_adjust;
1024
1025 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16);
1026 if (bpf_stack_adjust)
1027 mark_fp(ctx);
1028
1029 if (seen_reg(RV_REG_RA, ctx))
1030 stack_adjust += 8;
1031 stack_adjust += 8; /* RV_REG_FP */
1032 if (seen_reg(RV_REG_S1, ctx))
1033 stack_adjust += 8;
1034 if (seen_reg(RV_REG_S2, ctx))
1035 stack_adjust += 8;
1036 if (seen_reg(RV_REG_S3, ctx))
1037 stack_adjust += 8;
1038 if (seen_reg(RV_REG_S4, ctx))
1039 stack_adjust += 8;
1040 if (seen_reg(RV_REG_S5, ctx))
1041 stack_adjust += 8;
1042 if (seen_reg(RV_REG_S6, ctx))
1043 stack_adjust += 8;
1044
1045 stack_adjust = round_up(stack_adjust, 16);
1046 stack_adjust += bpf_stack_adjust;
1047
1048 store_offset = stack_adjust - 8;
1049
1050 /* First instruction is always setting the tail-call-counter
1051 * (TCC) register. This instruction is skipped for tail calls.
1052 */
1053 emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx);
1054
1055 emit(rv_addi(RV_REG_SP, RV_REG_SP, -stack_adjust), ctx);
1056
1057 if (seen_reg(RV_REG_RA, ctx)) {
1058 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_RA), ctx);
1059 store_offset -= 8;
1060 }
1061 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_FP), ctx);
1062 store_offset -= 8;
1063 if (seen_reg(RV_REG_S1, ctx)) {
1064 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S1), ctx);
1065 store_offset -= 8;
1066 }
1067 if (seen_reg(RV_REG_S2, ctx)) {
1068 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S2), ctx);
1069 store_offset -= 8;
1070 }
1071 if (seen_reg(RV_REG_S3, ctx)) {
1072 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S3), ctx);
1073 store_offset -= 8;
1074 }
1075 if (seen_reg(RV_REG_S4, ctx)) {
1076 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S4), ctx);
1077 store_offset -= 8;
1078 }
1079 if (seen_reg(RV_REG_S5, ctx)) {
1080 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S5), ctx);
1081 store_offset -= 8;
1082 }
1083 if (seen_reg(RV_REG_S6, ctx)) {
1084 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S6), ctx);
1085 store_offset -= 8;
1086 }
1087
1088 emit(rv_addi(RV_REG_FP, RV_REG_SP, stack_adjust), ctx);
1089
1090 if (bpf_stack_adjust)
1091 emit(rv_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust), ctx);
1092
1093 /* Program contains calls and tail calls, so RV_REG_TCC need
1094 * to be saved across calls.
1095 */
1096 if (seen_tail_call(ctx) && seen_call(ctx))
1097 emit(rv_addi(RV_REG_TCC_SAVED, RV_REG_TCC, 0), ctx);
1098
1099 ctx->stack_size = stack_adjust;
1100 }
1101
1102 void bpf_jit_build_epilogue(struct rv_jit_context *ctx)
1103 {
1104 __build_epilogue(false, ctx);
1105 }
1106
1107 void *bpf_jit_alloc_exec(unsigned long size)
1108 {
1109 return __vmalloc_node_range(size, PAGE_SIZE, BPF_JIT_REGION_START,
1110 BPF_JIT_REGION_END, GFP_KERNEL,
1111 PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE,
1112 __builtin_return_address(0));
1113 }
1114
1115 void bpf_jit_free_exec(void *addr)
1116 {
1117 return vfree(addr);
1118 }
Linux with added FPC-III support