?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
gpio: rcar: Fix runtime PM imbalance on error
[linux/fpc-iii.git] / arch / riscv / net / bpf_jit_comp64.c
blobcc1985d8750a1fd7ba12825553a0cfc60bfc91b9
1 // SPDX-License-Identifier: GPL-2.0
2 /* BPF JIT compiler for RV64G
3 *
4 * Copyright(c) 2019 Björn Töpel <bjorn.topel@gmail.com>
5 *
6 */
7
8 #include <linux/bpf.h>
9 #include <linux/filter.h>
10 #include "bpf_jit.h"
11
12 #define RV_REG_TCC RV_REG_A6
13 #define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */
14
15 static const int regmap[] = {
16 [BPF_REG_0] = RV_REG_A5,
17 [BPF_REG_1] = RV_REG_A0,
18 [BPF_REG_2] = RV_REG_A1,
19 [BPF_REG_3] = RV_REG_A2,
20 [BPF_REG_4] = RV_REG_A3,
21 [BPF_REG_5] = RV_REG_A4,
22 [BPF_REG_6] = RV_REG_S1,
23 [BPF_REG_7] = RV_REG_S2,
24 [BPF_REG_8] = RV_REG_S3,
25 [BPF_REG_9] = RV_REG_S4,
26 [BPF_REG_FP] = RV_REG_S5,
27 [BPF_REG_AX] = RV_REG_T0,
28 };
29
30 enum {
31 RV_CTX_F_SEEN_TAIL_CALL = 0,
32 RV_CTX_F_SEEN_CALL = RV_REG_RA,
33 RV_CTX_F_SEEN_S1 = RV_REG_S1,
34 RV_CTX_F_SEEN_S2 = RV_REG_S2,
35 RV_CTX_F_SEEN_S3 = RV_REG_S3,
36 RV_CTX_F_SEEN_S4 = RV_REG_S4,
37 RV_CTX_F_SEEN_S5 = RV_REG_S5,
38 RV_CTX_F_SEEN_S6 = RV_REG_S6,
39 };
40
41 static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx)
42 {
43 u8 reg = regmap[bpf_reg];
44
45 switch (reg) {
46 case RV_CTX_F_SEEN_S1:
47 case RV_CTX_F_SEEN_S2:
48 case RV_CTX_F_SEEN_S3:
49 case RV_CTX_F_SEEN_S4:
50 case RV_CTX_F_SEEN_S5:
51 case RV_CTX_F_SEEN_S6:
52 __set_bit(reg, &ctx->flags);
53 }
54 return reg;
55 };
56
57 static bool seen_reg(int reg, struct rv_jit_context *ctx)
58 {
59 switch (reg) {
60 case RV_CTX_F_SEEN_CALL:
61 case RV_CTX_F_SEEN_S1:
62 case RV_CTX_F_SEEN_S2:
63 case RV_CTX_F_SEEN_S3:
64 case RV_CTX_F_SEEN_S4:
65 case RV_CTX_F_SEEN_S5:
66 case RV_CTX_F_SEEN_S6:
67 return test_bit(reg, &ctx->flags);
68 }
69 return false;
70 }
71
72 static void mark_fp(struct rv_jit_context *ctx)
73 {
74 __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags);
75 }
76
77 static void mark_call(struct rv_jit_context *ctx)
78 {
79 __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags);
80 }
81
82 static bool seen_call(struct rv_jit_context *ctx)
83 {
84 return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags);
85 }
86
87 static void mark_tail_call(struct rv_jit_context *ctx)
88 {
89 __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags);
90 }
91
92 static bool seen_tail_call(struct rv_jit_context *ctx)
93 {
94 return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags);
95 }
96
97 static u8 rv_tail_call_reg(struct rv_jit_context *ctx)
98 {
99 mark_tail_call(ctx);
100
101 if (seen_call(ctx)) {
102 __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags);
103 return RV_REG_S6;
104 }
105 return RV_REG_A6;
106 }
107
108 static bool is_32b_int(s64 val)
109 {
110 return -(1L << 31) <= val && val < (1L << 31);
111 }
112
113 static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx)
114 {
115 /* Note that the immediate from the add is sign-extended,
116 * which means that we need to compensate this by adding 2^12,
117 * when the 12th bit is set. A simpler way of doing this, and
118 * getting rid of the check, is to just add 2**11 before the
119 * shift. The "Loading a 32-Bit constant" example from the
120 * "Computer Organization and Design, RISC-V edition" book by
121 * Patterson/Hennessy highlights this fact.
122 *
123 * This also means that we need to process LSB to MSB.
124 */
125 s64 upper = (val + (1 << 11)) >> 12, lower = val & 0xfff;
126 int shift;
127
128 if (is_32b_int(val)) {
129 if (upper)
130 emit(rv_lui(rd, upper), ctx);
131
132 if (!upper) {
133 emit(rv_addi(rd, RV_REG_ZERO, lower), ctx);
134 return;
135 }
136
137 emit(rv_addiw(rd, rd, lower), ctx);
138 return;
139 }
140
141 shift = __ffs(upper);
142 upper >>= shift;
143 shift += 12;
144
145 emit_imm(rd, upper, ctx);
146
147 emit(rv_slli(rd, rd, shift), ctx);
148 if (lower)
149 emit(rv_addi(rd, rd, lower), ctx);
150 }
151
152 static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx)
153 {
154 int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8;
155
156 if (seen_reg(RV_REG_RA, ctx)) {
157 emit(rv_ld(RV_REG_RA, store_offset, RV_REG_SP), ctx);
158 store_offset -= 8;
159 }
160 emit(rv_ld(RV_REG_FP, store_offset, RV_REG_SP), ctx);
161 store_offset -= 8;
162 if (seen_reg(RV_REG_S1, ctx)) {
163 emit(rv_ld(RV_REG_S1, store_offset, RV_REG_SP), ctx);
164 store_offset -= 8;
165 }
166 if (seen_reg(RV_REG_S2, ctx)) {
167 emit(rv_ld(RV_REG_S2, store_offset, RV_REG_SP), ctx);
168 store_offset -= 8;
169 }
170 if (seen_reg(RV_REG_S3, ctx)) {
171 emit(rv_ld(RV_REG_S3, store_offset, RV_REG_SP), ctx);
172 store_offset -= 8;
173 }
174 if (seen_reg(RV_REG_S4, ctx)) {
175 emit(rv_ld(RV_REG_S4, store_offset, RV_REG_SP), ctx);
176 store_offset -= 8;
177 }
178 if (seen_reg(RV_REG_S5, ctx)) {
179 emit(rv_ld(RV_REG_S5, store_offset, RV_REG_SP), ctx);
180 store_offset -= 8;
181 }
182 if (seen_reg(RV_REG_S6, ctx)) {
183 emit(rv_ld(RV_REG_S6, store_offset, RV_REG_SP), ctx);
184 store_offset -= 8;
185 }
186
187 emit(rv_addi(RV_REG_SP, RV_REG_SP, stack_adjust), ctx);
188 /* Set return value. */
189 if (!is_tail_call)
190 emit(rv_addi(RV_REG_A0, RV_REG_A5, 0), ctx);
191 emit(rv_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA,
192 is_tail_call ? 4 : 0), /* skip TCC init */
193 ctx);
194 }
195
196 static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff,
197 struct rv_jit_context *ctx)
198 {
199 switch (cond) {
200 case BPF_JEQ:
201 emit(rv_beq(rd, rs, rvoff >> 1), ctx);
202 return;
203 case BPF_JGT:
204 emit(rv_bltu(rs, rd, rvoff >> 1), ctx);
205 return;
206 case BPF_JLT:
207 emit(rv_bltu(rd, rs, rvoff >> 1), ctx);
208 return;
209 case BPF_JGE:
210 emit(rv_bgeu(rd, rs, rvoff >> 1), ctx);
211 return;
212 case BPF_JLE:
213 emit(rv_bgeu(rs, rd, rvoff >> 1), ctx);
214 return;
215 case BPF_JNE:
216 emit(rv_bne(rd, rs, rvoff >> 1), ctx);
217 return;
218 case BPF_JSGT:
219 emit(rv_blt(rs, rd, rvoff >> 1), ctx);
220 return;
221 case BPF_JSLT:
222 emit(rv_blt(rd, rs, rvoff >> 1), ctx);
223 return;
224 case BPF_JSGE:
225 emit(rv_bge(rd, rs, rvoff >> 1), ctx);
226 return;
227 case BPF_JSLE:
228 emit(rv_bge(rs, rd, rvoff >> 1), ctx);
229 }
230 }
231
232 static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff,
233 struct rv_jit_context *ctx)
234 {
235 s64 upper, lower;
236
237 if (is_13b_int(rvoff)) {
238 emit_bcc(cond, rd, rs, rvoff, ctx);
239 return;
240 }
241
242 /* Adjust for jal */
243 rvoff -= 4;
244
245 /* Transform, e.g.:
246 * bne rd,rs,foo
247 * to
248 * beq rd,rs,<.L1>
249 * (auipc foo)
250 * jal(r) foo
251 * .L1
252 */
253 cond = invert_bpf_cond(cond);
254 if (is_21b_int(rvoff)) {
255 emit_bcc(cond, rd, rs, 8, ctx);
256 emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx);
257 return;
258 }
259
260 /* 32b No need for an additional rvoff adjustment, since we
261 * get that from the auipc at PC', where PC = PC' + 4.
262 */
263 upper = (rvoff + (1 << 11)) >> 12;
264 lower = rvoff & 0xfff;
265
266 emit_bcc(cond, rd, rs, 12, ctx);
267 emit(rv_auipc(RV_REG_T1, upper), ctx);
268 emit(rv_jalr(RV_REG_ZERO, RV_REG_T1, lower), ctx);
269 }
270
271 static void emit_zext_32(u8 reg, struct rv_jit_context *ctx)
272 {
273 emit(rv_slli(reg, reg, 32), ctx);
274 emit(rv_srli(reg, reg, 32), ctx);
275 }
276
277 static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx)
278 {
279 int tc_ninsn, off, start_insn = ctx->ninsns;
280 u8 tcc = rv_tail_call_reg(ctx);
281
282 /* a0: &ctx
283 * a1: &array
284 * a2: index
285 *
286 * if (index >= array->map.max_entries)
287 * goto out;
288 */
289 tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] :
290 ctx->offset[0];
291 emit_zext_32(RV_REG_A2, ctx);
292
293 off = offsetof(struct bpf_array, map.max_entries);
294 if (is_12b_check(off, insn))
295 return -1;
296 emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx);
297 off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2;
298 emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx);
299
300 /* if (TCC-- < 0)
301 * goto out;
302 */
303 emit(rv_addi(RV_REG_T1, tcc, -1), ctx);
304 off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2;
305 emit_branch(BPF_JSLT, tcc, RV_REG_ZERO, off, ctx);
306
307 /* prog = array->ptrs[index];
308 * if (!prog)
309 * goto out;
310 */
311 emit(rv_slli(RV_REG_T2, RV_REG_A2, 3), ctx);
312 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_A1), ctx);
313 off = offsetof(struct bpf_array, ptrs);
314 if (is_12b_check(off, insn))
315 return -1;
316 emit(rv_ld(RV_REG_T2, off, RV_REG_T2), ctx);
317 off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2;
318 emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx);
319
320 /* goto *(prog->bpf_func + 4); */
321 off = offsetof(struct bpf_prog, bpf_func);
322 if (is_12b_check(off, insn))
323 return -1;
324 emit(rv_ld(RV_REG_T3, off, RV_REG_T2), ctx);
325 emit(rv_addi(RV_REG_TCC, RV_REG_T1, 0), ctx);
326 __build_epilogue(true, ctx);
327 return 0;
328 }
329
330 static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn,
331 struct rv_jit_context *ctx)
332 {
333 u8 code = insn->code;
334
335 switch (code) {
336 case BPF_JMP | BPF_JA:
337 case BPF_JMP | BPF_CALL:
338 case BPF_JMP | BPF_EXIT:
339 case BPF_JMP | BPF_TAIL_CALL:
340 break;
341 default:
342 *rd = bpf_to_rv_reg(insn->dst_reg, ctx);
343 }
344
345 if (code & (BPF_ALU | BPF_X) || code & (BPF_ALU64 | BPF_X) ||
346 code & (BPF_JMP | BPF_X) || code & (BPF_JMP32 | BPF_X) ||
347 code & BPF_LDX || code & BPF_STX)
348 *rs = bpf_to_rv_reg(insn->src_reg, ctx);
349 }
350
351 static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx)
352 {
353 emit(rv_addi(RV_REG_T2, *rd, 0), ctx);
354 emit_zext_32(RV_REG_T2, ctx);
355 emit(rv_addi(RV_REG_T1, *rs, 0), ctx);
356 emit_zext_32(RV_REG_T1, ctx);
357 *rd = RV_REG_T2;
358 *rs = RV_REG_T1;
359 }
360
361 static void emit_sext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx)
362 {
363 emit(rv_addiw(RV_REG_T2, *rd, 0), ctx);
364 emit(rv_addiw(RV_REG_T1, *rs, 0), ctx);
365 *rd = RV_REG_T2;
366 *rs = RV_REG_T1;
367 }
368
369 static void emit_zext_32_rd_t1(u8 *rd, struct rv_jit_context *ctx)
370 {
371 emit(rv_addi(RV_REG_T2, *rd, 0), ctx);
372 emit_zext_32(RV_REG_T2, ctx);
373 emit_zext_32(RV_REG_T1, ctx);
374 *rd = RV_REG_T2;
375 }
376
377 static void emit_sext_32_rd(u8 *rd, struct rv_jit_context *ctx)
378 {
379 emit(rv_addiw(RV_REG_T2, *rd, 0), ctx);
380 *rd = RV_REG_T2;
381 }
382
383 static void emit_jump_and_link(u8 rd, s64 rvoff, bool force_jalr,
384 struct rv_jit_context *ctx)
385 {
386 s64 upper, lower;
387
388 if (rvoff && is_21b_int(rvoff) && !force_jalr) {
389 emit(rv_jal(rd, rvoff >> 1), ctx);
390 return;
391 }
392
393 upper = (rvoff + (1 << 11)) >> 12;
394 lower = rvoff & 0xfff;
395 emit(rv_auipc(RV_REG_T1, upper), ctx);
396 emit(rv_jalr(rd, RV_REG_T1, lower), ctx);
397 }
398
399 static bool is_signed_bpf_cond(u8 cond)
400 {
401 return cond == BPF_JSGT || cond == BPF_JSLT ||
402 cond == BPF_JSGE || cond == BPF_JSLE;
403 }
404
405 static int emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx)
406 {
407 s64 off = 0;
408 u64 ip;
409 u8 rd;
410
411 if (addr && ctx->insns) {
412 ip = (u64)(long)(ctx->insns + ctx->ninsns);
413 off = addr - ip;
414 if (!is_32b_int(off)) {
415 pr_err("bpf-jit: target call addr %pK is out of range\n",
416 (void *)addr);
417 return -ERANGE;
418 }
419 }
420
421 emit_jump_and_link(RV_REG_RA, off, !fixed, ctx);
422 rd = bpf_to_rv_reg(BPF_REG_0, ctx);
423 emit(rv_addi(rd, RV_REG_A0, 0), ctx);
424 return 0;
425 }
426
427 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
428 bool extra_pass)
429 {
430 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 ||
431 BPF_CLASS(insn->code) == BPF_JMP;
432 int s, e, rvoff, i = insn - ctx->prog->insnsi;
433 struct bpf_prog_aux *aux = ctx->prog->aux;
434 u8 rd = -1, rs = -1, code = insn->code;
435 s16 off = insn->off;
436 s32 imm = insn->imm;
437
438 init_regs(&rd, &rs, insn, ctx);
439
440 switch (code) {
441 /* dst = src */
442 case BPF_ALU | BPF_MOV | BPF_X:
443 case BPF_ALU64 | BPF_MOV | BPF_X:
444 if (imm == 1) {
445 /* Special mov32 for zext */
446 emit_zext_32(rd, ctx);
447 break;
448 }
449 emit(is64 ? rv_addi(rd, rs, 0) : rv_addiw(rd, rs, 0), ctx);
450 if (!is64 && !aux->verifier_zext)
451 emit_zext_32(rd, ctx);
452 break;
453
454 /* dst = dst OP src */
455 case BPF_ALU | BPF_ADD | BPF_X:
456 case BPF_ALU64 | BPF_ADD | BPF_X:
457 emit(is64 ? rv_add(rd, rd, rs) : rv_addw(rd, rd, rs), ctx);
458 if (!is64 && !aux->verifier_zext)
459 emit_zext_32(rd, ctx);
460 break;
461 case BPF_ALU | BPF_SUB | BPF_X:
462 case BPF_ALU64 | BPF_SUB | BPF_X:
463 emit(is64 ? rv_sub(rd, rd, rs) : rv_subw(rd, rd, rs), ctx);
464 if (!is64 && !aux->verifier_zext)
465 emit_zext_32(rd, ctx);
466 break;
467 case BPF_ALU | BPF_AND | BPF_X:
468 case BPF_ALU64 | BPF_AND | BPF_X:
469 emit(rv_and(rd, rd, rs), ctx);
470 if (!is64 && !aux->verifier_zext)
471 emit_zext_32(rd, ctx);
472 break;
473 case BPF_ALU | BPF_OR | BPF_X:
474 case BPF_ALU64 | BPF_OR | BPF_X:
475 emit(rv_or(rd, rd, rs), ctx);
476 if (!is64 && !aux->verifier_zext)
477 emit_zext_32(rd, ctx);
478 break;
479 case BPF_ALU | BPF_XOR | BPF_X:
480 case BPF_ALU64 | BPF_XOR | BPF_X:
481 emit(rv_xor(rd, rd, rs), ctx);
482 if (!is64 && !aux->verifier_zext)
483 emit_zext_32(rd, ctx);
484 break;
485 case BPF_ALU | BPF_MUL | BPF_X:
486 case BPF_ALU64 | BPF_MUL | BPF_X:
487 emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx);
488 if (!is64 && !aux->verifier_zext)
489 emit_zext_32(rd, ctx);
490 break;
491 case BPF_ALU | BPF_DIV | BPF_X:
492 case BPF_ALU64 | BPF_DIV | BPF_X:
493 emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx);
494 if (!is64 && !aux->verifier_zext)
495 emit_zext_32(rd, ctx);
496 break;
497 case BPF_ALU | BPF_MOD | BPF_X:
498 case BPF_ALU64 | BPF_MOD | BPF_X:
499 emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx);
500 if (!is64 && !aux->verifier_zext)
501 emit_zext_32(rd, ctx);
502 break;
503 case BPF_ALU | BPF_LSH | BPF_X:
504 case BPF_ALU64 | BPF_LSH | BPF_X:
505 emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx);
506 if (!is64)
507 emit_zext_32(rd, ctx);
508 break;
509 case BPF_ALU | BPF_RSH | BPF_X:
510 case BPF_ALU64 | BPF_RSH | BPF_X:
511 emit(is64 ? rv_srl(rd, rd, rs) : rv_srlw(rd, rd, rs), ctx);
512 if (!is64 && !aux->verifier_zext)
513 emit_zext_32(rd, ctx);
514 break;
515 case BPF_ALU | BPF_ARSH | BPF_X:
516 case BPF_ALU64 | BPF_ARSH | BPF_X:
517 emit(is64 ? rv_sra(rd, rd, rs) : rv_sraw(rd, rd, rs), ctx);
518 if (!is64 && !aux->verifier_zext)
519 emit_zext_32(rd, ctx);
520 break;
521
522 /* dst = -dst */
523 case BPF_ALU | BPF_NEG:
524 case BPF_ALU64 | BPF_NEG:
525 emit(is64 ? rv_sub(rd, RV_REG_ZERO, rd) :
526 rv_subw(rd, RV_REG_ZERO, rd), ctx);
527 if (!is64 && !aux->verifier_zext)
528 emit_zext_32(rd, ctx);
529 break;
530
531 /* dst = BSWAP##imm(dst) */
532 case BPF_ALU | BPF_END | BPF_FROM_LE:
533 {
534 int shift = 64 - imm;
535
536 emit(rv_slli(rd, rd, shift), ctx);
537 emit(rv_srli(rd, rd, shift), ctx);
538 break;
539 }
540 case BPF_ALU | BPF_END | BPF_FROM_BE:
541 emit(rv_addi(RV_REG_T2, RV_REG_ZERO, 0), ctx);
542
543 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
544 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
545 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
546 emit(rv_srli(rd, rd, 8), ctx);
547 if (imm == 16)
548 goto out_be;
549
550 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
551 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
552 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
553 emit(rv_srli(rd, rd, 8), ctx);
554
555 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
556 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
557 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
558 emit(rv_srli(rd, rd, 8), ctx);
559 if (imm == 32)
560 goto out_be;
561
562 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
563 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
564 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
565 emit(rv_srli(rd, rd, 8), ctx);
566
567 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
568 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
569 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
570 emit(rv_srli(rd, rd, 8), ctx);
571
572 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
573 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
574 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
575 emit(rv_srli(rd, rd, 8), ctx);
576
577 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
578 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
579 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
580 emit(rv_srli(rd, rd, 8), ctx);
581 out_be:
582 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
583 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
584
585 emit(rv_addi(rd, RV_REG_T2, 0), ctx);
586 break;
587
588 /* dst = imm */
589 case BPF_ALU | BPF_MOV | BPF_K:
590 case BPF_ALU64 | BPF_MOV | BPF_K:
591 emit_imm(rd, imm, ctx);
592 if (!is64 && !aux->verifier_zext)
593 emit_zext_32(rd, ctx);
594 break;
595
596 /* dst = dst OP imm */
597 case BPF_ALU | BPF_ADD | BPF_K:
598 case BPF_ALU64 | BPF_ADD | BPF_K:
599 if (is_12b_int(imm)) {
600 emit(is64 ? rv_addi(rd, rd, imm) :
601 rv_addiw(rd, rd, imm), ctx);
602 } else {
603 emit_imm(RV_REG_T1, imm, ctx);
604 emit(is64 ? rv_add(rd, rd, RV_REG_T1) :
605 rv_addw(rd, rd, RV_REG_T1), ctx);
606 }
607 if (!is64 && !aux->verifier_zext)
608 emit_zext_32(rd, ctx);
609 break;
610 case BPF_ALU | BPF_SUB | BPF_K:
611 case BPF_ALU64 | BPF_SUB | BPF_K:
612 if (is_12b_int(-imm)) {
613 emit(is64 ? rv_addi(rd, rd, -imm) :
614 rv_addiw(rd, rd, -imm), ctx);
615 } else {
616 emit_imm(RV_REG_T1, imm, ctx);
617 emit(is64 ? rv_sub(rd, rd, RV_REG_T1) :
618 rv_subw(rd, rd, RV_REG_T1), ctx);
619 }
620 if (!is64 && !aux->verifier_zext)
621 emit_zext_32(rd, ctx);
622 break;
623 case BPF_ALU | BPF_AND | BPF_K:
624 case BPF_ALU64 | BPF_AND | BPF_K:
625 if (is_12b_int(imm)) {
626 emit(rv_andi(rd, rd, imm), ctx);
627 } else {
628 emit_imm(RV_REG_T1, imm, ctx);
629 emit(rv_and(rd, rd, RV_REG_T1), ctx);
630 }
631 if (!is64 && !aux->verifier_zext)
632 emit_zext_32(rd, ctx);
633 break;
634 case BPF_ALU | BPF_OR | BPF_K:
635 case BPF_ALU64 | BPF_OR | BPF_K:
636 if (is_12b_int(imm)) {
637 emit(rv_ori(rd, rd, imm), ctx);
638 } else {
639 emit_imm(RV_REG_T1, imm, ctx);
640 emit(rv_or(rd, rd, RV_REG_T1), ctx);
641 }
642 if (!is64 && !aux->verifier_zext)
643 emit_zext_32(rd, ctx);
644 break;
645 case BPF_ALU | BPF_XOR | BPF_K:
646 case BPF_ALU64 | BPF_XOR | BPF_K:
647 if (is_12b_int(imm)) {
648 emit(rv_xori(rd, rd, imm), ctx);
649 } else {
650 emit_imm(RV_REG_T1, imm, ctx);
651 emit(rv_xor(rd, rd, RV_REG_T1), ctx);
652 }
653 if (!is64 && !aux->verifier_zext)
654 emit_zext_32(rd, ctx);
655 break;
656 case BPF_ALU | BPF_MUL | BPF_K:
657 case BPF_ALU64 | BPF_MUL | BPF_K:
658 emit_imm(RV_REG_T1, imm, ctx);
659 emit(is64 ? rv_mul(rd, rd, RV_REG_T1) :
660 rv_mulw(rd, rd, RV_REG_T1), ctx);
661 if (!is64 && !aux->verifier_zext)
662 emit_zext_32(rd, ctx);
663 break;
664 case BPF_ALU | BPF_DIV | BPF_K:
665 case BPF_ALU64 | BPF_DIV | BPF_K:
666 emit_imm(RV_REG_T1, imm, ctx);
667 emit(is64 ? rv_divu(rd, rd, RV_REG_T1) :
668 rv_divuw(rd, rd, RV_REG_T1), ctx);
669 if (!is64 && !aux->verifier_zext)
670 emit_zext_32(rd, ctx);
671 break;
672 case BPF_ALU | BPF_MOD | BPF_K:
673 case BPF_ALU64 | BPF_MOD | BPF_K:
674 emit_imm(RV_REG_T1, imm, ctx);
675 emit(is64 ? rv_remu(rd, rd, RV_REG_T1) :
676 rv_remuw(rd, rd, RV_REG_T1), ctx);
677 if (!is64 && !aux->verifier_zext)
678 emit_zext_32(rd, ctx);
679 break;
680 case BPF_ALU | BPF_LSH | BPF_K:
681 case BPF_ALU64 | BPF_LSH | BPF_K:
682 emit(is64 ? rv_slli(rd, rd, imm) : rv_slliw(rd, rd, imm), ctx);
683 if (!is64)
684 emit_zext_32(rd, ctx);
685 break;
686 case BPF_ALU | BPF_RSH | BPF_K:
687 case BPF_ALU64 | BPF_RSH | BPF_K:
688 emit(is64 ? rv_srli(rd, rd, imm) : rv_srliw(rd, rd, imm), ctx);
689 if (!is64)
690 emit_zext_32(rd, ctx);
691 break;
692 case BPF_ALU | BPF_ARSH | BPF_K:
693 case BPF_ALU64 | BPF_ARSH | BPF_K:
694 emit(is64 ? rv_srai(rd, rd, imm) : rv_sraiw(rd, rd, imm), ctx);
695 if (!is64)
696 emit_zext_32(rd, ctx);
697 break;
698
699 /* JUMP off */
700 case BPF_JMP | BPF_JA:
701 rvoff = rv_offset(i, off, ctx);
702 emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx);
703 break;
704
705 /* IF (dst COND src) JUMP off */
706 case BPF_JMP | BPF_JEQ | BPF_X:
707 case BPF_JMP32 | BPF_JEQ | BPF_X:
708 case BPF_JMP | BPF_JGT | BPF_X:
709 case BPF_JMP32 | BPF_JGT | BPF_X:
710 case BPF_JMP | BPF_JLT | BPF_X:
711 case BPF_JMP32 | BPF_JLT | BPF_X:
712 case BPF_JMP | BPF_JGE | BPF_X:
713 case BPF_JMP32 | BPF_JGE | BPF_X:
714 case BPF_JMP | BPF_JLE | BPF_X:
715 case BPF_JMP32 | BPF_JLE | BPF_X:
716 case BPF_JMP | BPF_JNE | BPF_X:
717 case BPF_JMP32 | BPF_JNE | BPF_X:
718 case BPF_JMP | BPF_JSGT | BPF_X:
719 case BPF_JMP32 | BPF_JSGT | BPF_X:
720 case BPF_JMP | BPF_JSLT | BPF_X:
721 case BPF_JMP32 | BPF_JSLT | BPF_X:
722 case BPF_JMP | BPF_JSGE | BPF_X:
723 case BPF_JMP32 | BPF_JSGE | BPF_X:
724 case BPF_JMP | BPF_JSLE | BPF_X:
725 case BPF_JMP32 | BPF_JSLE | BPF_X:
726 case BPF_JMP | BPF_JSET | BPF_X:
727 case BPF_JMP32 | BPF_JSET | BPF_X:
728 rvoff = rv_offset(i, off, ctx);
729 if (!is64) {
730 s = ctx->ninsns;
731 if (is_signed_bpf_cond(BPF_OP(code)))
732 emit_sext_32_rd_rs(&rd, &rs, ctx);
733 else
734 emit_zext_32_rd_rs(&rd, &rs, ctx);
735 e = ctx->ninsns;
736
737 /* Adjust for extra insns */
738 rvoff -= (e - s) << 2;
739 }
740
741 if (BPF_OP(code) == BPF_JSET) {
742 /* Adjust for and */
743 rvoff -= 4;
744 emit(rv_and(RV_REG_T1, rd, rs), ctx);
745 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff,
746 ctx);
747 } else {
748 emit_branch(BPF_OP(code), rd, rs, rvoff, ctx);
749 }
750 break;
751
752 /* IF (dst COND imm) JUMP off */
753 case BPF_JMP | BPF_JEQ | BPF_K:
754 case BPF_JMP32 | BPF_JEQ | BPF_K:
755 case BPF_JMP | BPF_JGT | BPF_K:
756 case BPF_JMP32 | BPF_JGT | BPF_K:
757 case BPF_JMP | BPF_JLT | BPF_K:
758 case BPF_JMP32 | BPF_JLT | BPF_K:
759 case BPF_JMP | BPF_JGE | BPF_K:
760 case BPF_JMP32 | BPF_JGE | BPF_K:
761 case BPF_JMP | BPF_JLE | BPF_K:
762 case BPF_JMP32 | BPF_JLE | BPF_K:
763 case BPF_JMP | BPF_JNE | BPF_K:
764 case BPF_JMP32 | BPF_JNE | BPF_K:
765 case BPF_JMP | BPF_JSGT | BPF_K:
766 case BPF_JMP32 | BPF_JSGT | BPF_K:
767 case BPF_JMP | BPF_JSLT | BPF_K:
768 case BPF_JMP32 | BPF_JSLT | BPF_K:
769 case BPF_JMP | BPF_JSGE | BPF_K:
770 case BPF_JMP32 | BPF_JSGE | BPF_K:
771 case BPF_JMP | BPF_JSLE | BPF_K:
772 case BPF_JMP32 | BPF_JSLE | BPF_K:
773 case BPF_JMP | BPF_JSET | BPF_K:
774 case BPF_JMP32 | BPF_JSET | BPF_K:
775 rvoff = rv_offset(i, off, ctx);
776 s = ctx->ninsns;
777 emit_imm(RV_REG_T1, imm, ctx);
778 if (!is64) {
779 if (is_signed_bpf_cond(BPF_OP(code)))
780 emit_sext_32_rd(&rd, ctx);
781 else
782 emit_zext_32_rd_t1(&rd, ctx);
783 }
784 e = ctx->ninsns;
785
786 /* Adjust for extra insns */
787 rvoff -= (e - s) << 2;
788
789 if (BPF_OP(code) == BPF_JSET) {
790 /* Adjust for and */
791 rvoff -= 4;
792 emit(rv_and(RV_REG_T1, rd, RV_REG_T1), ctx);
793 emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff,
794 ctx);
795 } else {
796 emit_branch(BPF_OP(code), rd, RV_REG_T1, rvoff, ctx);
797 }
798 break;
799
800 /* function call */
801 case BPF_JMP | BPF_CALL:
802 {
803 bool fixed;
804 int ret;
805 u64 addr;
806
807 mark_call(ctx);
808 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, &addr,
809 &fixed);
810 if (ret < 0)
811 return ret;
812 ret = emit_call(fixed, addr, ctx);
813 if (ret)
814 return ret;
815 break;
816 }
817 /* tail call */
818 case BPF_JMP | BPF_TAIL_CALL:
819 if (emit_bpf_tail_call(i, ctx))
820 return -1;
821 break;
822
823 /* function return */
824 case BPF_JMP | BPF_EXIT:
825 if (i == ctx->prog->len - 1)
826 break;
827
828 rvoff = epilogue_offset(ctx);
829 emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx);
830 break;
831
832 /* dst = imm64 */
833 case BPF_LD | BPF_IMM | BPF_DW:
834 {
835 struct bpf_insn insn1 = insn[1];
836 u64 imm64;
837
838 imm64 = (u64)insn1.imm << 32 | (u32)imm;
839 emit_imm(rd, imm64, ctx);
840 return 1;
841 }
842
843 /* LDX: dst = *(size *)(src + off) */
844 case BPF_LDX | BPF_MEM | BPF_B:
845 if (is_12b_int(off)) {
846 emit(rv_lbu(rd, off, rs), ctx);
847 break;
848 }
849
850 emit_imm(RV_REG_T1, off, ctx);
851 emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx);
852 emit(rv_lbu(rd, 0, RV_REG_T1), ctx);
853 if (insn_is_zext(&insn[1]))
854 return 1;
855 break;
856 case BPF_LDX | BPF_MEM | BPF_H:
857 if (is_12b_int(off)) {
858 emit(rv_lhu(rd, off, rs), ctx);
859 break;
860 }
861
862 emit_imm(RV_REG_T1, off, ctx);
863 emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx);
864 emit(rv_lhu(rd, 0, RV_REG_T1), ctx);
865 if (insn_is_zext(&insn[1]))
866 return 1;
867 break;
868 case BPF_LDX | BPF_MEM | BPF_W:
869 if (is_12b_int(off)) {
870 emit(rv_lwu(rd, off, rs), ctx);
871 break;
872 }
873
874 emit_imm(RV_REG_T1, off, ctx);
875 emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx);
876 emit(rv_lwu(rd, 0, RV_REG_T1), ctx);
877 if (insn_is_zext(&insn[1]))
878 return 1;
879 break;
880 case BPF_LDX | BPF_MEM | BPF_DW:
881 if (is_12b_int(off)) {
882 emit(rv_ld(rd, off, rs), ctx);
883 break;
884 }
885
886 emit_imm(RV_REG_T1, off, ctx);
887 emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx);
888 emit(rv_ld(rd, 0, RV_REG_T1), ctx);
889 break;
890
891 /* ST: *(size *)(dst + off) = imm */
892 case BPF_ST | BPF_MEM | BPF_B:
893 emit_imm(RV_REG_T1, imm, ctx);
894 if (is_12b_int(off)) {
895 emit(rv_sb(rd, off, RV_REG_T1), ctx);
896 break;
897 }
898
899 emit_imm(RV_REG_T2, off, ctx);
900 emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx);
901 emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx);
902 break;
903
904 case BPF_ST | BPF_MEM | BPF_H:
905 emit_imm(RV_REG_T1, imm, ctx);
906 if (is_12b_int(off)) {
907 emit(rv_sh(rd, off, RV_REG_T1), ctx);
908 break;
909 }
910
911 emit_imm(RV_REG_T2, off, ctx);
912 emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx);
913 emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx);
914 break;
915 case BPF_ST | BPF_MEM | BPF_W:
916 emit_imm(RV_REG_T1, imm, ctx);
917 if (is_12b_int(off)) {
918 emit(rv_sw(rd, off, RV_REG_T1), ctx);
919 break;
920 }
921
922 emit_imm(RV_REG_T2, off, ctx);
923 emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx);
924 emit(rv_sw(RV_REG_T2, 0, RV_REG_T1), ctx);
925 break;
926 case BPF_ST | BPF_MEM | BPF_DW:
927 emit_imm(RV_REG_T1, imm, ctx);
928 if (is_12b_int(off)) {
929 emit(rv_sd(rd, off, RV_REG_T1), ctx);
930 break;
931 }
932
933 emit_imm(RV_REG_T2, off, ctx);
934 emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx);
935 emit(rv_sd(RV_REG_T2, 0, RV_REG_T1), ctx);
936 break;
937
938 /* STX: *(size *)(dst + off) = src */
939 case BPF_STX | BPF_MEM | BPF_B:
940 if (is_12b_int(off)) {
941 emit(rv_sb(rd, off, rs), ctx);
942 break;
943 }
944
945 emit_imm(RV_REG_T1, off, ctx);
946 emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx);
947 emit(rv_sb(RV_REG_T1, 0, rs), ctx);
948 break;
949 case BPF_STX | BPF_MEM | BPF_H:
950 if (is_12b_int(off)) {
951 emit(rv_sh(rd, off, rs), ctx);
952 break;
953 }
954
955 emit_imm(RV_REG_T1, off, ctx);
956 emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx);
957 emit(rv_sh(RV_REG_T1, 0, rs), ctx);
958 break;
959 case BPF_STX | BPF_MEM | BPF_W:
960 if (is_12b_int(off)) {
961 emit(rv_sw(rd, off, rs), ctx);
962 break;
963 }
964
965 emit_imm(RV_REG_T1, off, ctx);
966 emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx);
967 emit(rv_sw(RV_REG_T1, 0, rs), ctx);
968 break;
969 case BPF_STX | BPF_MEM | BPF_DW:
970 if (is_12b_int(off)) {
971 emit(rv_sd(rd, off, rs), ctx);
972 break;
973 }
974
975 emit_imm(RV_REG_T1, off, ctx);
976 emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx);
977 emit(rv_sd(RV_REG_T1, 0, rs), ctx);
978 break;
979 /* STX XADD: lock *(u32 *)(dst + off) += src */
980 case BPF_STX | BPF_XADD | BPF_W:
981 /* STX XADD: lock *(u64 *)(dst + off) += src */
982 case BPF_STX | BPF_XADD | BPF_DW:
983 if (off) {
984 if (is_12b_int(off)) {
985 emit(rv_addi(RV_REG_T1, rd, off), ctx);
986 } else {
987 emit_imm(RV_REG_T1, off, ctx);
988 emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx);
989 }
990
991 rd = RV_REG_T1;
992 }
993
994 emit(BPF_SIZE(code) == BPF_W ?
995 rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0) :
996 rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0), ctx);
997 break;
998 default:
999 pr_err("bpf-jit: unknown opcode %02x\n", code);
1000 return -EINVAL;
1001 }
1002
1003 return 0;
1004 }
1005
1006 void bpf_jit_build_prologue(struct rv_jit_context *ctx)
1007 {
1008 int stack_adjust = 0, store_offset, bpf_stack_adjust;
1009
1010 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16);
1011 if (bpf_stack_adjust)
1012 mark_fp(ctx);
1013
1014 if (seen_reg(RV_REG_RA, ctx))
1015 stack_adjust += 8;
1016 stack_adjust += 8; /* RV_REG_FP */
1017 if (seen_reg(RV_REG_S1, ctx))
1018 stack_adjust += 8;
1019 if (seen_reg(RV_REG_S2, ctx))
1020 stack_adjust += 8;
1021 if (seen_reg(RV_REG_S3, ctx))
1022 stack_adjust += 8;
1023 if (seen_reg(RV_REG_S4, ctx))
1024 stack_adjust += 8;
1025 if (seen_reg(RV_REG_S5, ctx))
1026 stack_adjust += 8;
1027 if (seen_reg(RV_REG_S6, ctx))
1028 stack_adjust += 8;
1029
1030 stack_adjust = round_up(stack_adjust, 16);
1031 stack_adjust += bpf_stack_adjust;
1032
1033 store_offset = stack_adjust - 8;
1034
1035 /* First instruction is always setting the tail-call-counter
1036 * (TCC) register. This instruction is skipped for tail calls.
1037 */
1038 emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx);
1039
1040 emit(rv_addi(RV_REG_SP, RV_REG_SP, -stack_adjust), ctx);
1041
1042 if (seen_reg(RV_REG_RA, ctx)) {
1043 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_RA), ctx);
1044 store_offset -= 8;
1045 }
1046 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_FP), ctx);
1047 store_offset -= 8;
1048 if (seen_reg(RV_REG_S1, ctx)) {
1049 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S1), ctx);
1050 store_offset -= 8;
1051 }
1052 if (seen_reg(RV_REG_S2, ctx)) {
1053 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S2), ctx);
1054 store_offset -= 8;
1055 }
1056 if (seen_reg(RV_REG_S3, ctx)) {
1057 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S3), ctx);
1058 store_offset -= 8;
1059 }
1060 if (seen_reg(RV_REG_S4, ctx)) {
1061 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S4), ctx);
1062 store_offset -= 8;
1063 }
1064 if (seen_reg(RV_REG_S5, ctx)) {
1065 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S5), ctx);
1066 store_offset -= 8;
1067 }
1068 if (seen_reg(RV_REG_S6, ctx)) {
1069 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S6), ctx);
1070 store_offset -= 8;
1071 }
1072
1073 emit(rv_addi(RV_REG_FP, RV_REG_SP, stack_adjust), ctx);
1074
1075 if (bpf_stack_adjust)
1076 emit(rv_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust), ctx);
1077
1078 /* Program contains calls and tail calls, so RV_REG_TCC need
1079 * to be saved across calls.
1080 */
1081 if (seen_tail_call(ctx) && seen_call(ctx))
1082 emit(rv_addi(RV_REG_TCC_SAVED, RV_REG_TCC, 0), ctx);
1083
1084 ctx->stack_size = stack_adjust;
1085 }
1086
1087 void bpf_jit_build_epilogue(struct rv_jit_context *ctx)
1088 {
1089 __build_epilogue(false, ctx);
1090 }
1091
1092 void *bpf_jit_alloc_exec(unsigned long size)
1093 {
1094 return __vmalloc_node_range(size, PAGE_SIZE, BPF_JIT_REGION_START,
1095 BPF_JIT_REGION_END, GFP_KERNEL,
1096 PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE,
1097 __builtin_return_address(0));
1098 }
1099
1100 void bpf_jit_free_exec(void *addr)
1101 {
1102 return vfree(addr);
1103 }
Linux with added FPC-III support