?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'for-linus-20190706' of git://git.kernel.dk/linux-block
[linux/fpc-iii.git] / arch / riscv / net / bpf_jit_comp.c
blob426d5c33ea9037e10fd7706dddb8ac18d5649c46
1 // SPDX-License-Identifier: GPL-2.0
2 /* BPF JIT compiler for RV64G
3 *
4 * Copyright(c) 2019 Björn Töpel <bjorn.topel@gmail.com>
5 *
6 */
7
8 #include <linux/bpf.h>
9 #include <linux/filter.h>
10 #include <asm/cacheflush.h>
11
12 enum {
13 RV_REG_ZERO = 0, /* The constant value 0 */
14 RV_REG_RA = 1, /* Return address */
15 RV_REG_SP = 2, /* Stack pointer */
16 RV_REG_GP = 3, /* Global pointer */
17 RV_REG_TP = 4, /* Thread pointer */
18 RV_REG_T0 = 5, /* Temporaries */
19 RV_REG_T1 = 6,
20 RV_REG_T2 = 7,
21 RV_REG_FP = 8,
22 RV_REG_S1 = 9, /* Saved registers */
23 RV_REG_A0 = 10, /* Function argument/return values */
24 RV_REG_A1 = 11, /* Function arguments */
25 RV_REG_A2 = 12,
26 RV_REG_A3 = 13,
27 RV_REG_A4 = 14,
28 RV_REG_A5 = 15,
29 RV_REG_A6 = 16,
30 RV_REG_A7 = 17,
31 RV_REG_S2 = 18, /* Saved registers */
32 RV_REG_S3 = 19,
33 RV_REG_S4 = 20,
34 RV_REG_S5 = 21,
35 RV_REG_S6 = 22,
36 RV_REG_S7 = 23,
37 RV_REG_S8 = 24,
38 RV_REG_S9 = 25,
39 RV_REG_S10 = 26,
40 RV_REG_S11 = 27,
41 RV_REG_T3 = 28, /* Temporaries */
42 RV_REG_T4 = 29,
43 RV_REG_T5 = 30,
44 RV_REG_T6 = 31,
45 };
46
47 #define RV_REG_TCC RV_REG_A6
48 #define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */
49
50 static const int regmap[] = {
51 [BPF_REG_0] = RV_REG_A5,
52 [BPF_REG_1] = RV_REG_A0,
53 [BPF_REG_2] = RV_REG_A1,
54 [BPF_REG_3] = RV_REG_A2,
55 [BPF_REG_4] = RV_REG_A3,
56 [BPF_REG_5] = RV_REG_A4,
57 [BPF_REG_6] = RV_REG_S1,
58 [BPF_REG_7] = RV_REG_S2,
59 [BPF_REG_8] = RV_REG_S3,
60 [BPF_REG_9] = RV_REG_S4,
61 [BPF_REG_FP] = RV_REG_S5,
62 [BPF_REG_AX] = RV_REG_T0,
63 };
64
65 enum {
66 RV_CTX_F_SEEN_TAIL_CALL = 0,
67 RV_CTX_F_SEEN_CALL = RV_REG_RA,
68 RV_CTX_F_SEEN_S1 = RV_REG_S1,
69 RV_CTX_F_SEEN_S2 = RV_REG_S2,
70 RV_CTX_F_SEEN_S3 = RV_REG_S3,
71 RV_CTX_F_SEEN_S4 = RV_REG_S4,
72 RV_CTX_F_SEEN_S5 = RV_REG_S5,
73 RV_CTX_F_SEEN_S6 = RV_REG_S6,
74 };
75
76 struct rv_jit_context {
77 struct bpf_prog *prog;
78 u32 *insns; /* RV insns */
79 int ninsns;
80 int epilogue_offset;
81 int *offset; /* BPF to RV */
82 unsigned long flags;
83 int stack_size;
84 };
85
86 struct rv_jit_data {
87 struct bpf_binary_header *header;
88 u8 *image;
89 struct rv_jit_context ctx;
90 };
91
92 static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx)
93 {
94 u8 reg = regmap[bpf_reg];
95
96 switch (reg) {
97 case RV_CTX_F_SEEN_S1:
98 case RV_CTX_F_SEEN_S2:
99 case RV_CTX_F_SEEN_S3:
100 case RV_CTX_F_SEEN_S4:
101 case RV_CTX_F_SEEN_S5:
102 case RV_CTX_F_SEEN_S6:
103 __set_bit(reg, &ctx->flags);
104 }
105 return reg;
106 };
107
108 static bool seen_reg(int reg, struct rv_jit_context *ctx)
109 {
110 switch (reg) {
111 case RV_CTX_F_SEEN_CALL:
112 case RV_CTX_F_SEEN_S1:
113 case RV_CTX_F_SEEN_S2:
114 case RV_CTX_F_SEEN_S3:
115 case RV_CTX_F_SEEN_S4:
116 case RV_CTX_F_SEEN_S5:
117 case RV_CTX_F_SEEN_S6:
118 return test_bit(reg, &ctx->flags);
119 }
120 return false;
121 }
122
123 static void mark_call(struct rv_jit_context *ctx)
124 {
125 __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags);
126 }
127
128 static bool seen_call(struct rv_jit_context *ctx)
129 {
130 return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags);
131 }
132
133 static void mark_tail_call(struct rv_jit_context *ctx)
134 {
135 __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags);
136 }
137
138 static bool seen_tail_call(struct rv_jit_context *ctx)
139 {
140 return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags);
141 }
142
143 static u8 rv_tail_call_reg(struct rv_jit_context *ctx)
144 {
145 mark_tail_call(ctx);
146
147 if (seen_call(ctx)) {
148 __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags);
149 return RV_REG_S6;
150 }
151 return RV_REG_A6;
152 }
153
154 static void emit(const u32 insn, struct rv_jit_context *ctx)
155 {
156 if (ctx->insns)
157 ctx->insns[ctx->ninsns] = insn;
158
159 ctx->ninsns++;
160 }
161
162 static u32 rv_r_insn(u8 funct7, u8 rs2, u8 rs1, u8 funct3, u8 rd, u8 opcode)
163 {
164 return (funct7 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
165 (rd << 7) | opcode;
166 }
167
168 static u32 rv_i_insn(u16 imm11_0, u8 rs1, u8 funct3, u8 rd, u8 opcode)
169 {
170 return (imm11_0 << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) |
171 opcode;
172 }
173
174 static u32 rv_s_insn(u16 imm11_0, u8 rs2, u8 rs1, u8 funct3, u8 opcode)
175 {
176 u8 imm11_5 = imm11_0 >> 5, imm4_0 = imm11_0 & 0x1f;
177
178 return (imm11_5 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
179 (imm4_0 << 7) | opcode;
180 }
181
182 static u32 rv_sb_insn(u16 imm12_1, u8 rs2, u8 rs1, u8 funct3, u8 opcode)
183 {
184 u8 imm12 = ((imm12_1 & 0x800) >> 5) | ((imm12_1 & 0x3f0) >> 4);
185 u8 imm4_1 = ((imm12_1 & 0xf) << 1) | ((imm12_1 & 0x400) >> 10);
186
187 return (imm12 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
188 (imm4_1 << 7) | opcode;
189 }
190
191 static u32 rv_u_insn(u32 imm31_12, u8 rd, u8 opcode)
192 {
193 return (imm31_12 << 12) | (rd << 7) | opcode;
194 }
195
196 static u32 rv_uj_insn(u32 imm20_1, u8 rd, u8 opcode)
197 {
198 u32 imm;
199
200 imm = (imm20_1 & 0x80000) | ((imm20_1 & 0x3ff) << 9) |
201 ((imm20_1 & 0x400) >> 2) | ((imm20_1 & 0x7f800) >> 11);
202
203 return (imm << 12) | (rd << 7) | opcode;
204 }
205
206 static u32 rv_amo_insn(u8 funct5, u8 aq, u8 rl, u8 rs2, u8 rs1,
207 u8 funct3, u8 rd, u8 opcode)
208 {
209 u8 funct7 = (funct5 << 2) | (aq << 1) | rl;
210
211 return rv_r_insn(funct7, rs2, rs1, funct3, rd, opcode);
212 }
213
214 static u32 rv_addiw(u8 rd, u8 rs1, u16 imm11_0)
215 {
216 return rv_i_insn(imm11_0, rs1, 0, rd, 0x1b);
217 }
218
219 static u32 rv_addi(u8 rd, u8 rs1, u16 imm11_0)
220 {
221 return rv_i_insn(imm11_0, rs1, 0, rd, 0x13);
222 }
223
224 static u32 rv_addw(u8 rd, u8 rs1, u8 rs2)
225 {
226 return rv_r_insn(0, rs2, rs1, 0, rd, 0x3b);
227 }
228
229 static u32 rv_add(u8 rd, u8 rs1, u8 rs2)
230 {
231 return rv_r_insn(0, rs2, rs1, 0, rd, 0x33);
232 }
233
234 static u32 rv_subw(u8 rd, u8 rs1, u8 rs2)
235 {
236 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x3b);
237 }
238
239 static u32 rv_sub(u8 rd, u8 rs1, u8 rs2)
240 {
241 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x33);
242 }
243
244 static u32 rv_and(u8 rd, u8 rs1, u8 rs2)
245 {
246 return rv_r_insn(0, rs2, rs1, 7, rd, 0x33);
247 }
248
249 static u32 rv_or(u8 rd, u8 rs1, u8 rs2)
250 {
251 return rv_r_insn(0, rs2, rs1, 6, rd, 0x33);
252 }
253
254 static u32 rv_xor(u8 rd, u8 rs1, u8 rs2)
255 {
256 return rv_r_insn(0, rs2, rs1, 4, rd, 0x33);
257 }
258
259 static u32 rv_mulw(u8 rd, u8 rs1, u8 rs2)
260 {
261 return rv_r_insn(1, rs2, rs1, 0, rd, 0x3b);
262 }
263
264 static u32 rv_mul(u8 rd, u8 rs1, u8 rs2)
265 {
266 return rv_r_insn(1, rs2, rs1, 0, rd, 0x33);
267 }
268
269 static u32 rv_divuw(u8 rd, u8 rs1, u8 rs2)
270 {
271 return rv_r_insn(1, rs2, rs1, 5, rd, 0x3b);
272 }
273
274 static u32 rv_divu(u8 rd, u8 rs1, u8 rs2)
275 {
276 return rv_r_insn(1, rs2, rs1, 5, rd, 0x33);
277 }
278
279 static u32 rv_remuw(u8 rd, u8 rs1, u8 rs2)
280 {
281 return rv_r_insn(1, rs2, rs1, 7, rd, 0x3b);
282 }
283
284 static u32 rv_remu(u8 rd, u8 rs1, u8 rs2)
285 {
286 return rv_r_insn(1, rs2, rs1, 7, rd, 0x33);
287 }
288
289 static u32 rv_sllw(u8 rd, u8 rs1, u8 rs2)
290 {
291 return rv_r_insn(0, rs2, rs1, 1, rd, 0x3b);
292 }
293
294 static u32 rv_sll(u8 rd, u8 rs1, u8 rs2)
295 {
296 return rv_r_insn(0, rs2, rs1, 1, rd, 0x33);
297 }
298
299 static u32 rv_srlw(u8 rd, u8 rs1, u8 rs2)
300 {
301 return rv_r_insn(0, rs2, rs1, 5, rd, 0x3b);
302 }
303
304 static u32 rv_srl(u8 rd, u8 rs1, u8 rs2)
305 {
306 return rv_r_insn(0, rs2, rs1, 5, rd, 0x33);
307 }
308
309 static u32 rv_sraw(u8 rd, u8 rs1, u8 rs2)
310 {
311 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x3b);
312 }
313
314 static u32 rv_sra(u8 rd, u8 rs1, u8 rs2)
315 {
316 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x33);
317 }
318
319 static u32 rv_lui(u8 rd, u32 imm31_12)
320 {
321 return rv_u_insn(imm31_12, rd, 0x37);
322 }
323
324 static u32 rv_slli(u8 rd, u8 rs1, u16 imm11_0)
325 {
326 return rv_i_insn(imm11_0, rs1, 1, rd, 0x13);
327 }
328
329 static u32 rv_andi(u8 rd, u8 rs1, u16 imm11_0)
330 {
331 return rv_i_insn(imm11_0, rs1, 7, rd, 0x13);
332 }
333
334 static u32 rv_ori(u8 rd, u8 rs1, u16 imm11_0)
335 {
336 return rv_i_insn(imm11_0, rs1, 6, rd, 0x13);
337 }
338
339 static u32 rv_xori(u8 rd, u8 rs1, u16 imm11_0)
340 {
341 return rv_i_insn(imm11_0, rs1, 4, rd, 0x13);
342 }
343
344 static u32 rv_slliw(u8 rd, u8 rs1, u16 imm11_0)
345 {
346 return rv_i_insn(imm11_0, rs1, 1, rd, 0x1b);
347 }
348
349 static u32 rv_srliw(u8 rd, u8 rs1, u16 imm11_0)
350 {
351 return rv_i_insn(imm11_0, rs1, 5, rd, 0x1b);
352 }
353
354 static u32 rv_srli(u8 rd, u8 rs1, u16 imm11_0)
355 {
356 return rv_i_insn(imm11_0, rs1, 5, rd, 0x13);
357 }
358
359 static u32 rv_sraiw(u8 rd, u8 rs1, u16 imm11_0)
360 {
361 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x1b);
362 }
363
364 static u32 rv_srai(u8 rd, u8 rs1, u16 imm11_0)
365 {
366 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x13);
367 }
368
369 static u32 rv_jal(u8 rd, u32 imm20_1)
370 {
371 return rv_uj_insn(imm20_1, rd, 0x6f);
372 }
373
374 static u32 rv_jalr(u8 rd, u8 rs1, u16 imm11_0)
375 {
376 return rv_i_insn(imm11_0, rs1, 0, rd, 0x67);
377 }
378
379 static u32 rv_beq(u8 rs1, u8 rs2, u16 imm12_1)
380 {
381 return rv_sb_insn(imm12_1, rs2, rs1, 0, 0x63);
382 }
383
384 static u32 rv_bltu(u8 rs1, u8 rs2, u16 imm12_1)
385 {
386 return rv_sb_insn(imm12_1, rs2, rs1, 6, 0x63);
387 }
388
389 static u32 rv_bgeu(u8 rs1, u8 rs2, u16 imm12_1)
390 {
391 return rv_sb_insn(imm12_1, rs2, rs1, 7, 0x63);
392 }
393
394 static u32 rv_bne(u8 rs1, u8 rs2, u16 imm12_1)
395 {
396 return rv_sb_insn(imm12_1, rs2, rs1, 1, 0x63);
397 }
398
399 static u32 rv_blt(u8 rs1, u8 rs2, u16 imm12_1)
400 {
401 return rv_sb_insn(imm12_1, rs2, rs1, 4, 0x63);
402 }
403
404 static u32 rv_bge(u8 rs1, u8 rs2, u16 imm12_1)
405 {
406 return rv_sb_insn(imm12_1, rs2, rs1, 5, 0x63);
407 }
408
409 static u32 rv_sb(u8 rs1, u16 imm11_0, u8 rs2)
410 {
411 return rv_s_insn(imm11_0, rs2, rs1, 0, 0x23);
412 }
413
414 static u32 rv_sh(u8 rs1, u16 imm11_0, u8 rs2)
415 {
416 return rv_s_insn(imm11_0, rs2, rs1, 1, 0x23);
417 }
418
419 static u32 rv_sw(u8 rs1, u16 imm11_0, u8 rs2)
420 {
421 return rv_s_insn(imm11_0, rs2, rs1, 2, 0x23);
422 }
423
424 static u32 rv_sd(u8 rs1, u16 imm11_0, u8 rs2)
425 {
426 return rv_s_insn(imm11_0, rs2, rs1, 3, 0x23);
427 }
428
429 static u32 rv_lbu(u8 rd, u16 imm11_0, u8 rs1)
430 {
431 return rv_i_insn(imm11_0, rs1, 4, rd, 0x03);
432 }
433
434 static u32 rv_lhu(u8 rd, u16 imm11_0, u8 rs1)
435 {
436 return rv_i_insn(imm11_0, rs1, 5, rd, 0x03);
437 }
438
439 static u32 rv_lwu(u8 rd, u16 imm11_0, u8 rs1)
440 {
441 return rv_i_insn(imm11_0, rs1, 6, rd, 0x03);
442 }
443
444 static u32 rv_ld(u8 rd, u16 imm11_0, u8 rs1)
445 {
446 return rv_i_insn(imm11_0, rs1, 3, rd, 0x03);
447 }
448
449 static u32 rv_amoadd_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
450 {
451 return rv_amo_insn(0, aq, rl, rs2, rs1, 2, rd, 0x2f);
452 }
453
454 static u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
455 {
456 return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f);
457 }
458
459 static bool is_12b_int(s64 val)
460 {
461 return -(1 << 11) <= val && val < (1 << 11);
462 }
463
464 static bool is_13b_int(s64 val)
465 {
466 return -(1 << 12) <= val && val < (1 << 12);
467 }
468
469 static bool is_21b_int(s64 val)
470 {
471 return -(1L << 20) <= val && val < (1L << 20);
472 }
473
474 static bool is_32b_int(s64 val)
475 {
476 return -(1L << 31) <= val && val < (1L << 31);
477 }
478
479 static int is_12b_check(int off, int insn)
480 {
481 if (!is_12b_int(off)) {
482 pr_err("bpf-jit: insn=%d offset=%d not supported yet!\n",
483 insn, (int)off);
484 return -1;
485 }
486 return 0;
487 }
488
489 static int is_13b_check(int off, int insn)
490 {
491 if (!is_13b_int(off)) {
492 pr_err("bpf-jit: insn=%d offset=%d not supported yet!\n",
493 insn, (int)off);
494 return -1;
495 }
496 return 0;
497 }
498
499 static int is_21b_check(int off, int insn)
500 {
501 if (!is_21b_int(off)) {
502 pr_err("bpf-jit: insn=%d offset=%d not supported yet!\n",
503 insn, (int)off);
504 return -1;
505 }
506 return 0;
507 }
508
509 static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx)
510 {
511 /* Note that the immediate from the add is sign-extended,
512 * which means that we need to compensate this by adding 2^12,
513 * when the 12th bit is set. A simpler way of doing this, and
514 * getting rid of the check, is to just add 2**11 before the
515 * shift. The "Loading a 32-Bit constant" example from the
516 * "Computer Organization and Design, RISC-V edition" book by
517 * Patterson/Hennessy highlights this fact.
518 *
519 * This also means that we need to process LSB to MSB.
520 */
521 s64 upper = (val + (1 << 11)) >> 12, lower = val & 0xfff;
522 int shift;
523
524 if (is_32b_int(val)) {
525 if (upper)
526 emit(rv_lui(rd, upper), ctx);
527
528 if (!upper) {
529 emit(rv_addi(rd, RV_REG_ZERO, lower), ctx);
530 return;
531 }
532
533 emit(rv_addiw(rd, rd, lower), ctx);
534 return;
535 }
536
537 shift = __ffs(upper);
538 upper >>= shift;
539 shift += 12;
540
541 emit_imm(rd, upper, ctx);
542
543 emit(rv_slli(rd, rd, shift), ctx);
544 if (lower)
545 emit(rv_addi(rd, rd, lower), ctx);
546 }
547
548 static int rv_offset(int bpf_to, int bpf_from, struct rv_jit_context *ctx)
549 {
550 int from = ctx->offset[bpf_from] - 1, to = ctx->offset[bpf_to];
551
552 return (to - from) << 2;
553 }
554
555 static int epilogue_offset(struct rv_jit_context *ctx)
556 {
557 int to = ctx->epilogue_offset, from = ctx->ninsns;
558
559 return (to - from) << 2;
560 }
561
562 static void __build_epilogue(u8 reg, struct rv_jit_context *ctx)
563 {
564 int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8;
565
566 if (seen_reg(RV_REG_RA, ctx)) {
567 emit(rv_ld(RV_REG_RA, store_offset, RV_REG_SP), ctx);
568 store_offset -= 8;
569 }
570 emit(rv_ld(RV_REG_FP, store_offset, RV_REG_SP), ctx);
571 store_offset -= 8;
572 if (seen_reg(RV_REG_S1, ctx)) {
573 emit(rv_ld(RV_REG_S1, store_offset, RV_REG_SP), ctx);
574 store_offset -= 8;
575 }
576 if (seen_reg(RV_REG_S2, ctx)) {
577 emit(rv_ld(RV_REG_S2, store_offset, RV_REG_SP), ctx);
578 store_offset -= 8;
579 }
580 if (seen_reg(RV_REG_S3, ctx)) {
581 emit(rv_ld(RV_REG_S3, store_offset, RV_REG_SP), ctx);
582 store_offset -= 8;
583 }
584 if (seen_reg(RV_REG_S4, ctx)) {
585 emit(rv_ld(RV_REG_S4, store_offset, RV_REG_SP), ctx);
586 store_offset -= 8;
587 }
588 if (seen_reg(RV_REG_S5, ctx)) {
589 emit(rv_ld(RV_REG_S5, store_offset, RV_REG_SP), ctx);
590 store_offset -= 8;
591 }
592 if (seen_reg(RV_REG_S6, ctx)) {
593 emit(rv_ld(RV_REG_S6, store_offset, RV_REG_SP), ctx);
594 store_offset -= 8;
595 }
596
597 emit(rv_addi(RV_REG_SP, RV_REG_SP, stack_adjust), ctx);
598 /* Set return value. */
599 emit(rv_addi(RV_REG_A0, RV_REG_A5, 0), ctx);
600 emit(rv_jalr(RV_REG_ZERO, reg, 0), ctx);
601 }
602
603 static void emit_zext_32(u8 reg, struct rv_jit_context *ctx)
604 {
605 emit(rv_slli(reg, reg, 32), ctx);
606 emit(rv_srli(reg, reg, 32), ctx);
607 }
608
609 static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx)
610 {
611 int tc_ninsn, off, start_insn = ctx->ninsns;
612 u8 tcc = rv_tail_call_reg(ctx);
613
614 /* a0: &ctx
615 * a1: &array
616 * a2: index
617 *
618 * if (index >= array->map.max_entries)
619 * goto out;
620 */
621 tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] :
622 ctx->offset[0];
623 emit_zext_32(RV_REG_A2, ctx);
624
625 off = offsetof(struct bpf_array, map.max_entries);
626 if (is_12b_check(off, insn))
627 return -1;
628 emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx);
629 off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2;
630 if (is_13b_check(off, insn))
631 return -1;
632 emit(rv_bgeu(RV_REG_A2, RV_REG_T1, off >> 1), ctx);
633
634 /* if (--TCC < 0)
635 * goto out;
636 */
637 emit(rv_addi(RV_REG_T1, tcc, -1), ctx);
638 off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2;
639 if (is_13b_check(off, insn))
640 return -1;
641 emit(rv_blt(RV_REG_T1, RV_REG_ZERO, off >> 1), ctx);
642
643 /* prog = array->ptrs[index];
644 * if (!prog)
645 * goto out;
646 */
647 emit(rv_slli(RV_REG_T2, RV_REG_A2, 3), ctx);
648 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_A1), ctx);
649 off = offsetof(struct bpf_array, ptrs);
650 if (is_12b_check(off, insn))
651 return -1;
652 emit(rv_ld(RV_REG_T2, off, RV_REG_T2), ctx);
653 off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2;
654 if (is_13b_check(off, insn))
655 return -1;
656 emit(rv_beq(RV_REG_T2, RV_REG_ZERO, off >> 1), ctx);
657
658 /* goto *(prog->bpf_func + 4); */
659 off = offsetof(struct bpf_prog, bpf_func);
660 if (is_12b_check(off, insn))
661 return -1;
662 emit(rv_ld(RV_REG_T3, off, RV_REG_T2), ctx);
663 emit(rv_addi(RV_REG_T3, RV_REG_T3, 4), ctx);
664 emit(rv_addi(RV_REG_TCC, RV_REG_T1, 0), ctx);
665 __build_epilogue(RV_REG_T3, ctx);
666 return 0;
667 }
668
669 static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn,
670 struct rv_jit_context *ctx)
671 {
672 u8 code = insn->code;
673
674 switch (code) {
675 case BPF_JMP | BPF_JA:
676 case BPF_JMP | BPF_CALL:
677 case BPF_JMP | BPF_EXIT:
678 case BPF_JMP | BPF_TAIL_CALL:
679 break;
680 default:
681 *rd = bpf_to_rv_reg(insn->dst_reg, ctx);
682 }
683
684 if (code & (BPF_ALU | BPF_X) || code & (BPF_ALU64 | BPF_X) ||
685 code & (BPF_JMP | BPF_X) || code & (BPF_JMP32 | BPF_X) ||
686 code & BPF_LDX || code & BPF_STX)
687 *rs = bpf_to_rv_reg(insn->src_reg, ctx);
688 }
689
690 static int rv_offset_check(int *rvoff, s16 off, int insn,
691 struct rv_jit_context *ctx)
692 {
693 *rvoff = rv_offset(insn + off, insn, ctx);
694 return is_13b_check(*rvoff, insn);
695 }
696
697 static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx)
698 {
699 emit(rv_addi(RV_REG_T2, *rd, 0), ctx);
700 emit_zext_32(RV_REG_T2, ctx);
701 emit(rv_addi(RV_REG_T1, *rs, 0), ctx);
702 emit_zext_32(RV_REG_T1, ctx);
703 *rd = RV_REG_T2;
704 *rs = RV_REG_T1;
705 }
706
707 static void emit_sext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx)
708 {
709 emit(rv_addiw(RV_REG_T2, *rd, 0), ctx);
710 emit(rv_addiw(RV_REG_T1, *rs, 0), ctx);
711 *rd = RV_REG_T2;
712 *rs = RV_REG_T1;
713 }
714
715 static void emit_zext_32_rd_t1(u8 *rd, struct rv_jit_context *ctx)
716 {
717 emit(rv_addi(RV_REG_T2, *rd, 0), ctx);
718 emit_zext_32(RV_REG_T2, ctx);
719 emit_zext_32(RV_REG_T1, ctx);
720 *rd = RV_REG_T2;
721 }
722
723 static void emit_sext_32_rd(u8 *rd, struct rv_jit_context *ctx)
724 {
725 emit(rv_addiw(RV_REG_T2, *rd, 0), ctx);
726 *rd = RV_REG_T2;
727 }
728
729 static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
730 bool extra_pass)
731 {
732 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 ||
733 BPF_CLASS(insn->code) == BPF_JMP;
734 int rvoff, i = insn - ctx->prog->insnsi;
735 u8 rd = -1, rs = -1, code = insn->code;
736 s16 off = insn->off;
737 s32 imm = insn->imm;
738
739 init_regs(&rd, &rs, insn, ctx);
740
741 switch (code) {
742 /* dst = src */
743 case BPF_ALU | BPF_MOV | BPF_X:
744 case BPF_ALU64 | BPF_MOV | BPF_X:
745 emit(is64 ? rv_addi(rd, rs, 0) : rv_addiw(rd, rs, 0), ctx);
746 if (!is64)
747 emit_zext_32(rd, ctx);
748 break;
749
750 /* dst = dst OP src */
751 case BPF_ALU | BPF_ADD | BPF_X:
752 case BPF_ALU64 | BPF_ADD | BPF_X:
753 emit(is64 ? rv_add(rd, rd, rs) : rv_addw(rd, rd, rs), ctx);
754 if (!is64)
755 emit_zext_32(rd, ctx);
756 break;
757 case BPF_ALU | BPF_SUB | BPF_X:
758 case BPF_ALU64 | BPF_SUB | BPF_X:
759 emit(is64 ? rv_sub(rd, rd, rs) : rv_subw(rd, rd, rs), ctx);
760 if (!is64)
761 emit_zext_32(rd, ctx);
762 break;
763 case BPF_ALU | BPF_AND | BPF_X:
764 case BPF_ALU64 | BPF_AND | BPF_X:
765 emit(rv_and(rd, rd, rs), ctx);
766 if (!is64)
767 emit_zext_32(rd, ctx);
768 break;
769 case BPF_ALU | BPF_OR | BPF_X:
770 case BPF_ALU64 | BPF_OR | BPF_X:
771 emit(rv_or(rd, rd, rs), ctx);
772 if (!is64)
773 emit_zext_32(rd, ctx);
774 break;
775 case BPF_ALU | BPF_XOR | BPF_X:
776 case BPF_ALU64 | BPF_XOR | BPF_X:
777 emit(rv_xor(rd, rd, rs), ctx);
778 if (!is64)
779 emit_zext_32(rd, ctx);
780 break;
781 case BPF_ALU | BPF_MUL | BPF_X:
782 case BPF_ALU64 | BPF_MUL | BPF_X:
783 emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx);
784 if (!is64)
785 emit_zext_32(rd, ctx);
786 break;
787 case BPF_ALU | BPF_DIV | BPF_X:
788 case BPF_ALU64 | BPF_DIV | BPF_X:
789 emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx);
790 if (!is64)
791 emit_zext_32(rd, ctx);
792 break;
793 case BPF_ALU | BPF_MOD | BPF_X:
794 case BPF_ALU64 | BPF_MOD | BPF_X:
795 emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx);
796 if (!is64)
797 emit_zext_32(rd, ctx);
798 break;
799 case BPF_ALU | BPF_LSH | BPF_X:
800 case BPF_ALU64 | BPF_LSH | BPF_X:
801 emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx);
802 if (!is64)
803 emit_zext_32(rd, ctx);
804 break;
805 case BPF_ALU | BPF_RSH | BPF_X:
806 case BPF_ALU64 | BPF_RSH | BPF_X:
807 emit(is64 ? rv_srl(rd, rd, rs) : rv_srlw(rd, rd, rs), ctx);
808 if (!is64)
809 emit_zext_32(rd, ctx);
810 break;
811 case BPF_ALU | BPF_ARSH | BPF_X:
812 case BPF_ALU64 | BPF_ARSH | BPF_X:
813 emit(is64 ? rv_sra(rd, rd, rs) : rv_sraw(rd, rd, rs), ctx);
814 if (!is64)
815 emit_zext_32(rd, ctx);
816 break;
817
818 /* dst = -dst */
819 case BPF_ALU | BPF_NEG:
820 case BPF_ALU64 | BPF_NEG:
821 emit(is64 ? rv_sub(rd, RV_REG_ZERO, rd) :
822 rv_subw(rd, RV_REG_ZERO, rd), ctx);
823 if (!is64)
824 emit_zext_32(rd, ctx);
825 break;
826
827 /* dst = BSWAP##imm(dst) */
828 case BPF_ALU | BPF_END | BPF_FROM_LE:
829 {
830 int shift = 64 - imm;
831
832 emit(rv_slli(rd, rd, shift), ctx);
833 emit(rv_srli(rd, rd, shift), ctx);
834 break;
835 }
836 case BPF_ALU | BPF_END | BPF_FROM_BE:
837 emit(rv_addi(RV_REG_T2, RV_REG_ZERO, 0), ctx);
838
839 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
840 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
841 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
842 emit(rv_srli(rd, rd, 8), ctx);
843 if (imm == 16)
844 goto out_be;
845
846 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
847 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
848 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
849 emit(rv_srli(rd, rd, 8), ctx);
850
851 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
852 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
853 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
854 emit(rv_srli(rd, rd, 8), ctx);
855 if (imm == 32)
856 goto out_be;
857
858 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
859 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
860 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
861 emit(rv_srli(rd, rd, 8), ctx);
862
863 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
864 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
865 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
866 emit(rv_srli(rd, rd, 8), ctx);
867
868 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
869 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
870 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
871 emit(rv_srli(rd, rd, 8), ctx);
872
873 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
874 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
875 emit(rv_slli(RV_REG_T2, RV_REG_T2, 8), ctx);
876 emit(rv_srli(rd, rd, 8), ctx);
877 out_be:
878 emit(rv_andi(RV_REG_T1, rd, 0xff), ctx);
879 emit(rv_add(RV_REG_T2, RV_REG_T2, RV_REG_T1), ctx);
880
881 emit(rv_addi(rd, RV_REG_T2, 0), ctx);
882 break;
883
884 /* dst = imm */
885 case BPF_ALU | BPF_MOV | BPF_K:
886 case BPF_ALU64 | BPF_MOV | BPF_K:
887 emit_imm(rd, imm, ctx);
888 if (!is64)
889 emit_zext_32(rd, ctx);
890 break;
891
892 /* dst = dst OP imm */
893 case BPF_ALU | BPF_ADD | BPF_K:
894 case BPF_ALU64 | BPF_ADD | BPF_K:
895 if (is_12b_int(imm)) {
896 emit(is64 ? rv_addi(rd, rd, imm) :
897 rv_addiw(rd, rd, imm), ctx);
898 } else {
899 emit_imm(RV_REG_T1, imm, ctx);
900 emit(is64 ? rv_add(rd, rd, RV_REG_T1) :
901 rv_addw(rd, rd, RV_REG_T1), ctx);
902 }
903 if (!is64)
904 emit_zext_32(rd, ctx);
905 break;
906 case BPF_ALU | BPF_SUB | BPF_K:
907 case BPF_ALU64 | BPF_SUB | BPF_K:
908 if (is_12b_int(-imm)) {
909 emit(is64 ? rv_addi(rd, rd, -imm) :
910 rv_addiw(rd, rd, -imm), ctx);
911 } else {
912 emit_imm(RV_REG_T1, imm, ctx);
913 emit(is64 ? rv_sub(rd, rd, RV_REG_T1) :
914 rv_subw(rd, rd, RV_REG_T1), ctx);
915 }
916 if (!is64)
917 emit_zext_32(rd, ctx);
918 break;
919 case BPF_ALU | BPF_AND | BPF_K:
920 case BPF_ALU64 | BPF_AND | BPF_K:
921 if (is_12b_int(imm)) {
922 emit(rv_andi(rd, rd, imm), ctx);
923 } else {
924 emit_imm(RV_REG_T1, imm, ctx);
925 emit(rv_and(rd, rd, RV_REG_T1), ctx);
926 }
927 if (!is64)
928 emit_zext_32(rd, ctx);
929 break;
930 case BPF_ALU | BPF_OR | BPF_K:
931 case BPF_ALU64 | BPF_OR | BPF_K:
932 if (is_12b_int(imm)) {
933 emit(rv_ori(rd, rd, imm), ctx);
934 } else {
935 emit_imm(RV_REG_T1, imm, ctx);
936 emit(rv_or(rd, rd, RV_REG_T1), ctx);
937 }
938 if (!is64)
939 emit_zext_32(rd, ctx);
940 break;
941 case BPF_ALU | BPF_XOR | BPF_K:
942 case BPF_ALU64 | BPF_XOR | BPF_K:
943 if (is_12b_int(imm)) {
944 emit(rv_xori(rd, rd, imm), ctx);
945 } else {
946 emit_imm(RV_REG_T1, imm, ctx);
947 emit(rv_xor(rd, rd, RV_REG_T1), ctx);
948 }
949 if (!is64)
950 emit_zext_32(rd, ctx);
951 break;
952 case BPF_ALU | BPF_MUL | BPF_K:
953 case BPF_ALU64 | BPF_MUL | BPF_K:
954 emit_imm(RV_REG_T1, imm, ctx);
955 emit(is64 ? rv_mul(rd, rd, RV_REG_T1) :
956 rv_mulw(rd, rd, RV_REG_T1), ctx);
957 if (!is64)
958 emit_zext_32(rd, ctx);
959 break;
960 case BPF_ALU | BPF_DIV | BPF_K:
961 case BPF_ALU64 | BPF_DIV | BPF_K:
962 emit_imm(RV_REG_T1, imm, ctx);
963 emit(is64 ? rv_divu(rd, rd, RV_REG_T1) :
964 rv_divuw(rd, rd, RV_REG_T1), ctx);
965 if (!is64)
966 emit_zext_32(rd, ctx);
967 break;
968 case BPF_ALU | BPF_MOD | BPF_K:
969 case BPF_ALU64 | BPF_MOD | BPF_K:
970 emit_imm(RV_REG_T1, imm, ctx);
971 emit(is64 ? rv_remu(rd, rd, RV_REG_T1) :
972 rv_remuw(rd, rd, RV_REG_T1), ctx);
973 if (!is64)
974 emit_zext_32(rd, ctx);
975 break;
976 case BPF_ALU | BPF_LSH | BPF_K:
977 case BPF_ALU64 | BPF_LSH | BPF_K:
978 emit(is64 ? rv_slli(rd, rd, imm) : rv_slliw(rd, rd, imm), ctx);
979 if (!is64)
980 emit_zext_32(rd, ctx);
981 break;
982 case BPF_ALU | BPF_RSH | BPF_K:
983 case BPF_ALU64 | BPF_RSH | BPF_K:
984 emit(is64 ? rv_srli(rd, rd, imm) : rv_srliw(rd, rd, imm), ctx);
985 if (!is64)
986 emit_zext_32(rd, ctx);
987 break;
988 case BPF_ALU | BPF_ARSH | BPF_K:
989 case BPF_ALU64 | BPF_ARSH | BPF_K:
990 emit(is64 ? rv_srai(rd, rd, imm) : rv_sraiw(rd, rd, imm), ctx);
991 if (!is64)
992 emit_zext_32(rd, ctx);
993 break;
994
995 /* JUMP off */
996 case BPF_JMP | BPF_JA:
997 rvoff = rv_offset(i + off, i, ctx);
998 if (!is_21b_int(rvoff)) {
999 pr_err("bpf-jit: insn=%d offset=%d not supported yet!\n",
1000 i, rvoff);
1001 return -1;
1002 }
1003
1004 emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx);
1005 break;
1006
1007 /* IF (dst COND src) JUMP off */
1008 case BPF_JMP | BPF_JEQ | BPF_X:
1009 case BPF_JMP32 | BPF_JEQ | BPF_X:
1010 if (rv_offset_check(&rvoff, off, i, ctx))
1011 return -1;
1012 if (!is64)
1013 emit_zext_32_rd_rs(&rd, &rs, ctx);
1014 emit(rv_beq(rd, rs, rvoff >> 1), ctx);
1015 break;
1016 case BPF_JMP | BPF_JGT | BPF_X:
1017 case BPF_JMP32 | BPF_JGT | BPF_X:
1018 if (rv_offset_check(&rvoff, off, i, ctx))
1019 return -1;
1020 if (!is64)
1021 emit_zext_32_rd_rs(&rd, &rs, ctx);
1022 emit(rv_bltu(rs, rd, rvoff >> 1), ctx);
1023 break;
1024 case BPF_JMP | BPF_JLT | BPF_X:
1025 case BPF_JMP32 | BPF_JLT | BPF_X:
1026 if (rv_offset_check(&rvoff, off, i, ctx))
1027 return -1;
1028 if (!is64)
1029 emit_zext_32_rd_rs(&rd, &rs, ctx);
1030 emit(rv_bltu(rd, rs, rvoff >> 1), ctx);
1031 break;
1032 case BPF_JMP | BPF_JGE | BPF_X:
1033 case BPF_JMP32 | BPF_JGE | BPF_X:
1034 if (rv_offset_check(&rvoff, off, i, ctx))
1035 return -1;
1036 if (!is64)
1037 emit_zext_32_rd_rs(&rd, &rs, ctx);
1038 emit(rv_bgeu(rd, rs, rvoff >> 1), ctx);
1039 break;
1040 case BPF_JMP | BPF_JLE | BPF_X:
1041 case BPF_JMP32 | BPF_JLE | BPF_X:
1042 if (rv_offset_check(&rvoff, off, i, ctx))
1043 return -1;
1044 if (!is64)
1045 emit_zext_32_rd_rs(&rd, &rs, ctx);
1046 emit(rv_bgeu(rs, rd, rvoff >> 1), ctx);
1047 break;
1048 case BPF_JMP | BPF_JNE | BPF_X:
1049 case BPF_JMP32 | BPF_JNE | BPF_X:
1050 if (rv_offset_check(&rvoff, off, i, ctx))
1051 return -1;
1052 if (!is64)
1053 emit_zext_32_rd_rs(&rd, &rs, ctx);
1054 emit(rv_bne(rd, rs, rvoff >> 1), ctx);
1055 break;
1056 case BPF_JMP | BPF_JSGT | BPF_X:
1057 case BPF_JMP32 | BPF_JSGT | BPF_X:
1058 if (rv_offset_check(&rvoff, off, i, ctx))
1059 return -1;
1060 if (!is64)
1061 emit_sext_32_rd_rs(&rd, &rs, ctx);
1062 emit(rv_blt(rs, rd, rvoff >> 1), ctx);
1063 break;
1064 case BPF_JMP | BPF_JSLT | BPF_X:
1065 case BPF_JMP32 | BPF_JSLT | BPF_X:
1066 if (rv_offset_check(&rvoff, off, i, ctx))
1067 return -1;
1068 if (!is64)
1069 emit_sext_32_rd_rs(&rd, &rs, ctx);
1070 emit(rv_blt(rd, rs, rvoff >> 1), ctx);
1071 break;
1072 case BPF_JMP | BPF_JSGE | BPF_X:
1073 case BPF_JMP32 | BPF_JSGE | BPF_X:
1074 if (rv_offset_check(&rvoff, off, i, ctx))
1075 return -1;
1076 if (!is64)
1077 emit_sext_32_rd_rs(&rd, &rs, ctx);
1078 emit(rv_bge(rd, rs, rvoff >> 1), ctx);
1079 break;
1080 case BPF_JMP | BPF_JSLE | BPF_X:
1081 case BPF_JMP32 | BPF_JSLE | BPF_X:
1082 if (rv_offset_check(&rvoff, off, i, ctx))
1083 return -1;
1084 if (!is64)
1085 emit_sext_32_rd_rs(&rd, &rs, ctx);
1086 emit(rv_bge(rs, rd, rvoff >> 1), ctx);
1087 break;
1088 case BPF_JMP | BPF_JSET | BPF_X:
1089 case BPF_JMP32 | BPF_JSET | BPF_X:
1090 if (rv_offset_check(&rvoff, off, i, ctx))
1091 return -1;
1092 if (!is64)
1093 emit_zext_32_rd_rs(&rd, &rs, ctx);
1094 emit(rv_and(RV_REG_T1, rd, rs), ctx);
1095 emit(rv_bne(RV_REG_T1, RV_REG_ZERO, rvoff >> 1), ctx);
1096 break;
1097
1098 /* IF (dst COND imm) JUMP off */
1099 case BPF_JMP | BPF_JEQ | BPF_K:
1100 case BPF_JMP32 | BPF_JEQ | BPF_K:
1101 if (rv_offset_check(&rvoff, off, i, ctx))
1102 return -1;
1103 emit_imm(RV_REG_T1, imm, ctx);
1104 if (!is64)
1105 emit_zext_32_rd_t1(&rd, ctx);
1106 emit(rv_beq(rd, RV_REG_T1, rvoff >> 1), ctx);
1107 break;
1108 case BPF_JMP | BPF_JGT | BPF_K:
1109 case BPF_JMP32 | BPF_JGT | BPF_K:
1110 if (rv_offset_check(&rvoff, off, i, ctx))
1111 return -1;
1112 emit_imm(RV_REG_T1, imm, ctx);
1113 if (!is64)
1114 emit_zext_32_rd_t1(&rd, ctx);
1115 emit(rv_bltu(RV_REG_T1, rd, rvoff >> 1), ctx);
1116 break;
1117 case BPF_JMP | BPF_JLT | BPF_K:
1118 case BPF_JMP32 | BPF_JLT | BPF_K:
1119 if (rv_offset_check(&rvoff, off, i, ctx))
1120 return -1;
1121 emit_imm(RV_REG_T1, imm, ctx);
1122 if (!is64)
1123 emit_zext_32_rd_t1(&rd, ctx);
1124 emit(rv_bltu(rd, RV_REG_T1, rvoff >> 1), ctx);
1125 break;
1126 case BPF_JMP | BPF_JGE | BPF_K:
1127 case BPF_JMP32 | BPF_JGE | BPF_K:
1128 if (rv_offset_check(&rvoff, off, i, ctx))
1129 return -1;
1130 emit_imm(RV_REG_T1, imm, ctx);
1131 if (!is64)
1132 emit_zext_32_rd_t1(&rd, ctx);
1133 emit(rv_bgeu(rd, RV_REG_T1, rvoff >> 1), ctx);
1134 break;
1135 case BPF_JMP | BPF_JLE | BPF_K:
1136 case BPF_JMP32 | BPF_JLE | BPF_K:
1137 if (rv_offset_check(&rvoff, off, i, ctx))
1138 return -1;
1139 emit_imm(RV_REG_T1, imm, ctx);
1140 if (!is64)
1141 emit_zext_32_rd_t1(&rd, ctx);
1142 emit(rv_bgeu(RV_REG_T1, rd, rvoff >> 1), ctx);
1143 break;
1144 case BPF_JMP | BPF_JNE | BPF_K:
1145 case BPF_JMP32 | BPF_JNE | BPF_K:
1146 if (rv_offset_check(&rvoff, off, i, ctx))
1147 return -1;
1148 emit_imm(RV_REG_T1, imm, ctx);
1149 if (!is64)
1150 emit_zext_32_rd_t1(&rd, ctx);
1151 emit(rv_bne(rd, RV_REG_T1, rvoff >> 1), ctx);
1152 break;
1153 case BPF_JMP | BPF_JSGT | BPF_K:
1154 case BPF_JMP32 | BPF_JSGT | BPF_K:
1155 if (rv_offset_check(&rvoff, off, i, ctx))
1156 return -1;
1157 emit_imm(RV_REG_T1, imm, ctx);
1158 if (!is64)
1159 emit_sext_32_rd(&rd, ctx);
1160 emit(rv_blt(RV_REG_T1, rd, rvoff >> 1), ctx);
1161 break;
1162 case BPF_JMP | BPF_JSLT | BPF_K:
1163 case BPF_JMP32 | BPF_JSLT | BPF_K:
1164 if (rv_offset_check(&rvoff, off, i, ctx))
1165 return -1;
1166 emit_imm(RV_REG_T1, imm, ctx);
1167 if (!is64)
1168 emit_sext_32_rd(&rd, ctx);
1169 emit(rv_blt(rd, RV_REG_T1, rvoff >> 1), ctx);
1170 break;
1171 case BPF_JMP | BPF_JSGE | BPF_K:
1172 case BPF_JMP32 | BPF_JSGE | BPF_K:
1173 if (rv_offset_check(&rvoff, off, i, ctx))
1174 return -1;
1175 emit_imm(RV_REG_T1, imm, ctx);
1176 if (!is64)
1177 emit_sext_32_rd(&rd, ctx);
1178 emit(rv_bge(rd, RV_REG_T1, rvoff >> 1), ctx);
1179 break;
1180 case BPF_JMP | BPF_JSLE | BPF_K:
1181 case BPF_JMP32 | BPF_JSLE | BPF_K:
1182 if (rv_offset_check(&rvoff, off, i, ctx))
1183 return -1;
1184 emit_imm(RV_REG_T1, imm, ctx);
1185 if (!is64)
1186 emit_sext_32_rd(&rd, ctx);
1187 emit(rv_bge(RV_REG_T1, rd, rvoff >> 1), ctx);
1188 break;
1189 case BPF_JMP | BPF_JSET | BPF_K:
1190 case BPF_JMP32 | BPF_JSET | BPF_K:
1191 if (rv_offset_check(&rvoff, off, i, ctx))
1192 return -1;
1193 emit_imm(RV_REG_T1, imm, ctx);
1194 if (!is64)
1195 emit_zext_32_rd_t1(&rd, ctx);
1196 emit(rv_and(RV_REG_T1, rd, RV_REG_T1), ctx);
1197 emit(rv_bne(RV_REG_T1, RV_REG_ZERO, rvoff >> 1), ctx);
1198 break;
1199
1200 /* function call */
1201 case BPF_JMP | BPF_CALL:
1202 {
1203 bool fixed;
1204 int i, ret;
1205 u64 addr;
1206
1207 mark_call(ctx);
1208 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, &addr,
1209 &fixed);
1210 if (ret < 0)
1211 return ret;
1212 if (fixed) {
1213 emit_imm(RV_REG_T1, addr, ctx);
1214 } else {
1215 i = ctx->ninsns;
1216 emit_imm(RV_REG_T1, addr, ctx);
1217 for (i = ctx->ninsns - i; i < 8; i++) {
1218 /* nop */
1219 emit(rv_addi(RV_REG_ZERO, RV_REG_ZERO, 0),
1220 ctx);
1221 }
1222 }
1223 emit(rv_jalr(RV_REG_RA, RV_REG_T1, 0), ctx);
1224 rd = bpf_to_rv_reg(BPF_REG_0, ctx);
1225 emit(rv_addi(rd, RV_REG_A0, 0), ctx);
1226 break;
1227 }
1228 /* tail call */
1229 case BPF_JMP | BPF_TAIL_CALL:
1230 if (emit_bpf_tail_call(i, ctx))
1231 return -1;
1232 break;
1233
1234 /* function return */
1235 case BPF_JMP | BPF_EXIT:
1236 if (i == ctx->prog->len - 1)
1237 break;
1238
1239 rvoff = epilogue_offset(ctx);
1240 if (is_21b_check(rvoff, i))
1241 return -1;
1242 emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx);
1243 break;
1244
1245 /* dst = imm64 */
1246 case BPF_LD | BPF_IMM | BPF_DW:
1247 {
1248 struct bpf_insn insn1 = insn[1];
1249 u64 imm64;
1250
1251 imm64 = (u64)insn1.imm << 32 | (u32)imm;
1252 emit_imm(rd, imm64, ctx);
1253 return 1;
1254 }
1255
1256 /* LDX: dst = *(size *)(src + off) */
1257 case BPF_LDX | BPF_MEM | BPF_B:
1258 if (is_12b_int(off)) {
1259 emit(rv_lbu(rd, off, rs), ctx);
1260 break;
1261 }
1262
1263 emit_imm(RV_REG_T1, off, ctx);
1264 emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx);
1265 emit(rv_lbu(rd, 0, RV_REG_T1), ctx);
1266 break;
1267 case BPF_LDX | BPF_MEM | BPF_H:
1268 if (is_12b_int(off)) {
1269 emit(rv_lhu(rd, off, rs), ctx);
1270 break;
1271 }
1272
1273 emit_imm(RV_REG_T1, off, ctx);
1274 emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx);
1275 emit(rv_lhu(rd, 0, RV_REG_T1), ctx);
1276 break;
1277 case BPF_LDX | BPF_MEM | BPF_W:
1278 if (is_12b_int(off)) {
1279 emit(rv_lwu(rd, off, rs), ctx);
1280 break;
1281 }
1282
1283 emit_imm(RV_REG_T1, off, ctx);
1284 emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx);
1285 emit(rv_lwu(rd, 0, RV_REG_T1), ctx);
1286 break;
1287 case BPF_LDX | BPF_MEM | BPF_DW:
1288 if (is_12b_int(off)) {
1289 emit(rv_ld(rd, off, rs), ctx);
1290 break;
1291 }
1292
1293 emit_imm(RV_REG_T1, off, ctx);
1294 emit(rv_add(RV_REG_T1, RV_REG_T1, rs), ctx);
1295 emit(rv_ld(rd, 0, RV_REG_T1), ctx);
1296 break;
1297
1298 /* ST: *(size *)(dst + off) = imm */
1299 case BPF_ST | BPF_MEM | BPF_B:
1300 emit_imm(RV_REG_T1, imm, ctx);
1301 if (is_12b_int(off)) {
1302 emit(rv_sb(rd, off, RV_REG_T1), ctx);
1303 break;
1304 }
1305
1306 emit_imm(RV_REG_T2, off, ctx);
1307 emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx);
1308 emit(rv_sb(RV_REG_T2, 0, RV_REG_T1), ctx);
1309 break;
1310
1311 case BPF_ST | BPF_MEM | BPF_H:
1312 emit_imm(RV_REG_T1, imm, ctx);
1313 if (is_12b_int(off)) {
1314 emit(rv_sh(rd, off, RV_REG_T1), ctx);
1315 break;
1316 }
1317
1318 emit_imm(RV_REG_T2, off, ctx);
1319 emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx);
1320 emit(rv_sh(RV_REG_T2, 0, RV_REG_T1), ctx);
1321 break;
1322 case BPF_ST | BPF_MEM | BPF_W:
1323 emit_imm(RV_REG_T1, imm, ctx);
1324 if (is_12b_int(off)) {
1325 emit(rv_sw(rd, off, RV_REG_T1), ctx);
1326 break;
1327 }
1328
1329 emit_imm(RV_REG_T2, off, ctx);
1330 emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx);
1331 emit(rv_sw(RV_REG_T2, 0, RV_REG_T1), ctx);
1332 break;
1333 case BPF_ST | BPF_MEM | BPF_DW:
1334 emit_imm(RV_REG_T1, imm, ctx);
1335 if (is_12b_int(off)) {
1336 emit(rv_sd(rd, off, RV_REG_T1), ctx);
1337 break;
1338 }
1339
1340 emit_imm(RV_REG_T2, off, ctx);
1341 emit(rv_add(RV_REG_T2, RV_REG_T2, rd), ctx);
1342 emit(rv_sd(RV_REG_T2, 0, RV_REG_T1), ctx);
1343 break;
1344
1345 /* STX: *(size *)(dst + off) = src */
1346 case BPF_STX | BPF_MEM | BPF_B:
1347 if (is_12b_int(off)) {
1348 emit(rv_sb(rd, off, rs), ctx);
1349 break;
1350 }
1351
1352 emit_imm(RV_REG_T1, off, ctx);
1353 emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx);
1354 emit(rv_sb(RV_REG_T1, 0, rs), ctx);
1355 break;
1356 case BPF_STX | BPF_MEM | BPF_H:
1357 if (is_12b_int(off)) {
1358 emit(rv_sh(rd, off, rs), ctx);
1359 break;
1360 }
1361
1362 emit_imm(RV_REG_T1, off, ctx);
1363 emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx);
1364 emit(rv_sh(RV_REG_T1, 0, rs), ctx);
1365 break;
1366 case BPF_STX | BPF_MEM | BPF_W:
1367 if (is_12b_int(off)) {
1368 emit(rv_sw(rd, off, rs), ctx);
1369 break;
1370 }
1371
1372 emit_imm(RV_REG_T1, off, ctx);
1373 emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx);
1374 emit(rv_sw(RV_REG_T1, 0, rs), ctx);
1375 break;
1376 case BPF_STX | BPF_MEM | BPF_DW:
1377 if (is_12b_int(off)) {
1378 emit(rv_sd(rd, off, rs), ctx);
1379 break;
1380 }
1381
1382 emit_imm(RV_REG_T1, off, ctx);
1383 emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx);
1384 emit(rv_sd(RV_REG_T1, 0, rs), ctx);
1385 break;
1386 /* STX XADD: lock *(u32 *)(dst + off) += src */
1387 case BPF_STX | BPF_XADD | BPF_W:
1388 /* STX XADD: lock *(u64 *)(dst + off) += src */
1389 case BPF_STX | BPF_XADD | BPF_DW:
1390 if (off) {
1391 if (is_12b_int(off)) {
1392 emit(rv_addi(RV_REG_T1, rd, off), ctx);
1393 } else {
1394 emit_imm(RV_REG_T1, off, ctx);
1395 emit(rv_add(RV_REG_T1, RV_REG_T1, rd), ctx);
1396 }
1397
1398 rd = RV_REG_T1;
1399 }
1400
1401 emit(BPF_SIZE(code) == BPF_W ?
1402 rv_amoadd_w(RV_REG_ZERO, rs, rd, 0, 0) :
1403 rv_amoadd_d(RV_REG_ZERO, rs, rd, 0, 0), ctx);
1404 break;
1405 default:
1406 pr_err("bpf-jit: unknown opcode %02x\n", code);
1407 return -EINVAL;
1408 }
1409
1410 return 0;
1411 }
1412
1413 static void build_prologue(struct rv_jit_context *ctx)
1414 {
1415 int stack_adjust = 0, store_offset, bpf_stack_adjust;
1416
1417 if (seen_reg(RV_REG_RA, ctx))
1418 stack_adjust += 8;
1419 stack_adjust += 8; /* RV_REG_FP */
1420 if (seen_reg(RV_REG_S1, ctx))
1421 stack_adjust += 8;
1422 if (seen_reg(RV_REG_S2, ctx))
1423 stack_adjust += 8;
1424 if (seen_reg(RV_REG_S3, ctx))
1425 stack_adjust += 8;
1426 if (seen_reg(RV_REG_S4, ctx))
1427 stack_adjust += 8;
1428 if (seen_reg(RV_REG_S5, ctx))
1429 stack_adjust += 8;
1430 if (seen_reg(RV_REG_S6, ctx))
1431 stack_adjust += 8;
1432
1433 stack_adjust = round_up(stack_adjust, 16);
1434 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16);
1435 stack_adjust += bpf_stack_adjust;
1436
1437 store_offset = stack_adjust - 8;
1438
1439 /* First instruction is always setting the tail-call-counter
1440 * (TCC) register. This instruction is skipped for tail calls.
1441 */
1442 emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx);
1443
1444 emit(rv_addi(RV_REG_SP, RV_REG_SP, -stack_adjust), ctx);
1445
1446 if (seen_reg(RV_REG_RA, ctx)) {
1447 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_RA), ctx);
1448 store_offset -= 8;
1449 }
1450 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_FP), ctx);
1451 store_offset -= 8;
1452 if (seen_reg(RV_REG_S1, ctx)) {
1453 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S1), ctx);
1454 store_offset -= 8;
1455 }
1456 if (seen_reg(RV_REG_S2, ctx)) {
1457 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S2), ctx);
1458 store_offset -= 8;
1459 }
1460 if (seen_reg(RV_REG_S3, ctx)) {
1461 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S3), ctx);
1462 store_offset -= 8;
1463 }
1464 if (seen_reg(RV_REG_S4, ctx)) {
1465 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S4), ctx);
1466 store_offset -= 8;
1467 }
1468 if (seen_reg(RV_REG_S5, ctx)) {
1469 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S5), ctx);
1470 store_offset -= 8;
1471 }
1472 if (seen_reg(RV_REG_S6, ctx)) {
1473 emit(rv_sd(RV_REG_SP, store_offset, RV_REG_S6), ctx);
1474 store_offset -= 8;
1475 }
1476
1477 emit(rv_addi(RV_REG_FP, RV_REG_SP, stack_adjust), ctx);
1478
1479 if (bpf_stack_adjust)
1480 emit(rv_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust), ctx);
1481
1482 /* Program contains calls and tail calls, so RV_REG_TCC need
1483 * to be saved across calls.
1484 */
1485 if (seen_tail_call(ctx) && seen_call(ctx))
1486 emit(rv_addi(RV_REG_TCC_SAVED, RV_REG_TCC, 0), ctx);
1487
1488 ctx->stack_size = stack_adjust;
1489 }
1490
1491 static void build_epilogue(struct rv_jit_context *ctx)
1492 {
1493 __build_epilogue(RV_REG_RA, ctx);
1494 }
1495
1496 static int build_body(struct rv_jit_context *ctx, bool extra_pass)
1497 {
1498 const struct bpf_prog *prog = ctx->prog;
1499 int i;
1500
1501 for (i = 0; i < prog->len; i++) {
1502 const struct bpf_insn *insn = &prog->insnsi[i];
1503 int ret;
1504
1505 ret = emit_insn(insn, ctx, extra_pass);
1506 if (ret > 0) {
1507 i++;
1508 if (ctx->insns == NULL)
1509 ctx->offset[i] = ctx->ninsns;
1510 continue;
1511 }
1512 if (ctx->insns == NULL)
1513 ctx->offset[i] = ctx->ninsns;
1514 if (ret)
1515 return ret;
1516 }
1517 return 0;
1518 }
1519
1520 static void bpf_fill_ill_insns(void *area, unsigned int size)
1521 {
1522 memset(area, 0, size);
1523 }
1524
1525 static void bpf_flush_icache(void *start, void *end)
1526 {
1527 flush_icache_range((unsigned long)start, (unsigned long)end);
1528 }
1529
1530 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
1531 {
1532 bool tmp_blinded = false, extra_pass = false;
1533 struct bpf_prog *tmp, *orig_prog = prog;
1534 struct rv_jit_data *jit_data;
1535 struct rv_jit_context *ctx;
1536 unsigned int image_size;
1537
1538 if (!prog->jit_requested)
1539 return orig_prog;
1540
1541 tmp = bpf_jit_blind_constants(prog);
1542 if (IS_ERR(tmp))
1543 return orig_prog;
1544 if (tmp != prog) {
1545 tmp_blinded = true;
1546 prog = tmp;
1547 }
1548
1549 jit_data = prog->aux->jit_data;
1550 if (!jit_data) {
1551 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
1552 if (!jit_data) {
1553 prog = orig_prog;
1554 goto out;
1555 }
1556 prog->aux->jit_data = jit_data;
1557 }
1558
1559 ctx = &jit_data->ctx;
1560
1561 if (ctx->offset) {
1562 extra_pass = true;
1563 image_size = sizeof(u32) * ctx->ninsns;
1564 goto skip_init_ctx;
1565 }
1566
1567 ctx->prog = prog;
1568 ctx->offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL);
1569 if (!ctx->offset) {
1570 prog = orig_prog;
1571 goto out_offset;
1572 }
1573
1574 /* First pass generates the ctx->offset, but does not emit an image. */
1575 if (build_body(ctx, extra_pass)) {
1576 prog = orig_prog;
1577 goto out_offset;
1578 }
1579 build_prologue(ctx);
1580 ctx->epilogue_offset = ctx->ninsns;
1581 build_epilogue(ctx);
1582
1583 /* Allocate image, now that we know the size. */
1584 image_size = sizeof(u32) * ctx->ninsns;
1585 jit_data->header = bpf_jit_binary_alloc(image_size, &jit_data->image,
1586 sizeof(u32),
1587 bpf_fill_ill_insns);
1588 if (!jit_data->header) {
1589 prog = orig_prog;
1590 goto out_offset;
1591 }
1592
1593 /* Second, real pass, that acutally emits the image. */
1594 ctx->insns = (u32 *)jit_data->image;
1595 skip_init_ctx:
1596 ctx->ninsns = 0;
1597
1598 build_prologue(ctx);
1599 if (build_body(ctx, extra_pass)) {
1600 bpf_jit_binary_free(jit_data->header);
1601 prog = orig_prog;
1602 goto out_offset;
1603 }
1604 build_epilogue(ctx);
1605
1606 if (bpf_jit_enable > 1)
1607 bpf_jit_dump(prog->len, image_size, 2, ctx->insns);
1608
1609 prog->bpf_func = (void *)ctx->insns;
1610 prog->jited = 1;
1611 prog->jited_len = image_size;
1612
1613 bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns);
1614
1615 if (!prog->is_func || extra_pass) {
1616 out_offset:
1617 kfree(ctx->offset);
1618 kfree(jit_data);
1619 prog->aux->jit_data = NULL;
1620 }
1621 out:
1622 if (tmp_blinded)
1623 bpf_jit_prog_release_other(prog, prog == orig_prog ?
1624 tmp : orig_prog);
1625 return prog;
1626 }
Linux with added FPC-III support