?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
spi: spi-fsl-qspi: Fix return value check of devm_ioremap() in probe
[linux/fpc-iii.git] / arch / riscv / net / bpf_jit_comp32.c
blob302934177760f7481c9a2a0ad73427ac69e6fc0a
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * BPF JIT compiler for RV32G
4 *
5 * Copyright (c) 2020 Luke Nelson <luke.r.nels@gmail.com>
6 * Copyright (c) 2020 Xi Wang <xi.wang@gmail.com>
7 *
8 * The code is based on the BPF JIT compiler for RV64G by Björn Töpel and
9 * the BPF JIT compiler for 32-bit ARM by Shubham Bansal and Mircea Gherzan.
10 */
11
12 #include <linux/bpf.h>
13 #include <linux/filter.h>
14 #include "bpf_jit.h"
15
16 enum {
17 /* Stack layout - these are offsets from (top of stack - 4). */
18 BPF_R6_HI,
19 BPF_R6_LO,
20 BPF_R7_HI,
21 BPF_R7_LO,
22 BPF_R8_HI,
23 BPF_R8_LO,
24 BPF_R9_HI,
25 BPF_R9_LO,
26 BPF_AX_HI,
27 BPF_AX_LO,
28 /* Stack space for BPF_REG_6 through BPF_REG_9 and BPF_REG_AX. */
29 BPF_JIT_SCRATCH_REGS,
30 };
31
32 #define STACK_OFFSET(k) (-4 - ((k) * 4))
33
34 #define TMP_REG_1 (MAX_BPF_JIT_REG + 0)
35 #define TMP_REG_2 (MAX_BPF_JIT_REG + 1)
36
37 #define RV_REG_TCC RV_REG_T6
38 #define RV_REG_TCC_SAVED RV_REG_S7
39
40 static const s8 bpf2rv32[][2] = {
41 /* Return value from in-kernel function, and exit value from eBPF. */
42 [BPF_REG_0] = {RV_REG_S2, RV_REG_S1},
43 /* Arguments from eBPF program to in-kernel function. */
44 [BPF_REG_1] = {RV_REG_A1, RV_REG_A0},
45 [BPF_REG_2] = {RV_REG_A3, RV_REG_A2},
46 [BPF_REG_3] = {RV_REG_A5, RV_REG_A4},
47 [BPF_REG_4] = {RV_REG_A7, RV_REG_A6},
48 [BPF_REG_5] = {RV_REG_S4, RV_REG_S3},
49 /*
50 * Callee-saved registers that in-kernel function will preserve.
51 * Stored on the stack.
52 */
53 [BPF_REG_6] = {STACK_OFFSET(BPF_R6_HI), STACK_OFFSET(BPF_R6_LO)},
54 [BPF_REG_7] = {STACK_OFFSET(BPF_R7_HI), STACK_OFFSET(BPF_R7_LO)},
55 [BPF_REG_8] = {STACK_OFFSET(BPF_R8_HI), STACK_OFFSET(BPF_R8_LO)},
56 [BPF_REG_9] = {STACK_OFFSET(BPF_R9_HI), STACK_OFFSET(BPF_R9_LO)},
57 /* Read-only frame pointer to access BPF stack. */
58 [BPF_REG_FP] = {RV_REG_S6, RV_REG_S5},
59 /* Temporary register for blinding constants. Stored on the stack. */
60 [BPF_REG_AX] = {STACK_OFFSET(BPF_AX_HI), STACK_OFFSET(BPF_AX_LO)},
61 /*
62 * Temporary registers used by the JIT to operate on registers stored
63 * on the stack. Save t0 and t1 to be used as temporaries in generated
64 * code.
65 */
66 [TMP_REG_1] = {RV_REG_T3, RV_REG_T2},
67 [TMP_REG_2] = {RV_REG_T5, RV_REG_T4},
68 };
69
70 static s8 hi(const s8 *r)
71 {
72 return r[0];
73 }
74
75 static s8 lo(const s8 *r)
76 {
77 return r[1];
78 }
79
80 static void emit_imm(const s8 rd, s32 imm, struct rv_jit_context *ctx)
81 {
82 u32 upper = (imm + (1 << 11)) >> 12;
83 u32 lower = imm & 0xfff;
84
85 if (upper) {
86 emit(rv_lui(rd, upper), ctx);
87 emit(rv_addi(rd, rd, lower), ctx);
88 } else {
89 emit(rv_addi(rd, RV_REG_ZERO, lower), ctx);
90 }
91 }
92
93 static void emit_imm32(const s8 *rd, s32 imm, struct rv_jit_context *ctx)
94 {
95 /* Emit immediate into lower bits. */
96 emit_imm(lo(rd), imm, ctx);
97
98 /* Sign-extend into upper bits. */
99 if (imm >= 0)
100 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
101 else
102 emit(rv_addi(hi(rd), RV_REG_ZERO, -1), ctx);
103 }
104
105 static void emit_imm64(const s8 *rd, s32 imm_hi, s32 imm_lo,
106 struct rv_jit_context *ctx)
107 {
108 emit_imm(lo(rd), imm_lo, ctx);
109 emit_imm(hi(rd), imm_hi, ctx);
110 }
111
112 static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx)
113 {
114 int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 4;
115 const s8 *r0 = bpf2rv32[BPF_REG_0];
116
117 store_offset -= 4 * BPF_JIT_SCRATCH_REGS;
118
119 /* Set return value if not tail call. */
120 if (!is_tail_call) {
121 emit(rv_addi(RV_REG_A0, lo(r0), 0), ctx);
122 emit(rv_addi(RV_REG_A1, hi(r0), 0), ctx);
123 }
124
125 /* Restore callee-saved registers. */
126 emit(rv_lw(RV_REG_RA, store_offset - 0, RV_REG_SP), ctx);
127 emit(rv_lw(RV_REG_FP, store_offset - 4, RV_REG_SP), ctx);
128 emit(rv_lw(RV_REG_S1, store_offset - 8, RV_REG_SP), ctx);
129 emit(rv_lw(RV_REG_S2, store_offset - 12, RV_REG_SP), ctx);
130 emit(rv_lw(RV_REG_S3, store_offset - 16, RV_REG_SP), ctx);
131 emit(rv_lw(RV_REG_S4, store_offset - 20, RV_REG_SP), ctx);
132 emit(rv_lw(RV_REG_S5, store_offset - 24, RV_REG_SP), ctx);
133 emit(rv_lw(RV_REG_S6, store_offset - 28, RV_REG_SP), ctx);
134 emit(rv_lw(RV_REG_S7, store_offset - 32, RV_REG_SP), ctx);
135
136 emit(rv_addi(RV_REG_SP, RV_REG_SP, stack_adjust), ctx);
137
138 if (is_tail_call) {
139 /*
140 * goto *(t0 + 4);
141 * Skips first instruction of prologue which initializes tail
142 * call counter. Assumes t0 contains address of target program,
143 * see emit_bpf_tail_call.
144 */
145 emit(rv_jalr(RV_REG_ZERO, RV_REG_T0, 4), ctx);
146 } else {
147 emit(rv_jalr(RV_REG_ZERO, RV_REG_RA, 0), ctx);
148 }
149 }
150
151 static bool is_stacked(s8 reg)
152 {
153 return reg < 0;
154 }
155
156 static const s8 *bpf_get_reg64(const s8 *reg, const s8 *tmp,
157 struct rv_jit_context *ctx)
158 {
159 if (is_stacked(hi(reg))) {
160 emit(rv_lw(hi(tmp), hi(reg), RV_REG_FP), ctx);
161 emit(rv_lw(lo(tmp), lo(reg), RV_REG_FP), ctx);
162 reg = tmp;
163 }
164 return reg;
165 }
166
167 static void bpf_put_reg64(const s8 *reg, const s8 *src,
168 struct rv_jit_context *ctx)
169 {
170 if (is_stacked(hi(reg))) {
171 emit(rv_sw(RV_REG_FP, hi(reg), hi(src)), ctx);
172 emit(rv_sw(RV_REG_FP, lo(reg), lo(src)), ctx);
173 }
174 }
175
176 static const s8 *bpf_get_reg32(const s8 *reg, const s8 *tmp,
177 struct rv_jit_context *ctx)
178 {
179 if (is_stacked(lo(reg))) {
180 emit(rv_lw(lo(tmp), lo(reg), RV_REG_FP), ctx);
181 reg = tmp;
182 }
183 return reg;
184 }
185
186 static void bpf_put_reg32(const s8 *reg, const s8 *src,
187 struct rv_jit_context *ctx)
188 {
189 if (is_stacked(lo(reg))) {
190 emit(rv_sw(RV_REG_FP, lo(reg), lo(src)), ctx);
191 if (!ctx->prog->aux->verifier_zext)
192 emit(rv_sw(RV_REG_FP, hi(reg), RV_REG_ZERO), ctx);
193 } else if (!ctx->prog->aux->verifier_zext) {
194 emit(rv_addi(hi(reg), RV_REG_ZERO, 0), ctx);
195 }
196 }
197
198 static void emit_jump_and_link(u8 rd, s32 rvoff, bool force_jalr,
199 struct rv_jit_context *ctx)
200 {
201 s32 upper, lower;
202
203 if (rvoff && is_21b_int(rvoff) && !force_jalr) {
204 emit(rv_jal(rd, rvoff >> 1), ctx);
205 return;
206 }
207
208 upper = (rvoff + (1 << 11)) >> 12;
209 lower = rvoff & 0xfff;
210 emit(rv_auipc(RV_REG_T1, upper), ctx);
211 emit(rv_jalr(rd, RV_REG_T1, lower), ctx);
212 }
213
214 static void emit_alu_i64(const s8 *dst, s32 imm,
215 struct rv_jit_context *ctx, const u8 op)
216 {
217 const s8 *tmp1 = bpf2rv32[TMP_REG_1];
218 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
219
220 switch (op) {
221 case BPF_MOV:
222 emit_imm32(rd, imm, ctx);
223 break;
224 case BPF_AND:
225 if (is_12b_int(imm)) {
226 emit(rv_andi(lo(rd), lo(rd), imm), ctx);
227 } else {
228 emit_imm(RV_REG_T0, imm, ctx);
229 emit(rv_and(lo(rd), lo(rd), RV_REG_T0), ctx);
230 }
231 if (imm >= 0)
232 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
233 break;
234 case BPF_OR:
235 if (is_12b_int(imm)) {
236 emit(rv_ori(lo(rd), lo(rd), imm), ctx);
237 } else {
238 emit_imm(RV_REG_T0, imm, ctx);
239 emit(rv_or(lo(rd), lo(rd), RV_REG_T0), ctx);
240 }
241 if (imm < 0)
242 emit(rv_ori(hi(rd), RV_REG_ZERO, -1), ctx);
243 break;
244 case BPF_XOR:
245 if (is_12b_int(imm)) {
246 emit(rv_xori(lo(rd), lo(rd), imm), ctx);
247 } else {
248 emit_imm(RV_REG_T0, imm, ctx);
249 emit(rv_xor(lo(rd), lo(rd), RV_REG_T0), ctx);
250 }
251 if (imm < 0)
252 emit(rv_xori(hi(rd), hi(rd), -1), ctx);
253 break;
254 case BPF_LSH:
255 if (imm >= 32) {
256 emit(rv_slli(hi(rd), lo(rd), imm - 32), ctx);
257 emit(rv_addi(lo(rd), RV_REG_ZERO, 0), ctx);
258 } else if (imm == 0) {
259 /* Do nothing. */
260 } else {
261 emit(rv_srli(RV_REG_T0, lo(rd), 32 - imm), ctx);
262 emit(rv_slli(hi(rd), hi(rd), imm), ctx);
263 emit(rv_or(hi(rd), RV_REG_T0, hi(rd)), ctx);
264 emit(rv_slli(lo(rd), lo(rd), imm), ctx);
265 }
266 break;
267 case BPF_RSH:
268 if (imm >= 32) {
269 emit(rv_srli(lo(rd), hi(rd), imm - 32), ctx);
270 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
271 } else if (imm == 0) {
272 /* Do nothing. */
273 } else {
274 emit(rv_slli(RV_REG_T0, hi(rd), 32 - imm), ctx);
275 emit(rv_srli(lo(rd), lo(rd), imm), ctx);
276 emit(rv_or(lo(rd), RV_REG_T0, lo(rd)), ctx);
277 emit(rv_srli(hi(rd), hi(rd), imm), ctx);
278 }
279 break;
280 case BPF_ARSH:
281 if (imm >= 32) {
282 emit(rv_srai(lo(rd), hi(rd), imm - 32), ctx);
283 emit(rv_srai(hi(rd), hi(rd), 31), ctx);
284 } else if (imm == 0) {
285 /* Do nothing. */
286 } else {
287 emit(rv_slli(RV_REG_T0, hi(rd), 32 - imm), ctx);
288 emit(rv_srli(lo(rd), lo(rd), imm), ctx);
289 emit(rv_or(lo(rd), RV_REG_T0, lo(rd)), ctx);
290 emit(rv_srai(hi(rd), hi(rd), imm), ctx);
291 }
292 break;
293 }
294
295 bpf_put_reg64(dst, rd, ctx);
296 }
297
298 static void emit_alu_i32(const s8 *dst, s32 imm,
299 struct rv_jit_context *ctx, const u8 op)
300 {
301 const s8 *tmp1 = bpf2rv32[TMP_REG_1];
302 const s8 *rd = bpf_get_reg32(dst, tmp1, ctx);
303
304 switch (op) {
305 case BPF_MOV:
306 emit_imm(lo(rd), imm, ctx);
307 break;
308 case BPF_ADD:
309 if (is_12b_int(imm)) {
310 emit(rv_addi(lo(rd), lo(rd), imm), ctx);
311 } else {
312 emit_imm(RV_REG_T0, imm, ctx);
313 emit(rv_add(lo(rd), lo(rd), RV_REG_T0), ctx);
314 }
315 break;
316 case BPF_SUB:
317 if (is_12b_int(-imm)) {
318 emit(rv_addi(lo(rd), lo(rd), -imm), ctx);
319 } else {
320 emit_imm(RV_REG_T0, imm, ctx);
321 emit(rv_sub(lo(rd), lo(rd), RV_REG_T0), ctx);
322 }
323 break;
324 case BPF_AND:
325 if (is_12b_int(imm)) {
326 emit(rv_andi(lo(rd), lo(rd), imm), ctx);
327 } else {
328 emit_imm(RV_REG_T0, imm, ctx);
329 emit(rv_and(lo(rd), lo(rd), RV_REG_T0), ctx);
330 }
331 break;
332 case BPF_OR:
333 if (is_12b_int(imm)) {
334 emit(rv_ori(lo(rd), lo(rd), imm), ctx);
335 } else {
336 emit_imm(RV_REG_T0, imm, ctx);
337 emit(rv_or(lo(rd), lo(rd), RV_REG_T0), ctx);
338 }
339 break;
340 case BPF_XOR:
341 if (is_12b_int(imm)) {
342 emit(rv_xori(lo(rd), lo(rd), imm), ctx);
343 } else {
344 emit_imm(RV_REG_T0, imm, ctx);
345 emit(rv_xor(lo(rd), lo(rd), RV_REG_T0), ctx);
346 }
347 break;
348 case BPF_LSH:
349 if (is_12b_int(imm)) {
350 emit(rv_slli(lo(rd), lo(rd), imm), ctx);
351 } else {
352 emit_imm(RV_REG_T0, imm, ctx);
353 emit(rv_sll(lo(rd), lo(rd), RV_REG_T0), ctx);
354 }
355 break;
356 case BPF_RSH:
357 if (is_12b_int(imm)) {
358 emit(rv_srli(lo(rd), lo(rd), imm), ctx);
359 } else {
360 emit_imm(RV_REG_T0, imm, ctx);
361 emit(rv_srl(lo(rd), lo(rd), RV_REG_T0), ctx);
362 }
363 break;
364 case BPF_ARSH:
365 if (is_12b_int(imm)) {
366 emit(rv_srai(lo(rd), lo(rd), imm), ctx);
367 } else {
368 emit_imm(RV_REG_T0, imm, ctx);
369 emit(rv_sra(lo(rd), lo(rd), RV_REG_T0), ctx);
370 }
371 break;
372 }
373
374 bpf_put_reg32(dst, rd, ctx);
375 }
376
377 static void emit_alu_r64(const s8 *dst, const s8 *src,
378 struct rv_jit_context *ctx, const u8 op)
379 {
380 const s8 *tmp1 = bpf2rv32[TMP_REG_1];
381 const s8 *tmp2 = bpf2rv32[TMP_REG_2];
382 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
383 const s8 *rs = bpf_get_reg64(src, tmp2, ctx);
384
385 switch (op) {
386 case BPF_MOV:
387 emit(rv_addi(lo(rd), lo(rs), 0), ctx);
388 emit(rv_addi(hi(rd), hi(rs), 0), ctx);
389 break;
390 case BPF_ADD:
391 if (rd == rs) {
392 emit(rv_srli(RV_REG_T0, lo(rd), 31), ctx);
393 emit(rv_slli(hi(rd), hi(rd), 1), ctx);
394 emit(rv_or(hi(rd), RV_REG_T0, hi(rd)), ctx);
395 emit(rv_slli(lo(rd), lo(rd), 1), ctx);
396 } else {
397 emit(rv_add(lo(rd), lo(rd), lo(rs)), ctx);
398 emit(rv_sltu(RV_REG_T0, lo(rd), lo(rs)), ctx);
399 emit(rv_add(hi(rd), hi(rd), hi(rs)), ctx);
400 emit(rv_add(hi(rd), hi(rd), RV_REG_T0), ctx);
401 }
402 break;
403 case BPF_SUB:
404 emit(rv_sub(RV_REG_T1, hi(rd), hi(rs)), ctx);
405 emit(rv_sltu(RV_REG_T0, lo(rd), lo(rs)), ctx);
406 emit(rv_sub(hi(rd), RV_REG_T1, RV_REG_T0), ctx);
407 emit(rv_sub(lo(rd), lo(rd), lo(rs)), ctx);
408 break;
409 case BPF_AND:
410 emit(rv_and(lo(rd), lo(rd), lo(rs)), ctx);
411 emit(rv_and(hi(rd), hi(rd), hi(rs)), ctx);
412 break;
413 case BPF_OR:
414 emit(rv_or(lo(rd), lo(rd), lo(rs)), ctx);
415 emit(rv_or(hi(rd), hi(rd), hi(rs)), ctx);
416 break;
417 case BPF_XOR:
418 emit(rv_xor(lo(rd), lo(rd), lo(rs)), ctx);
419 emit(rv_xor(hi(rd), hi(rd), hi(rs)), ctx);
420 break;
421 case BPF_MUL:
422 emit(rv_mul(RV_REG_T0, hi(rs), lo(rd)), ctx);
423 emit(rv_mul(hi(rd), hi(rd), lo(rs)), ctx);
424 emit(rv_mulhu(RV_REG_T1, lo(rd), lo(rs)), ctx);
425 emit(rv_add(hi(rd), hi(rd), RV_REG_T0), ctx);
426 emit(rv_mul(lo(rd), lo(rd), lo(rs)), ctx);
427 emit(rv_add(hi(rd), hi(rd), RV_REG_T1), ctx);
428 break;
429 case BPF_LSH:
430 emit(rv_addi(RV_REG_T0, lo(rs), -32), ctx);
431 emit(rv_blt(RV_REG_T0, RV_REG_ZERO, 8), ctx);
432 emit(rv_sll(hi(rd), lo(rd), RV_REG_T0), ctx);
433 emit(rv_addi(lo(rd), RV_REG_ZERO, 0), ctx);
434 emit(rv_jal(RV_REG_ZERO, 16), ctx);
435 emit(rv_addi(RV_REG_T1, RV_REG_ZERO, 31), ctx);
436 emit(rv_srli(RV_REG_T0, lo(rd), 1), ctx);
437 emit(rv_sub(RV_REG_T1, RV_REG_T1, lo(rs)), ctx);
438 emit(rv_srl(RV_REG_T0, RV_REG_T0, RV_REG_T1), ctx);
439 emit(rv_sll(hi(rd), hi(rd), lo(rs)), ctx);
440 emit(rv_or(hi(rd), RV_REG_T0, hi(rd)), ctx);
441 emit(rv_sll(lo(rd), lo(rd), lo(rs)), ctx);
442 break;
443 case BPF_RSH:
444 emit(rv_addi(RV_REG_T0, lo(rs), -32), ctx);
445 emit(rv_blt(RV_REG_T0, RV_REG_ZERO, 8), ctx);
446 emit(rv_srl(lo(rd), hi(rd), RV_REG_T0), ctx);
447 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
448 emit(rv_jal(RV_REG_ZERO, 16), ctx);
449 emit(rv_addi(RV_REG_T1, RV_REG_ZERO, 31), ctx);
450 emit(rv_slli(RV_REG_T0, hi(rd), 1), ctx);
451 emit(rv_sub(RV_REG_T1, RV_REG_T1, lo(rs)), ctx);
452 emit(rv_sll(RV_REG_T0, RV_REG_T0, RV_REG_T1), ctx);
453 emit(rv_srl(lo(rd), lo(rd), lo(rs)), ctx);
454 emit(rv_or(lo(rd), RV_REG_T0, lo(rd)), ctx);
455 emit(rv_srl(hi(rd), hi(rd), lo(rs)), ctx);
456 break;
457 case BPF_ARSH:
458 emit(rv_addi(RV_REG_T0, lo(rs), -32), ctx);
459 emit(rv_blt(RV_REG_T0, RV_REG_ZERO, 8), ctx);
460 emit(rv_sra(lo(rd), hi(rd), RV_REG_T0), ctx);
461 emit(rv_srai(hi(rd), hi(rd), 31), ctx);
462 emit(rv_jal(RV_REG_ZERO, 16), ctx);
463 emit(rv_addi(RV_REG_T1, RV_REG_ZERO, 31), ctx);
464 emit(rv_slli(RV_REG_T0, hi(rd), 1), ctx);
465 emit(rv_sub(RV_REG_T1, RV_REG_T1, lo(rs)), ctx);
466 emit(rv_sll(RV_REG_T0, RV_REG_T0, RV_REG_T1), ctx);
467 emit(rv_srl(lo(rd), lo(rd), lo(rs)), ctx);
468 emit(rv_or(lo(rd), RV_REG_T0, lo(rd)), ctx);
469 emit(rv_sra(hi(rd), hi(rd), lo(rs)), ctx);
470 break;
471 case BPF_NEG:
472 emit(rv_sub(lo(rd), RV_REG_ZERO, lo(rd)), ctx);
473 emit(rv_sltu(RV_REG_T0, RV_REG_ZERO, lo(rd)), ctx);
474 emit(rv_sub(hi(rd), RV_REG_ZERO, hi(rd)), ctx);
475 emit(rv_sub(hi(rd), hi(rd), RV_REG_T0), ctx);
476 break;
477 }
478
479 bpf_put_reg64(dst, rd, ctx);
480 }
481
482 static void emit_alu_r32(const s8 *dst, const s8 *src,
483 struct rv_jit_context *ctx, const u8 op)
484 {
485 const s8 *tmp1 = bpf2rv32[TMP_REG_1];
486 const s8 *tmp2 = bpf2rv32[TMP_REG_2];
487 const s8 *rd = bpf_get_reg32(dst, tmp1, ctx);
488 const s8 *rs = bpf_get_reg32(src, tmp2, ctx);
489
490 switch (op) {
491 case BPF_MOV:
492 emit(rv_addi(lo(rd), lo(rs), 0), ctx);
493 break;
494 case BPF_ADD:
495 emit(rv_add(lo(rd), lo(rd), lo(rs)), ctx);
496 break;
497 case BPF_SUB:
498 emit(rv_sub(lo(rd), lo(rd), lo(rs)), ctx);
499 break;
500 case BPF_AND:
501 emit(rv_and(lo(rd), lo(rd), lo(rs)), ctx);
502 break;
503 case BPF_OR:
504 emit(rv_or(lo(rd), lo(rd), lo(rs)), ctx);
505 break;
506 case BPF_XOR:
507 emit(rv_xor(lo(rd), lo(rd), lo(rs)), ctx);
508 break;
509 case BPF_MUL:
510 emit(rv_mul(lo(rd), lo(rd), lo(rs)), ctx);
511 break;
512 case BPF_DIV:
513 emit(rv_divu(lo(rd), lo(rd), lo(rs)), ctx);
514 break;
515 case BPF_MOD:
516 emit(rv_remu(lo(rd), lo(rd), lo(rs)), ctx);
517 break;
518 case BPF_LSH:
519 emit(rv_sll(lo(rd), lo(rd), lo(rs)), ctx);
520 break;
521 case BPF_RSH:
522 emit(rv_srl(lo(rd), lo(rd), lo(rs)), ctx);
523 break;
524 case BPF_ARSH:
525 emit(rv_sra(lo(rd), lo(rd), lo(rs)), ctx);
526 break;
527 case BPF_NEG:
528 emit(rv_sub(lo(rd), RV_REG_ZERO, lo(rd)), ctx);
529 break;
530 }
531
532 bpf_put_reg32(dst, rd, ctx);
533 }
534
535 static int emit_branch_r64(const s8 *src1, const s8 *src2, s32 rvoff,
536 struct rv_jit_context *ctx, const u8 op)
537 {
538 int e, s = ctx->ninsns;
539 const s8 *tmp1 = bpf2rv32[TMP_REG_1];
540 const s8 *tmp2 = bpf2rv32[TMP_REG_2];
541
542 const s8 *rs1 = bpf_get_reg64(src1, tmp1, ctx);
543 const s8 *rs2 = bpf_get_reg64(src2, tmp2, ctx);
544
545 /*
546 * NO_JUMP skips over the rest of the instructions and the
547 * emit_jump_and_link, meaning the BPF branch is not taken.
548 * JUMP skips directly to the emit_jump_and_link, meaning
549 * the BPF branch is taken.
550 *
551 * The fallthrough case results in the BPF branch being taken.
552 */
553 #define NO_JUMP(idx) (6 + (2 * (idx)))
554 #define JUMP(idx) (2 + (2 * (idx)))
555
556 switch (op) {
557 case BPF_JEQ:
558 emit(rv_bne(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
559 emit(rv_bne(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
560 break;
561 case BPF_JGT:
562 emit(rv_bgtu(hi(rs1), hi(rs2), JUMP(2)), ctx);
563 emit(rv_bltu(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
564 emit(rv_bleu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
565 break;
566 case BPF_JLT:
567 emit(rv_bltu(hi(rs1), hi(rs2), JUMP(2)), ctx);
568 emit(rv_bgtu(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
569 emit(rv_bgeu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
570 break;
571 case BPF_JGE:
572 emit(rv_bgtu(hi(rs1), hi(rs2), JUMP(2)), ctx);
573 emit(rv_bltu(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
574 emit(rv_bltu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
575 break;
576 case BPF_JLE:
577 emit(rv_bltu(hi(rs1), hi(rs2), JUMP(2)), ctx);
578 emit(rv_bgtu(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
579 emit(rv_bgtu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
580 break;
581 case BPF_JNE:
582 emit(rv_bne(hi(rs1), hi(rs2), JUMP(1)), ctx);
583 emit(rv_beq(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
584 break;
585 case BPF_JSGT:
586 emit(rv_bgt(hi(rs1), hi(rs2), JUMP(2)), ctx);
587 emit(rv_blt(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
588 emit(rv_bleu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
589 break;
590 case BPF_JSLT:
591 emit(rv_blt(hi(rs1), hi(rs2), JUMP(2)), ctx);
592 emit(rv_bgt(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
593 emit(rv_bgeu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
594 break;
595 case BPF_JSGE:
596 emit(rv_bgt(hi(rs1), hi(rs2), JUMP(2)), ctx);
597 emit(rv_blt(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
598 emit(rv_bltu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
599 break;
600 case BPF_JSLE:
601 emit(rv_blt(hi(rs1), hi(rs2), JUMP(2)), ctx);
602 emit(rv_bgt(hi(rs1), hi(rs2), NO_JUMP(1)), ctx);
603 emit(rv_bgtu(lo(rs1), lo(rs2), NO_JUMP(0)), ctx);
604 break;
605 case BPF_JSET:
606 emit(rv_and(RV_REG_T0, hi(rs1), hi(rs2)), ctx);
607 emit(rv_bne(RV_REG_T0, RV_REG_ZERO, JUMP(2)), ctx);
608 emit(rv_and(RV_REG_T0, lo(rs1), lo(rs2)), ctx);
609 emit(rv_beq(RV_REG_T0, RV_REG_ZERO, NO_JUMP(0)), ctx);
610 break;
611 }
612
613 #undef NO_JUMP
614 #undef JUMP
615
616 e = ctx->ninsns;
617 /* Adjust for extra insns. */
618 rvoff -= (e - s) << 2;
619 emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx);
620 return 0;
621 }
622
623 static int emit_bcc(u8 op, u8 rd, u8 rs, int rvoff, struct rv_jit_context *ctx)
624 {
625 int e, s = ctx->ninsns;
626 bool far = false;
627 int off;
628
629 if (op == BPF_JSET) {
630 /*
631 * BPF_JSET is a special case: it has no inverse so we always
632 * treat it as a far branch.
633 */
634 far = true;
635 } else if (!is_13b_int(rvoff)) {
636 op = invert_bpf_cond(op);
637 far = true;
638 }
639
640 /*
641 * For a far branch, the condition is negated and we jump over the
642 * branch itself, and the two instructions from emit_jump_and_link.
643 * For a near branch, just use rvoff.
644 */
645 off = far ? 6 : (rvoff >> 1);
646
647 switch (op) {
648 case BPF_JEQ:
649 emit(rv_beq(rd, rs, off), ctx);
650 break;
651 case BPF_JGT:
652 emit(rv_bgtu(rd, rs, off), ctx);
653 break;
654 case BPF_JLT:
655 emit(rv_bltu(rd, rs, off), ctx);
656 break;
657 case BPF_JGE:
658 emit(rv_bgeu(rd, rs, off), ctx);
659 break;
660 case BPF_JLE:
661 emit(rv_bleu(rd, rs, off), ctx);
662 break;
663 case BPF_JNE:
664 emit(rv_bne(rd, rs, off), ctx);
665 break;
666 case BPF_JSGT:
667 emit(rv_bgt(rd, rs, off), ctx);
668 break;
669 case BPF_JSLT:
670 emit(rv_blt(rd, rs, off), ctx);
671 break;
672 case BPF_JSGE:
673 emit(rv_bge(rd, rs, off), ctx);
674 break;
675 case BPF_JSLE:
676 emit(rv_ble(rd, rs, off), ctx);
677 break;
678 case BPF_JSET:
679 emit(rv_and(RV_REG_T0, rd, rs), ctx);
680 emit(rv_beq(RV_REG_T0, RV_REG_ZERO, off), ctx);
681 break;
682 }
683
684 if (far) {
685 e = ctx->ninsns;
686 /* Adjust for extra insns. */
687 rvoff -= (e - s) << 2;
688 emit_jump_and_link(RV_REG_ZERO, rvoff, true, ctx);
689 }
690 return 0;
691 }
692
693 static int emit_branch_r32(const s8 *src1, const s8 *src2, s32 rvoff,
694 struct rv_jit_context *ctx, const u8 op)
695 {
696 int e, s = ctx->ninsns;
697 const s8 *tmp1 = bpf2rv32[TMP_REG_1];
698 const s8 *tmp2 = bpf2rv32[TMP_REG_2];
699
700 const s8 *rs1 = bpf_get_reg32(src1, tmp1, ctx);
701 const s8 *rs2 = bpf_get_reg32(src2, tmp2, ctx);
702
703 e = ctx->ninsns;
704 /* Adjust for extra insns. */
705 rvoff -= (e - s) << 2;
706
707 if (emit_bcc(op, lo(rs1), lo(rs2), rvoff, ctx))
708 return -1;
709
710 return 0;
711 }
712
713 static void emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx)
714 {
715 const s8 *r0 = bpf2rv32[BPF_REG_0];
716 const s8 *r5 = bpf2rv32[BPF_REG_5];
717 u32 upper = ((u32)addr + (1 << 11)) >> 12;
718 u32 lower = addr & 0xfff;
719
720 /* R1-R4 already in correct registers---need to push R5 to stack. */
721 emit(rv_addi(RV_REG_SP, RV_REG_SP, -16), ctx);
722 emit(rv_sw(RV_REG_SP, 0, lo(r5)), ctx);
723 emit(rv_sw(RV_REG_SP, 4, hi(r5)), ctx);
724
725 /* Backup TCC. */
726 emit(rv_addi(RV_REG_TCC_SAVED, RV_REG_TCC, 0), ctx);
727
728 /*
729 * Use lui/jalr pair to jump to absolute address. Don't use emit_imm as
730 * the number of emitted instructions should not depend on the value of
731 * addr.
732 */
733 emit(rv_lui(RV_REG_T1, upper), ctx);
734 emit(rv_jalr(RV_REG_RA, RV_REG_T1, lower), ctx);
735
736 /* Restore TCC. */
737 emit(rv_addi(RV_REG_TCC, RV_REG_TCC_SAVED, 0), ctx);
738
739 /* Set return value and restore stack. */
740 emit(rv_addi(lo(r0), RV_REG_A0, 0), ctx);
741 emit(rv_addi(hi(r0), RV_REG_A1, 0), ctx);
742 emit(rv_addi(RV_REG_SP, RV_REG_SP, 16), ctx);
743 }
744
745 static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx)
746 {
747 /*
748 * R1 -> &ctx
749 * R2 -> &array
750 * R3 -> index
751 */
752 int tc_ninsn, off, start_insn = ctx->ninsns;
753 const s8 *arr_reg = bpf2rv32[BPF_REG_2];
754 const s8 *idx_reg = bpf2rv32[BPF_REG_3];
755
756 tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] :
757 ctx->offset[0];
758
759 /* max_entries = array->map.max_entries; */
760 off = offsetof(struct bpf_array, map.max_entries);
761 if (is_12b_check(off, insn))
762 return -1;
763 emit(rv_lw(RV_REG_T1, off, lo(arr_reg)), ctx);
764
765 /*
766 * if (index >= max_entries)
767 * goto out;
768 */
769 off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2;
770 emit_bcc(BPF_JGE, lo(idx_reg), RV_REG_T1, off, ctx);
771
772 /*
773 * if ((temp_tcc = tcc - 1) < 0)
774 * goto out;
775 */
776 emit(rv_addi(RV_REG_T1, RV_REG_TCC, -1), ctx);
777 off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2;
778 emit_bcc(BPF_JSLT, RV_REG_T1, RV_REG_ZERO, off, ctx);
779
780 /*
781 * prog = array->ptrs[index];
782 * if (!prog)
783 * goto out;
784 */
785 emit(rv_slli(RV_REG_T0, lo(idx_reg), 2), ctx);
786 emit(rv_add(RV_REG_T0, RV_REG_T0, lo(arr_reg)), ctx);
787 off = offsetof(struct bpf_array, ptrs);
788 if (is_12b_check(off, insn))
789 return -1;
790 emit(rv_lw(RV_REG_T0, off, RV_REG_T0), ctx);
791 off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2;
792 emit_bcc(BPF_JEQ, RV_REG_T0, RV_REG_ZERO, off, ctx);
793
794 /*
795 * tcc = temp_tcc;
796 * goto *(prog->bpf_func + 4);
797 */
798 off = offsetof(struct bpf_prog, bpf_func);
799 if (is_12b_check(off, insn))
800 return -1;
801 emit(rv_lw(RV_REG_T0, off, RV_REG_T0), ctx);
802 emit(rv_addi(RV_REG_TCC, RV_REG_T1, 0), ctx);
803 /* Epilogue jumps to *(t0 + 4). */
804 __build_epilogue(true, ctx);
805 return 0;
806 }
807
808 static int emit_load_r64(const s8 *dst, const s8 *src, s16 off,
809 struct rv_jit_context *ctx, const u8 size)
810 {
811 const s8 *tmp1 = bpf2rv32[TMP_REG_1];
812 const s8 *tmp2 = bpf2rv32[TMP_REG_2];
813 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
814 const s8 *rs = bpf_get_reg64(src, tmp2, ctx);
815
816 emit_imm(RV_REG_T0, off, ctx);
817 emit(rv_add(RV_REG_T0, RV_REG_T0, lo(rs)), ctx);
818
819 switch (size) {
820 case BPF_B:
821 emit(rv_lbu(lo(rd), 0, RV_REG_T0), ctx);
822 if (!ctx->prog->aux->verifier_zext)
823 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
824 break;
825 case BPF_H:
826 emit(rv_lhu(lo(rd), 0, RV_REG_T0), ctx);
827 if (!ctx->prog->aux->verifier_zext)
828 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
829 break;
830 case BPF_W:
831 emit(rv_lw(lo(rd), 0, RV_REG_T0), ctx);
832 if (!ctx->prog->aux->verifier_zext)
833 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
834 break;
835 case BPF_DW:
836 emit(rv_lw(lo(rd), 0, RV_REG_T0), ctx);
837 emit(rv_lw(hi(rd), 4, RV_REG_T0), ctx);
838 break;
839 }
840
841 bpf_put_reg64(dst, rd, ctx);
842 return 0;
843 }
844
845 static int emit_store_r64(const s8 *dst, const s8 *src, s16 off,
846 struct rv_jit_context *ctx, const u8 size,
847 const u8 mode)
848 {
849 const s8 *tmp1 = bpf2rv32[TMP_REG_1];
850 const s8 *tmp2 = bpf2rv32[TMP_REG_2];
851 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
852 const s8 *rs = bpf_get_reg64(src, tmp2, ctx);
853
854 if (mode == BPF_XADD && size != BPF_W)
855 return -1;
856
857 emit_imm(RV_REG_T0, off, ctx);
858 emit(rv_add(RV_REG_T0, RV_REG_T0, lo(rd)), ctx);
859
860 switch (size) {
861 case BPF_B:
862 emit(rv_sb(RV_REG_T0, 0, lo(rs)), ctx);
863 break;
864 case BPF_H:
865 emit(rv_sh(RV_REG_T0, 0, lo(rs)), ctx);
866 break;
867 case BPF_W:
868 switch (mode) {
869 case BPF_MEM:
870 emit(rv_sw(RV_REG_T0, 0, lo(rs)), ctx);
871 break;
872 case BPF_XADD:
873 emit(rv_amoadd_w(RV_REG_ZERO, lo(rs), RV_REG_T0, 0, 0),
874 ctx);
875 break;
876 }
877 break;
878 case BPF_DW:
879 emit(rv_sw(RV_REG_T0, 0, lo(rs)), ctx);
880 emit(rv_sw(RV_REG_T0, 4, hi(rs)), ctx);
881 break;
882 }
883
884 return 0;
885 }
886
887 static void emit_rev16(const s8 rd, struct rv_jit_context *ctx)
888 {
889 emit(rv_slli(rd, rd, 16), ctx);
890 emit(rv_slli(RV_REG_T1, rd, 8), ctx);
891 emit(rv_srli(rd, rd, 8), ctx);
892 emit(rv_add(RV_REG_T1, rd, RV_REG_T1), ctx);
893 emit(rv_srli(rd, RV_REG_T1, 16), ctx);
894 }
895
896 static void emit_rev32(const s8 rd, struct rv_jit_context *ctx)
897 {
898 emit(rv_addi(RV_REG_T1, RV_REG_ZERO, 0), ctx);
899 emit(rv_andi(RV_REG_T0, rd, 255), ctx);
900 emit(rv_add(RV_REG_T1, RV_REG_T1, RV_REG_T0), ctx);
901 emit(rv_slli(RV_REG_T1, RV_REG_T1, 8), ctx);
902 emit(rv_srli(rd, rd, 8), ctx);
903 emit(rv_andi(RV_REG_T0, rd, 255), ctx);
904 emit(rv_add(RV_REG_T1, RV_REG_T1, RV_REG_T0), ctx);
905 emit(rv_slli(RV_REG_T1, RV_REG_T1, 8), ctx);
906 emit(rv_srli(rd, rd, 8), ctx);
907 emit(rv_andi(RV_REG_T0, rd, 255), ctx);
908 emit(rv_add(RV_REG_T1, RV_REG_T1, RV_REG_T0), ctx);
909 emit(rv_slli(RV_REG_T1, RV_REG_T1, 8), ctx);
910 emit(rv_srli(rd, rd, 8), ctx);
911 emit(rv_andi(RV_REG_T0, rd, 255), ctx);
912 emit(rv_add(RV_REG_T1, RV_REG_T1, RV_REG_T0), ctx);
913 emit(rv_addi(rd, RV_REG_T1, 0), ctx);
914 }
915
916 static void emit_zext64(const s8 *dst, struct rv_jit_context *ctx)
917 {
918 const s8 *rd;
919 const s8 *tmp1 = bpf2rv32[TMP_REG_1];
920
921 rd = bpf_get_reg64(dst, tmp1, ctx);
922 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
923 bpf_put_reg64(dst, rd, ctx);
924 }
925
926 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
927 bool extra_pass)
928 {
929 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 ||
930 BPF_CLASS(insn->code) == BPF_JMP;
931 int s, e, rvoff, i = insn - ctx->prog->insnsi;
932 u8 code = insn->code;
933 s16 off = insn->off;
934 s32 imm = insn->imm;
935
936 const s8 *dst = bpf2rv32[insn->dst_reg];
937 const s8 *src = bpf2rv32[insn->src_reg];
938 const s8 *tmp1 = bpf2rv32[TMP_REG_1];
939 const s8 *tmp2 = bpf2rv32[TMP_REG_2];
940
941 switch (code) {
942 case BPF_ALU64 | BPF_MOV | BPF_X:
943
944 case BPF_ALU64 | BPF_ADD | BPF_X:
945 case BPF_ALU64 | BPF_ADD | BPF_K:
946
947 case BPF_ALU64 | BPF_SUB | BPF_X:
948 case BPF_ALU64 | BPF_SUB | BPF_K:
949
950 case BPF_ALU64 | BPF_AND | BPF_X:
951 case BPF_ALU64 | BPF_OR | BPF_X:
952 case BPF_ALU64 | BPF_XOR | BPF_X:
953
954 case BPF_ALU64 | BPF_MUL | BPF_X:
955 case BPF_ALU64 | BPF_MUL | BPF_K:
956
957 case BPF_ALU64 | BPF_LSH | BPF_X:
958 case BPF_ALU64 | BPF_RSH | BPF_X:
959 case BPF_ALU64 | BPF_ARSH | BPF_X:
960 if (BPF_SRC(code) == BPF_K) {
961 emit_imm32(tmp2, imm, ctx);
962 src = tmp2;
963 }
964 emit_alu_r64(dst, src, ctx, BPF_OP(code));
965 break;
966
967 case BPF_ALU64 | BPF_NEG:
968 emit_alu_r64(dst, tmp2, ctx, BPF_OP(code));
969 break;
970
971 case BPF_ALU64 | BPF_DIV | BPF_X:
972 case BPF_ALU64 | BPF_DIV | BPF_K:
973 case BPF_ALU64 | BPF_MOD | BPF_X:
974 case BPF_ALU64 | BPF_MOD | BPF_K:
975 goto notsupported;
976
977 case BPF_ALU64 | BPF_MOV | BPF_K:
978 case BPF_ALU64 | BPF_AND | BPF_K:
979 case BPF_ALU64 | BPF_OR | BPF_K:
980 case BPF_ALU64 | BPF_XOR | BPF_K:
981 case BPF_ALU64 | BPF_LSH | BPF_K:
982 case BPF_ALU64 | BPF_RSH | BPF_K:
983 case BPF_ALU64 | BPF_ARSH | BPF_K:
984 emit_alu_i64(dst, imm, ctx, BPF_OP(code));
985 break;
986
987 case BPF_ALU | BPF_MOV | BPF_X:
988 if (imm == 1) {
989 /* Special mov32 for zext. */
990 emit_zext64(dst, ctx);
991 break;
992 }
993 /* Fallthrough. */
994
995 case BPF_ALU | BPF_ADD | BPF_X:
996 case BPF_ALU | BPF_SUB | BPF_X:
997 case BPF_ALU | BPF_AND | BPF_X:
998 case BPF_ALU | BPF_OR | BPF_X:
999 case BPF_ALU | BPF_XOR | BPF_X:
1000
1001 case BPF_ALU | BPF_MUL | BPF_X:
1002 case BPF_ALU | BPF_MUL | BPF_K:
1003
1004 case BPF_ALU | BPF_DIV | BPF_X:
1005 case BPF_ALU | BPF_DIV | BPF_K:
1006
1007 case BPF_ALU | BPF_MOD | BPF_X:
1008 case BPF_ALU | BPF_MOD | BPF_K:
1009
1010 case BPF_ALU | BPF_LSH | BPF_X:
1011 case BPF_ALU | BPF_RSH | BPF_X:
1012 case BPF_ALU | BPF_ARSH | BPF_X:
1013 if (BPF_SRC(code) == BPF_K) {
1014 emit_imm32(tmp2, imm, ctx);
1015 src = tmp2;
1016 }
1017 emit_alu_r32(dst, src, ctx, BPF_OP(code));
1018 break;
1019
1020 case BPF_ALU | BPF_MOV | BPF_K:
1021 case BPF_ALU | BPF_ADD | BPF_K:
1022 case BPF_ALU | BPF_SUB | BPF_K:
1023 case BPF_ALU | BPF_AND | BPF_K:
1024 case BPF_ALU | BPF_OR | BPF_K:
1025 case BPF_ALU | BPF_XOR | BPF_K:
1026 case BPF_ALU | BPF_LSH | BPF_K:
1027 case BPF_ALU | BPF_RSH | BPF_K:
1028 case BPF_ALU | BPF_ARSH | BPF_K:
1029 /*
1030 * mul,div,mod are handled in the BPF_X case since there are
1031 * no RISC-V I-type equivalents.
1032 */
1033 emit_alu_i32(dst, imm, ctx, BPF_OP(code));
1034 break;
1035
1036 case BPF_ALU | BPF_NEG:
1037 /*
1038 * src is ignored---choose tmp2 as a dummy register since it
1039 * is not on the stack.
1040 */
1041 emit_alu_r32(dst, tmp2, ctx, BPF_OP(code));
1042 break;
1043
1044 case BPF_ALU | BPF_END | BPF_FROM_LE:
1045 {
1046 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
1047
1048 switch (imm) {
1049 case 16:
1050 emit(rv_slli(lo(rd), lo(rd), 16), ctx);
1051 emit(rv_srli(lo(rd), lo(rd), 16), ctx);
1052 /* Fallthrough. */
1053 case 32:
1054 if (!ctx->prog->aux->verifier_zext)
1055 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
1056 break;
1057 case 64:
1058 /* Do nothing. */
1059 break;
1060 default:
1061 pr_err("bpf-jit: BPF_END imm %d invalid\n", imm);
1062 return -1;
1063 }
1064
1065 bpf_put_reg64(dst, rd, ctx);
1066 break;
1067 }
1068
1069 case BPF_ALU | BPF_END | BPF_FROM_BE:
1070 {
1071 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
1072
1073 switch (imm) {
1074 case 16:
1075 emit_rev16(lo(rd), ctx);
1076 if (!ctx->prog->aux->verifier_zext)
1077 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
1078 break;
1079 case 32:
1080 emit_rev32(lo(rd), ctx);
1081 if (!ctx->prog->aux->verifier_zext)
1082 emit(rv_addi(hi(rd), RV_REG_ZERO, 0), ctx);
1083 break;
1084 case 64:
1085 /* Swap upper and lower halves. */
1086 emit(rv_addi(RV_REG_T0, lo(rd), 0), ctx);
1087 emit(rv_addi(lo(rd), hi(rd), 0), ctx);
1088 emit(rv_addi(hi(rd), RV_REG_T0, 0), ctx);
1089
1090 /* Swap each half. */
1091 emit_rev32(lo(rd), ctx);
1092 emit_rev32(hi(rd), ctx);
1093 break;
1094 default:
1095 pr_err("bpf-jit: BPF_END imm %d invalid\n", imm);
1096 return -1;
1097 }
1098
1099 bpf_put_reg64(dst, rd, ctx);
1100 break;
1101 }
1102
1103 case BPF_JMP | BPF_JA:
1104 rvoff = rv_offset(i, off, ctx);
1105 emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx);
1106 break;
1107
1108 case BPF_JMP | BPF_CALL:
1109 {
1110 bool fixed;
1111 int ret;
1112 u64 addr;
1113
1114 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, &addr,
1115 &fixed);
1116 if (ret < 0)
1117 return ret;
1118 emit_call(fixed, addr, ctx);
1119 break;
1120 }
1121
1122 case BPF_JMP | BPF_TAIL_CALL:
1123 if (emit_bpf_tail_call(i, ctx))
1124 return -1;
1125 break;
1126
1127 case BPF_JMP | BPF_JEQ | BPF_X:
1128 case BPF_JMP | BPF_JEQ | BPF_K:
1129 case BPF_JMP32 | BPF_JEQ | BPF_X:
1130 case BPF_JMP32 | BPF_JEQ | BPF_K:
1131
1132 case BPF_JMP | BPF_JNE | BPF_X:
1133 case BPF_JMP | BPF_JNE | BPF_K:
1134 case BPF_JMP32 | BPF_JNE | BPF_X:
1135 case BPF_JMP32 | BPF_JNE | BPF_K:
1136
1137 case BPF_JMP | BPF_JLE | BPF_X:
1138 case BPF_JMP | BPF_JLE | BPF_K:
1139 case BPF_JMP32 | BPF_JLE | BPF_X:
1140 case BPF_JMP32 | BPF_JLE | BPF_K:
1141
1142 case BPF_JMP | BPF_JLT | BPF_X:
1143 case BPF_JMP | BPF_JLT | BPF_K:
1144 case BPF_JMP32 | BPF_JLT | BPF_X:
1145 case BPF_JMP32 | BPF_JLT | BPF_K:
1146
1147 case BPF_JMP | BPF_JGE | BPF_X:
1148 case BPF_JMP | BPF_JGE | BPF_K:
1149 case BPF_JMP32 | BPF_JGE | BPF_X:
1150 case BPF_JMP32 | BPF_JGE | BPF_K:
1151
1152 case BPF_JMP | BPF_JGT | BPF_X:
1153 case BPF_JMP | BPF_JGT | BPF_K:
1154 case BPF_JMP32 | BPF_JGT | BPF_X:
1155 case BPF_JMP32 | BPF_JGT | BPF_K:
1156
1157 case BPF_JMP | BPF_JSLE | BPF_X:
1158 case BPF_JMP | BPF_JSLE | BPF_K:
1159 case BPF_JMP32 | BPF_JSLE | BPF_X:
1160 case BPF_JMP32 | BPF_JSLE | BPF_K:
1161
1162 case BPF_JMP | BPF_JSLT | BPF_X:
1163 case BPF_JMP | BPF_JSLT | BPF_K:
1164 case BPF_JMP32 | BPF_JSLT | BPF_X:
1165 case BPF_JMP32 | BPF_JSLT | BPF_K:
1166
1167 case BPF_JMP | BPF_JSGE | BPF_X:
1168 case BPF_JMP | BPF_JSGE | BPF_K:
1169 case BPF_JMP32 | BPF_JSGE | BPF_X:
1170 case BPF_JMP32 | BPF_JSGE | BPF_K:
1171
1172 case BPF_JMP | BPF_JSGT | BPF_X:
1173 case BPF_JMP | BPF_JSGT | BPF_K:
1174 case BPF_JMP32 | BPF_JSGT | BPF_X:
1175 case BPF_JMP32 | BPF_JSGT | BPF_K:
1176
1177 case BPF_JMP | BPF_JSET | BPF_X:
1178 case BPF_JMP | BPF_JSET | BPF_K:
1179 case BPF_JMP32 | BPF_JSET | BPF_X:
1180 case BPF_JMP32 | BPF_JSET | BPF_K:
1181 rvoff = rv_offset(i, off, ctx);
1182 if (BPF_SRC(code) == BPF_K) {
1183 s = ctx->ninsns;
1184 emit_imm32(tmp2, imm, ctx);
1185 src = tmp2;
1186 e = ctx->ninsns;
1187 rvoff -= (e - s) << 2;
1188 }
1189
1190 if (is64)
1191 emit_branch_r64(dst, src, rvoff, ctx, BPF_OP(code));
1192 else
1193 emit_branch_r32(dst, src, rvoff, ctx, BPF_OP(code));
1194 break;
1195
1196 case BPF_JMP | BPF_EXIT:
1197 if (i == ctx->prog->len - 1)
1198 break;
1199
1200 rvoff = epilogue_offset(ctx);
1201 emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx);
1202 break;
1203
1204 case BPF_LD | BPF_IMM | BPF_DW:
1205 {
1206 struct bpf_insn insn1 = insn[1];
1207 s32 imm_lo = imm;
1208 s32 imm_hi = insn1.imm;
1209 const s8 *rd = bpf_get_reg64(dst, tmp1, ctx);
1210
1211 emit_imm64(rd, imm_hi, imm_lo, ctx);
1212 bpf_put_reg64(dst, rd, ctx);
1213 return 1;
1214 }
1215
1216 case BPF_LDX | BPF_MEM | BPF_B:
1217 case BPF_LDX | BPF_MEM | BPF_H:
1218 case BPF_LDX | BPF_MEM | BPF_W:
1219 case BPF_LDX | BPF_MEM | BPF_DW:
1220 if (emit_load_r64(dst, src, off, ctx, BPF_SIZE(code)))
1221 return -1;
1222 break;
1223
1224 case BPF_ST | BPF_MEM | BPF_B:
1225 case BPF_ST | BPF_MEM | BPF_H:
1226 case BPF_ST | BPF_MEM | BPF_W:
1227 case BPF_ST | BPF_MEM | BPF_DW:
1228
1229 case BPF_STX | BPF_MEM | BPF_B:
1230 case BPF_STX | BPF_MEM | BPF_H:
1231 case BPF_STX | BPF_MEM | BPF_W:
1232 case BPF_STX | BPF_MEM | BPF_DW:
1233 case BPF_STX | BPF_XADD | BPF_W:
1234 if (BPF_CLASS(code) == BPF_ST) {
1235 emit_imm32(tmp2, imm, ctx);
1236 src = tmp2;
1237 }
1238
1239 if (emit_store_r64(dst, src, off, ctx, BPF_SIZE(code),
1240 BPF_MODE(code)))
1241 return -1;
1242 break;
1243
1244 /* No hardware support for 8-byte atomics in RV32. */
1245 case BPF_STX | BPF_XADD | BPF_DW:
1246 /* Fallthrough. */
1247
1248 notsupported:
1249 pr_info_once("bpf-jit: not supported: opcode %02x ***\n", code);
1250 return -EFAULT;
1251
1252 default:
1253 pr_err("bpf-jit: unknown opcode %02x\n", code);
1254 return -EINVAL;
1255 }
1256
1257 return 0;
1258 }
1259
1260 void bpf_jit_build_prologue(struct rv_jit_context *ctx)
1261 {
1262 /* Make space to save 9 registers: ra, fp, s1--s7. */
1263 int stack_adjust = 9 * sizeof(u32), store_offset, bpf_stack_adjust;
1264 const s8 *fp = bpf2rv32[BPF_REG_FP];
1265 const s8 *r1 = bpf2rv32[BPF_REG_1];
1266
1267 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16);
1268 stack_adjust += bpf_stack_adjust;
1269
1270 store_offset = stack_adjust - 4;
1271
1272 stack_adjust += 4 * BPF_JIT_SCRATCH_REGS;
1273
1274 /*
1275 * The first instruction sets the tail-call-counter (TCC) register.
1276 * This instruction is skipped by tail calls.
1277 */
1278 emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx);
1279
1280 emit(rv_addi(RV_REG_SP, RV_REG_SP, -stack_adjust), ctx);
1281
1282 /* Save callee-save registers. */
1283 emit(rv_sw(RV_REG_SP, store_offset - 0, RV_REG_RA), ctx);
1284 emit(rv_sw(RV_REG_SP, store_offset - 4, RV_REG_FP), ctx);
1285 emit(rv_sw(RV_REG_SP, store_offset - 8, RV_REG_S1), ctx);
1286 emit(rv_sw(RV_REG_SP, store_offset - 12, RV_REG_S2), ctx);
1287 emit(rv_sw(RV_REG_SP, store_offset - 16, RV_REG_S3), ctx);
1288 emit(rv_sw(RV_REG_SP, store_offset - 20, RV_REG_S4), ctx);
1289 emit(rv_sw(RV_REG_SP, store_offset - 24, RV_REG_S5), ctx);
1290 emit(rv_sw(RV_REG_SP, store_offset - 28, RV_REG_S6), ctx);
1291 emit(rv_sw(RV_REG_SP, store_offset - 32, RV_REG_S7), ctx);
1292
1293 /* Set fp: used as the base address for stacked BPF registers. */
1294 emit(rv_addi(RV_REG_FP, RV_REG_SP, stack_adjust), ctx);
1295
1296 /* Set up BPF stack pointer. */
1297 emit(rv_addi(lo(fp), RV_REG_SP, bpf_stack_adjust), ctx);
1298 emit(rv_addi(hi(fp), RV_REG_ZERO, 0), ctx);
1299
1300 /* Set up context pointer. */
1301 emit(rv_addi(lo(r1), RV_REG_A0, 0), ctx);
1302 emit(rv_addi(hi(r1), RV_REG_ZERO, 0), ctx);
1303
1304 ctx->stack_size = stack_adjust;
1305 }
1306
1307 void bpf_jit_build_epilogue(struct rv_jit_context *ctx)
1308 {
1309 __build_epilogue(false, ctx);
1310 }
Linux with added FPC-III support