arch/riscv/net/bpf_jit.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  * Common functionality for RV32 and RV64 BPF JIT compilers
   4  *
   5  * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com>
   6  *
   7  */
   8
   9 #ifndef _BPF_JIT_H
  10 #define _BPF_JIT_H
  11
  12 #include <linux/bpf.h>
  13 #include <linux/filter.h>
  14 #include <asm/cacheflush.h>
  15
  16 enum {
  17         RV_REG_ZERO =   0,      /* The constant value 0 */
  18         RV_REG_RA =     1,      /* Return address */
  19         RV_REG_SP =     2,      /* Stack pointer */
  20         RV_REG_GP =     3,      /* Global pointer */
  21         RV_REG_TP =     4,      /* Thread pointer */
  22         RV_REG_T0 =     5,      /* Temporaries */
  23         RV_REG_T1 =     6,
  24         RV_REG_T2 =     7,
  25         RV_REG_FP =     8,      /* Saved register/frame pointer */
  26         RV_REG_S1 =     9,      /* Saved register */
  27         RV_REG_A0 =     10,     /* Function argument/return values */
  28         RV_REG_A1 =     11,     /* Function arguments */
  29         RV_REG_A2 =     12,
  30         RV_REG_A3 =     13,
  31         RV_REG_A4 =     14,
  32         RV_REG_A5 =     15,
  33         RV_REG_A6 =     16,
  34         RV_REG_A7 =     17,
  35         RV_REG_S2 =     18,     /* Saved registers */
  36         RV_REG_S3 =     19,
  37         RV_REG_S4 =     20,
  38         RV_REG_S5 =     21,
  39         RV_REG_S6 =     22,
  40         RV_REG_S7 =     23,
  41         RV_REG_S8 =     24,
  42         RV_REG_S9 =     25,
  43         RV_REG_S10 =    26,
  44         RV_REG_S11 =    27,
  45         RV_REG_T3 =     28,     /* Temporaries */
  46         RV_REG_T4 =     29,
  47         RV_REG_T5 =     30,
  48         RV_REG_T6 =     31,
  49 };
  50
  51 struct rv_jit_context {
  52         struct bpf_prog *prog;
  53         u32 *insns;             /* RV insns */
  54         int ninsns;
  55         int epilogue_offset;
  56         int *offset;            /* BPF to RV */
  57         unsigned long flags;
  58         int stack_size;
  59 };
  60
  61 struct rv_jit_data {
  62         struct bpf_binary_header *header;
  63         u8 *image;
  64         struct rv_jit_context ctx;
  65 };
  66
  67 static inline void bpf_fill_ill_insns(void *area, unsigned int size)
  68 {
  69         memset(area, 0, size);
  70 }
  71
  72 static inline void bpf_flush_icache(void *start, void *end)
  73 {
  74         flush_icache_range((unsigned long)start, (unsigned long)end);
  75 }
  76
  77 static inline void emit(const u32 insn, struct rv_jit_context *ctx)
  78 {
  79         if (ctx->insns)
  80                 ctx->insns[ctx->ninsns] = insn;
  81
  82         ctx->ninsns++;
  83 }
  84
  85 static inline int epilogue_offset(struct rv_jit_context *ctx)
  86 {
  87         int to = ctx->epilogue_offset, from = ctx->ninsns;
  88
  89         return (to - from) << 2;
  90 }
  91
  92 /* Return -1 or inverted cond. */
  93 static inline int invert_bpf_cond(u8 cond)
  94 {
  95         switch (cond) {
  96         case BPF_JEQ:
  97                 return BPF_JNE;
  98         case BPF_JGT:
  99                 return BPF_JLE;
 100         case BPF_JLT:
 101                 return BPF_JGE;
 102         case BPF_JGE:
 103                 return BPF_JLT;
 104         case BPF_JLE:
 105                 return BPF_JGT;
 106         case BPF_JNE:
 107                 return BPF_JEQ;
 108         case BPF_JSGT:
 109                 return BPF_JSLE;
 110         case BPF_JSLT:
 111                 return BPF_JSGE;
 112         case BPF_JSGE:
 113                 return BPF_JSLT;
 114         case BPF_JSLE:
 115                 return BPF_JSGT;
 116         }
 117         return -1;
 118 }
 119
 120 static inline bool is_12b_int(long val)
 121 {
 122         return -(1L << 11) <= val && val < (1L << 11);
 123 }
 124
 125 static inline int is_12b_check(int off, int insn)
 126 {
 127         if (!is_12b_int(off)) {
 128                 pr_err("bpf-jit: insn=%d 12b < offset=%d not supported yet!\n",
 129                        insn, (int)off);
 130                 return -1;
 131         }
 132         return 0;
 133 }
 134
 135 static inline bool is_13b_int(long val)
 136 {
 137         return -(1L << 12) <= val && val < (1L << 12);
 138 }
 139
 140 static inline bool is_21b_int(long val)
 141 {
 142         return -(1L << 20) <= val && val < (1L << 20);
 143 }
 144
 145 static inline int rv_offset(int insn, int off, struct rv_jit_context *ctx)
 146 {
 147         int from, to;
 148
 149         off++; /* BPF branch is from PC+1, RV is from PC */
 150         from = (insn > 0) ? ctx->offset[insn - 1] : 0;
 151         to = (insn + off > 0) ? ctx->offset[insn + off - 1] : 0;
 152         return (to - from) << 2;
 153 }
 154
 155 /* Instruction formats. */
 156
 157 static inline u32 rv_r_insn(u8 funct7, u8 rs2, u8 rs1, u8 funct3, u8 rd,
 158                             u8 opcode)
 159 {
 160         return (funct7 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
 161                 (rd << 7) | opcode;
 162 }
 163
 164 static inline u32 rv_i_insn(u16 imm11_0, u8 rs1, u8 funct3, u8 rd, u8 opcode)
 165 {
 166         return (imm11_0 << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) |
 167                 opcode;
 168 }
 169
 170 static inline u32 rv_s_insn(u16 imm11_0, u8 rs2, u8 rs1, u8 funct3, u8 opcode)
 171 {
 172         u8 imm11_5 = imm11_0 >> 5, imm4_0 = imm11_0 & 0x1f;
 173
 174         return (imm11_5 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
 175                 (imm4_0 << 7) | opcode;
 176 }
 177
 178 static inline u32 rv_b_insn(u16 imm12_1, u8 rs2, u8 rs1, u8 funct3, u8 opcode)
 179 {
 180         u8 imm12 = ((imm12_1 & 0x800) >> 5) | ((imm12_1 & 0x3f0) >> 4);
 181         u8 imm4_1 = ((imm12_1 & 0xf) << 1) | ((imm12_1 & 0x400) >> 10);
 182
 183         return (imm12 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
 184                 (imm4_1 << 7) | opcode;
 185 }
 186
 187 static inline u32 rv_u_insn(u32 imm31_12, u8 rd, u8 opcode)
 188 {
 189         return (imm31_12 << 12) | (rd << 7) | opcode;
 190 }
 191
 192 static inline u32 rv_j_insn(u32 imm20_1, u8 rd, u8 opcode)
 193 {
 194         u32 imm;
 195
 196         imm = (imm20_1 & 0x80000) | ((imm20_1 & 0x3ff) << 9) |
 197                 ((imm20_1 & 0x400) >> 2) | ((imm20_1 & 0x7f800) >> 11);
 198
 199         return (imm << 12) | (rd << 7) | opcode;
 200 }
 201
 202 static inline u32 rv_amo_insn(u8 funct5, u8 aq, u8 rl, u8 rs2, u8 rs1,
 203                               u8 funct3, u8 rd, u8 opcode)
 204 {
 205         u8 funct7 = (funct5 << 2) | (aq << 1) | rl;
 206
 207         return rv_r_insn(funct7, rs2, rs1, funct3, rd, opcode);
 208 }
 209
 210 /* Instructions shared by both RV32 and RV64. */
 211
 212 static inline u32 rv_addi(u8 rd, u8 rs1, u16 imm11_0)
 213 {
 214         return rv_i_insn(imm11_0, rs1, 0, rd, 0x13);
 215 }
 216
 217 static inline u32 rv_andi(u8 rd, u8 rs1, u16 imm11_0)
 218 {
 219         return rv_i_insn(imm11_0, rs1, 7, rd, 0x13);
 220 }
 221
 222 static inline u32 rv_ori(u8 rd, u8 rs1, u16 imm11_0)
 223 {
 224         return rv_i_insn(imm11_0, rs1, 6, rd, 0x13);
 225 }
 226
 227 static inline u32 rv_xori(u8 rd, u8 rs1, u16 imm11_0)
 228 {
 229         return rv_i_insn(imm11_0, rs1, 4, rd, 0x13);
 230 }
 231
 232 static inline u32 rv_slli(u8 rd, u8 rs1, u16 imm11_0)
 233 {
 234         return rv_i_insn(imm11_0, rs1, 1, rd, 0x13);
 235 }
 236
 237 static inline u32 rv_srli(u8 rd, u8 rs1, u16 imm11_0)
 238 {
 239         return rv_i_insn(imm11_0, rs1, 5, rd, 0x13);
 240 }
 241
 242 static inline u32 rv_srai(u8 rd, u8 rs1, u16 imm11_0)
 243 {
 244         return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x13);
 245 }
 246
 247 static inline u32 rv_lui(u8 rd, u32 imm31_12)
 248 {
 249         return rv_u_insn(imm31_12, rd, 0x37);
 250 }
 251
 252 static inline u32 rv_auipc(u8 rd, u32 imm31_12)
 253 {
 254         return rv_u_insn(imm31_12, rd, 0x17);
 255 }
 256
 257 static inline u32 rv_add(u8 rd, u8 rs1, u8 rs2)
 258 {
 259         return rv_r_insn(0, rs2, rs1, 0, rd, 0x33);
 260 }
 261
 262 static inline u32 rv_sub(u8 rd, u8 rs1, u8 rs2)
 263 {
 264         return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x33);
 265 }
 266
 267 static inline u32 rv_sltu(u8 rd, u8 rs1, u8 rs2)
 268 {
 269         return rv_r_insn(0, rs2, rs1, 3, rd, 0x33);
 270 }
 271
 272 static inline u32 rv_and(u8 rd, u8 rs1, u8 rs2)
 273 {
 274         return rv_r_insn(0, rs2, rs1, 7, rd, 0x33);
 275 }
 276
 277 static inline u32 rv_or(u8 rd, u8 rs1, u8 rs2)
 278 {
 279         return rv_r_insn(0, rs2, rs1, 6, rd, 0x33);
 280 }
 281
 282 static inline u32 rv_xor(u8 rd, u8 rs1, u8 rs2)
 283 {
 284         return rv_r_insn(0, rs2, rs1, 4, rd, 0x33);
 285 }
 286
 287 static inline u32 rv_sll(u8 rd, u8 rs1, u8 rs2)
 288 {
 289         return rv_r_insn(0, rs2, rs1, 1, rd, 0x33);
 290 }
 291
 292 static inline u32 rv_srl(u8 rd, u8 rs1, u8 rs2)
 293 {
 294         return rv_r_insn(0, rs2, rs1, 5, rd, 0x33);
 295 }
 296
 297 static inline u32 rv_sra(u8 rd, u8 rs1, u8 rs2)
 298 {
 299         return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x33);
 300 }
 301
 302 static inline u32 rv_mul(u8 rd, u8 rs1, u8 rs2)
 303 {
 304         return rv_r_insn(1, rs2, rs1, 0, rd, 0x33);
 305 }
 306
 307 static inline u32 rv_mulhu(u8 rd, u8 rs1, u8 rs2)
 308 {
 309         return rv_r_insn(1, rs2, rs1, 3, rd, 0x33);
 310 }
 311
 312 static inline u32 rv_divu(u8 rd, u8 rs1, u8 rs2)
 313 {
 314         return rv_r_insn(1, rs2, rs1, 5, rd, 0x33);
 315 }
 316
 317 static inline u32 rv_remu(u8 rd, u8 rs1, u8 rs2)
 318 {
 319         return rv_r_insn(1, rs2, rs1, 7, rd, 0x33);
 320 }
 321
 322 static inline u32 rv_jal(u8 rd, u32 imm20_1)
 323 {
 324         return rv_j_insn(imm20_1, rd, 0x6f);
 325 }
 326
 327 static inline u32 rv_jalr(u8 rd, u8 rs1, u16 imm11_0)
 328 {
 329         return rv_i_insn(imm11_0, rs1, 0, rd, 0x67);
 330 }
 331
 332 static inline u32 rv_beq(u8 rs1, u8 rs2, u16 imm12_1)
 333 {
 334         return rv_b_insn(imm12_1, rs2, rs1, 0, 0x63);
 335 }
 336
 337 static inline u32 rv_bne(u8 rs1, u8 rs2, u16 imm12_1)
 338 {
 339         return rv_b_insn(imm12_1, rs2, rs1, 1, 0x63);
 340 }
 341
 342 static inline u32 rv_bltu(u8 rs1, u8 rs2, u16 imm12_1)
 343 {
 344         return rv_b_insn(imm12_1, rs2, rs1, 6, 0x63);
 345 }
 346
 347 static inline u32 rv_bgtu(u8 rs1, u8 rs2, u16 imm12_1)
 348 {
 349         return rv_bltu(rs2, rs1, imm12_1);
 350 }
 351
 352 static inline u32 rv_bgeu(u8 rs1, u8 rs2, u16 imm12_1)
 353 {
 354         return rv_b_insn(imm12_1, rs2, rs1, 7, 0x63);
 355 }
 356
 357 static inline u32 rv_bleu(u8 rs1, u8 rs2, u16 imm12_1)
 358 {
 359         return rv_bgeu(rs2, rs1, imm12_1);
 360 }
 361
 362 static inline u32 rv_blt(u8 rs1, u8 rs2, u16 imm12_1)
 363 {
 364         return rv_b_insn(imm12_1, rs2, rs1, 4, 0x63);
 365 }
 366
 367 static inline u32 rv_bgt(u8 rs1, u8 rs2, u16 imm12_1)
 368 {
 369         return rv_blt(rs2, rs1, imm12_1);
 370 }
 371
 372 static inline u32 rv_bge(u8 rs1, u8 rs2, u16 imm12_1)
 373 {
 374         return rv_b_insn(imm12_1, rs2, rs1, 5, 0x63);
 375 }
 376
 377 static inline u32 rv_ble(u8 rs1, u8 rs2, u16 imm12_1)
 378 {
 379         return rv_bge(rs2, rs1, imm12_1);
 380 }
 381
 382 static inline u32 rv_lw(u8 rd, u16 imm11_0, u8 rs1)
 383 {
 384         return rv_i_insn(imm11_0, rs1, 2, rd, 0x03);
 385 }
 386
 387 static inline u32 rv_lbu(u8 rd, u16 imm11_0, u8 rs1)
 388 {
 389         return rv_i_insn(imm11_0, rs1, 4, rd, 0x03);
 390 }
 391
 392 static inline u32 rv_lhu(u8 rd, u16 imm11_0, u8 rs1)
 393 {
 394         return rv_i_insn(imm11_0, rs1, 5, rd, 0x03);
 395 }
 396
 397 static inline u32 rv_sb(u8 rs1, u16 imm11_0, u8 rs2)
 398 {
 399         return rv_s_insn(imm11_0, rs2, rs1, 0, 0x23);
 400 }
 401
 402 static inline u32 rv_sh(u8 rs1, u16 imm11_0, u8 rs2)
 403 {
 404         return rv_s_insn(imm11_0, rs2, rs1, 1, 0x23);
 405 }
 406
 407 static inline u32 rv_sw(u8 rs1, u16 imm11_0, u8 rs2)
 408 {
 409         return rv_s_insn(imm11_0, rs2, rs1, 2, 0x23);
 410 }
 411
 412 static inline u32 rv_amoadd_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
 413 {
 414         return rv_amo_insn(0, aq, rl, rs2, rs1, 2, rd, 0x2f);
 415 }
 416
 417 /*
 418  * RV64-only instructions.
 419  *
 420  * These instructions are not available on RV32.  Wrap them below a #if to
 421  * ensure that the RV32 JIT doesn't emit any of these instructions.
 422  */
 423
 424 #if __riscv_xlen == 64
 425
 426 static inline u32 rv_addiw(u8 rd, u8 rs1, u16 imm11_0)
 427 {
 428         return rv_i_insn(imm11_0, rs1, 0, rd, 0x1b);
 429 }
 430
 431 static inline u32 rv_slliw(u8 rd, u8 rs1, u16 imm11_0)
 432 {
 433         return rv_i_insn(imm11_0, rs1, 1, rd, 0x1b);
 434 }
 435
 436 static inline u32 rv_srliw(u8 rd, u8 rs1, u16 imm11_0)
 437 {
 438         return rv_i_insn(imm11_0, rs1, 5, rd, 0x1b);
 439 }
 440
 441 static inline u32 rv_sraiw(u8 rd, u8 rs1, u16 imm11_0)
 442 {
 443         return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x1b);
 444 }
 445
 446 static inline u32 rv_addw(u8 rd, u8 rs1, u8 rs2)
 447 {
 448         return rv_r_insn(0, rs2, rs1, 0, rd, 0x3b);
 449 }
 450
 451 static inline u32 rv_subw(u8 rd, u8 rs1, u8 rs2)
 452 {
 453         return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x3b);
 454 }
 455
 456 static inline u32 rv_sllw(u8 rd, u8 rs1, u8 rs2)
 457 {
 458         return rv_r_insn(0, rs2, rs1, 1, rd, 0x3b);
 459 }
 460
 461 static inline u32 rv_srlw(u8 rd, u8 rs1, u8 rs2)
 462 {
 463         return rv_r_insn(0, rs2, rs1, 5, rd, 0x3b);
 464 }
 465
 466 static inline u32 rv_sraw(u8 rd, u8 rs1, u8 rs2)
 467 {
 468         return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x3b);
 469 }
 470
 471 static inline u32 rv_mulw(u8 rd, u8 rs1, u8 rs2)
 472 {
 473         return rv_r_insn(1, rs2, rs1, 0, rd, 0x3b);
 474 }
 475
 476 static inline u32 rv_divuw(u8 rd, u8 rs1, u8 rs2)
 477 {
 478         return rv_r_insn(1, rs2, rs1, 5, rd, 0x3b);
 479 }
 480
 481 static inline u32 rv_remuw(u8 rd, u8 rs1, u8 rs2)
 482 {
 483         return rv_r_insn(1, rs2, rs1, 7, rd, 0x3b);
 484 }
 485
 486 static inline u32 rv_ld(u8 rd, u16 imm11_0, u8 rs1)
 487 {
 488         return rv_i_insn(imm11_0, rs1, 3, rd, 0x03);
 489 }
 490
 491 static inline u32 rv_lwu(u8 rd, u16 imm11_0, u8 rs1)
 492 {
 493         return rv_i_insn(imm11_0, rs1, 6, rd, 0x03);
 494 }
 495
 496 static inline u32 rv_sd(u8 rs1, u16 imm11_0, u8 rs2)
 497 {
 498         return rv_s_insn(imm11_0, rs2, rs1, 3, 0x23);
 499 }
 500
 501 static inline u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
 502 {
 503         return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f);
 504 }
 505
 506 #endif /* __riscv_xlen == 64 */
 507
 508 void bpf_jit_build_prologue(struct rv_jit_context *ctx);
 509 void bpf_jit_build_epilogue(struct rv_jit_context *ctx);
 510
 511 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
 512                       bool extra_pass);
 513
 514 #endif /* _BPF_JIT_H */