Merge tag 'trace-printf-v6.13' of git://git.kernel.org/pub/scm/linux/kernel/git/trace...

[drm/drm-misc.git] / arch / arc / net / bpf_jit_core.c

// SPDX-License-Identifier: GPL-2.0
/*
 * The back-end-agnostic part of Just-In-Time compiler for eBPF bytecode.
 *
 * Copyright (c) 2024 Synopsys Inc.
 * Author: Shahab Vahedi <shahab@synopsys.com>
 */
#include <linux/bug.h>
#include "bpf_jit.h"

/*
 * Check for the return value. A pattern used often in this file.
 * There must be a "ret" variable of type "int" in the scope.
 */
#define CHECK_RET(cmd)                  \
        do {                            \
                ret = (cmd);            \
                if (ret < 0)            \
                        return ret;     \
        } while (0)

#ifdef ARC_BPF_JIT_DEBUG
/* Dumps bytes in /var/log/messages at KERN_INFO level (4). */
static void dump_bytes(const u8 *buf, u32 len, const char *header)
{
        u8 line[64];
        size_t i, j;

        pr_info("-----------------[ %s ]-----------------\n", header);

        for (i = 0, j = 0; i < len; i++) {
                /* Last input byte? */
                if (i == len - 1) {
                        j += scnprintf(line + j, 64 - j, "0x%02x", buf[i]);
                        pr_info("%s\n", line);
                        break;
                }
                /* End of line? */
                else if (i % 8 == 7) {
                        j += scnprintf(line + j, 64 - j, "0x%02x", buf[i]);
                        pr_info("%s\n", line);
                        j = 0;
                } else {
                        j += scnprintf(line + j, 64 - j, "0x%02x, ", buf[i]);
                }
        }
}
#endif /* ARC_BPF_JIT_DEBUG */

/********************* JIT context ***********************/

/*
 * buf:         Translated instructions end up here.
 * len:         The length of whole block in bytes.
 * index:       The offset at which the _next_ instruction may be put.
 */
struct jit_buffer {
        u8      *buf;
        u32     len;
        u32     index;
};

/*
 * This is a subset of "struct jit_context" that its information is deemed
 * necessary for the next extra pass to come.
 *
 * bpf_header:  Needed to finally lock the region.
 * bpf2insn:    Used to find the translation for instructions of interest.
 *
 * Things like "jit.buf" and "jit.len" can be retrieved respectively from
 * "prog->bpf_func" and "prog->jited_len".
 */
struct arc_jit_data {
        struct bpf_binary_header *bpf_header;
        u32                      *bpf2insn;
};

/*
 * The JIT pertinent context that is used by different functions.
 *
 * prog:                The current eBPF program being handled.
 * orig_prog:           The original eBPF program before any possible change.
 * jit:                 The JIT buffer and its length.
 * bpf_header:          The JITed program header. "jit.buf" points inside it.
 * emit:                If set, opcodes are written to memory; else, a dry-run.
 * do_zext:             If true, 32-bit sub-regs must be zero extended.
 * bpf2insn:            Maps BPF insn indices to their counterparts in jit.buf.
 * bpf2insn_valid:      Indicates if "bpf2ins" is populated with the mappings.
 * jit_data:            A piece of memory to transfer data to the next pass.
 * arc_regs_clobbered:  Each bit status determines if that arc reg is clobbered.
 * save_blink:          Whether ARC's "blink" register needs to be saved.
 * frame_size:          Derived from "prog->aux->stack_depth".
 * epilogue_offset:     Used by early "return"s in the code to jump here.
 * need_extra_pass:     A forecast if an "extra_pass" will occur.
 * is_extra_pass:       Indicates if the current pass is an extra pass.
 * user_bpf_prog:       True, if VM opcodes come from a real program.
 * blinded:             True if "constant blinding" step returned a new "prog".
 * success:             Indicates if the whole JIT went OK.
 */
struct jit_context {
        struct bpf_prog                 *prog;
        struct bpf_prog                 *orig_prog;
        struct jit_buffer               jit;
        struct bpf_binary_header        *bpf_header;
        bool                            emit;
        bool                            do_zext;
        u32                             *bpf2insn;
        bool                            bpf2insn_valid;
        struct arc_jit_data             *jit_data;
        u32                             arc_regs_clobbered;
        bool                            save_blink;
        u16                             frame_size;
        u32                             epilogue_offset;
        bool                            need_extra_pass;
        bool                            is_extra_pass;
        bool                            user_bpf_prog;
        bool                            blinded;
        bool                            success;
};

/*
 * If we're in ARC_BPF_JIT_DEBUG mode and the debug level is right, dump the
 * input BPF stream. "bpf_jit_dump()" is not fully suited for this purpose.
 */
static void vm_dump(const struct bpf_prog *prog)
{
#ifdef ARC_BPF_JIT_DEBUG
        if (bpf_jit_enable > 1)
                dump_bytes((u8 *)prog->insns, 8 * prog->len, " VM  ");
#endif
}

/*
 * If the right level of debug is set, dump the bytes. There are 2 variants
 * of this function:
 *
 * 1. Use the standard bpf_jit_dump() which is meant only for JITed code.
 * 2. Use the dump_bytes() to match its "vm_dump()" instance.
 */
static void jit_dump(const struct jit_context *ctx)
{
#ifdef ARC_BPF_JIT_DEBUG
        u8 header[8];
#endif
        const int pass = ctx->is_extra_pass ? 2 : 1;

        if (bpf_jit_enable <= 1 || !ctx->prog->jited)
                return;

#ifdef ARC_BPF_JIT_DEBUG
        scnprintf(header, sizeof(header), "JIT:%d", pass);
        dump_bytes(ctx->jit.buf, ctx->jit.len, header);
        pr_info("\n");
#else
        bpf_jit_dump(ctx->prog->len, ctx->jit.len, pass, ctx->jit.buf);
#endif
}

/* Initialise the context so there's no garbage. */
static int jit_ctx_init(struct jit_context *ctx, struct bpf_prog *prog)
{
        memset(ctx, 0, sizeof(*ctx));

        ctx->orig_prog = prog;

        /* If constant blinding was requested but failed, scram. */
        ctx->prog = bpf_jit_blind_constants(prog);
        if (IS_ERR(ctx->prog))
                return PTR_ERR(ctx->prog);
        ctx->blinded = (ctx->prog != ctx->orig_prog);

        /* If the verifier doesn't zero-extend, then we have to do it. */
        ctx->do_zext = !ctx->prog->aux->verifier_zext;

        ctx->is_extra_pass = ctx->prog->jited;
        ctx->user_bpf_prog = ctx->prog->is_func;

        return 0;
}

/*
 * Only after the first iteration of normal pass (the dry-run),
 * there are valid offsets in ctx->bpf2insn array.
 */
static inline bool offsets_available(const struct jit_context *ctx)
{
        return ctx->bpf2insn_valid;
}

/*
 * "*mem" should be freed when there is no "extra pass" to come,
 * or the compilation terminated abruptly. A few of such memory
 * allocations are: ctx->jit_data and ctx->bpf2insn.
 */
static inline void maybe_free(struct jit_context *ctx, void **mem)
{
        if (*mem) {
                if (!ctx->success || !ctx->need_extra_pass) {
                        kfree(*mem);
                        *mem = NULL;
                }
        }
}

/*
 * Free memories based on the status of the context.
 *
 * A note about "bpf_header": On successful runs, "bpf_header" is
 * not freed, because "jit.buf", a sub-array of it, is returned as
 * the "bpf_func". However, "bpf_header" is lost and nothing points
 * to it. This should not cause a leakage, because apparently
 * "bpf_header" can be revived by "bpf_jit_binary_hdr()". This is
 * how "bpf_jit_free()" in "kernel/bpf/core.c" releases the memory.
 */
static void jit_ctx_cleanup(struct jit_context *ctx)
{
        if (ctx->blinded) {
                /* if all went well, release the orig_prog. */
                if (ctx->success)
                        bpf_jit_prog_release_other(ctx->prog, ctx->orig_prog);
                else
                        bpf_jit_prog_release_other(ctx->orig_prog, ctx->prog);
        }

        maybe_free(ctx, (void **)&ctx->bpf2insn);
        maybe_free(ctx, (void **)&ctx->jit_data);

        if (!ctx->bpf2insn)
                ctx->bpf2insn_valid = false;

        /* Freeing "bpf_header" is enough. "jit.buf" is a sub-array of it. */
        if (!ctx->success && ctx->bpf_header) {
                bpf_jit_binary_free(ctx->bpf_header);
                ctx->bpf_header = NULL;
                ctx->jit.buf    = NULL;
                ctx->jit.index  = 0;
                ctx->jit.len    = 0;
        }

        ctx->emit = false;
        ctx->do_zext = false;
}

/*
 * Analyse the register usage and record the frame size.
 * The register usage is determined by consulting the back-end.
 */
static void analyze_reg_usage(struct jit_context *ctx)
{
        size_t i;
        u32 usage = 0;
        const struct bpf_insn *insn = ctx->prog->insnsi;

        for (i = 0; i < ctx->prog->len; i++) {
                u8 bpf_reg;
                bool call;

                bpf_reg = insn[i].dst_reg;
                call = (insn[i].code == (BPF_JMP | BPF_CALL)) ? true : false;
                usage |= mask_for_used_regs(bpf_reg, call);
        }

        ctx->arc_regs_clobbered = usage;
        ctx->frame_size = ctx->prog->aux->stack_depth;
}

/* Verify that no instruction will be emitted when there is no buffer. */
static inline int jit_buffer_check(const struct jit_context *ctx)
{
        if (ctx->emit) {
                if (!ctx->jit.buf) {
                        pr_err("bpf-jit: inconsistence state; no "
                               "buffer to emit instructions.\n");
                        return -EINVAL;
                } else if (ctx->jit.index > ctx->jit.len) {
                        pr_err("bpf-jit: estimated JIT length is less "
                               "than the emitted instructions.\n");
                        return -EFAULT;
                }
        }
        return 0;
}

/* On a dry-run (emit=false), "jit.len" is growing gradually. */
static inline void jit_buffer_update(struct jit_context *ctx, u32 n)
{
        if (!ctx->emit)
                ctx->jit.len += n;
        else
                ctx->jit.index += n;
}

/* Based on "emit", determine the address where instructions are emitted. */
static inline u8 *effective_jit_buf(const struct jit_context *ctx)
{
        return ctx->emit ? (ctx->jit.buf + ctx->jit.index) : NULL;
}

/* Prologue based on context variables set by "analyze_reg_usage()". */
static int handle_prologue(struct jit_context *ctx)
{
        int ret;
        u8 *buf = effective_jit_buf(ctx);
        u32 len = 0;

        CHECK_RET(jit_buffer_check(ctx));

        len = arc_prologue(buf, ctx->arc_regs_clobbered, ctx->frame_size);
        jit_buffer_update(ctx, len);

        return 0;
}

/* The counter part for "handle_prologue()". */
static int handle_epilogue(struct jit_context *ctx)
{
        int ret;
        u8 *buf = effective_jit_buf(ctx);
        u32 len = 0;

        CHECK_RET(jit_buffer_check(ctx));

        len = arc_epilogue(buf, ctx->arc_regs_clobbered, ctx->frame_size);
        jit_buffer_update(ctx, len);

        return 0;
}

/* Tell which number of the BPF instruction we are dealing with. */
static inline s32 get_index_for_insn(const struct jit_context *ctx,
                                     const struct bpf_insn *insn)
{
        return (insn - ctx->prog->insnsi);
}

/*
 * In most of the cases, the "offset" is read from "insn->off". However,
 * if it is an unconditional BPF_JMP32, then it comes from "insn->imm".
 *
 * (Courtesy of "cpu=v4" support)
 */
static inline s32 get_offset(const struct bpf_insn *insn)
{
        if ((BPF_CLASS(insn->code) == BPF_JMP32) &&
            (BPF_OP(insn->code) == BPF_JA))
                return insn->imm;
        else
                return insn->off;
}

/*
 * Determine to which number of the BPF instruction we're jumping to.
 *
 * The "offset" is interpreted as the "number" of BPF instructions
 * from the _next_ BPF instruction. e.g.:
 *
 *  4 means 4 instructions after  the next insn
 *  0 means 0 instructions after  the next insn -> fallthrough.
 * -1 means 1 instruction  before the next insn -> jmp to current insn.
 *
 *  Another way to look at this, "offset" is the number of instructions
 *  that exist between the current instruction and the target instruction.
 *
 *  It is worth noting that a "mov r,i64", which is 16-byte long, is
 *  treated as two instructions long, therefore "offset" needn't be
 *  treated specially for those. Everything is uniform.
 */
static inline s32 get_target_index_for_insn(const struct jit_context *ctx,
                                            const struct bpf_insn *insn)
{
        return (get_index_for_insn(ctx, insn) + 1) + get_offset(insn);
}

/* Is there an immediate operand encoded in the "insn"? */
static inline bool has_imm(const struct bpf_insn *insn)
{
        return BPF_SRC(insn->code) == BPF_K;
}

/* Is the last BPF instruction? */
static inline bool is_last_insn(const struct bpf_prog *prog, u32 idx)
{
        return idx == (prog->len - 1);
}

/*
 * Invocation of this function, conditionally signals the need for
 * an extra pass. The conditions that must be met are:
 *
 * 1. The current pass itself shouldn't be an extra pass.
 * 2. The stream of bytes being JITed must come from a user program.
 */
static inline void set_need_for_extra_pass(struct jit_context *ctx)
{
        if (!ctx->is_extra_pass)
                ctx->need_extra_pass = ctx->user_bpf_prog;
}

/*
 * Check if the "size" is valid and then transfer the control to
 * the back-end for the swap.
 */
static int handle_swap(u8 *buf, u8 rd, u8 size, u8 endian,
                       bool force, bool do_zext, u8 *len)
{
        /* Sanity check on the size. */
        switch (size) {
        case 16:
        case 32:
        case 64:
                break;
        default:
                pr_err("bpf-jit: invalid size for swap.\n");
                return -EINVAL;
        }

        *len = gen_swap(buf, rd, size, endian, force, do_zext);

        return 0;
}

/* Checks if the (instruction) index is in valid range. */
static inline bool check_insn_idx_valid(const struct jit_context *ctx,
                                        const s32 idx)
{
        return (idx >= 0 && idx < ctx->prog->len);
}

/*
 * Decouple the back-end from BPF by converting BPF conditions
 * to internal enum. ARC_CC_* start from 0 and are used as index
 * to an array. BPF_J* usage must end after this conversion.
 */
static int bpf_cond_to_arc(const u8 op, u8 *arc_cc)
{
        switch (op) {
        case BPF_JA:
                *arc_cc = ARC_CC_AL;
                break;
        case BPF_JEQ:
                *arc_cc = ARC_CC_EQ;
                break;
        case BPF_JGT:
                *arc_cc = ARC_CC_UGT;
                break;
        case BPF_JGE:
                *arc_cc = ARC_CC_UGE;
                break;
        case BPF_JSET:
                *arc_cc = ARC_CC_SET;
                break;
        case BPF_JNE:
                *arc_cc = ARC_CC_NE;
                break;
        case BPF_JSGT:
                *arc_cc = ARC_CC_SGT;
                break;
        case BPF_JSGE:
                *arc_cc = ARC_CC_SGE;
                break;
        case BPF_JLT:
                *arc_cc = ARC_CC_ULT;
                break;
        case BPF_JLE:
                *arc_cc = ARC_CC_ULE;
                break;
        case BPF_JSLT:
                *arc_cc = ARC_CC_SLT;
                break;
        case BPF_JSLE:
                *arc_cc = ARC_CC_SLE;
                break;
        default:
                pr_err("bpf-jit: can't handle condition 0x%02X\n", op);
                return -EINVAL;
        }
        return 0;
}

/*
 * Check a few things for a supposedly "jump" instruction:
 *
 * 0. "insn" is a "jump" instruction, but not the "call/exit" variant.
 * 1. The current "insn" index is in valid range.
 * 2. The index of target instruction is in valid range.
 */
static int check_bpf_jump(const struct jit_context *ctx,
                          const struct bpf_insn *insn)
{
        const u8 class = BPF_CLASS(insn->code);
        const u8 op = BPF_OP(insn->code);

        /* Must be a jmp(32) instruction that is not a "call/exit". */
        if ((class != BPF_JMP && class != BPF_JMP32) ||
            (op == BPF_CALL || op == BPF_EXIT)) {
                pr_err("bpf-jit: not a jump instruction.\n");
                return -EINVAL;
        }

        if (!check_insn_idx_valid(ctx, get_index_for_insn(ctx, insn))) {
                pr_err("bpf-jit: the bpf jump insn is not in prog.\n");
                return -EINVAL;
        }

        if (!check_insn_idx_valid(ctx, get_target_index_for_insn(ctx, insn))) {
                pr_err("bpf-jit: bpf jump label is out of range.\n");
                return -EINVAL;
        }

        return 0;
}

/*
 * Based on input "insn", consult "ctx->bpf2insn" to get the
 * related index (offset) of the translation in JIT stream.
 */
static u32 get_curr_jit_off(const struct jit_context *ctx,
                            const struct bpf_insn *insn)
{
        const s32 idx = get_index_for_insn(ctx, insn);
#ifdef ARC_BPF_JIT_DEBUG
        BUG_ON(!offsets_available(ctx) || !check_insn_idx_valid(ctx, idx));
#endif
        return ctx->bpf2insn[idx];
}

/*
 * The input "insn" must be a jump instruction.
 *
 * Based on input "insn", consult "ctx->bpf2insn" to get the
 * related JIT index (offset) of "target instruction" that
 * "insn" would jump to.
 */
static u32 get_targ_jit_off(const struct jit_context *ctx,
                            const struct bpf_insn *insn)
{
        const s32 tidx = get_target_index_for_insn(ctx, insn);
#ifdef ARC_BPF_JIT_DEBUG
        BUG_ON(!offsets_available(ctx) || !check_insn_idx_valid(ctx, tidx));
#endif
        return ctx->bpf2insn[tidx];
}

/*
 * This function will return 0 for a feasible jump.
 *
 * Consult the back-end to check if it finds it feasible to emit
 * the necessary instructions based on "cond" and the displacement
 * between the "from_off" and the "to_off".
 */
static int feasible_jit_jump(u32 from_off, u32 to_off, u8 cond, bool j32)
{
        int ret = 0;

        if (j32) {
                if (!check_jmp_32(from_off, to_off, cond))
                        ret = -EFAULT;
        } else {
                if (!check_jmp_64(from_off, to_off, cond))
                        ret = -EFAULT;
        }

        if (ret != 0)
                pr_err("bpf-jit: the JIT displacement is not OK.\n");

        return ret;
}

/*
 * This jump handler performs the following steps:
 *
 * 1. Compute ARC's internal condition code from BPF's
 * 2. Determine the bitness of the operation (32 vs. 64)
 * 3. Sanity check on BPF stream
 * 4. Sanity check on what is supposed to be JIT's displacement
 * 5. And finally, emit the necessary instructions
 *
 * The last two steps are performed through the back-end.
 * The value of steps 1 and 2 are necessary inputs for the back-end.
 */
static int handle_jumps(const struct jit_context *ctx,
                        const struct bpf_insn *insn,
                        u8 *len)
{
        u8 cond;
        int ret = 0;
        u8 *buf = effective_jit_buf(ctx);
        const bool j32 = (BPF_CLASS(insn->code) == BPF_JMP32) ? true : false;
        const u8 rd = insn->dst_reg;
        u8 rs = insn->src_reg;
        u32 curr_off = 0, targ_off = 0;

        *len = 0;

        /* Map the BPF condition to internal enum. */
        CHECK_RET(bpf_cond_to_arc(BPF_OP(insn->code), &cond));

        /* Sanity check on the BPF byte stream. */
        CHECK_RET(check_bpf_jump(ctx, insn));

        /*
         * Move the immediate into a temporary register _now_ for 2 reasons:
         *
         * 1. "gen_jmp_{32,64}()" deal with operands in registers.
         *
         * 2. The "len" parameter will grow so that the current jit offset
         *    (curr_off) will have increased to a point where the necessary
         *    instructions can be inserted by "gen_jmp_{32,64}()".
         */
        if (has_imm(insn) && cond != ARC_CC_AL) {
                if (j32) {
                        *len += mov_r32_i32(BUF(buf, *len), JIT_REG_TMP,
                                            insn->imm);
                } else {
                        *len += mov_r64_i32(BUF(buf, *len), JIT_REG_TMP,
                                            insn->imm);
                }
                rs = JIT_REG_TMP;
        }

        /* If the offsets are known, check if the branch can occur. */
        if (offsets_available(ctx)) {
                curr_off = get_curr_jit_off(ctx, insn) + *len;
                targ_off = get_targ_jit_off(ctx, insn);

                /* Sanity check on the back-end side. */
                CHECK_RET(feasible_jit_jump(curr_off, targ_off, cond, j32));
        }

        if (j32) {
                *len += gen_jmp_32(BUF(buf, *len), rd, rs, cond,
                                   curr_off, targ_off);
        } else {
                *len += gen_jmp_64(BUF(buf, *len), rd, rs, cond,
                                   curr_off, targ_off);
        }

        return ret;
}

/* Jump to translated epilogue address. */
static int handle_jmp_epilogue(struct jit_context *ctx,
                               const struct bpf_insn *insn, u8 *len)
{
        u8 *buf = effective_jit_buf(ctx);
        u32 curr_off = 0, epi_off = 0;

        /* Check the offset only if the data is available. */
        if (offsets_available(ctx)) {
                curr_off = get_curr_jit_off(ctx, insn);
                epi_off = ctx->epilogue_offset;

                if (!check_jmp_64(curr_off, epi_off, ARC_CC_AL)) {
                        pr_err("bpf-jit: epilogue offset is not valid.\n");
                        return -EINVAL;
                }
        }

        /* Jump to "epilogue offset" (rd and rs don't matter). */
        *len = gen_jmp_64(buf, 0, 0, ARC_CC_AL, curr_off, epi_off);

        return 0;
}

/* Try to get the resolved address and generate the instructions. */
static int handle_call(struct jit_context *ctx,
                       const struct bpf_insn *insn,
                       u8 *len)
{
        int  ret;
        bool in_kernel_func, fixed = false;
        u64  addr = 0;
        u8  *buf = effective_jit_buf(ctx);

        ret = bpf_jit_get_func_addr(ctx->prog, insn, ctx->is_extra_pass,
                                    &addr, &fixed);
        if (ret < 0) {
                pr_err("bpf-jit: can't get the address for call.\n");
                return ret;
        }
        in_kernel_func = (fixed ? true : false);

        /* No valuable address retrieved (yet). */
        if (!fixed && !addr)
                set_need_for_extra_pass(ctx);

        *len = gen_func_call(buf, (ARC_ADDR)addr, in_kernel_func);

        if (insn->src_reg != BPF_PSEUDO_CALL) {
                /* Assigning ABI's return reg to JIT's return reg. */
                *len += arc_to_bpf_return(BUF(buf, *len));
        }

        return 0;
}

/*
 * Try to generate instructions for loading a 64-bit immediate.
 * These sort of instructions are usually associated with the 64-bit
 * relocations: R_BPF_64_64. Therefore, signal the need for an extra
 * pass if the circumstances are right.
 */
static int handle_ld_imm64(struct jit_context *ctx,
                           const struct bpf_insn *insn,
                           u8 *len)
{
        const s32 idx = get_index_for_insn(ctx, insn);
        u8 *buf = effective_jit_buf(ctx);

        /* We're about to consume 2 VM instructions. */
        if (is_last_insn(ctx->prog, idx)) {
                pr_err("bpf-jit: need more data for 64-bit immediate.\n");
                return -EINVAL;
        }

        *len = mov_r64_i64(buf, insn->dst_reg, insn->imm, (insn + 1)->imm);

        if (bpf_pseudo_func(insn))
                set_need_for_extra_pass(ctx);

        return 0;
}

/*
 * Handles one eBPF instruction at a time. To make this function faster,
 * it does not call "jit_buffer_check()". Else, it would call it for every
 * instruction. As a result, it should not be invoked directly. Only
 * "handle_body()", that has already executed the "check", may call this
 * function.
 *
 * If the "ret" value is negative, something has went wrong. Else,
 * it mostly holds the value 0 and rarely 1. Number 1 signals
 * the loop in "handle_body()" to skip the next instruction, because
 * it has been consumed as part of a 64-bit immediate value.
 */
static int handle_insn(struct jit_context *ctx, u32 idx)
{
        const struct bpf_insn *insn = &ctx->prog->insnsi[idx];
        const u8  code = insn->code;
        const u8  dst  = insn->dst_reg;
        const u8  src  = insn->src_reg;
        const s16 off  = insn->off;
        const s32 imm  = insn->imm;
        u8 *buf = effective_jit_buf(ctx);
        u8  len = 0;
        int ret = 0;

        switch (code) {
        /* dst += src (32-bit) */
        case BPF_ALU | BPF_ADD | BPF_X:
                len = add_r32(buf, dst, src);
                break;
        /* dst += imm (32-bit) */
        case BPF_ALU | BPF_ADD | BPF_K:
                len = add_r32_i32(buf, dst, imm);
                break;
        /* dst -= src (32-bit) */
        case BPF_ALU | BPF_SUB | BPF_X:
                len = sub_r32(buf, dst, src);
                break;
        /* dst -= imm (32-bit) */
        case BPF_ALU | BPF_SUB | BPF_K:
                len = sub_r32_i32(buf, dst, imm);
                break;
        /* dst = -dst (32-bit) */
        case BPF_ALU | BPF_NEG:
                len = neg_r32(buf, dst);
                break;
        /* dst *= src (32-bit) */
        case BPF_ALU | BPF_MUL | BPF_X:
                len = mul_r32(buf, dst, src);
                break;
        /* dst *= imm (32-bit) */
        case BPF_ALU | BPF_MUL | BPF_K:
                len = mul_r32_i32(buf, dst, imm);
                break;
        /* dst /= src (32-bit) */
        case BPF_ALU | BPF_DIV | BPF_X:
                len = div_r32(buf, dst, src, off == 1);
                break;
        /* dst /= imm (32-bit) */
        case BPF_ALU | BPF_DIV | BPF_K:
                len = div_r32_i32(buf, dst, imm, off == 1);
                break;
        /* dst %= src (32-bit) */
        case BPF_ALU | BPF_MOD | BPF_X:
                len = mod_r32(buf, dst, src, off == 1);
                break;
        /* dst %= imm (32-bit) */
        case BPF_ALU | BPF_MOD | BPF_K:
                len = mod_r32_i32(buf, dst, imm, off == 1);
                break;
        /* dst &= src (32-bit) */
        case BPF_ALU | BPF_AND | BPF_X:
                len = and_r32(buf, dst, src);
                break;
        /* dst &= imm (32-bit) */
        case BPF_ALU | BPF_AND | BPF_K:
                len = and_r32_i32(buf, dst, imm);
                break;
        /* dst |= src (32-bit) */
        case BPF_ALU | BPF_OR | BPF_X:
                len = or_r32(buf, dst, src);
                break;
        /* dst |= imm (32-bit) */
        case BPF_ALU | BPF_OR | BPF_K:
                len = or_r32_i32(buf, dst, imm);
                break;
        /* dst ^= src (32-bit) */
        case BPF_ALU | BPF_XOR | BPF_X:
                len = xor_r32(buf, dst, src);
                break;
        /* dst ^= imm (32-bit) */
        case BPF_ALU | BPF_XOR | BPF_K:
                len = xor_r32_i32(buf, dst, imm);
                break;
        /* dst <<= src (32-bit) */
        case BPF_ALU | BPF_LSH | BPF_X:
                len = lsh_r32(buf, dst, src);
                break;
        /* dst <<= imm (32-bit) */
        case BPF_ALU | BPF_LSH | BPF_K:
                len = lsh_r32_i32(buf, dst, imm);
                break;
        /* dst >>= src (32-bit) [unsigned] */
        case BPF_ALU | BPF_RSH | BPF_X:
                len = rsh_r32(buf, dst, src);
                break;
        /* dst >>= imm (32-bit) [unsigned] */
        case BPF_ALU | BPF_RSH | BPF_K:
                len = rsh_r32_i32(buf, dst, imm);
                break;
        /* dst >>= src (32-bit) [signed] */
        case BPF_ALU | BPF_ARSH | BPF_X:
                len = arsh_r32(buf, dst, src);
                break;
        /* dst >>= imm (32-bit) [signed] */
        case BPF_ALU | BPF_ARSH | BPF_K:
                len = arsh_r32_i32(buf, dst, imm);
                break;
        /* dst = src (32-bit) */
        case BPF_ALU | BPF_MOV | BPF_X:
                len = mov_r32(buf, dst, src, (u8)off);
                break;
        /* dst = imm32 (32-bit) */
        case BPF_ALU | BPF_MOV | BPF_K:
                len = mov_r32_i32(buf, dst, imm);
                break;
        /* dst = swap(dst) */
        case BPF_ALU   | BPF_END | BPF_FROM_LE:
        case BPF_ALU   | BPF_END | BPF_FROM_BE:
        case BPF_ALU64 | BPF_END | BPF_FROM_LE: {
                CHECK_RET(handle_swap(buf, dst, imm, BPF_SRC(code),
                                      BPF_CLASS(code) == BPF_ALU64,
                                      ctx->do_zext, &len));
                break;
        }
        /* dst += src (64-bit) */
        case BPF_ALU64 | BPF_ADD | BPF_X:
                len = add_r64(buf, dst, src);
                break;
        /* dst += imm32 (64-bit) */
        case BPF_ALU64 | BPF_ADD | BPF_K:
                len = add_r64_i32(buf, dst, imm);
                break;
        /* dst -= src (64-bit) */
        case BPF_ALU64 | BPF_SUB | BPF_X:
                len = sub_r64(buf, dst, src);
                break;
        /* dst -= imm32 (64-bit) */
        case BPF_ALU64 | BPF_SUB | BPF_K:
                len = sub_r64_i32(buf, dst, imm);
                break;
        /* dst = -dst (64-bit) */
        case BPF_ALU64 | BPF_NEG:
                len = neg_r64(buf, dst);
                break;
        /* dst *= src (64-bit) */
        case BPF_ALU64 | BPF_MUL | BPF_X:
                len = mul_r64(buf, dst, src);
                break;
        /* dst *= imm32 (64-bit) */
        case BPF_ALU64 | BPF_MUL | BPF_K:
                len = mul_r64_i32(buf, dst, imm);
                break;
        /* dst &= src (64-bit) */
        case BPF_ALU64 | BPF_AND | BPF_X:
                len = and_r64(buf, dst, src);
                break;
        /* dst &= imm32 (64-bit) */
        case BPF_ALU64 | BPF_AND | BPF_K:
                len = and_r64_i32(buf, dst, imm);
                break;
        /* dst |= src (64-bit) */
        case BPF_ALU64 | BPF_OR | BPF_X:
                len = or_r64(buf, dst, src);
                break;
        /* dst |= imm32 (64-bit) */
        case BPF_ALU64 | BPF_OR | BPF_K:
                len = or_r64_i32(buf, dst, imm);
                break;
        /* dst ^= src (64-bit) */
        case BPF_ALU64 | BPF_XOR | BPF_X:
                len = xor_r64(buf, dst, src);
                break;
        /* dst ^= imm32 (64-bit) */
        case BPF_ALU64 | BPF_XOR | BPF_K:
                len = xor_r64_i32(buf, dst, imm);
                break;
        /* dst <<= src (64-bit) */
        case BPF_ALU64 | BPF_LSH | BPF_X:
                len = lsh_r64(buf, dst, src);
                break;
        /* dst <<= imm32 (64-bit) */
        case BPF_ALU64 | BPF_LSH | BPF_K:
                len = lsh_r64_i32(buf, dst, imm);
                break;
        /* dst >>= src (64-bit) [unsigned] */
        case BPF_ALU64 | BPF_RSH | BPF_X:
                len = rsh_r64(buf, dst, src);
                break;
        /* dst >>= imm32 (64-bit) [unsigned] */
        case BPF_ALU64 | BPF_RSH | BPF_K:
                len = rsh_r64_i32(buf, dst, imm);
                break;
        /* dst >>= src (64-bit) [signed] */
        case BPF_ALU64 | BPF_ARSH | BPF_X:
                len = arsh_r64(buf, dst, src);
                break;
        /* dst >>= imm32 (64-bit) [signed] */
        case BPF_ALU64 | BPF_ARSH | BPF_K:
                len = arsh_r64_i32(buf, dst, imm);
                break;
        /* dst = src (64-bit) */
        case BPF_ALU64 | BPF_MOV | BPF_X:
                len = mov_r64(buf, dst, src, (u8)off);
                break;
        /* dst = imm32 (sign extend to 64-bit) */
        case BPF_ALU64 | BPF_MOV | BPF_K:
                len = mov_r64_i32(buf, dst, imm);
                break;
        /* dst = imm64 */
        case BPF_LD | BPF_DW | BPF_IMM:
                CHECK_RET(handle_ld_imm64(ctx, insn, &len));
                /* Tell the loop to skip the next instruction. */
                ret = 1;
                break;
        /* dst = *(size *)(src + off) */
        case BPF_LDX | BPF_MEM | BPF_W:
        case BPF_LDX | BPF_MEM | BPF_H:
        case BPF_LDX | BPF_MEM | BPF_B:
        case BPF_LDX | BPF_MEM | BPF_DW:
                len = load_r(buf, dst, src, off, BPF_SIZE(code), false);
                break;
        case BPF_LDX | BPF_MEMSX | BPF_W:
        case BPF_LDX | BPF_MEMSX | BPF_H:
        case BPF_LDX | BPF_MEMSX | BPF_B:
                len = load_r(buf, dst, src, off, BPF_SIZE(code), true);
                break;
        /* *(size *)(dst + off) = src */
        case BPF_STX | BPF_MEM | BPF_W:
        case BPF_STX | BPF_MEM | BPF_H:
        case BPF_STX | BPF_MEM | BPF_B:
        case BPF_STX | BPF_MEM | BPF_DW:
                len = store_r(buf, src, dst, off, BPF_SIZE(code));
                break;
        case BPF_ST | BPF_MEM | BPF_W:
        case BPF_ST | BPF_MEM | BPF_H:
        case BPF_ST | BPF_MEM | BPF_B:
        case BPF_ST | BPF_MEM | BPF_DW:
                len = store_i(buf, imm, dst, off, BPF_SIZE(code));
                break;
        case BPF_JMP   | BPF_JA:
        case BPF_JMP   | BPF_JEQ  | BPF_X:
        case BPF_JMP   | BPF_JEQ  | BPF_K:
        case BPF_JMP   | BPF_JNE  | BPF_X:
        case BPF_JMP   | BPF_JNE  | BPF_K:
        case BPF_JMP   | BPF_JSET | BPF_X:
        case BPF_JMP   | BPF_JSET | BPF_K:
        case BPF_JMP   | BPF_JGT  | BPF_X:
        case BPF_JMP   | BPF_JGT  | BPF_K:
        case BPF_JMP   | BPF_JGE  | BPF_X:
        case BPF_JMP   | BPF_JGE  | BPF_K:
        case BPF_JMP   | BPF_JSGT | BPF_X:
        case BPF_JMP   | BPF_JSGT | BPF_K:
        case BPF_JMP   | BPF_JSGE | BPF_X:
        case BPF_JMP   | BPF_JSGE | BPF_K:
        case BPF_JMP   | BPF_JLT  | BPF_X:
        case BPF_JMP   | BPF_JLT  | BPF_K:
        case BPF_JMP   | BPF_JLE  | BPF_X:
        case BPF_JMP   | BPF_JLE  | BPF_K:
        case BPF_JMP   | BPF_JSLT | BPF_X:
        case BPF_JMP   | BPF_JSLT | BPF_K:
        case BPF_JMP   | BPF_JSLE | BPF_X:
        case BPF_JMP   | BPF_JSLE | BPF_K:
        case BPF_JMP32 | BPF_JA:
        case BPF_JMP32 | BPF_JEQ  | BPF_X:
        case BPF_JMP32 | BPF_JEQ  | BPF_K:
        case BPF_JMP32 | BPF_JNE  | BPF_X:
        case BPF_JMP32 | BPF_JNE  | BPF_K:
        case BPF_JMP32 | BPF_JSET | BPF_X:
        case BPF_JMP32 | BPF_JSET | BPF_K:
        case BPF_JMP32 | BPF_JGT  | BPF_X:
        case BPF_JMP32 | BPF_JGT  | BPF_K:
        case BPF_JMP32 | BPF_JGE  | BPF_X:
        case BPF_JMP32 | BPF_JGE  | BPF_K:
        case BPF_JMP32 | BPF_JSGT | BPF_X:
        case BPF_JMP32 | BPF_JSGT | BPF_K:
        case BPF_JMP32 | BPF_JSGE | BPF_X:
        case BPF_JMP32 | BPF_JSGE | BPF_K:
        case BPF_JMP32 | BPF_JLT  | BPF_X:
        case BPF_JMP32 | BPF_JLT  | BPF_K:
        case BPF_JMP32 | BPF_JLE  | BPF_X:
        case BPF_JMP32 | BPF_JLE  | BPF_K:
        case BPF_JMP32 | BPF_JSLT | BPF_X:
        case BPF_JMP32 | BPF_JSLT | BPF_K:
        case BPF_JMP32 | BPF_JSLE | BPF_X:
        case BPF_JMP32 | BPF_JSLE | BPF_K:
                CHECK_RET(handle_jumps(ctx, insn, &len));
                break;
        case BPF_JMP | BPF_CALL:
                CHECK_RET(handle_call(ctx, insn, &len));
                break;

        case BPF_JMP | BPF_EXIT:
                /* If this is the last instruction, epilogue will follow. */
                if (is_last_insn(ctx->prog, idx))
                        break;
                CHECK_RET(handle_jmp_epilogue(ctx, insn, &len));
                break;
        default:
                pr_err("bpf-jit: can't handle instruction code 0x%02X\n", code);
                return -EOPNOTSUPP;
        }

        if (BPF_CLASS(code) == BPF_ALU) {
                /*
                 * Skip the "swap" instructions. Even 64-bit swaps are of type
                 * BPF_ALU (and not BPF_ALU64). Therefore, for the swaps, one
                 * has to look at the "size" of the operations rather than the
                 * ALU type. "gen_swap()" specifically takes care of that.
                 */
                if (BPF_OP(code) != BPF_END && ctx->do_zext)
                        len += zext(BUF(buf, len), dst);
        }

        jit_buffer_update(ctx, len);

        return ret;
}

static int handle_body(struct jit_context *ctx)
{
        int ret;
        bool populate_bpf2insn = false;
        const struct bpf_prog *prog = ctx->prog;

        CHECK_RET(jit_buffer_check(ctx));

        /*
         * Record the mapping for the instructions during the dry-run.
         * Doing it this way allows us to have the mapping ready for
         * the jump instructions during the real compilation phase.
         */
        if (!ctx->emit)
                populate_bpf2insn = true;

        for (u32 i = 0; i < prog->len; i++) {
                /* During the dry-run, jit.len grows gradually per BPF insn. */
                if (populate_bpf2insn)
                        ctx->bpf2insn[i] = ctx->jit.len;

                CHECK_RET(handle_insn(ctx, i));
                if (ret > 0) {
                        /* "ret" is 1 if two (64-bit) chunks were consumed. */
                        ctx->bpf2insn[i + 1] = ctx->bpf2insn[i];
                        i++;
                }
        }

        /* If bpf2insn had to be populated, then it is done at this point. */
        if (populate_bpf2insn)
                ctx->bpf2insn_valid = true;

        return 0;
}

/*
 * Initialize the memory with "unimp_s" which is the mnemonic for
 * "unimplemented" instruction and always raises an exception.
 *
 * The instruction is 2 bytes. If "size" is odd, there is not much
 * that can be done about the last byte in "area". Because, the
 * CPU always fetches instructions in two bytes. Therefore, the
 * byte beyond the last one is going to accompany it during a
 * possible fetch. In the most likely case of a little endian
 * system, that beyond-byte will become the major opcode and
 * we have no control over its initialisation.
 */
static void fill_ill_insn(void *area, unsigned int size)
{
        const u16 unimp_s = 0x79e0;

        if (size & 1) {
                *((u8 *)area + (size - 1)) = 0xff;
                size -= 1;
        }

        memset16(area, unimp_s, size >> 1);
}

/* Piece of memory that can be allocated at the beginning of jit_prepare(). */
static int jit_prepare_early_mem_alloc(struct jit_context *ctx)
{
        ctx->bpf2insn = kcalloc(ctx->prog->len, sizeof(ctx->jit.len),
                                GFP_KERNEL);

        if (!ctx->bpf2insn) {
                pr_err("bpf-jit: could not allocate memory for "
                       "mapping of the instructions.\n");
                return -ENOMEM;
        }

        return 0;
}

/*
 * Memory allocations that rely on parameters known at the end of
 * jit_prepare().
 */
static int jit_prepare_final_mem_alloc(struct jit_context *ctx)
{
        const size_t alignment = sizeof(u32);

        ctx->bpf_header = bpf_jit_binary_alloc(ctx->jit.len, &ctx->jit.buf,
                                               alignment, fill_ill_insn);
        if (!ctx->bpf_header) {
                pr_err("bpf-jit: could not allocate memory for translation.\n");
                return -ENOMEM;
        }

        if (ctx->need_extra_pass) {
                ctx->jit_data = kzalloc(sizeof(*ctx->jit_data), GFP_KERNEL);
                if (!ctx->jit_data)
                        return -ENOMEM;
        }

        return 0;
}

/*
 * The first phase of the translation without actually emitting any
 * instruction. It helps in getting a forecast on some aspects, such
 * as the length of the whole program or where the epilogue starts.
 *
 * Whenever the necessary parameters are known, memories are allocated.
 */
static int jit_prepare(struct jit_context *ctx)
{
        int ret;

        /* Dry run. */
        ctx->emit = false;

        CHECK_RET(jit_prepare_early_mem_alloc(ctx));

        /* Get the length of prologue section after some register analysis. */
        analyze_reg_usage(ctx);
        CHECK_RET(handle_prologue(ctx));

        CHECK_RET(handle_body(ctx));

        /* Record at which offset epilogue begins. */
        ctx->epilogue_offset = ctx->jit.len;

        /* Process the epilogue section now. */
        CHECK_RET(handle_epilogue(ctx));

        CHECK_RET(jit_prepare_final_mem_alloc(ctx));

        return 0;
}

/*
 * jit_compile() is the real compilation phase. jit_prepare() is
 * invoked before jit_compile() as a dry-run to make sure everything
 * will go OK and allocate the necessary memory.
 *
 * In the end, jit_compile() checks if it has produced the same number
 * of instructions as jit_prepare() would.
 */
static int jit_compile(struct jit_context *ctx)
{
        int ret;

        /* Let there be code. */
        ctx->emit = true;

        CHECK_RET(handle_prologue(ctx));

        CHECK_RET(handle_body(ctx));

        CHECK_RET(handle_epilogue(ctx));

        if (ctx->jit.index != ctx->jit.len) {
                pr_err("bpf-jit: divergence between the phases; "
                       "%u vs. %u (bytes).\n",
                       ctx->jit.len, ctx->jit.index);
                return -EFAULT;
        }

        return 0;
}

/*
 * Calling this function implies a successful JIT. A successful
 * translation is signaled by setting the right parameters:
 *
 * prog->jited=1, prog->jited_len=..., prog->bpf_func=...
 */
static int jit_finalize(struct jit_context *ctx)
{
        struct bpf_prog *prog = ctx->prog;

        /* We're going to need this information for the "do_extra_pass()". */
        if (ctx->need_extra_pass) {
                ctx->jit_data->bpf_header = ctx->bpf_header;
                ctx->jit_data->bpf2insn = ctx->bpf2insn;
                prog->aux->jit_data = (void *)ctx->jit_data;
        } else {
                /*
                 * If things seem finalised, then mark the JITed memory
                 * as R-X and flush it.
                 */
                if (bpf_jit_binary_lock_ro(ctx->bpf_header)) {
                        pr_err("bpf-jit: Could not lock the JIT memory.\n");
                        return -EFAULT;
                }
                flush_icache_range((unsigned long)ctx->bpf_header,
                                   (unsigned long)
                                   BUF(ctx->jit.buf, ctx->jit.len));
                prog->aux->jit_data = NULL;
                bpf_prog_fill_jited_linfo(prog, ctx->bpf2insn);
        }

        ctx->success = true;
        prog->bpf_func = (void *)ctx->jit.buf;
        prog->jited_len = ctx->jit.len;
        prog->jited = 1;

        jit_ctx_cleanup(ctx);
        jit_dump(ctx);

        return 0;
}

/*
 * A lenient verification for the existence of JIT context in "prog".
 * Apparently the JIT internals, namely jit_subprogs() in bpf/verifier.c,
 * may request for a second compilation although nothing needs to be done.
 */
static inline int check_jit_context(const struct bpf_prog *prog)
{
        if (!prog->aux->jit_data) {
                pr_notice("bpf-jit: no jit data for the extra pass.\n");
                return 1;
        } else {
                return 0;
        }
}

/* Reuse the previous pass's data. */
static int jit_resume_context(struct jit_context *ctx)
{
        struct arc_jit_data *jdata =
                (struct arc_jit_data *)ctx->prog->aux->jit_data;

        if (!jdata) {
                pr_err("bpf-jit: no jit data for the extra pass.\n");
                return -EINVAL;
        }

        ctx->jit.buf = (u8 *)ctx->prog->bpf_func;
        ctx->jit.len = ctx->prog->jited_len;
        ctx->bpf_header = jdata->bpf_header;
        ctx->bpf2insn = (u32 *)jdata->bpf2insn;
        ctx->bpf2insn_valid = ctx->bpf2insn ? true : false;
        ctx->jit_data = jdata;

        return 0;
}

/*
 * Patch in the new addresses. The instructions of interest are:
 *
 * - call
 * - ld r64, imm64
 *
 * For "call"s, it resolves the addresses one more time through the
 * handle_call().
 *
 * For 64-bit immediate loads, it just retranslates them, because the BPF
 * core in kernel might have changed the value since the normal pass.
 */
static int jit_patch_relocations(struct jit_context *ctx)
{
        const u8 bpf_opc_call = BPF_JMP | BPF_CALL;
        const u8 bpf_opc_ldi64 = BPF_LD | BPF_DW | BPF_IMM;
        const struct bpf_prog *prog = ctx->prog;
        int ret;

        ctx->emit = true;
        for (u32 i = 0; i < prog->len; i++) {
                const struct bpf_insn *insn = &prog->insnsi[i];
                u8 dummy;
                /*
                 * Adjust "ctx.jit.index", so "gen_*()" functions below
                 * can use it for their output addresses.
                 */
                ctx->jit.index = ctx->bpf2insn[i];

                if (insn->code == bpf_opc_call) {
                        CHECK_RET(handle_call(ctx, insn, &dummy));
                } else if (insn->code == bpf_opc_ldi64) {
                        CHECK_RET(handle_ld_imm64(ctx, insn, &dummy));
                        /* Skip the next instruction. */
                        ++i;
                }
        }
        return 0;
}

/*
 * A normal pass that involves a "dry-run" phase, jit_prepare(),
 * to get the necessary data for the real compilation phase,
 * jit_compile().
 */
static struct bpf_prog *do_normal_pass(struct bpf_prog *prog)
{
        struct jit_context ctx;

        /* Bail out if JIT is disabled. */
        if (!prog->jit_requested)
                return prog;

        if (jit_ctx_init(&ctx, prog)) {
                jit_ctx_cleanup(&ctx);
                return prog;
        }

        /* Get the lengths and allocate buffer. */
        if (jit_prepare(&ctx)) {
                jit_ctx_cleanup(&ctx);
                return prog;
        }

        if (jit_compile(&ctx)) {
                jit_ctx_cleanup(&ctx);
                return prog;
        }

        if (jit_finalize(&ctx)) {
                jit_ctx_cleanup(&ctx);
                return prog;
        }

        return ctx.prog;
}

/*
 * If there are multi-function BPF programs that call each other,
 * their translated addresses are not known all at once. Therefore,
 * an extra pass is needed to consult the bpf_jit_get_func_addr()
 * again to get the newly translated addresses in order to resolve
 * the "call"s.
 */
static struct bpf_prog *do_extra_pass(struct bpf_prog *prog)
{
        struct jit_context ctx;

        /* Skip if there's no context to resume from. */
        if (check_jit_context(prog))
                return prog;

        if (jit_ctx_init(&ctx, prog)) {
                jit_ctx_cleanup(&ctx);
                return prog;
        }

        if (jit_resume_context(&ctx)) {
                jit_ctx_cleanup(&ctx);
                return prog;
        }

        if (jit_patch_relocations(&ctx)) {
                jit_ctx_cleanup(&ctx);
                return prog;
        }

        if (jit_finalize(&ctx)) {
                jit_ctx_cleanup(&ctx);
                return prog;
        }

        return ctx.prog;
}

/*
 * This function may be invoked twice for the same stream of BPF
 * instructions. The "extra pass" happens, when there are
 * (re)locations involved that their addresses are not known
 * during the first run.
 */
struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
{
        vm_dump(prog);

        /* Was this program already translated? */
        if (!prog->jited)
                return do_normal_pass(prog);
        else
                return do_extra_pass(prog);

        return prog;
}