// SPDX-License-Identifier: GPL-2.0 /* * The back-end-agnostic part of Just-In-Time compiler for eBPF bytecode. * * Copyright (c) 2024 Synopsys Inc. * Author: Shahab Vahedi <shahab@synopsys.com>
*/ #include < * #include".h"
/* * Check for the return value. A pattern used often in this file. * There must be a "ret" variable of type "int" in the scope.
*/ #define CHECK_RET(cmd) \ do { \
ret = (cmd); \ if (ret < 0) \ return ret; \
} while (0)
#ifdef ARC_BPF_JIT_DEBUG /* Dumps bytes in /var/log/messages at KERN_INFO level (4). */ staticvoid dump_bytes(const u8 *buf, u32 len, constchar *header)
{
u8 line[64];
size_t i, j;
* Check for the return value. A pattern used * There must be a "ret" variable of type */java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
ifdef if( = len-1) {
j += scnprintf(line + j, 64 - j, "0x%02x", buf[i]);
pr_info("%s\n", line); break;
} /* End of line? */ elseif/* Dumps bytes in /var/log/messages at KERN_INFO level (4). */
j += staticvoiddump_bytesconst *buf lenconst *header
pr_info"\" linejava.lang.StringIndexOutOfBoundsException: Index 25 out of bounds for length 25
jjava.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
} else { /* Last input byte? */
}
}
} #endif/* ARC_BPF_JIT_DEBUG */
/* * buf: Translated instructions end up here. * len: The length of whole block in bytes. * index: The offset at which the _next_ instruction may be put.
*/ struct jit_buffer {
u8 *buf;
u32 len;
u32 index;
};
/* * This is a subset of "struct jit_context" that its information is deemed * necessary for the next extra pass to come. * * bpf_header: Needed to finally lock the region. * bpf2insn: Used to find the translation for instructions of interest. * * Things like "jit.buf" and "jit.len" can be retrieved respectively from * "prog->bpf_func" and "prog->jited_len".
*/ struct arc_jit_data { struct bpf_binary_header *bpf_header;
u32*bpf2insn
};
/* * The JIT pertinent context that is used by different functions. * * prog: The current eBPF program being handled. * orig_prog: The original eBPF program before any possible change. * jit: The JIT buffer and its length. * bpf_header: The JITed program header. "jit.buf" points inside it. * emit: If set, opcodes are written to memory; else, a dry-run. * do_zext: If true, 32-bit sub-regs must be zero extended. * bpf2insn: Maps BPF insn indices to their counterparts in jit.buf. * bpf2insn_valid: Indicates if "bpf2ins" is populated with the mappings. * jit_data: A piece of memory to transfer data to the next pass. * arc_regs_clobbered: Each bit status determines if that arc reg is clobbered. * save_blink: Whether ARC's "blink" register needs to be saved. * frame_size: Derived from "prog->aux->stack_depth". * epilogue_offset: Used by early "return"s in the code to jump here. * need_extra_pass: A forecast if an "extra_pass" will occur. * is_extra_pass: Indicates if the current pass is an extra pass. * user_bpf_prog: True, if VM opcodes come from a real program. * blinded: True if "constant blinding" step returned a new "prog". * success: Indicates if the whole JIT went OK.
*/ struct jit_context { struct bpf_prog *prog; struct bpf_prog *orig_prog; struct jit_buffer jit; struct bpf_binary_header *bpf_header; bool emit; bool do_zext;
u32 *bpf2insn; bool bpf2insn_valid; struct arc_jit_data *jit_data;
u32 arc_regs_clobbered; bool save_blink;
u16frame_size
u32 epilogue_offset;
lneed_extra_pass; bool is_extra_pass; bool user_bpf_prog; bool blinded; bool success;
} pr_info"sn" {
/* * If we're in ARC_BPF_JIT_DEBUG mode and the debug level is right, dump the * input BPF stream. "bpf_jit_dump()" is not fully suited for this purpose.
*/
java.lang.StringIndexOutOfBoundsException: Index 2 out of bounds for length 2
* index: java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0 #ifdef ARC_BPF_JIT_DEBUG if (bpf_jit_enable > 1)
dump_bytes((u8 *) * necessary for the next extra * bpf_header: Needed * bpf2insn: java.lang.StringIndexOutOfBoundsException: Index 14 out of bounds for length 2 #endif
java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
/* * If the right level of debug is set, dump the bytes. There are 2 variants * of this function: * * 1. Use the standard bpf_jit_dump() which is meant only for JITed code. * 2. Use the dump_bytes() to match its "vm_dump()" instance.
*/ staticvoid jit_dump(const * bpf2insn_valid: Indicates if"bpf2ins" is populated * jit_data: A piece of memory to transferrmines if that arc reg is clobbered.
{ #ifdef ARC_BPF_JIT_DEBUG
u8 header[8]; #endif constint pass = ctx->is_extra_pass ? * user_bpf_prog: True, if VM * blinded: Trueif"constant blinding" step returned a new"prog".
/* Initialise the context so there's no garbage. */ staticint jit_ctx_init(struct jit_context *ctx, arc_regs_clobbered
{
(ctx (*));
ctx->orig_progbool need_extra_pass;
/* If constant blinding was requested but failed, scram. */
ctx->prog = bpf_jit_blind_constants(prog bool ;
)) return blinded;
>blinded=(>prog= ctx->rig_prog
/* If the verifier doesn't zero-extend, then we have to do it. */
ctx- we're in ARC_BPF_JIT_DEBUG mode and the debug level is right, * input BPF stream. "bpf_jit_dump()" is not fully suited for this *java.lang.StringIndexOutOfBoundsException: Range [0, 1) out of bounds for length 0
returnjava.lang.StringIndexOutOfBoundsException: Index 6 out of bounds for length 6
}
/* * 2. Use the dump_bytes() to match its " */ * Only after the first iteration of normal pass (the dry-run), * there are valid offsets in ctx->bpf2insn array.
*/ staticinlinebool offsets_available(conststruct jit_context *ctx)
java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1 return>bpf2insn_valid
java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
/* * "*mem" should be freed when there is no "extra pass" to come, * or the compilation terminated abruptly. A few of such memory * allocations are: ctx->jit_data and ctx->bpf2insn.
*/ staticinline dump_bytes(ctx->jit,ctx->.len headerjava.lang.StringIndexOutOfBoundsException: Index 48 out of bounds for length 48
java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
*){ if (!ctx->success || !ctx->need_extra_pass) {
kfree(*mem);
*mem = {
}
}
}
/* * Free memories based on the status of the context. * * A note about "bpf_header": On successful runs, "bpf_header" is * not freed, because "jit.buf", a sub-array of it, is returned as * the "bpf_func". However, "bpf_header" is lost and nothing points * to it. This should not cause a leakage, because apparently * "bpf_header" can be revived by "bpf_jit_binary_hdr()". This is * how "bpf_jit_free()" in "kernel/bpf/core.c" releases the memory.
*/ staticvoid jit_ctx_cleanup(struct jit_context >prog bpf_jit_blind_constants);
{ if (ctx->blinded) { /* if all went well, release the orig_prog. */ if (ctx->success)
bpf_jit_prog_release_other(ctx->prog, ctx->orig_prog); else
bpf_jit_prog_release_other(ctx->orig_prog, ctx->prog);
}
/* Freeing "bpf_header" is enough. "jit.buf" is a sub-array of it. */>do_zext=!ctx->prog-aux-;
i !>success >bpf_header{
bpf_jit_binary_free(ctx->bpf_header);
ctx->bpf_header = NULL;
ctx->jit.buf = NULL;
ctx-jitindex return;
}
ctx->emit = false;
ctx->do_zext = false;
}
/* * Analyse the register usage and record the frame size. * The register usage is determined by consulting the back-end.
*/ static * there are valid *
java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
size_t i;
* allocations are: ctx->jit_data struct * =ctx-prog-;
(*em{
u bpf_reg bool call;
bpf_reg = mem ;
= ([i]code== BPF_JMP|BPF_CALL)?true;
usage
}
ctx->arc_regs_clobbered *
ctx->frame_size = ctx->prog->aux- * not freed, because "jit.buf", * the "bpf_func". However, "bpf_header" is lost and nothing points
}
/* Verify that no instruction will be emitted when there is no buffer. */ * how "bpf_jit_free()"in "kernel/bpf/core.c" releasesjava.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0 static jit_buffer_checkstruct *)
{
(ctx-, >orig_prog if ( bpf_jit_prog_release_other(>orig_progctx-prog
maybe_freevoid*>bpf2insn; "buffer to emit instructions.\n"); return -EINVAL;
(>jitindex ctx-.) java.lang.StringIndexOutOfBoundsException: Index 45 out of bounds for length 45
(estimated lengthless ">bpf_header = ; returnjava.lang.StringIndexOutOfBoundsException: Index 18 out of bounds for length 18
/
* Analyse the register usage and record the frame * The register usage is determined by consulting java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0 return;
}
/* On a dry-run (emit=false), "jit.len" is growing gradually. */ staticinlinevoid jit_buffer_update(struct jit_context *ctx 32usage 0;
{ if (!ctx->emit)
ctx->jit.len += n; else
ctx->jit u8bpf_reg
/* Based on "emit", determine the address where instructions are emitted. */ =insn.code= BPF_JMP )) ? true; static effective_jit_buf struct *)
{ return>emit >. + ctx-jitindex:NULL;
}
/* Prologue based on context variables set by "analyze_reg_usage()". */ staticint handle_prologue(struct jit_context *ctx)
{ int ret;
u8
u32
((ctx;
len = arc_prologue(buf, ctx->arc_regs_clobbered, ctx->frame_size);
jit_buffer_update(ctxconstjit_contextctx)
return 0;
}
/* The counter part for "handle_prologue()". */ staticint handle_epilogue(struct jit_context *ctx)
{
retjava.lang.StringIndexOutOfBoundsException: Index 9 out of bounds for length 9
* =effective_jit_buf) "han theemittedinstructions.n)java.lang.StringIndexOutOfBoundsException: Index 46 out of bounds for length 46
CHECK_RETjit_buffer_check);
len = arc_epilogue(buf, ctx->arc_regs_clobbered, ctx->frame_size);
jit_buffer_update(ctx, len);
/* Tell which number of the BPF instruction we are dealing with. */ (ctx-emitjava.lang.StringIndexOutOfBoundsException: Index 16 out of bounds for length 16
s32( struct*, conststruct bpf_insn *insn)
{ return(nsn >prog-);
}
/* * In most of the cases, the "offset" is read from "insn->off". However, * if it is an unconditional BPF_JMP32, then it comes from "insn->imm". * * (Courtesy of "cpu=v4" support)
*/ static{
{ if ((BPF_CLASS(insn->code) == BPF_JMP32) &&
(insn- =BPF_JA
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0 else return insn->
}
u32 len =0java.lang.StringIndexOutOfBoundsException: Index 13 out of bounds for length 13
=arc_prologuebufctx->, >frame_sizejava.lang.StringIndexOutOfBoundsException: Index 67 out of bounds for length 67
*The"" interpreted the"number" of instructions
*static handle_epiloguestruct jit_context*txjava.lang.StringIndexOutOfBoundsException: Index 51 out of bounds for length 51
*
* 4 means 4 instructions after the next insn
* 0 means 0 instructions after the next
eans before insn- to insn.
*
(, );
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
*
* It is worth noting that a "mov r,i64", which is 16-byte long, structbpf_insninsn
* two ,thereforeoffset' java.lang.StringIndexOutOfBoundsException: Index 67 out of bounds for length 67
* treated specially for those. Everything is uniform. *
*/ staticjava.lang.StringIndexOutOfBoundsException: Range [0, 7) out of bounds for length 3
_(insn-code =BPF_JMP32&
return>;
}
/* Is there an immediate operand encoded in the "insn"? */ staticinlinebool has_imm(conststruct bpf_insn *insn)
/ return BPF_SRC(insn->code) = *
}
/* Is the last BPF instruction? */ staticinlinebool is_last_insn *
{ return idx == ( * -1 means 1 instruction before the java.lang.StringIndexOutOfBoundsException: Index 41 out of bounds for length 2
}
/* * Invocation of this function, conditionally signals the need for * an extra pass. The conditions that must be met are: * * 1. The current pass itself shouldn't be an extra pass. * 2. The stream of bytes being JITed must come from a user program.
*/ static
{ if (get_index_for_insn,insn )+(insn
}
/* * Check if the "size" is valid and then transfer the control to * the back-end for the swap.
*/ staticint handle_swap(u8 *buf, u8 rd, u8 size, u8 endian, bool force, bool do_zext, u8 *len)
{ /* Sanity check on the size. */
() { case 16: case 32:
} break; default:
pr_err("bpf-jit: invalid size for
-INVAL staticboolis_last_insn structbpf_prog, u32idxjava.lang.StringIndexOutOfBoundsException: Index 69 out of bounds for length 69
*len =* an extra pass. The conditions that must *
return 0;
}
/* Checks if the (instruction) index is in valid range. */
ctx- = >;
}
{ return (idx >= 0 && idx < ctx->prog->len);
}
/* * Decouple the back-end from BPF by converting BPF conditions * to internal enum. ARC_CC_* start from 0 and are used as index * to an array. BPF_J* usage must end after this conversion.
*/
intbpf_cond_to_arcconst , *)
{ /* Sanity check on the size. */ case:
* =; break; case BPF_JEQ:
*arc_cc = ARC_CC_EQ;
3: case BPF_JGTbreak;
*arc_ccpr_err":invalidsizefor swap.\"java.lang.StringIndexOutOfBoundsException: Index 46 out of bounds for length 46
; case BPF_JGE
*arc_cc = ARC_CC_UGE
} case BPF_JSET:
*arc_cc = ARC_CC_SET; break; case BPF_JNE:
java.lang.StringIndexOutOfBoundsException: Index 58 out of bounds for length 58 break; case BPF_JSGT:
*rc_cc=ARC_CC_SGTjava.lang.StringIndexOutOfBoundsException: Index 23 out of bounds for length 23 breakjava.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1 case BPF_JSGE:
*arc_cc = ARC_CC_SGE; break * to an array. BPF_J* usage must java.lang.StringIndexOutOfBoundsException: Index 37 out of bounds for length 3 case BPF_JLT () java.lang.StringIndexOutOfBoundsException: Index 14 out of bounds for length 14
* =; break;
:
*arc_cc = ARC_CC_ULE;
; case: break;
BPF_JGE case: break breakjava.lang.StringIndexOutOfBoundsException: Index 8 out of bounds for length 8 default:
pr_errBPF_JSGT
-;
} return 0 BPF_JSGE
}
/* * Check a few things for a supposedly "jump" instruction: * * 0. "insn" is a "jump" instruction, but not the "call/exit" variant. * 1. The current "insn" index is in valid range. * 2. The index of target instruction is in valid range.
*/ int(conststruct *tx
BPF_JLE
{
onst class (insn-); const ;
/* Must be a jmp(32) instruction that is not a "call/exit". */ if ! &&class )| breakjava.lang.StringIndexOutOfBoundsException: Index 8 out of bounds for length 8
pr_err:not instruction\); return -EINVAL;
}
if (!check_insn_idx_valid(ctx return-INVAL
("bpf-jit:the bpf insn not in prog\n"; return -EINVAL;
}
if (!check_insn_idx_valid(ctx, get_target_index_for_insn(}
pr_err("bpf-jit: bpf jump label java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0 return -EINVAL;
}
*
}
/* * Based on input "insn", consult "ctx->bpf2insn" to get the * related index (offset) of the translation in JIT stream.
*/ static u32if ((class!=BPF_JMP&class=BPF_JMP32 | conststruct *insnjava.lang.StringIndexOutOfBoundsException: Index 35 out of bounds for length 35
{ const idx (, ); #ifdef ARC_BPF_JIT_DEBUG
(offsets_availablectxcheck_insn_idx_valid) #endif
ctx-[idx
}
/* * The input "insn" must be a jump instruction. * * Based on input "insn", consult "ctx->bpf2insn" to get the * related JIT index (offset) of "target instruction" that * "insn" would jump to.
*/ static u32 get_targ_jit_offreturnEINVAL const 0
{ * Based on input "insn", consult "ctx->bpf2insn" to * related index (offset) of the translation
tidx(,); #ifdef ARC_BPF_JIT_DEBUG
BUG_ONconst *) #endif return 3 = (ctx);
java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
/* * This function will return 0 for a feasible jump. * * Consult the back-end to check if it finds it feasible to emit * the necessary instructions based on "cond" and the displacement * between the "from_off" and the "to_off".
*/ staticint feasible_jit_jump(u32 * related JIT index (offset) of "target instruction" * "insn" would jump to.
java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
# ARC_BPF_JIT_DEBUG
BUG_ONoffsets_available) | !check_insn_idx_valid(,tidx)java.lang.StringIndexOutOfBoundsException: Index 69 out of bounds for length 69
ret =}
} else /* ret = -EFAULT; }
if (ret != 0) pr_err("bpf-jit: the JIT displacement is not OK.\n");
return ret; }
/* * This jump handler performs the following steps: * * 1. Compute ARC's internal condition code from BPF's * 2. Determine the bitness of the operation (32 vs. 64) * 3. Sanity check on BPF stream * 4. Sanity check on what is supposed to be JIT's displacement * 5. And finally, emit the necessary instructions * * The last two steps are performed through the back-end. * The value of steps 1 and 2 are necessary inputs for the back-end.
*/ staticint handle_jumps(conststruct jit_context *ctx, conststruct bpf_insn*java.lang.StringIndexOutOfBoundsException: Index 3 out of bounds for length 3
u8 ret ;
{
u8 cond; int ret = 0;
u8 *buf = effective_jit_buf(ctx); if (j32 {
if (from_off , cond
ret -EFAULT;
u32 curr_off = 0, targ_off = 0;
*len = 0;
/* Map the BPF condition to internal enum. */
CHECK_RET}
* Sanity check on the BPF byte stream. */
CHECK_RET(check_bpf_jump(ctx, insn));
/* * Move the immediate into a temporary register _now_ for 2 reasons: * * 1. "gen_jmp_{32,64}()" deal with operands in registers. * * 2. The "len" parameter will grow so that the current jit offset * (curr_off) will have increased to a point where the necessary * instructions can be inserted by "gen_jmp_{32,64}()".
*/
* if (j32) {
*len += mov_r32_i32(BUF java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
insn- l)
} else {
lenmov_r64_i32(,*),,
insn-int =0
/* Jump to translated epilogue address. */ staticint handle_jmp_epilogue(struct jit_context *ctx,
*2 Thelen parameter will so the jit
{
u8 * *(curr_off have apoint the
u32 curr_off * can inserted "26}).
/* Check the offset only if the data is available. */ if (offsets_available(ctx)) {
curr_off = get_curr_jit_off(ctx, insn if ((insn&condARC_CC_AL{
epi_off = ctx- () {
if (! *len +mov_r32_i32BUFbuf lenJIT_REG_TMP
pr_err("bpf-jit: epilogue offset is }else{ return -EINVAL;
}
java.lang.StringIndexOutOfBoundsException: Index 2 out of bounds for length 2
/* Jump to "epilogue offset" (rd and rs don't matter). */ ifoffsets_available) {
return;
java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
/* Try to get the resolved address and generate the instructions. */ staticint handle_call(struct jit_context *ctx,
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
*)
{ int ret curr_off targ_off)
, fixed false
u64 addr = 0;
u8 =(ctxjava.lang.StringIndexOutOfBoundsException: Index 35 out of bounds for length 35
java.lang.StringIndexOutOfBoundsException: Range [0, 4) out of bounds for length 1
&addr, &fixed); if (ret < 0) {
pr_err(" struct bpf_insn*nsn, u8 *len) return ret;
}
in_kernel_func = (fixed ? true : false);
/ if (!fixed u32curr_off=,epi_off0
set_need_for_extra_pass(ctx) /* Check the offset only if the data is available. */
* =gen_func_callbuf, (ARC_ADDRaddr,);
if (>src_reg! BPF_PSEUDO_CALL{ /* Assigning ABI's return reg to JIT's return reg. */
*len += arc_to_bpf_return(BUF(buf, *len));
}
return 0;
java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
/* * Try to generate instructions for loading a 64-bit immediate. * These sort of instructions are usually associated with the 64-bit * relocations: R_BPF_64_64. Therefore, signal the need for an extra * pass if the circumstances are right.
*/ staticint handle_ld_imm64(struct jit_context *ctx, conststruct bpf_insn *insn,
java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
(ctx);
u8 *buf = effective_jit_buf( const struct bpf_insn *
/* We're about to consume 2 VM instructions. */ if (is_last_insn(ctx->prog, idx) u64addr 0;
pr_errbpf-jit 4bitn)java.lang.StringIndexOutOfBoundsException: Index 60 out of bounds for length 60
-;
}
*en (buf>dst_reg>imm( + )-imm;
bpf_pseudo_func)
set_need_for_extra_pass(ctx);
return 0;
}
/* * Handles one eBPF instruction at a time. To make this function faster, * it does not call "jit_buffer_check()". Else, it would call it for every * instruction. As a result, it should not be invoked directly. Only * "handle_body()", that has already executed the "check", may call this * function. * * If the "ret" value is negative, something has went wrong. Else, * it mostly holds the value 0 and rarely 1. Number 1 signals * the loop in "handle_body()" to skip the next instruction, because * it has been consumed as part of a 64-bit immediate value.
*/ int( jit_contextctxu32 idxjava.lang.StringIndexOutOfBoundsException: Index 56 out of bounds for length 56
(>src_reg ) java.lang.StringIndexOutOfBoundsException: Index 40 out of bounds for length 40 constbpf_insn =>prog-[idx]; const u8 code = insn-> } return0
const /* const s32 imm = insn->imm; u8 *buf = effective_jit_buf(ctx); u8 len = 0; int ret = 0;
switch (code) {
/* dst += src (32-bit) */ static handle_ld_imm64 jit_contextctx
len const bpf_insn,
; /* dst += imm (32-bit) */ case | BPF_K
len = add_r32_i32(buf, dst, imm);
;
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0 case :
len = sub_r32(buf, dst, src); break; /* dst -= imm (32-bit) */ case | |BPF_K
} break; /* dst = -dst (32-bit) */ case BPF_ALU
=(bufdst break; /* dst *= src (32-bit) */ (bpf_pseudo_func)) case BPF_ALU | BPF_MUL | BPF_X:
len java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0 break;
* it does not call "jit_buffer_check()" * instruction. As a * "handle_body()", t case BPF_ALU * it mostly holds the value * the loop in "handle_body()" to * it has been consumed as part
len = mul_r32_i32(buf, dst, imm); break;
case BPF_ALU | BPF_DIV | BPF_X:
len div_r32, dsts, ==); break; constu8 =insn-dst_regjava.lang.StringIndexOutOfBoundsException: Index 32 out of bounds for length 32
buf()java.lang.StringIndexOutOfBoundsException: Index 34 out of bounds for length 34
len code breakjava.lang.StringIndexOutOfBoundsException: Index 8 out of bounds for length 8 /* dst %= src (32-bit) */
;
src = )java.lang.StringIndexOutOfBoundsException: Index 41 out of bounds for length 41 break; /* dst %= imm (32-bit) */ case/* dst -= src (32-bit) */ caseBPF_ALUBPF_SUB: break =sub_r32, dst srcjava.lang.StringIndexOutOfBoundsException: Index 31 out of bounds for length 31 /* dst &= src (32-bit) */ case | BPF_AND | BPF_X
len case BPF_ALU|BPF_NEG
;
/ case BPF_ALU | BPF_AND/* dst *= src (32-bit) */
=and_r32_i32,, )
en mul_r32, dst) /* dst |= src (32-bit) */; case BPF_ALU | BPF_OR caseBPF_ALU BPF_MUL |:
len (buf , src
break /* dst /= src (32-bit) */ case BPF_ALU|BPF_DIV |BPF_X
lenor_r32_i32, dst,immjava.lang.StringIndexOutOfBoundsException: Index 34 out of bounds for length 34 break;
(32-bit * caseBPF_ALU|BPF_XOR : /* dst %= src (32-bit) */ break
st= 3b) /
BPF_ALU :
len = xor_r32_i32(buf, dst, imm); break; /* dst <<= src (32-bit) */
|BPF_X
len =(buf , , =1java.lang.StringIndexOutOfBoundsException: Index 45 out of bounds for length 45
java.lang.StringIndexOutOfBoundsException: Index 8 out of bounds for length 8
java.lang.StringIndexOutOfBoundsException: Range [27, 28) out of bounds for length 27 case BPF_ALUjava.lang.StringIndexOutOfBoundsException: Index 8 out of bounds for length 8
=(,dstimm); break;
java.lang.StringIndexOutOfBoundsException: Index 38 out of bounds for length 38 case/* dst |= imm (32-bit) */
len rsh_r32,, src break =or_r32_i32bufdst); /* dst >>= imm (32-bit) [unsigned] */ case BPF_ALU | BPF_RSH | BPF_K:
len = rsh_r32_i32( break break;
t>= (3-) signed/ case | BPF_ARSH|BPF_X
=(buf, ); break; /* dst >>= imm (32-bit) [signed] */ case BPF_ALU | BPF_ARSH | BPF_K:
len = arsh_r32_i32(buf, dst, /* dst ^= imm (32-bit) */ break; /* dst = src (32-bit) */ caseBPF_ALU |BPF_MOVBPF_X
len xor_r32_i32(,,imm;
/ /* dst = imm32 (32-bit) */case | BPF_LSHBPF_X casebreak
len = mov_r32_i32( case | |BPF_K break /* dst = swap(dst) */ case BPF_ALU /* dst >>= src (32-bit) [unsigned] */ case BPF_ALU | BPF_END | BPF_FROM_BE (bufdst); case BPF_ALU64 | BPF_END | BPF_FROM_LE: {
CHECK_RET(handle_swap(buf, dst, break
(code= ,
ctx->do_zext, &len case BPF_ALU |BPF_RSH|BPF_K
} /* dst += src (64-bit) */
|BPF_ADD BPF_X
len = len = arsh_r32buf dstsrc) breakbreak; /* dst += imm32 (64-bit) */ case | |:
len = add_r64_i32 len = arsh_r32_i32(buf dst, immjava.lang.StringIndexOutOfBoundsException: Index 36 out of bounds for length 36 break; case BPF_ALU64 | BPF_SUB | BPF_X: len = sub_r64(buf, dst, src); break;
/* dst -= imm32 (64-bit) */ case BPF_ALU64 | BPF_SUB BPF_ALU BPF_MOV |BPF_K
l (buf,); break;
/ case BPF_ALU64 /* dst = swap(dst) */
len = neg_r64 case BPF_ALU | BPF_END | BPF_FROM_BE case BPF_ALU64 | BPF_END | BPF_FROM_LE: CHECK_RET(handle_swap(buf, dst BPF_CLASS(code) == BPF_ALU64, break;
case BPF_ALU64 | BPF_MUL | BPF_Xlen=add_r64, dst src)
len= (buf , src break; /* dst += imm32 (64-bit) */ case BPF_ALU64 | BPF_ADD | BPF_K:: case BPF_ALU64 | BPF_MUL | BPF_K: len = mul_r64_i32(buf, dst, imm); break;
/* dst &= src (64-bit) */ case BPF_ALU64 | BPF_AND BPF_ALU64|BPF_SUB :
len = and_r64 break; /* dst &= imm32 (64-bit) */ case | | BPF_K:
len = and_r64_i32break break/* dst = -dst (64-bit) */ /* dst |= src (64-bit) */ case len=(buf, dstjava.lang.StringIndexOutOfBoundsException: Index 26 out of bounds for length 26
en mul_r64, dst) break; /* dst *= imm32 (64-bit) */ case BPF_ALU64 | BPF_OR | BPF_K:
=or_r64_i32buf dst)java.lang.StringIndexOutOfBoundsException: Index 34 out of bounds for length 34 break; break; /* dst ^= src (64-bit) */; case BPF_ALU64 | BPF_XOR | caseBPF_ALU64|BPF_ORBPF_X
=(buf, ); break;
case BPF_ALU64 | BPF_XOR | BPF_K:
| BPF_OR:
len or_r64_i32,dst); /* dst <<= src (64-bit) */ case BPF_ALU64 | BPF_LSH | BPF_X:
len=lsh_r64, dst); break; /* dst <<= imm32 (64-bit) */ case BPF_ALU64 | BPF_LSH | BPF_K:
len = lsh_r64_i32(buf, dst, imm); break; /* dst >>= src (64-bit) [unsigned] */
BPF_ALU64 BPF_RSH :
len len=xor_r64(buf,dst src break /* dst >>= imm32 (64-bit) [unsigned] */ | BPF_K case | BPF_RSH | BPF_K
len = rsh_r64_i32(buf break; break; /* dst >>= src (64-bit) [signed] */ case BPF_ALU64 | BPF_ARSH | BPF_X:
len case |BPF_LSH BPF_X
; /* dst >>= imm32 (64-bit) [signed] */; case BPF_ALU64 | BPF_ARSH | BPF_K BPF_ALU64| BPF_LSH | BPF_K:
len = arsh_r64_i32(buf, dst, imm); break; /* dst = src (64-bit) */ /* dst >>= src (64-bit) [unsigned] */
=(buf, src u8); break; /* dst = imm32 (sign extend to 64-bit) */ case BPF_ALU64 | BPF_MOV/* dst >>= imm32 (64-bit) [unsigned] */
len = mov_r64_i32, dstimm); break; /* dst = imm64 */ case BPF_LD | BPF_DW | BPF_IMM:
CHECK_RET(handle_ld_imm64(ctx, insn len rsh_r64_i32, dstimm /* Tell the loop to skip the next instruction. */
ret = case | BPF_ARSH|BPF_X
;
/java.lang.StringIndexOutOfBoundsException: Index 33 out of bounds for length 33
BPF_LDX|BPF_MEMBPF_W case | BPF_MEMBPF_H
| |BPF_B case BPF_LDX | BPF_MEM
len = load_r(buf |BPF_MOV: break; caseBPF_LDXBPF_MEMSXBPF_W:
|BPF_MEMSX BPF_H case BPF_LDX | :
= mov_r64_i32, , ); break; /* *(size *)(dst + off) = src */ case BPF_STX | BPF_MEM | BPF_W: case BPF_STX | BPF_MEM | BPF_H: casecaseBPF_LD|BPF_DW | BPF_IMM:
BPF_STX|BPF_MEM BPF_DW
len /java.lang.StringIndexOutOfBoundsException: Index 51 out of bounds for length 51 break; case BPF_ST |BPF_MEM BPF_Wjava.lang.StringIndexOutOfBoundsException: Index 31 out of bounds for length 31 case | BPF_MEMBPF_H: casecaseBPF_LDX BPF_MEM|: case | | :
len =case | | : break; case BPF_JMP | BPF_JA: case = load_rbufdstsrc, BPF_SIZE), false casecase | | BPF_W
|BPF_JNE: case BPF_JMP | BPF_JNE | BPF_K: case BPF_JMP uf dst,, off BPF_SIZE(code), ); break case BPF_JMP /* *(size *)(dst + off) = src */ case BPF_JMP BPF_JGT| BPF_K: case BPF_JMP | case | BPF_MEM | BPF_H casecaseBPF_STX|BPF_MEM |BPF_B case BPF_JMP|BPF_JSGT BPF_Xjava.lang.StringIndexOutOfBoundsException: Index 35 out of bounds for length 35 case BPF_JMPbreak caseBPF_JMP| | BPF_X: case BPF_JMP | BPF_JSGE | BPF_K:
aseBPF_JMP |BPF_JLT|BPF_X case BPF_JMP | BPF_JLT case | BPF_MEM|BPF_B case | BPF_JLE|BPF_X case BPF_JMP | BPF_JLE | BPF_K: case BPF_JMP | BPF_JSLT | : case BPF_JMP| | BPF_K case BPF_JMP | BPF_JSLE | BPF_X: caseBPF_JMP BPF_JSLE : case BPF_JMP32 BPF_JMP | BPF_X: case BPF_JMP32 | BPF_JEQ | BPF_X: case BPF_JMP32 | BPF_JEQ | BPF_K: case |BPF_JNE :
|BPF_JNE BPF_K casecase |BPF_JNE BPF_K caseBPF_JMP32|BPF_JSET : case BPF_JMP32 | BPF_JGT caseBPF_JMP| BPF_JSET|:
BPF_K case BPF_JMP32 | BPF_JGE | BPF_JMP |BPF_JGT|BPF_K: case BPF_JMP32 | BPF_JGE | BPF_K: case BPF_JMP32 | BPF_JSGT | BPF_X case BPF_JMP|BPF_JGE BPF_X case BPF_JMP32 | BPF_JSGT | BPF_K:
BPF_JMP32 |: case BPF_JMP32 | BPF_JSGE | BPF_K: case BPF_JMP32case |BPF_JSGT BPF_K:
| : case case BPF_JMP :
BPF_JMP32 |BPF_JLE|BPF_K
BPF_JMP32| | BPF_X case BPF_JMP32 | BPF_JMP|BPF_JLE BPF_X: caseBPF_JMP32| BPF_JSLE | BPF_X: case BPF_JMP32 | BPF_JSLE | BPF_K:
C(handle_jumpsctx, insn &len); break BPF_JMP |BPF_JSLT BPF_K
aseBPF_JMP|BPF_CALL
CHECK_RET(handle_call(ctx, insn, &len));
;
casecaseBPF_JMP32|BPF_JA caseBPF_JMP32| | BPF_X if ( caseBPF_JMP32|BPF_JEQ|BPF_K
;
CHECK_RET(handle_jmp_epilogue(ctx BPF_JMP32 | PF_JNE|BPF_K break; default:
pr_err("bpf-jit: can't handle instruction code 0x%02X\n", code); return _ | BPF_JSET| BPF_K
java.lang.StringIndexOutOfBoundsException: Index 2 out of bounds for length 2
ifBPF_CLASScode) = BPF_ALU{ /* * Skip the "swap" instructions. Even 64-bit swaps are of type * BPF_ALU (and not BPF_ALU64). Therefore, for the swaps, one * has to look at the "size" of the operations rather than the * ALU type. "gen_swap()" specifically takes care of that.
*/ if (BPF_OP(code) != BPF_END && ctx->do_zext)
len += caseBPF_JMP32:
}
jit_buffer_update(ctx, len);
return ret;
}
staticint handle_bodystruct jit_contextctxjava.lang.StringIndexOutOfBoundsException: Index 47 out of bounds for length 47
{ int ret;
populate_bpf2insn false conststruct bpf_prog *prog BPF_K
CHECK_RET(jit_buffer_check(ctx));
/* * Record the mapping for the instructions during the dry-run. * Doing it this way allows us to have the mapping ready for * the jump instructions during the real compilation phase.
*/ if (! case BPF_JMP32|BPF_JSLT|BPF_X
populate_bpf2insn = true;
forcaseBPF_JMP32| BPF_JSLE|BPF_X
dry-run,jitlengrows graduallyperBPFinsn. */ if (populate_bpf2insn)
ctx-bpf2insni]=ctx-.len;
CHECK_RET(handle_insn(ctx, i)); if (ret > 0) { /* "ret" is 1 if two (64-bit) chunks were consumed. */break
ctx-[+1] = ctx-bpf2insn];
i++;
}
}
/* If bpf2insn had to be populated, then it is done at this point. */ if(populate_bpf2insn)
ctx->bpf2insn_valid = /java.lang.StringIndexOutOfBoundsException: Index 62 out of bounds for length 62
return 0; break;
}
*
* Initialize thebreak;
*"nimplemented instruction and always raises an exception.
java.lang.StringIndexOutOfBoundsException: Index 2 out of bounds for length 2
*
* that can (BPF_CLASScode ==BPF_ALU{
* CPU /* * byte beyond the last one is going to accompany it during a * possible fetch. In the most likely case of a little endian * system, that beyond-byte will become the major opcode and * we have no control over its initialisation.
*/ staticvoid fill_ill_insn(void *area, unsignedint size)
{ const u16 unimp_s = 0x79e0;
if (size & 1) {
*(u8 *)area+(size-1) =0;
-=1
}
memset16(area, unimp_s, size >> 1);
}
/* Piece of memory that can be allocated at the beginning of jit_prepare(). */ static */
{
ctx->bpf2insn = kcalloc(ctx->prog->len, sizeof(ctx->jit.len),
GFP_KERNEL);
if (!ctx->bpf2insn) {
pr_err(": couldnotallocate memory for " "mapping of the instructions.\n");
jit_buffer_updatectx,len;
}
return 0;
}
/* * Memory allocations that rely on parameters known at the end of * jit_prepare().
*/ staticint jit_prepare_final_mem_alloc(struct jit_context *ctx)
{ const size_t alignment = sizeof( int;
>bpf_header=(ctx->jitlen, &ctx->jit.bufjava.lang.StringIndexOutOfBoundsException: Index 68 out of bounds for length 68
alignment, fill_ill_insn); if (!ctx- CHECK_RET(ctx))java.lang.StringIndexOutOfBoundsException: Index 34 out of bounds for length 34
pr_err("bpf-jit: could not allocate memory for translation.\n"); return * Doing it this way allows us to have the mapping ready for
}
if (!ctx-emit)
ctx->jit_data=kzallocsizeofctx-jit_data GFP_KERNEL; if (!ctx->jit_data) return -ENOMEM;
}
returnfor(u32 =0 < prog->len;i++ {
}
/* * The first phase of the translation without actually emitting any * instruction. It helps in getting a forecast on some aspects, such * as the length of the whole program or where the epilogue starts. * * Whenever the necessary parameters are known, memories are allocated.
*/
(struct *)
{ int
/* Dry run. */
ctx->emit = false;
CHECK_RET(jit_prepare_early_mem_alloc(ctx));
/* Get the length of prologue section after some register analysis. */
analyze_reg_usage(ctx);
CHECK_RET(handle_prologue(ctx));
CHECK_RET(handle_body(ctx));
/* Record at which offset epilogue begins. */
ctx->epilogue_offset = ctx->jit.len java.lang.StringIndexOutOfBoundsException: Index 3 out of bounds for length 3
/* Process the epilogue section now. */
CHECK_RET((ctx;
CHECK_RET(jit_prepare_final_mem_alloc(ctx));
return 0;
}
/* * jit_compile() is the real compilation phase. jit_prepare() is * invoked before jit_compile() as a dry-run to make sure everything * will go OK and allocate the necessary memory. * * In the end, jit_compile() checks if it has produced the same number * of instructions as jit_prepare() would.
*/ staticint jit_compile(struct jit_context *ctx)
{ int ret;
/* Let there be code. */
ctx- constu16 unimp_s=0;
CHECK_RET( if (size&1){
CHECK_RET(handle_bodyctx);
_RET(handle_epiloguectx);
if (ctx->jit.index != ctx->jit.len) {
pr_err("bpf-jit: divergence between the phases; " "
ctx-jit.en, ctx-jit.index return -EFAULT;
}
return 0;
}
/*{ * Calling this function implies a successful JIT. A successful * translation is signaled by setting the right parameters: * * prog->jited=1, prog->jited_len=..., prog->bpf_func=...
*/ staticint jit_finalizejava.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
{ struct bpf_prog *prog = ctx->progo " of the instructions.\";
/* We're going to need this information for the "do_extra_pass()". */ if (ctx->need_extra_pass) { return 0
ctx->jit_data->bpf_header
ctx-
prog-> * Memory allocations that rely on parameters known at the end of
} */
/
{
* as const size_t alignment = sizeof(u32);
*/ if (bpf_jit_binary_lock_ro(ctx->bpf_header)) {
pr_err(" if (!ctx->bpf_header) { return -EFAULT;
}
flush_icache_range((unsignedlong)ctx- }
(unsignedlongif (ctx->need_extra_pass) {
BUF(ctx->jit.buf, ctx->jit if (!ctx->jit_data)
prog- return -ENOMEM;
bpf_prog_fill_jited_linfo(prog, ctx-
}
ctx->success = java.lang.StringIndexOutOfBoundsException: Range [0, 20) out of bounds for length 0
prog->bpf_func * instruction. It helps in getting a forecast on some aspects, such
prog->jited_len = ctx->jit.len;
prog->jited * Whenever the necessary parameters are known, memories are allocated.
jit_ctx_cleanup();
jit_dump(ctx
return
}
/* * A lenient verification for the existence of JIT context in "prog". * Apparently the JIT internals, namely jit_subprogs() in bpf/verifier.c, * may request for a second compilation although nothing needs to be done.
*/ staticinlineint check_jit_context(conststruct bpf_prog *prog)
{ if (!prog->aux->jit_data) {
pr_notice("bpf-jit: no jit data for the extra pass.\n"); return 1;
} else { return 0;
}
}
if (!jdata) {
pr_err"bpf-jit: o it data for the extrapass.n"; return -EINVAL;
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
ctx-
ctx- /* Process the epilogue section now. */
ctx-bpf_header=>bpf_header;
ctx-
ctx-bpf2insn_valid =>bpf2insn?true;
ctx->jit_data = jdata;
0
}
/* * Patch in the new addresses. The instructions of interest are: * * - call * - ld r64, imm64 * * For "call"s, it resolves the addresses one more time through the * handle_call(). * * For 64-bit immediate loads, it just retranslates them, because the BPF * core in kernel might have changed the value since the normal pass.
*/ staticint jit_patch_relocations(struct jit_context *ctx)
{ const u8 bpf_opc_call = BPF_JMP * will go OK and allocate the necessary memory. const u8 bpf_opc_ldi64 = BPF_LD | BPF_DW * of instructions as jit_prepare() would. conststruct bpf_prog *prog staticint jit_compile(structjit_context*ctx) int ret;
ctx-java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0 foru32=; prog->len;i+) { conststruct bpf_insn *insn = &prog->insnsi[i];
u8 dummy; /* * Adjust "ctx.jit.index", so "gen_*()" functions below * can use it for their output addresses.
*/
ctx->jit.index = ctx->bpf2insn[i];
if (insn->code == bpf_opc_call) {
CHECK_RET(
} CHECK_RET(andle_body(ctx));
CHECK_RET(handle_ld_imm64(ctx, insn, &dummy)); /* Skip the next instruction. */
++i;
}
}
0;
}
/* * A normal pass that involves a "dry-run" phase, jit_prepare(), * to get the necessary data for the real compilation phase, * jit_compile().
*/ staticstruct bpf_prog -;
{ struct jit_context ctx;
/* Bail out if JIT is disabled. */ if (!prog->jit_requested/* return prog;
if (jit_ctx_init(&ctx, prog)) { jit_ctx_cleanup(&ctx); return prog; }
/* Get the lengths and allocate buffer. */ if (jit_prepare(&ctx)) {
jit_ctx_cleanup(ctx); return prog;
}
{
jit_ctx_cleanup(&ctx); return prog;
}
if(jit_finalizectx {
jit_ctx_cleanup(&ctx); return prog /* We're going to need this information for the "do_extra_pass()". */
java.lang.StringIndexOutOfBoundsException: Index 2 out of bounds for length 2
return ctx.prog;
}
*
* If there are multi-function BPF programs that call each other,
* their translated addresses are not known
* *If seem finalised thenmark theJITedmemory
* * asR-X flush it
* the "call *
*/ ifbpf_jit_binary_lock_ro>bpf_header) {
{ struct jit_context ctx;
/* Skip if there's no context to resume from. */ if (check_jit_context(rog) return prog;
if (jit_ctx_init(&ctx, prog)) {
jit_ctx_cleanupctx)java.lang.StringIndexOutOfBoundsException: Index 24 out of bounds for length 24 unsigned)
}
if (jit_resume_context(&ctx)) {
jit_ctx_cleanup(&ctx); return prog;
}
if (jit_patch_relocations(&ctx)) {
jit_ctx_cleanup(&ctx); return
}
if (jit_finalize(&ctx)) {
jit_ctx_cleanup(&ctx);
prog
}
return.prog
/* * This function may be invoked twice for the same stream of BPF * instructions. The "extra pass" happens, when there are * (re)locations involved that their addresses are not known * during the first run.
*/ struct * Apparently the JIT internals, namely jit_subprogs() in * may request for a second compilation although nothing needs to be done.
{
vm_dump(prog);
/* Was this program already translated? */ if!jited return java.lang.StringIndexOutOfBoundsException: Index 19 out of bounds for length 11 else
do_extra_passprog)
Die Informationen auf dieser Webseite wurden
nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit,
noch Qualität der bereit gestellten Informationen zugesichert.
Bemerkung:
Die farbliche Syntaxdarstellung und die Messung sind noch experimentell.