// SPDX-License-Identifier: GPL-2.0 /* * BPF Jit compiler for s390. * * Minimum build requirements: * * - HAVE_MARCH_Z196_FEATURES: laal, laalg * - HAVE_MARCH_Z10_FEATURES: msfi, cgrj, clgrj * - HAVE_MARCH_Z9_109_FEATURES: alfi, llilf, clfi, oilf, nilf * - 64BIT * * Copyright IBM Corp. 2012,2015 * * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> * Michael Holzheu <holzheu@linux.vnet.ibm.com> #define KMSG_COMPONENT "bpf_jit" #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt#include <linux/netdevice.h>#include <linux/filter.h>#include <linux/init.h>#include <linux/bpf.h>#include <linux/mm.h>#include <linux/kernel.h>#include <asm /cacheflush.h>#include <asm /extable.h>#include <asm /dis.h>#include <asm /facility.h>#include <asm /nospec-branch.h>#include <asm /set_memory.h>#include <asm /text-patching.h>#include <asm /unwind.h>struct bpf_jit {/* Flags to remember seen eBPF instructions */ /* Mask to remember which registers are used */ /* Array with relative instruction addresses */ /* Start of program */ int size; /* Size of program and literal pool */ int size_prg; /* Size of program */ int prg; /* Current position in program */ int lit32_start; /* Start of 32-bit literal pool */ int lit32; /* Current position in 32-bit literal pool */ int lit64_start; /* Start of 64-bit literal pool */ int lit64; /* Current position in 64-bit literal pool */ int base_ip; /* Base address for literal pool */ int exit_ip; /* Address of exit */ int tail_call_start; /* Tail call start offset */ int excnt; /* Number of exception table entries */ int prologue_plt_ret; /* Return address for prologue hotpatch PLT */ int prologue_plt; /* Start of prologue hotpatch PLT */ int kern_arena; /* Pool offset of kernel arena address */ /* User arena address */ /* Offset of struct bpf_prog from %r15 */ #define SEEN_MEM BIT(0) /* use mem[] for temporary storage */ #define SEEN_LITERAL BIT(1) /* code uses literals */ #define SEEN_FUNC BIT(2) /* calls C functions */ #define SEEN_STACK (SEEN_FUNC | SEEN_MEM)#define NVREGS 0xffc0 /* %r6-%r15 */ /* * s390 registers #define REG_W0 (MAX_BPF_JIT_REG + 0) /* Work register 1 (even) */ #define REG_W1 (MAX_BPF_JIT_REG + 1) /* Work register 2 (odd) */ #define REG_L (MAX_BPF_JIT_REG + 2) /* Literal pool register */ #define REG_15 (MAX_BPF_JIT_REG + 3) /* Register 15 */ #define REG_0 REG_W0 /* Register 0 */ #define REG_1 REG_W1 /* Register 1 */ #define REG_2 BPF_REG_1 /* Register 2 */ #define REG_3 BPF_REG_2 /* Register 3 */ #define REG_4 BPF_REG_3 /* Register 4 */ #define REG_7 BPF_REG_6 /* Register 7 */ #define REG_8 BPF_REG_7 /* Register 8 */ #define REG_14 BPF_REG_0 /* Register 14 */ /* * Mapping of BPF registers to s390 registers static const int reg2hex[] = {/* Return code */ /* Function parameters */ /* Call saved registers */ /* BPF stack pointer */ /* Register for blinding */ /* Work registers for s390x backend */ static inline u32 reg(u32 dst_reg, u32 src_reg)return reg2hex[dst_reg] << 4 | reg2hex[src_reg];static inline u32 reg_high(u32 reg)return reg2hex[reg] << 4;static inline void reg_set_seen(struct bpf_jit *jit, u32 b1)if (r1 >= 6 && r1 <= 15)static s32 off_to_pcrel(struct bpf_jit *jit, u32 off)return off - jit->prg;static s64 ptr_to_pcrel(struct bpf_jit *jit, const void *ptr)if (jit->prg_buf)return (const u8 *)ptr - ((const u8 *)jit->prg_buf + jit->prg);return 0;#define REG_SET_SEEN(b1) \/* * EMIT macros for code generation #define _EMIT2(op) \if (jit->prg_buf) \#define EMIT2(op, b1, b2) \#define _EMIT4(op) \if (jit->prg_buf) \#define EMIT4(op, b1, b2) \#define EMIT4_RRF(op, b1, b2, b3) \#define _EMIT4_DISP(op, disp) \unsigned int __disp = (disp) & 0xfff; \#define EMIT4_DISP(op, b1, b2, disp) \#define EMIT4_IMM(op, b1, imm) \unsigned int __imm = (imm) & 0xffff; \#define EMIT4_PCREL(op, pcrel) \long __pcrel = ((pcrel) >> 1) & 0xffff; \#define EMIT4_PCREL_RIC(op, mask, target) \int __rel = off_to_pcrel(jit, target) / 2; \#define _EMIT6(op1, op2) \if (jit->prg_buf) { \#define _EMIT6_DISP(op1, op2, disp) \unsigned int __disp = (disp) & 0xfff; \#define _EMIT6_DISP_LH(op1, op2, disp) \unsigned int __disp_h = _disp & 0xff000; \unsigned int __disp_l = _disp & 0x00fff; \#define EMIT6_DISP_LH(op1, op2, b1, b2, b3, disp) \#define EMIT6_PCREL_RIEB(op1, op2, b1, b2, mask, target) \unsigned int rel = off_to_pcrel(jit, target) / 2; \#define EMIT6_PCREL_RIEC(op1, op2, b1, imm, mask, target) \unsigned int rel = off_to_pcrel(jit, target) / 2; \unsigned long ) (imm)) > 0xff); \#define EMIT6_PCREL(op1, op2, b1, b2, i, off, mask) \int rel = off_to_pcrel(jit, addrs[(i) + (off) + 1]) / 2;\static void emit6_pcrel_ril(struct bpf_jit *jit, u32 op, s64 pcrel)static void emit6_pcrel_rilb(struct bpf_jit *jit, u32 op, u8 b, s64 pcrel)#define EMIT6_PCREL_RILB(op, b, target) \#define EMIT6_PCREL_RILB_PTR(op, b, target_ptr) \static void emit6_pcrel_rilc(struct bpf_jit *jit, u32 op, u8 mask, s64 pcrel)#define EMIT6_PCREL_RILC(op, mask, target) \#define EMIT6_PCREL_RILC_PTR(op, mask, target_ptr) \#define _EMIT6_IMM(op, imm) \unsigned int __imm = (imm); \#define EMIT6_IMM(op, b1, imm) \#define _EMIT_CONST_U32(val) \unsigned int ret; \if (jit->prg_buf) \#define EMIT_CONST_U32(val) \#define _EMIT_CONST_U64(val) \unsigned int ret; \if (jit->prg_buf) \#define EMIT_CONST_U64(val) \#define EMIT_ZERO(b1) \if (!fp->aux->verifier_zext) { \/* llgfr %dst,%dst (zero extend to 64 bit) */ \ /* * Return whether this is the first pass. The first pass is special, since we * don't know any sizes yet, and thus must be conservative. static bool is_first_pass(struct bpf_jit *jit)return jit->size == 0;/* * Return whether this is the code generation pass. The code generation pass is * special, since we should change as little as possible. static bool is_codegen_pass(struct bpf_jit *jit)return jit->prg_buf;/* * Return whether "rel" can be encoded as a short PC-relative offset static bool is_valid_rel(int rel)return rel >= -65536 && rel <= 65534;/* * Return whether "off" can be reached using a short PC-relative offset static bool can_use_rel(struct bpf_jit *jit, int off)return is_valid_rel(off - jit->prg);/* * Return whether given displacement can be encoded using * Long-Displacement Facility static bool is_valid_ldisp(int disp)return disp >= -524288 && disp <= 524287;/* * Return whether the next 32-bit literal pool entry can be referenced using * Long-Displacement Facility static bool can_use_ldisp_for_lit32(struct bpf_jit *jit)return is_valid_ldisp(jit->lit32 - jit->base_ip);/* * Return whether the next 64-bit literal pool entry can be referenced using * Long-Displacement Facility static bool can_use_ldisp_for_lit64(struct bpf_jit *jit)return is_valid_ldisp(jit->lit64 - jit->base_ip);/* * Fill whole space with illegal instructions static void jit_fill_hole(void *area, unsigned int size)/* * Caller-allocated part of the frame. * Thanks to packed stack, its otherwise unused initial part can be used for * the BPF stack and for the next frame. struct prog_frame {/* BPF stack starts here and grows towards 0 */ /* r6 - r15 */ /* * Save registers from "rs" (register start) to "re" (register end) on stack static void save_regs(struct bpf_jit *jit, u32 rs, u32 re)struct prog_frame, r6) + (rs - 6) * 8;if (rs == re)/* stg %rs,off(%r15) */ else /* stmg %rs,%re,off(%r15) */ /* * Restore registers from "rs" (register start) to "re" (register end) on stack static void restore_regs(struct bpf_jit *jit, u32 rs, u32 re)struct prog_frame, r6) + (rs - 6) * 8;if (rs == re)/* lg %rs,off(%r15) */ else /* lmg %rs,%re,off(%r15) */ /* * Return first seen register (from start) static int get_start(u16 seen_regs, int start)int i;for (i = start; i <= 15; i++) {if (seen_regs & (1 << i))return i;return 0;/* * Return last seen register (from start) (gap >= 2) static int get_end(u16 seen_regs, int start)int i;for (i = start; i < 15; i++) {if (!(seen_regs & (3 << i)))return i - 1;return (seen_regs & (1 << 15)) ? 15 : 14;#define REGS_SAVE 1#define REGS_RESTORE 0/* * Save and restore clobbered registers (6-15) on stack. * We save/restore registers in chunks with gap >= 2 registers. static void save_restore_regs(struct bpf_jit *jit, int op, u16 extra_regs)const int last = 15, save_restore_size = 6;int re = 6, rs;if (is_first_pass(jit)) {/* * We don't know yet which registers are used. Reserve space * conservatively. return ;do {if (!rs)break ;if (op == REGS_SAVE)else while (re <= last);static void bpf_skip(struct bpf_jit *jit, int size)if (size >= 6 && !is_valid_rel(size)) {/* brcl 0xf,size */ else if (size >= 4 && is_valid_rel(size)) {/* brc 0xf,size */ while (size >= 2) {/* bcr 0,%0 */ /* * PLT for hotpatchable calls. The calling convention is the same as for the * ftrace hotpatch trampolines: %r0 is return address, %r1 is clobbered. struct bpf_plt {char code[16];void *ret;void *target;extern const struct bpf_plt bpf_plt;asm (".pushsection .rodata\n" " .balign 8\n" "bpf_plt:\n" " lgrl %r0,bpf_plt_ret\n" " lgrl %r1,bpf_plt_target\n" " br %r1\n" " .balign 8\n" "bpf_plt_ret: .quad 0\n" "bpf_plt_target: .quad 0\n" " .popsection\n" static void bpf_jit_plt(struct bpf_plt *plt, void *ret, void *target)sizeof (*plt));/* * (target == NULL) implies that the branch to this PLT entry was * patched and became a no-op. However, some CPU could have jumped * to this PLT entry before patching and may be still executing it. * * Since the intention in this case is to make the PLT entry a no-op, * make the target point to the return label instead of NULL. /* * Emit function prologue * * Save registers and create stack frame if necessary. * Stack frame layout is described by struct prog_frame. static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp)sizeof (struct prog_frame) != STACK_FRAME_OVERHEAD);/* No-op for hotpatching */ /* brcl 0,prologue_plt */ if (!bpf_is_subprog(fp)) {/* Initialize the tail call counter in the main program. */ /* xc tail_call_cnt(4,%r15),tail_call_cnt(%r15) */ struct prog_frame, tail_call_cnt),struct prog_frame, tail_call_cnt));else {/* * Skip the tail call counter initialization in subprograms. * Insert nops in order to have tail_call_start at a * predictable offset. /* Tail calls have to skip above initialization */ if (fp->aux->exception_cb) {/* * Switch stack, the new address is in the 2nd parameter. * * Arrange the restoration of %r6-%r15 in the epilogue. * Do not restore them now, the prog does not need them. /* lgr %r15,%r3 */ else {/* Save registers */ /* Setup literal pool */ if (is_first_pass(jit) || (jit->seen & SEEN_LITERAL)) {if (!is_first_pass(jit) &&/* basr %l,0 */ else {/* larl %l,lit32_start */ /* Setup stack and backchain */ if (is_first_pass(jit) || (jit->seen & SEEN_STACK)) {/* lgr %w1,%r15 (backchain) */ /* la %bfp,unused_end(%r15) (BPF frame pointer) */ struct prog_frame, unused));/* aghi %r15,-frame_off */ /* stg %w1,backchain(%r15) */ struct prog_frame, backchain));/* * Jump using a register either directly or via an expoline thunk #define EMIT_JUMP_REG(reg) do { \if (nospec_uses_trampoline()) \/* brcl 0xf,__s390_indirect_jump_rN */ \ ## reg); \else \/* br %rN */ \ while (0)/* * Call r1 either directly or via __s390_indirect_jump_r1 thunk static void call_r1(struct bpf_jit *jit)if (nospec_uses_trampoline())/* brasl %r14,__s390_indirect_jump_r1 */ else /* basr %r14,%r1 */ /* * Function epilogue static void bpf_jit_epilogue(struct bpf_jit *jit)/* Load exit code: lgr %r2,%b0 */ /* Restore registers */ if (jit->prg_buf)struct bpf_plt *)(jit->prg_buf + jit->prg),sizeof (struct bpf_plt);bool ex_handler_bpf(const struct exception_table_entry *x, struct pt_regs *regs)if (x->data != -1)return true ;/* * A single BPF probe instruction struct bpf_jit_probe {int prg; /* JITed instruction offset */ int nop_prg; /* JITed nop offset */ int reg; /* Register to clear on exception */ int arena_reg; /* Register to use for arena addressing */ static void bpf_jit_probe_init(struct bpf_jit_probe *probe)/* * Handlers of certain exceptions leave psw.addr pointing to the instruction * directly after the failing one. Therefore, create two exception table * entries and also add a nop in case two probing instructions come directly * after each other. static void bpf_jit_probe_emit_nop(struct bpf_jit *jit,struct bpf_jit_probe *probe)if (probe->prg == -1 || probe->nop_prg != -1)/* The probe is not armed or nop is already emitted. */ return ;/* bcr 0,%0 */ static void bpf_jit_probe_load_pre(struct bpf_jit *jit, struct bpf_insn *insn,struct bpf_jit_probe *probe)if (BPF_MODE(insn->code) != BPF_PROBE_MEM &&return ;if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) {/* lgrl %r1,kern_arena */ static void bpf_jit_probe_store_pre(struct bpf_jit *jit, struct bpf_insn *insn,struct bpf_jit_probe *probe)if (BPF_MODE(insn->code) != BPF_PROBE_MEM32)return ;/* lgrl %r1,kern_arena */ static void bpf_jit_probe_atomic_pre(struct bpf_jit *jit,struct bpf_insn *insn,struct bpf_jit_probe *probe)if (BPF_MODE(insn->code) != BPF_PROBE_ATOMIC)return ;/* lgrl %r1,kern_arena */ /* agr %r1,%dst */ static int bpf_jit_probe_post(struct bpf_jit *jit, struct bpf_prog *fp,struct bpf_jit_probe *probe)struct exception_table_entry *ex;int i, prg;if (probe->prg == -1)/* The probe is not armed. */ return 0;if (!fp->aux->extable)/* Do nothing during early JIT passes. */ return 0;if (WARN_ON_ONCE(probe->prg + insn_length(*insn) != probe->nop_prg))/* JIT bug - gap between probe and nop instructions. */ return -1;for (i = 0; i < 2; i++) {if (WARN_ON_ONCE(jit->excnt >= fp->aux->num_exentries))/* Verifier bug - not enough entries. */ return -1;/* Add extable entries for probe and nop instructions. */ if (WARN_ON_ONCE(delta < INT_MIN || delta > INT_MAX))/* JIT bug - code and extable must be close. */ return -1;/* * Land on the current instruction. Note that the extable * infrastructure ignores the fixup field; it is handled by * ex_handler_bpf(). if (WARN_ON_ONCE(delta < INT_MIN || delta > INT_MAX))/* JIT bug - landing pad and extable must be close. */ return -1;return 0;/* * Sign-extend the register if necessary static int sign_extend(struct bpf_jit *jit, int r, u8 size, u8 flags)if (!(flags & BTF_FMODEL_SIGNED_ARG))return 0;switch (size) {case 1:/* lgbr %r,%r */ return 0;case 2:/* lghr %r,%r */ return 0;case 4:/* lgfr %r,%r */ return 0;case 8:return 0;default :return -1;/* * Compile one eBPF instruction into s390x code * * NOTE: Use noinline because for gcov (-fprofile-arcs) gcc allocates a lot of * stack space for the large switch statement. static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,int i, bool extra_pass)struct bpf_insn *insn = &fp->insnsi[i];struct bpf_jit_probe probe;int last, insn_count = 1;unsigned int mask;int err;switch (insn->code) {/* * BPF_MOV case BPF_ALU | BPF_MOV | BPF_X:switch (insn->off) {case 0: /* DST = (u32) SRC */ /* llgfr %dst,%src */ if (insn_is_zext(&insn[1]))break ;case 8: /* DST = (u32)(s8) SRC */ /* lbr %dst,%src */ /* llgfr %dst,%dst */ break ;case 16: /* DST = (u32)(s16) SRC */ /* lhr %dst,%src */ /* llgfr %dst,%dst */ break ;break ;case BPF_ALU64 | BPF_MOV | BPF_X:if (insn_is_cast_user(insn)) {int patch_brc;/* ltgr %dst,%src */ /* brc 8,0f */ /* iihf %dst,user_arena>>32 */ /* 0: */ if (jit->prg_buf)break ;switch (insn->off) {case 0: /* DST = SRC */ /* lgr %dst,%src */ break ;case 8: /* DST = (s8) SRC */ /* lgbr %dst,%src */ break ;case 16: /* DST = (s16) SRC */ /* lghr %dst,%src */ break ;case 32: /* DST = (s32) SRC */ /* lgfr %dst,%src */ break ;break ;case BPF_ALU | BPF_MOV | BPF_K: /* dst = (u32) imm */ /* llilf %dst,imm */ if (insn_is_zext(&insn[1]))break ;case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = imm */ /* lgfi %dst,imm */ break ;/* * BPF_LD 64 case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */ /* 16 byte instruction that uses two 'struct bpf_insn' */ /* lgrl %dst,imm */ break ;/* * BPF_ADD case BPF_ALU | BPF_ADD | BPF_X: /* dst = (u32) dst + (u32) src */ /* ar %dst,%src */ break ;case BPF_ALU64 | BPF_ADD | BPF_X: /* dst = dst + src */ /* agr %dst,%src */ break ;case BPF_ALU | BPF_ADD | BPF_K: /* dst = (u32) dst + (u32) imm */ if (imm != 0) {/* alfi %dst,imm */ break ;case BPF_ALU64 | BPF_ADD | BPF_K: /* dst = dst + imm */ if (!imm)break ;/* agfi %dst,imm */ break ;/* * BPF_SUB case BPF_ALU | BPF_SUB | BPF_X: /* dst = (u32) dst - (u32) src */ /* sr %dst,%src */ break ;case BPF_ALU64 | BPF_SUB | BPF_X: /* dst = dst - src */ /* sgr %dst,%src */ break ;case BPF_ALU | BPF_SUB | BPF_K: /* dst = (u32) dst - (u32) imm */ if (imm != 0) {/* alfi %dst,-imm */ break ;case BPF_ALU64 | BPF_SUB | BPF_K: /* dst = dst - imm */ if (!imm)break ;if (imm == -0x80000000) {/* algfi %dst,0x80000000 */ else {/* agfi %dst,-imm */ break ;/* * BPF_MUL case BPF_ALU | BPF_MUL | BPF_X: /* dst = (u32) dst * (u32) src */ /* msr %dst,%src */ break ;case BPF_ALU64 | BPF_MUL | BPF_X: /* dst = dst * src */ /* msgr %dst,%src */ break ;case BPF_ALU | BPF_MUL | BPF_K: /* dst = (u32) dst * (u32) imm */ if (imm != 1) {/* msfi %r5,imm */ break ;case BPF_ALU64 | BPF_MUL | BPF_K: /* dst = dst * imm */ if (imm == 1)break ;/* msgfi %dst,imm */ break ;/* * BPF_DIV / BPF_MOD case BPF_ALU | BPF_DIV | BPF_X:case BPF_ALU | BPF_MOD | BPF_X:int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;switch (off) {case 0: /* dst = (u32) dst {/,%} (u32) src */ /* xr %w0,%w0 */ /* lr %w1,%dst */ /* dlr %w0,%src */ break ;case 1: /* dst = (u32) ((s32) dst {/,%} (s32) src) */ /* lgfr %r1,%dst */ /* dsgfr %r0,%src */ break ;/* llgfr %dst,%rc */ if (insn_is_zext(&insn[1]))break ;case BPF_ALU64 | BPF_DIV | BPF_X:case BPF_ALU64 | BPF_MOD | BPF_X:int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;switch (off) {case 0: /* dst = dst {/,%} src */ /* lghi %w0,0 */ /* lgr %w1,%dst */ /* dlgr %w0,%src */ break ;case 1: /* dst = (s64) dst {/,%} (s64) src */ /* lgr %w1,%dst */ /* dsgr %w0,%src */ break ;/* lgr %dst,%rc */ break ;case BPF_ALU | BPF_DIV | BPF_K:case BPF_ALU | BPF_MOD | BPF_K:int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;if (imm == 1) {if (BPF_OP(insn->code) == BPF_MOD)/* lghi %dst,0 */ else break ;if (!is_first_pass(jit) && can_use_ldisp_for_lit32(jit)) {switch (off) {case 0: /* dst = (u32) dst {/,%} (u32) imm */ /* xr %w0,%w0 */ /* lr %w1,%dst */ /* dl %w0,<d(imm)>(%l) */ break ;case 1: /* dst = (s32) dst {/,%} (s32) imm */ /* lgfr %r1,%dst */ /* dsgf %r0,<d(imm)>(%l) */ break ;else {switch (off) {case 0: /* dst = (u32) dst {/,%} (u32) imm */ /* xr %w0,%w0 */ /* lr %w1,%dst */ /* lrl %dst,imm */ /* dlr %w0,%dst */ break ;case 1: /* dst = (s32) dst {/,%} (s32) imm */ /* lgfr %w1,%dst */ /* lgfrl %dst,imm */ /* dsgr %w0,%dst */ break ;/* llgfr %dst,%rc */ if (insn_is_zext(&insn[1]))break ;case BPF_ALU64 | BPF_DIV | BPF_K:case BPF_ALU64 | BPF_MOD | BPF_K:int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0;if (imm == 1) {if (BPF_OP(insn->code) == BPF_MOD)/* lhgi %dst,0 */ break ;if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {switch (off) {case 0: /* dst = dst {/,%} imm */ /* lghi %w0,0 */ /* lgr %w1,%dst */ /* dlg %w0,<d(imm)>(%l) */ break ;case 1: /* dst = (s64) dst {/,%} (s64) imm */ /* lgr %w1,%dst */ /* dsg %w0,<d(imm)>(%l) */ break ;else {switch (off) {case 0: /* dst = dst {/,%} imm */ /* lghi %w0,0 */ /* lgr %w1,%dst */ /* lgrl %dst,imm */ /* dlgr %w0,%dst */ break ;case 1: /* dst = (s64) dst {/,%} (s64) imm */ /* lgr %w1,%dst */ /* lgrl %dst,imm */ /* dsgr %w0,%dst */ break ;/* lgr %dst,%rc */ break ;/* * BPF_AND case BPF_ALU | BPF_AND | BPF_X: /* dst = (u32) dst & (u32) src */ /* nr %dst,%src */ break ;case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */ /* ngr %dst,%src */ break ;case BPF_ALU | BPF_AND | BPF_K: /* dst = (u32) dst & (u32) imm */ /* nilf %dst,imm */ break ;case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */ if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {/* ng %dst,<d(imm)>(%l) */ else {/* lgrl %w0,imm */ /* ngr %dst,%w0 */ break ;/* * BPF_OR case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */ /* or %dst,%src */ break ;case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */ /* ogr %dst,%src */ break ;case BPF_ALU | BPF_OR | BPF_K: /* dst = (u32) dst | (u32) imm */ /* oilf %dst,imm */ break ;case BPF_ALU64 | BPF_OR | BPF_K: /* dst = dst | imm */ if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {/* og %dst,<d(imm)>(%l) */ else {/* lgrl %w0,imm */ /* ogr %dst,%w0 */ break ;/* * BPF_XOR case BPF_ALU | BPF_XOR | BPF_X: /* dst = (u32) dst ^ (u32) src */ /* xr %dst,%src */ break ;case BPF_ALU64 | BPF_XOR | BPF_X: /* dst = dst ^ src */ /* xgr %dst,%src */ break ;case BPF_ALU | BPF_XOR | BPF_K: /* dst = (u32) dst ^ (u32) imm */ if (imm != 0) {/* xilf %dst,imm */ break ;case BPF_ALU64 | BPF_XOR | BPF_K: /* dst = dst ^ imm */ if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {/* xg %dst,<d(imm)>(%l) */ else {/* lgrl %w0,imm */ /* xgr %dst,%w0 */ break ;/* * BPF_LSH case BPF_ALU | BPF_LSH | BPF_X: /* dst = (u32) dst << (u32) src */ /* sll %dst,0(%src) */ break ;case BPF_ALU64 | BPF_LSH | BPF_X: /* dst = dst << src */ /* sllg %dst,%dst,0(%src) */ break ;case BPF_ALU | BPF_LSH | BPF_K: /* dst = (u32) dst << (u32) imm */ if (imm != 0) {/* sll %dst,imm(%r0) */ break ;case BPF_ALU64 | BPF_LSH | BPF_K: /* dst = dst << imm */ if (imm == 0)break ;/* sllg %dst,%dst,imm(%r0) */ break ;/* * BPF_RSH case BPF_ALU | BPF_RSH | BPF_X: /* dst = (u32) dst >> (u32) src */ /* srl %dst,0(%src) */ break ;case BPF_ALU64 | BPF_RSH | BPF_X: /* dst = dst >> src */ /* srlg %dst,%dst,0(%src) */ break ;case BPF_ALU | BPF_RSH | BPF_K: /* dst = (u32) dst >> (u32) imm */ if (imm != 0) {/* srl %dst,imm(%r0) */ break ;case BPF_ALU64 | BPF_RSH | BPF_K: /* dst = dst >> imm */ if (imm == 0)break ;/* srlg %dst,%dst,imm(%r0) */ break ;/* * BPF_ARSH case BPF_ALU | BPF_ARSH | BPF_X: /* ((s32) dst) >>= src */ /* sra %dst,%dst,0(%src) */ break ;case BPF_ALU64 | BPF_ARSH | BPF_X: /* ((s64) dst) >>= src */ /* srag %dst,%dst,0(%src) */ break ;case BPF_ALU | BPF_ARSH | BPF_K: /* ((s32) dst >> imm */ if (imm != 0) {/* sra %dst,imm(%r0) */ break ;case BPF_ALU64 | BPF_ARSH | BPF_K: /* ((s64) dst) >>= imm */ if (imm == 0)break ;/* srag %dst,%dst,imm(%r0) */ break ;/* * BPF_NEG case BPF_ALU | BPF_NEG: /* dst = (u32) -dst */ /* lcr %dst,%dst */ break ;case BPF_ALU64 | BPF_NEG: /* dst = -dst */ /* lcgr %dst,%dst */ break ;/* * BPF_FROM_BE/LE case BPF_ALU | BPF_END | BPF_FROM_BE:/* s390 is big endian, therefore only clear high order bytes */ switch (imm) {case 16: /* dst = (u16) cpu_to_be16(dst) */ /* llghr %dst,%dst */ if (insn_is_zext(&insn[1]))break ;case 32: /* dst = (u32) cpu_to_be32(dst) */ if (!fp->aux->verifier_zext)/* llgfr %dst,%dst */ break ;case 64: /* dst = (u64) cpu_to_be64(dst) */ break ;break ;case BPF_ALU | BPF_END | BPF_FROM_LE:case BPF_ALU64 | BPF_END | BPF_FROM_LE:switch (imm) {case 16: /* dst = (u16) cpu_to_le16(dst) */ /* lrvr %dst,%dst */ /* srl %dst,16(%r0) */ /* llghr %dst,%dst */ if (insn_is_zext(&insn[1]))break ;case 32: /* dst = (u32) cpu_to_le32(dst) */ /* lrvr %dst,%dst */ if (!fp->aux->verifier_zext)/* llgfr %dst,%dst */ break ;case 64: /* dst = (u64) cpu_to_le64(dst) */ /* lrvgr %dst,%dst */ break ;break ;/* * BPF_NOSPEC (speculation barrier) case BPF_ST | BPF_NOSPEC:break ;/* * BPF_ST(X) case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src_reg */ case BPF_STX | BPF_PROBE_MEM32 | BPF_B:/* stcy %src,off(%dst,%arena) */ if (err < 0)return err;break ;case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */ case BPF_STX | BPF_PROBE_MEM32 | BPF_H:/* sthy %src,off(%dst,%arena) */ if (err < 0)return err;break ;case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */ case BPF_STX | BPF_PROBE_MEM32 | BPF_W:/* sty %src,off(%dst,%arena) */ if (err < 0)return err;break ;case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */ case BPF_STX | BPF_PROBE_MEM32 | BPF_DW:/* stg %src,off(%dst,%arena) */ if (err < 0)return err;break ;case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */ case BPF_ST | BPF_PROBE_MEM32 | BPF_B:/* lhi %w0,imm */ /* stcy %w0,off(%dst,%arena) */ if (err < 0)return err;break ;case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */ case BPF_ST | BPF_PROBE_MEM32 | BPF_H:/* lhi %w0,imm */ /* sthy %w0,off(%dst,%arena) */ if (err < 0)return err;break ;case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */ case BPF_ST | BPF_PROBE_MEM32 | BPF_W:/* llilf %w0,imm */ /* sty %w0,off(%dst,%arena) */ if (err < 0)return err;break ;case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */ case BPF_ST | BPF_PROBE_MEM32 | BPF_DW:/* lgfi %w0,imm */ /* stg %w0,off(%dst,%arena) */ if (err < 0)return err;break ;/* * BPF_ATOMIC case BPF_STX | BPF_ATOMIC | BPF_DW:case BPF_STX | BPF_ATOMIC | BPF_W:case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW:case BPF_STX | BPF_PROBE_ATOMIC | BPF_W:bool is32 = BPF_SIZE(insn->code) == BPF_W;/* * Unlike loads and stores, atomics have only a base register, * but no index register. For the non-arena case, simply use * %dst as a base. For the arena case, use the work register * %r1: first, load the arena base into it, and then add %dst * to it. switch (insn->imm) {#define EMIT_ATOMIC(op32, op64) do { \/* {op32|op64} {%w0|%src},%src,off(%arena) */ \ if (err < 0) \return err; \if (insn->imm & BPF_FETCH) { \/* bcr 14,0 - see atomic_fetch_{add,and,or,xor}() */ \ if (is32) \while (0)case BPF_ADD:case BPF_ADD | BPF_FETCH:/* {laal|laalg} */ break ;case BPF_AND:case BPF_AND | BPF_FETCH:/* {lan|lang} */ break ;case BPF_OR:case BPF_OR | BPF_FETCH:/* {lao|laog} */ break ;case BPF_XOR:case BPF_XOR | BPF_FETCH:/* {lax|laxg} */ break ;#undef EMIT_ATOMICcase BPF_XCHG: {struct bpf_jit_probe load_probe = probe;int loop_start;/* {ly|lg} %w0,off(%arena) */ /* Reuse {ly|lg}'s arena_reg for {csy|csg}. */ if (load_probe.prg != -1) {/* 0: {csy|csg} %w0,%src,off(%arena) */ /* brc 4,0b */ /* {llgfr|lgr} %src,%w0 */ /* Both probes should land here on exception. */ if (err < 0)return err;if (err < 0)return err;if (is32 && insn_is_zext(&insn[1]))break ;case BPF_CMPXCHG:/* 0: {csy|csg} %b0,%src,off(%arena) */ if (err < 0)return err;break ;default :"Unknown atomic operation %02x\n" , insn->imm);return -1;break ;/* * BPF_LDX case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */ case BPF_LDX | BPF_PROBE_MEM | BPF_B:case BPF_LDX | BPF_PROBE_MEM32 | BPF_B:/* llgc %dst,off(%src,%arena) */ if (err < 0)return err;if (insn_is_zext(&insn[1]))break ;case BPF_LDX | BPF_MEMSX | BPF_B: /* dst = *(s8 *)(ul) (src + off) */ case BPF_LDX | BPF_PROBE_MEMSX | BPF_B:/* lgb %dst,off(%src) */ if (err < 0)return err;break ;case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */ case BPF_LDX | BPF_PROBE_MEM | BPF_H:case BPF_LDX | BPF_PROBE_MEM32 | BPF_H:/* llgh %dst,off(%src,%arena) */ if (err < 0)return err;if (insn_is_zext(&insn[1]))break ;case BPF_LDX | BPF_MEMSX | BPF_H: /* dst = *(s16 *)(ul) (src + off) */ case BPF_LDX | BPF_PROBE_MEMSX | BPF_H:/* lgh %dst,off(%src) */ if (err < 0)return err;break ;case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */ case BPF_LDX | BPF_PROBE_MEM | BPF_W:case BPF_LDX | BPF_PROBE_MEM32 | BPF_W:/* llgf %dst,off(%src) */ if (err < 0)return err;if (insn_is_zext(&insn[1]))break ;case BPF_LDX | BPF_MEMSX | BPF_W: /* dst = *(s32 *)(ul) (src + off) */ case BPF_LDX | BPF_PROBE_MEMSX | BPF_W:/* lgf %dst,off(%src) */ if (err < 0)return err;break ;case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */ case BPF_LDX | BPF_PROBE_MEM | BPF_DW:case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW:/* lg %dst,off(%src,%arena) */ if (err < 0)return err;break ;/* * BPF_JMP / CALL case BPF_JMP | BPF_CALL:const struct btf_func_model *m;bool func_addr_fixed;int j, ret;if (ret < 0)return -1;/* * Copy the tail call counter to where the callee expects it. if (insn->src_reg == BPF_PSEUDO_CALL)/* * mvc tail_call_cnt(4,%r15), * frame_off+tail_call_cnt(%r15) struct prog_frame,struct prog_frame,/* Sign-extend the kfunc arguments. */ if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {if (!m)return -1;for (j = 0; j < m->nr_args; j++) {if (sign_extend(jit, BPF_REG_1 + j,return -1;/* lgrl %w1,func */ /* %r1() */ /* lgr %b0,%r2: load return value into %b0 */ /* * Copy the potentially updated tail call counter back. if (insn->src_reg == BPF_PSEUDO_CALL)/* * mvc frame_off+tail_call_cnt(%r15), * tail_call_cnt(4,%r15) struct prog_frame,struct prog_frame,break ;case BPF_JMP | BPF_TAIL_CALL: {int patch_1_clrj, patch_2_clij, patch_3_brc;/* * Implicit input: * B1: pointer to ctx * B2: pointer to bpf_array * B3: index in bpf_array * * if (index >= array->map.max_entries) * goto out; /* llgf %w1,map.max_entries(%b2) */ struct bpf_array, map.max_entries));/* if ((u32)%b3 >= (u32)%w1) goto out; */ /* clrj %b3,%w1,0xa,out */ /* * if (tail_call_cnt++ >= MAX_TAIL_CALL_CNT) * goto out; struct prog_frame, tail_call_cnt);/* lhi %w0,1 */ /* laal %w1,%w0,off(%r15) */ /* clij %w1,MAX_TAIL_CALL_CNT-1,0x2,out */ /* * prog = array->ptrs[index]; * if (prog == NULL) * goto out; /* llgfr %r1,%b3: %r1 = (u32) index */ /* sllg %r1,%r1,3: %r1 *= 8 */ /* ltg %r1,prog(%b2,%r1) */ struct bpf_array, ptrs));/* brc 0x8,out */ /* * Restore registers before calling function /* * goto *(prog->bpf_func + tail_call_start); /* lg %r1,bpf_func(%r1) */ struct bpf_prog, bpf_func));if (nospec_uses_trampoline()) {/* aghi %r1,tail_call_start */ /* brcl 0xf,__s390_indirect_jump_r1 */ else {/* bc 0xf,tail_call_start(%r1) */ /* out: */ if (jit->prg_buf) {break ;case BPF_JMP | BPF_EXIT: /* return b0 */ if (last)break ;if (!is_first_pass(jit) && can_use_rel(jit, jit->exit_ip))/* brc 0xf, <exit> */ else /* brcl 0xf, <exit> */ break ;/* * Branch relative (number of skipped instructions) to offset on * condition. * * Condition code to mask mapping: * * CC | Description | Mask * ------------------------------ * 0 | Operands equal | 8 * 1 | First operand low | 4 * 2 | First operand high | 2 * 3 | Unused | 1 * * For s390x relative branches: ip = ip + off_bytes * For BPF relative branches: insn = insn + off_insns + 1 * * For example for s390x with offset 0 we jump to the branch * instruction itself (loop) and for BPF with offset 0 we * branch to the instruction behind the branch. case BPF_JMP32 | BPF_JA: /* if (true) */ case BPF_JMP | BPF_JA: /* if (true) */ /* j */ goto branch_oc;case BPF_JMP | BPF_JSGT | BPF_K: /* ((s64) dst > (s64) imm) */ case BPF_JMP32 | BPF_JSGT | BPF_K: /* ((s32) dst > (s32) imm) */ /* jh */ goto branch_ks;case BPF_JMP | BPF_JSLT | BPF_K: /* ((s64) dst < (s64) imm) */ case BPF_JMP32 | BPF_JSLT | BPF_K: /* ((s32) dst < (s32) imm) */ /* jl */ goto branch_ks;case BPF_JMP | BPF_JSGE | BPF_K: /* ((s64) dst >= (s64) imm) */ case BPF_JMP32 | BPF_JSGE | BPF_K: /* ((s32) dst >= (s32) imm) */ /* jhe */ goto branch_ks;case BPF_JMP | BPF_JSLE | BPF_K: /* ((s64) dst <= (s64) imm) */ case BPF_JMP32 | BPF_JSLE | BPF_K: /* ((s32) dst <= (s32) imm) */ /* jle */ goto branch_ks;case BPF_JMP | BPF_JGT | BPF_K: /* (dst_reg > imm) */ case BPF_JMP32 | BPF_JGT | BPF_K: /* ((u32) dst_reg > (u32) imm) */ /* jh */ goto branch_ku;case BPF_JMP | BPF_JLT | BPF_K: /* (dst_reg < imm) */ case BPF_JMP32 | BPF_JLT | BPF_K: /* ((u32) dst_reg < (u32) imm) */ /* jl */ goto branch_ku;case BPF_JMP | BPF_JGE | BPF_K: /* (dst_reg >= imm) */ case BPF_JMP32 | BPF_JGE | BPF_K: /* ((u32) dst_reg >= (u32) imm) */ /* jhe */ goto branch_ku;case BPF_JMP | BPF_JLE | BPF_K: /* (dst_reg <= imm) */ case BPF_JMP32 | BPF_JLE | BPF_K: /* ((u32) dst_reg <= (u32) imm) */ /* jle */ goto branch_ku;case BPF_JMP | BPF_JNE | BPF_K: /* (dst_reg != imm) */ case BPF_JMP32 | BPF_JNE | BPF_K: /* ((u32) dst_reg != (u32) imm) */ /* jne */ goto branch_ku;case BPF_JMP | BPF_JEQ | BPF_K: /* (dst_reg == imm) */ case BPF_JMP32 | BPF_JEQ | BPF_K: /* ((u32) dst_reg == (u32) imm) */ /* je */ goto branch_ku;case BPF_JMP | BPF_JSET | BPF_K: /* (dst_reg & imm) */ case BPF_JMP32 | BPF_JSET | BPF_K: /* ((u32) dst_reg & (u32) imm) */ /* jnz */ if (BPF_CLASS(insn->code) == BPF_JMP32) {/* llilf %w1,imm (load zero extend imm) */ /* nr %w1,%dst */ else {/* lgfi %w1,imm (load sign extend imm) */ /* ngr %w1,%dst */ goto branch_oc;case BPF_JMP | BPF_JSGT | BPF_X: /* ((s64) dst > (s64) src) */ case BPF_JMP32 | BPF_JSGT | BPF_X: /* ((s32) dst > (s32) src) */ /* jh */ goto branch_xs;case BPF_JMP | BPF_JSLT | BPF_X: /* ((s64) dst < (s64) src) */ case BPF_JMP32 | BPF_JSLT | BPF_X: /* ((s32) dst < (s32) src) */ /* jl */ goto branch_xs;case BPF_JMP | BPF_JSGE | BPF_X: /* ((s64) dst >= (s64) src) */ case BPF_JMP32 | BPF_JSGE | BPF_X: /* ((s32) dst >= (s32) src) */ /* jhe */ goto branch_xs;case BPF_JMP | BPF_JSLE | BPF_X: /* ((s64) dst <= (s64) src) */ case BPF_JMP32 | BPF_JSLE | BPF_X: /* ((s32) dst <= (s32) src) */ /* jle */ goto branch_xs;case BPF_JMP | BPF_JGT | BPF_X: /* (dst > src) */ case BPF_JMP32 | BPF_JGT | BPF_X: /* ((u32) dst > (u32) src) */ /* jh */ goto branch_xu;case BPF_JMP | BPF_JLT | BPF_X: /* (dst < src) */ case BPF_JMP32 | BPF_JLT | BPF_X: /* ((u32) dst < (u32) src) */ /* jl */ goto branch_xu;case BPF_JMP | BPF_JGE | BPF_X: /* (dst >= src) */ case BPF_JMP32 | BPF_JGE | BPF_X: /* ((u32) dst >= (u32) src) */ /* jhe */ goto branch_xu;case BPF_JMP | BPF_JLE | BPF_X: /* (dst <= src) */ case BPF_JMP32 | BPF_JLE | BPF_X: /* ((u32) dst <= (u32) src) */ /* jle */ goto branch_xu;case BPF_JMP | BPF_JNE | BPF_X: /* (dst != src) */ case BPF_JMP32 | BPF_JNE | BPF_X: /* ((u32) dst != (u32) src) */ /* jne */ goto branch_xu;case BPF_JMP | BPF_JEQ | BPF_X: /* (dst == src) */ case BPF_JMP32 | BPF_JEQ | BPF_X: /* ((u32) dst == (u32) src) */ /* je */ goto branch_xu;case BPF_JMP | BPF_JSET | BPF_X: /* (dst & src) */ case BPF_JMP32 | BPF_JSET | BPF_X: /* ((u32) dst & (u32) src) */ bool is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;/* jnz */ /* nrk or ngrk %w1,%dst,%src */ goto branch_oc;/* cfi or cgfi %dst,imm */ if (!is_first_pass(jit) &&/* brc mask,off */ else {/* brcl mask,off */ break ;/* lgfi %w1,imm (load sign extend imm) */ goto branch_xu;if (!is_first_pass(jit) &&/* crj or cgrj %dst,%src,mask,off */ else {/* cr or cgr %dst,%src */ if (is_jmp32)else /* brcl mask,off */ break ;if (!is_first_pass(jit) &&/* clrj or clgrj %dst,%src,mask,off */ else {/* clr or clgr %dst,%src */ if (is_jmp32)else /* brcl mask,off */ break ;if (!is_first_pass(jit) &&/* brc mask,off */ else {/* brcl mask,off */ break ;default : /* too complex, give up */ "Unknown opcode %02x\n" , insn->code);return -1;return insn_count;/* * Return whether new i-th instruction address does not violate any invariant static bool bpf_is_new_addr_sane(struct bpf_jit *jit, int i)/* On the first pass anything goes */ if (is_first_pass(jit))return true ;/* The codegen pass must not change anything */ if (is_codegen_pass(jit))return jit->addrs[i] == jit->prg;/* Passes in between must not increase code size */ return jit->addrs[i] >= jit->prg;/* * Update the address of i-th instruction static int bpf_set_addr(struct bpf_jit *jit, int i)int delta;if (is_codegen_pass(jit)) {if (delta < 0)if (WARN_ON_ONCE(!bpf_is_new_addr_sane(jit, i)))return -1;return 0;/* * Compile eBPF program into s390x code static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp,bool extra_pass)int i, insn_count, lit32_size, lit64_size;if (is_first_pass(jit) || (jit->seen & SEEN_STACK))sizeof (struct prog_frame) -struct prog_frame, unused) +else if (kern_arena)if (bpf_set_addr(jit, 0) < 0)return -1;for (i = 0; i < fp->len; i += insn_count) {if (insn_count < 0)return -1;/* Next instruction address */ if (bpf_set_addr(jit, i + insn_count) < 0)return -1;if (lit32_size)if (lit64_size)if (WARN_ON_ONCE(fp->aux->extable &&/* Verifier bug - too many entries. */ return -1;return 0;bool bpf_jit_needs_zext(void )return true ;struct s390_jit_data {struct bpf_binary_header *header;struct bpf_jit ctx;int pass;static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit,struct bpf_prog *fp)struct bpf_binary_header *header;struct bpf_insn *insn;int i;for (i = 0; i < fp->len; i++) {if (BPF_CLASS(insn->code) == BPF_STX &&/* * bpf_jit_insn() emits a load and a compare-and-swap, * both of which need to be probed. /* We need two entries per insn. */ struct exception_table_entry));sizeof (struct exception_table_entry);if (!header)return NULL;struct exception_table_entry *)return header;/* * Compile eBPF program "fp" struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)struct bpf_prog *tmp, *orig_fp = fp;struct bpf_binary_header *header;struct s390_jit_data *jit_data;bool tmp_blinded = false ;bool extra_pass = false ;struct bpf_jit jit;int pass;if (!fp->jit_requested)return orig_fp;/* * If blinding was requested and we failed during blinding, * we must fall back to the interpreter. if (IS_ERR(tmp))return orig_fp;if (tmp != fp) {true ;if (!jit_data) {sizeof (*jit_data), GFP_KERNEL);if (!jit_data) {goto out;if (jit_data->ctx.addrs) {true ;goto skip_init_ctx;sizeof (jit));sizeof (*jit.addrs), GFP_KERNEL);if (jit.addrs == NULL) {goto free_addrs;/* * Three initial passes: * - 1/2: Determine clobbered registers * - 3: Calculate program size and addrs array for (pass = 1; pass <= 3; pass++) {if (bpf_jit_prog(&jit, fp, extra_pass)) {goto free_addrs;/* * Final pass: Allocate and generate program if (!header) {goto free_addrs;if (bpf_jit_prog(&jit, fp, extra_pass)) {goto free_addrs;if (bpf_jit_enable > 1) {if (!fp->is_func || extra_pass) {if (bpf_jit_binary_lock_ro(header)) {goto free_addrs;else {void *) jit.prg_buf;if (!fp->is_func || extra_pass) {if (tmp_blinded)return fp;bool bpf_jit_supports_kfunc_call(void )return true ;bool bpf_jit_supports_far_kfunc_call(void )return true ;
Messung V0.5 C=93 H=91 G=91
¤ Dauer der Verarbeitung: 0.52 Sekunden
¤ *© Formatika GbR, Deutschland
