// SPDX-License-Identifier: GPL-2.0-or-later /* * Linux Socket Filter - Kernel level socket filtering * * Based on the design of the Berkeley Packet Filter . The new * internal format has been designed by PLUMgrid : * * Copyright ( c ) 2011 - 2014 PLUMgrid , http : //plumgrid.com * * Authors : * * Jay Schulist < jschlst @ samba . org > * Alexei Starovoitov < ast @ plumgrid . com > * Daniel Borkmann < dborkman @ redhat . com > * * Andi Kleen - Fix a few bad bugs and races . * Kris Katterjohn - Added many additional checks in bpf_check_classic ( ) #include <uapi/linux/btf.h>#include <linux/filter.h>#include <linux/skbuff.h>#include <linux/vmalloc.h>#include <linux/prandom.h>#include <linux/bpf.h>#include <linux/btf.h>#include <linux/objtool.h>#include <linux/overflow.h>#include <linux/rbtree_latch.h>#include <linux/kallsyms.h>#include <linux/rcupdate.h>#include <linux/perf_event.h>#include <linux/extable.h>#include <linux/log2.h>#include <linux/bpf_verifier.h>#include <linux/nodemask.h>#include <linux/nospec.h>#include <linux/bpf_mem_alloc.h>#include <linux/memcontrol.h>#include <linux/execmem.h>#include <asm /barrier.h>#include <linux/unaligned.h>/* Registers */ #define BPF_R0 regs[BPF_REG_0]#define BPF_R1 regs[BPF_REG_1]#define BPF_R2 regs[BPF_REG_2]#define BPF_R3 regs[BPF_REG_3]#define BPF_R4 regs[BPF_REG_4]#define BPF_R5 regs[BPF_REG_5]#define BPF_R6 regs[BPF_REG_6]#define BPF_R7 regs[BPF_REG_7]#define BPF_R8 regs[BPF_REG_8]#define BPF_R9 regs[BPF_REG_9]#define BPF_R10 regs[BPF_REG_10]/* Named registers */ #define DST regs[insn->dst_reg]#define SRC regs[insn->src_reg]#define FP regs[BPF_REG_FP]#define AX regs[BPF_REG_AX]#define ARG1 regs[BPF_REG_ARG1]#define CTX regs[BPF_REG_CTX]#define OFF insn->off#define IMM insn->immstruct bpf_mem_alloc bpf_global_ma;bool bpf_global_ma_set;/* No hurry in this branch * * Exported for the bpf jit load helper . void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb, int k, unsigned int size)if (k >= SKF_NET_OFF) {else if (k >= SKF_LL_OFF) {if (unlikely(!skb_mac_header_was_set(skb)))return NULL;if (ptr >= skb->head && ptr + size <= skb_tail_pointer(skb))return ptr;return NULL;/* tell bpf programs that include vmlinux.h kernel's PAGE_SIZE */ enum page_size_enum {struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flags)struct bpf_prog_aux *aux;struct bpf_prog *fp;if (fp == NULL)return NULL;sizeof (*aux), bpf_memcg_flags(GFP_KERNEL | gfp_extra_flags));if (aux == NULL) {return NULL;int , bpf_memcg_flags(GFP_KERNEL | gfp_extra_flags));if (!fp->active) {return NULL;#ifdef CONFIG_CGROUP_BPF#endif #ifdef CONFIG_FINEIBT#endif #ifdef CONFIG_BPF_SYSCALL#endif return fp;struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags)struct bpf_prog *prog;int cpu;if (!prog)return NULL;struct bpf_prog_stats, gfp_flags);if (!prog->stats) {return NULL;struct bpf_prog_stats *pstats;return prog;int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog)if (!prog->aux->nr_linfo || !prog->jit_requested)return 0 ;sizeof (*prog->aux->jited_linfo),if (!prog->aux->jited_linfo)return -ENOMEM;return 0 ;void bpf_prog_jit_attempt_done(struct bpf_prog *prog)if (prog->aux->jited_linfo &&0 ])) {/* The jit engine is responsible to provide an array * for insn_off to the jited_off mapping ( insn_to_jit_off ) . * * The idx to this array is the insn_off . Hence , the insn_off * here is relative to the prog itself instead of the main prog . * This array has one entry for each xlated bpf insn . * * jited_off is the byte off to the end of the jited insn . * * Hence , with * insn_start : * The first bpf insn off of the prog . The insn off * here is relative to the main prog . * e . g . if prog is a subprog , insn_start > 0 * linfo_idx : * The prog ' s idx to prog - > aux - > linfo and jited_linfo * * jited_linfo [ linfo_idx ] = prog - > bpf_func * * For i > linfo_idx , * * jited_linfo [ i ] = prog - > bpf_func + * insn_to_jit_off [ linfo [ i ] . insn_off - insn_start - 1 ] void bpf_prog_fill_jited_linfo(struct bpf_prog *prog,const u32 *insn_to_jit_off)const struct bpf_line_info *linfo;void **jited_linfo;if (!prog->aux->jited_linfo || prog->aux->func_idx > prog->aux->func_cnt)/* Userspace did not provide linfo */ return ;0 ].insn_off;0 ] = prog->bpf_func;for (i = 1 ; i < nr_linfo && linfo[i].insn_off < insn_end; i++)/* The verifier ensures that linfo[i].insn_off is * strictly increasing 1 ];struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size,struct bpf_prog *fp;if (pages <= fp_old->pages)return fp_old;if (fp) {/* We keep fp->aux from fp_old around in the new * reallocated structure . return fp;void __bpf_prog_free(struct bpf_prog *fp)if (fp->aux) {int bpf_prog_calc_tag(struct bpf_prog *fp)const u32 bits_offset = SHA1_BLOCK_SIZE - sizeof (__be64);struct bpf_insn *dst;bool was_ld_map;if (!raw)return -ENOMEM;0 , sizeof (ws));/* We need to take out the map fd for the digest calculation * since they are unstable from user space side . void *)raw;for (i = 0 , was_ld_map = false ; i < fp->len; i++) {if (!was_ld_map &&true ;0 ;else if (was_ld_map &&0 &&0 &&0 &&0 ) {false ;0 ;else {false ;0 , raw_size - psize);0 x80;if (bsize - psize >= sizeof (__be64)) {sizeof (__be64));else {1 ) << 3 );while (blocks--) {for (i = 0 ; i < SHA1_DIGEST_WORDS; i++)sizeof (fp->tag));return 0 ;static int bpf_adj_delta_to_imm(struct bpf_insn *insn, u32 pos, s32 end_old,const bool probe_pass)const s64 imm_min = S32_MIN, imm_max = S32_MAX;if (curr < pos && curr + imm + 1 >= end_old)else if (curr >= end_new && curr + imm + 1 < end_new)if (imm < imm_min || imm > imm_max)return -ERANGE;if (!probe_pass)return 0 ;static int bpf_adj_delta_to_off(struct bpf_insn *insn, u32 pos, s32 end_old,const bool probe_pass)if (insn->code == (BPF_JMP32 | BPF_JA)) {else {if (curr < pos && curr + off + 1 >= end_old)else if (curr >= end_new && curr + off + 1 < end_new)if (off < off_min || off > off_max)return -ERANGE;if (!probe_pass) {if (insn->code == (BPF_JMP32 | BPF_JA))else return 0 ;static int bpf_adj_branches(struct bpf_prog *prog, u32 pos, s32 end_old,const bool probe_pass)0 );struct bpf_insn *insn = prog->insnsi;int ret = 0 ;for (i = 0 ; i < insn_cnt; i++, insn++) {/* In the probing pass we still operate on the original, * unpatched image in order to check overflows before we * do any other adjustments . Therefore skip the patchlet . if (probe_pass && i == pos) {if (bpf_pseudo_func(insn)) {if (ret)return ret;continue ;if ((BPF_CLASS(code) != BPF_JMP &&continue ;/* Adjust offset of jmps if we cross patch boundaries. */ if (BPF_OP(code) == BPF_CALL) {if (insn->src_reg != BPF_PSEUDO_CALL)continue ;else {if (ret)break ;return ret;static void bpf_adj_linfo(struct bpf_prog *prog, u32 off, u32 delta)struct bpf_line_info *linfo;if (!nr_linfo || !delta)return ;for (i = 0 ; i < nr_linfo; i++)if (off < linfo[i].insn_off)break ;/* Push all off < linfo[i].insn_off by delta */ for (; i < nr_linfo; i++)struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,const struct bpf_insn *patch, u32 len)1 ;const u32 cnt_max = S16_MAX;struct bpf_prog *prog_adj;int err;/* Since our patchlet doesn't expand the image, we're done. */ if (insn_delta == 0 ) {sizeof (*patch));return prog;/* Reject anything that would potentially let the insn->off * target overflow when we have excessive program expansions . * We need to probe here before we do any reallocation where * we afterwards may not fail anymore . if (insn_adj_cnt > cnt_max &&1 , off + len, true )))return ERR_PTR(err);/* Several new instructions need to be inserted. Make room * for them . Likely , there ' s no need for a new allocation as * last page could have large enough tailroom . if (!prog_adj)return ERR_PTR(-ENOMEM);/* Patching happens in 3 steps: * * 1 ) Move over tail of insnsi from next instruction onwards , * so we can patch the single target insn with one or more * new ones ( patching is always from 1 to n insns , n > 0 ) . * 2 ) Inject new instructions at the target location . * 3 ) Adjust branch offsets if necessary . 1 ,sizeof (*patch) * insn_rest);sizeof (*patch) * len);/* We are guaranteed to not fail at this point, otherwise * the ship has sailed to reverse to the original state . An * overflow cannot happen at this point . 1 , off + len, false ));return prog_adj;int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt)int err;/* Branch offsets can't overflow when program is shrinking, no need * to call bpf_adj_branches ( . . . , true ) here sizeof (struct bpf_insn) * (prog->len - off - cnt));false );return err;static void bpf_prog_kallsyms_del_subprogs(struct bpf_prog *fp)int i;for (i = 0 ; i < fp->aux->real_func_cnt; i++)void bpf_prog_kallsyms_del_all(struct bpf_prog *fp)#ifdef CONFIG_BPF_JIT/* All BPF JIT sysctl knobs here. */ int bpf_jit_enable __read_mostly = IS_BUILTIN(CONFIG_BPF_JIT_DEFAULT_ON);int bpf_jit_kallsyms __read_mostly = IS_BUILTIN(CONFIG_BPF_JIT_DEFAULT_ON);int bpf_jit_harden __read_mostly;long bpf_jit_limit __read_mostly;long bpf_jit_limit_max __read_mostly;static void struct bpf_prog *prog)unsigned long ) prog->bpf_func;static void struct bpf_prog *prog)char *sym = prog->aux->ksym.name;const char *end = sym + KSYM_NAME_LEN;const struct btf_type *type;const char *func_name;sizeof ("bpf_prog_" ) +sizeof (prog->tag) * 2 +/* name has been null terminated. * We should need + 1 for the ' _ ' preceding * the name . However , the null character * is double counted between the name and the * sizeof ( " bpf_prog_ " ) above , so we omit * the + 1 here . sizeof (prog->aux->name) > KSYM_NAME_LEN);"bpf_prog_" );sizeof (prog->tag));/* prog->aux->name will be ignored if full btf name is available */ if (prog->aux->func_info_cnt && prog->aux->func_idx < prog->aux->func_info_cnt) {"_%s" , func_name);return ;if (prog->aux->name[0 ])"_%s" , prog->aux->name);else 0 ;static unsigned long bpf_get_ksym_start(struct latch_tree_node *n)return container_of(n, struct bpf_ksym, tnode)->start;static __always_inline bool bpf_tree_less(struct latch_tree_node *a,struct latch_tree_node *b)return bpf_get_ksym_start(a) < bpf_get_ksym_start(b);static __always_inline int bpf_tree_comp(void *key, struct latch_tree_node *n)unsigned long val = (unsigned long )key;const struct bpf_ksym *ksym;struct bpf_ksym, tnode);if (val < ksym->start)return -1 ;/* Ensure that we detect return addresses as part of the program, when * the final instruction is a call for a program part of the stack * trace . Therefore , do val > ksym - > end instead of val > = ksym - > end . if (val > ksym->end)return 1 ;return 0 ;static const struct latch_tree_ops bpf_tree_ops = {static DEFINE_SPINLOCK(bpf_lock);static LIST_HEAD(bpf_kallsyms);static struct latch_tree_root bpf_tree __cacheline_aligned;void bpf_ksym_add(struct bpf_ksym *ksym)static void __bpf_ksym_del(struct bpf_ksym *ksym)if (list_empty(&ksym->lnode))return ;void bpf_ksym_del(struct bpf_ksym *ksym)static bool bpf_prog_kallsyms_candidate(const struct bpf_prog *fp)return fp->jited && !bpf_prog_was_classic(fp);void bpf_prog_kallsyms_add(struct bpf_prog *fp)if (!bpf_prog_kallsyms_candidate(fp) ||return ;true ;#ifdef CONFIG_FINEIBT/* * When FineIBT , code in the _ _ cfi_foo ( ) symbols can get executed * and hence unwinder needs help . if (cfi_mode != CFI_FINEIBT)return ;"__cfi_%s" , fp->aux->ksym.name);unsigned long ) fp->bpf_func - 16 ;unsigned long ) fp->bpf_func;#endif void bpf_prog_kallsyms_del(struct bpf_prog *fp)if (!bpf_prog_kallsyms_candidate(fp))return ;#ifdef CONFIG_FINEIBTif (cfi_mode != CFI_FINEIBT)return ;#endif static struct bpf_ksym *bpf_ksym_find(unsigned long addr)struct latch_tree_node *n;void *)addr, &bpf_tree, &bpf_tree_ops);return n ? container_of(n, struct bpf_ksym, tnode) : NULL;int __bpf_address_lookup(unsigned long addr, unsigned long *size,unsigned long *off, char *sym)struct bpf_ksym *ksym;int ret = 0 ;if (ksym) {unsigned long symbol_start = ksym->start;unsigned long symbol_end = ksym->end;if (size)if (off)return ret;bool is_bpf_text_address(unsigned long addr)bool ret;return ret;struct bpf_prog *bpf_prog_ksym_find(unsigned long addr)struct bpf_ksym *ksym;return ksym && ksym->prog ?struct bpf_prog_aux, ksym)->prog :const struct exception_table_entry *search_bpf_extables(unsigned long addr)const struct exception_table_entry *e = NULL;struct bpf_prog *prog;if (!prog)goto out;if (!prog->aux->num_exentries)goto out;return e;int bpf_get_kallsym(unsigned int symnum, unsigned long *value, char *type,char *sym)struct bpf_ksym *ksym;unsigned int it = 0 ;int ret = -ERANGE;if (!bpf_jit_kallsyms_enabled())return ret;if (it++ != symnum)continue ;0 ;break ;return ret;int bpf_jit_add_poke_descriptor(struct bpf_prog *prog,struct bpf_jit_poke_descriptor *poke)struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab;static const u32 poke_tab_max = 1024 ;1 ;if (size > poke_tab_max)return -ENOSPC;if (poke->tailcall_target || poke->tailcall_target_stable ||return -EINVAL;switch (poke->reason) {case BPF_POKE_REASON_TAIL_CALL:if (!poke->tail_call.map)return -EINVAL;break ;default :return -EINVAL;sizeof (*poke), GFP_KERNEL);if (!tab)return -ENOMEM;sizeof (*poke));return slot;/* * BPF program pack allocator . * * Most BPF programs are pretty small . Allocating a hole page for each * program is sometime a waste . Many small bpf program also adds pressure * to instruction TLB . To solve this issue , we introduce a BPF program pack * allocator . The prog_pack allocator uses HPAGE_PMD_SIZE page ( 2 MB on x86 ) * to host BPF programs . #define BPF_PROG_CHUNK_SHIFT 6 #define BPF_PROG_CHUNK_SIZE (1 << BPF_PROG_CHUNK_SHIFT)#define BPF_PROG_CHUNK_MASK (~(BPF_PROG_CHUNK_SIZE - 1 ))struct bpf_prog_pack {struct list_head list;void *ptr;unsigned long bitmap[];void bpf_jit_fill_hole_with_zero(void *area, unsigned int size)0 , size);#define BPF_PROG_SIZE_TO_NBITS(size) (round_up(size, BPF_PROG_CHUNK_SIZE) / BPF_PROG_CHUNK_SIZE)static DEFINE_MUTEX(pack_mutex);static LIST_HEAD(pack_list);/* PMD_SIZE is not available in some special config, e.g. ARCH=arm with * CONFIG_MMU = n . Use PAGE_SIZE in these cases . #ifdef PMD_SIZE/* PMD_SIZE is really big for some archs. It doesn't make sense to * reserve too much memory in one allocation . Hardcode BPF_PROG_PACK_SIZE to * 2 MiB * num_possible_nodes ( ) . On most architectures PMD_SIZE will be * greater than or equal to 2 MB . #define BPF_PROG_PACK_SIZE (SZ_2M * num_possible_nodes())#else #define BPF_PROG_PACK_SIZE PAGE_SIZE#endif #define BPF_PROG_CHUNK_COUNT (BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE)static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_insns)struct bpf_prog_pack *pack ;int err;pack = kzalloc(struct_size(pack , bitmap, BITS_TO_LONGS(BPF_PROG_CHUNK_COUNT)),if (!pack )return NULL;pack ->ptr = bpf_jit_alloc_exec(BPF_PROG_PACK_SIZE);if (!pack ->ptr)goto out;pack ->ptr, BPF_PROG_PACK_SIZE);pack ->bitmap, BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE);pack ->ptr);unsigned long )pack ->ptr,if (err)goto out;pack ->list, &pack_list);return pack ;pack ->ptr);pack );return NULL;void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns)unsigned int nbits = BPF_PROG_SIZE_TO_NBITS(size);struct bpf_prog_pack *pack ;unsigned long pos;void *ptr = NULL;if (size > BPF_PROG_PACK_SIZE) {if (ptr) {int err;unsigned long )ptr,if (err) {goto out;pack , &pack_list, list) {pack ->bitmap, BPF_PROG_CHUNK_COUNT, 0 ,0 );if (pos < BPF_PROG_CHUNK_COUNT)goto found_free_area;pack = alloc_new_pack(bpf_fill_ill_insns);if (!pack )goto out;0 ;pack ->bitmap, pos, nbits);void *)(pack ->ptr) + (pos << BPF_PROG_CHUNK_SHIFT);return ptr;void bpf_prog_pack_free(void *ptr, u32 size)struct bpf_prog_pack *pack = NULL, *tmp;unsigned int nbits;unsigned long pos;if (size > BPF_PROG_PACK_SIZE) {goto out;if (ptr >= tmp->ptr && (tmp->ptr + BPF_PROG_PACK_SIZE) > ptr) {pack = tmp;break ;if (WARN_ONCE(!pack , "bpf_prog_pack bug\n" ))goto out;unsigned long )ptr - (unsigned long )pack ->ptr) >> BPF_PROG_CHUNK_SHIFT;"bpf_prog_pack bug: missing bpf_arch_text_invalidate?\n" );pack ->bitmap, pos, nbits);if (bitmap_find_next_zero_area(pack ->bitmap, BPF_PROG_CHUNK_COUNT, 0 ,0 ) == 0 ) {pack ->list);pack ->ptr);pack );static atomic_long_t bpf_jit_current;/* Can be overridden by an arch's JIT compiler if it has a custom, * dedicated BPF backend memory area , or if neither of the two * below apply . void )#if defined (MODULES_VADDR)return MODULES_END - MODULES_VADDR;#else return VMALLOC_END - VMALLOC_START;#endif static int __init bpf_jit_charge_init(void )/* Only used as heuristic here to derive limit. */ 1 ,return 0 ;int bpf_jit_charge_modmem(u32 size)if (atomic_long_add_return(size, &bpf_jit_current) > READ_ONCE(bpf_jit_limit)) {if (!bpf_capable()) {return -EPERM;return 0 ;void bpf_jit_uncharge_modmem(u32 size)void *__weak bpf_jit_alloc_exec(unsigned long size)return execmem_alloc(EXECMEM_BPF, size);void __weak bpf_jit_free_exec(void *addr)struct bpf_binary_header *unsigned int proglen, u8 **image_ptr,unsigned int alignment,struct bpf_binary_header *hdr;/* Most of BPF filters are really small, but if some of them * fill a page , allow at least 128 extra bytes to insert a * random section of illegal instructions . sizeof (*hdr) + 128 , PAGE_SIZE);if (bpf_jit_charge_modmem(size))return NULL;if (!hdr) {return NULL;/* Fill space with illegal/arch-dep instructions. */ unsigned int , size - (proglen + sizeof (*hdr)),sizeof (*hdr));1 );/* Leave a random number of instructions before BPF code. */ return hdr;void bpf_jit_binary_free(struct bpf_binary_header *hdr)/* Allocate jit binary from bpf_prog_pack allocator. * Since the allocated memory is RO + X , the JIT engine cannot write directly * to the memory . To solve this problem , a RW buffer is also allocated at * as the same time . The JIT engine should calculate offsets based on the * RO memory address , but write JITed program to the RW buffer . Once the * JIT engine finishes , it calls bpf_jit_binary_pack_finalize , which copies * the JITed program to the RO memory . struct bpf_binary_header *unsigned int proglen, u8 **image_ptr,unsigned int alignment,struct bpf_binary_header **rw_header,struct bpf_binary_header *ro_header;/* add 16 bytes for a random section of illegal instructions */ sizeof (*ro_header) + 16 , BPF_PROG_CHUNK_SIZE);if (bpf_jit_charge_modmem(size))return NULL;if (!ro_header) {return NULL;if (!*rw_header) {return NULL;/* Fill space with illegal/arch-dep instructions. */ unsigned int , size - (proglen + sizeof (*ro_header)),sizeof (*ro_header));1 );return ro_header;/* Copy JITed text from rw_header to its final location, the ro_header. */ int bpf_jit_binary_pack_finalize(struct bpf_binary_header *ro_header,struct bpf_binary_header *rw_header)void *ptr;if (IS_ERR(ptr)) {return PTR_ERR(ptr);return 0 ;/* bpf_jit_binary_pack_free is called in two different scenarios: * 1 ) when the program is freed after ; * 2 ) when the JIT engine fails ( before bpf_jit_binary_pack_finalize ) . * For case 2 ) , we need to free both the RO memory and the RW buffer . * * bpf_jit_binary_pack_free requires proper ro_header - > size . However , * bpf_jit_binary_pack_alloc does not set it . Therefore , ro_header - > size * must be set with either bpf_jit_binary_pack_finalize ( normal path ) or * bpf_arch_text_copy ( when jit fails ) . void bpf_jit_binary_pack_free(struct bpf_binary_header *ro_header,struct bpf_binary_header *rw_header)struct bpf_binary_header *const struct bpf_prog *fp)unsigned long real_start = (unsigned long )fp->bpf_func;unsigned long addr;return (void *)addr;static inline struct bpf_binary_header *const struct bpf_prog *fp)unsigned long real_start = (unsigned long )fp->bpf_func;unsigned long addr;return (void *)addr;/* This symbol is only overridden by archs that have different * requirements than the usual eBPF JITs , f . e . when they only * implement cBPF JIT , do not set images read - only , etc . void __weak bpf_jit_free(struct bpf_prog *fp)if (fp->jited) {struct bpf_binary_header *hdr = bpf_jit_binary_hdr(fp);int bpf_jit_get_func_addr(const struct bpf_prog *prog,const struct bpf_insn *insn, bool extra_pass,bool *func_addr_fixed)int err;if (!*func_addr_fixed) {/* Place-holder address till the last pass has collected * all addresses for JITed subprograms in which case we * can pick them up from prog - > aux . if (!extra_pass)else if (prog->aux->func &&0 && off < prog->aux->real_func_cnt)else return -EINVAL;else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL &&if (err)return err;else {/* Address of a BPF helper call. Since part of the core * kernel , it ' s always at a fixed location . _ _ bpf_call_base * and the helper with imm relative to it are both in core * kernel . unsigned long )addr;return 0 ;const char *bpf_jit_get_prog_name(struct bpf_prog *prog)if (prog->aux->ksym.prog)return prog->aux->ksym.name;return prog->aux->name;static int bpf_jit_blind_insn(const struct bpf_insn *from,const struct bpf_insn *aux,struct bpf_insn *to_buff,bool emit_zext)struct bpf_insn *to = to_buff;1 != MAX_BPF_JIT_REG);1 != MAX_BPF_JIT_REG);/* Constraints on AX register: * * AX register is inaccessible from user space . It is mapped in * all JITs , and used here for constant blinding rewrites . It is * typically " stateless " meaning its contents are only valid within * the executed instruction , but not across several instructions . * There are a few exceptions however which are further detailed * below . * * Constant blinding is only used by JITs , not in the interpreter . * The interpreter uses AX in some occasions as a local temporary * register e . g . in DIV or MOD instructions . * * In restricted circumstances , the verifier can also use the AX * register for rewrites as long as they do not interfere with * the above cases ! if (from->dst_reg == BPF_REG_AX || from->src_reg == BPF_REG_AX)goto out;if (from->imm == 0 &&goto out;switch (from->code) {case BPF_ALU | BPF_ADD | BPF_K:case BPF_ALU | BPF_SUB | BPF_K:case BPF_ALU | BPF_AND | BPF_K:case BPF_ALU | BPF_OR | BPF_K:case BPF_ALU | BPF_XOR | BPF_K:case BPF_ALU | BPF_MUL | BPF_K:case BPF_ALU | BPF_MOV | BPF_K:case BPF_ALU | BPF_DIV | BPF_K:case BPF_ALU | BPF_MOD | BPF_K:break ;case BPF_ALU64 | BPF_ADD | BPF_K:case BPF_ALU64 | BPF_SUB | BPF_K:case BPF_ALU64 | BPF_AND | BPF_K:case BPF_ALU64 | BPF_OR | BPF_K:case BPF_ALU64 | BPF_XOR | BPF_K:case BPF_ALU64 | BPF_MUL | BPF_K:case BPF_ALU64 | BPF_MOV | BPF_K:case BPF_ALU64 | BPF_DIV | BPF_K:case BPF_ALU64 | BPF_MOD | BPF_K:break ;case BPF_JMP | BPF_JEQ | BPF_K:case BPF_JMP | BPF_JNE | BPF_K:case BPF_JMP | BPF_JGT | BPF_K:case BPF_JMP | BPF_JLT | BPF_K:case BPF_JMP | BPF_JGE | BPF_K:case BPF_JMP | BPF_JLE | BPF_K:case BPF_JMP | BPF_JSGT | BPF_K:case BPF_JMP | BPF_JSLT | BPF_K:case BPF_JMP | BPF_JSGE | BPF_K:case BPF_JMP | BPF_JSLE | BPF_K:case BPF_JMP | BPF_JSET | BPF_K:/* Accommodate for extra offset in case of a backjump. */ if (off < 0 )2 ;break ;case BPF_JMP32 | BPF_JEQ | BPF_K:case BPF_JMP32 | BPF_JNE | BPF_K:case BPF_JMP32 | BPF_JGT | BPF_K:case BPF_JMP32 | BPF_JLT | BPF_K:case BPF_JMP32 | BPF_JGE | BPF_K:case BPF_JMP32 | BPF_JLE | BPF_K:case BPF_JMP32 | BPF_JSGT | BPF_K:case BPF_JMP32 | BPF_JSLT | BPF_K:case BPF_JMP32 | BPF_JSGE | BPF_K:case BPF_JMP32 | BPF_JSLE | BPF_K:case BPF_JMP32 | BPF_JSET | BPF_K:/* Accommodate for extra offset in case of a backjump. */ if (off < 0 )2 ;break ;case BPF_LD | BPF_IMM | BPF_DW:1 ].imm);32 );0 ].dst_reg, BPF_REG_AX);break ;case 0 : /* Part 2 of BPF_LD | BPF_IMM | BPF_DW. */ 0 ].imm);if (emit_zext)0 ].dst_reg, BPF_REG_AX);break ;case BPF_ST | BPF_MEM | BPF_DW:case BPF_ST | BPF_MEM | BPF_W:case BPF_ST | BPF_MEM | BPF_H:case BPF_ST | BPF_MEM | BPF_B:break ;return to - to_buff;static struct bpf_prog *bpf_prog_clone_create(struct bpf_prog *fp_other,struct bpf_prog *fp;if (fp != NULL) {/* aux->prog still points to the fp_other one, so * when promoting the clone to the real program , * this still needs to be adapted . return fp;static void bpf_prog_clone_free(struct bpf_prog *fp)/* aux was stolen by the other clone, so we cannot free * it from this path ! It will be freed eventually by the * other program on release . * * At this point , we don ' t need a deferred release since * clone is guaranteed to not be locked . void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other)/* We have to repoint aux->prog to self, as we don't * know whether fp here is the clone or the original . struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog)struct bpf_insn insn_buff[16 ], aux[2 ];struct bpf_prog *clone, *tmp;int insn_delta, insn_cnt;struct bpf_insn *insn;int i, rewritten;if (!prog->blinding_requested || prog->blinded)return prog;if (!clone)return ERR_PTR(-ENOMEM);for (i = 0 ; i < insn_cnt; i++, insn++) {if (bpf_pseudo_func(insn)) {/* ld_imm64 with an address of bpf subprog is not * a user controlled constant . Don ' t randomize it , * since it will conflict with jit_subprogs ( ) logic . continue ;/* We temporarily need to hold the original ld64 insn * so that we can still access the first part in the * second blinding run . if (insn[0 ].code == (BPF_LD | BPF_IMM | BPF_DW) &&1 ].code == 0 )sizeof (aux));if (!rewritten)continue ;if (IS_ERR(tmp)) {/* Patching may have repointed aux->prog during * realloc from the original one , so we need to * fix it up here on error . return tmp;1 ;/* Walk new program and skip insns we just inserted. */ 1 ;return clone;#endif /* CONFIG_BPF_JIT */ /* Base function for offset calculation. Needs to go into .text section, * therefore keeping it non - static as well ; will also be used by JITs * anyway later on , so do not let the compiler omit it . This also needs * to go into kallsyms for correlation from e . g . bpftool , so naming * must not change . return 0 ;/* All UAPI available opcodes. */ #define BPF_INSN_MAP(INSN_2, INSN_3) \/* 32 bit ALU operations. */ \ /* Register based. */ \ AND , X), \OR , X), \XOR , X), \/* Immediate based. */ \ AND , K), \OR , K), \XOR , K), \/* 64 bit ALU operations. */ \ /* Register based. */ \ AND , X), \OR , X), \XOR , X), \/* Immediate based. */ \ AND , K), \OR , K), \XOR , K), \/* Call instruction. */ \ /* Exit instruction. */ \ EXIT ), \/* 32-bit Jump instructions. */ \ /* Register based. */ \ /* Immediate based. */ \ /* Jump instructions. */ \ /* Register based. */ \ /* Immediate based. */ \ /* Atomic operations. */ \ /* Store instructions. */ \ /* Register based. */ \ /* Immediate based. */ \ /* Load instructions. */ \ /* Register based. */ \ /* Immediate based. */ \ bool bpf_opcode_in_insntable(u8 code)#define BPF_INSN_2_TBL(x, y) [BPF_## x | BPF_## y] = true #define BPF_INSN_3_TBL(x, y, z) [BPF_## x | BPF_## y | BPF_## z] = true static const bool public_insntable[256 ] = {0 ... 255 ] = false ,/* Now overwrite non-defaults ... */ /* UAPI exposed, but rewritten opcodes. cBPF carry-over. */ true ,true ,true ,true ,true ,true ,true ,#undef BPF_INSN_3_TBL#undef BPF_INSN_2_TBLreturn public_insntable[code];#ifndef CONFIG_BPF_JIT_ALWAYS_ON/** * _ _ _ bpf_prog_run - run eBPF program on a given context * @ regs : is the array of MAX_BPF_EXT_REG eBPF pseudo - registers * @ insn : is the array of eBPF instructions * * Decode and execute eBPF instructions . * * Return : whatever value is in % BPF_R0 at program exit static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn)#define BPF_INSN_2_LBL(x, y) [BPF_## x | BPF_## y] = &&x## _## y#define BPF_INSN_3_LBL(x, y, z) [BPF_## x | BPF_## y | BPF_## z] = &&x## _## y## _## zstatic const void * const jumptable[256 ] __annotate_jump_table = {0 ... 255 ] = &&default_label,/* Now overwrite non-defaults ... */ /* Non-UAPI available opcodes. */ #undef BPF_INSN_3_LBL#undef BPF_INSN_2_LBL0 ;#define CONT ({ insn++; goto select_insn; })#define CONT_JMP ({ insn++; goto select_insn; })goto *jumptable[insn->code];/* Explicitly mask the register-based shift amounts with 63 or 31 * to avoid undefined behavior . Normally this won ' t affect the * generated code , for example , in case of native 64 bit archs such * as x86 - 64 or arm64 , the compiler is optimizing the AND away for * the interpreter . In case of JITs , each of the JIT backends compiles * the BPF shift operations to machine instructions which produce * implementation - defined results in such a case ; the resulting * contents of the register may be arbitrary , but program behaviour * as a whole remains defined . In other words , in case of JIT backends , * the AND must / not / be added to the emitted LSH / RSH / ARSH translation . /* ALU (shifts) */ #define SHT(OPCODE, OP) \## OPCODE## _X: \63 ); \## OPCODE## _X: \31 ); \## OPCODE## _K: \## OPCODE## _K: \/* ALU (rest) */ #define ALU(OPCODE, OP) \## OPCODE## _X: \## OPCODE## _X: \## OPCODE## _K: \## OPCODE## _K: \AND , &)OR , |)XOR , ^)#undef SHT#undef ALUswitch (OFF) {case 0 :break ;case 8 :break ;case 16 :break ;switch (OFF) {case 0 :break ;case 8 :break ;case 16 :break ;case 32 :break ;0 ].imm | ((u64) (u32) insn[1 ].imm) << 32 ;31 ));63 );switch (OFF) {case 0 :break ;case 1 :break ;switch (OFF) {case 0 :break ;case 1 :if ((s32)DST < 0 )else break ;switch (OFF) {case 0 :break ;case 1 :break ;switch (OFF) {case 0 :break ;case 1 :if ((s32)DST < 0 )else break ;switch (OFF) {case 0 :break ;case 1 :break ;switch (OFF) {case 0 :break ;case 1 :if (((s32)DST < 0 ) == ((s32)SRC < 0 ))else break ;switch (OFF) {case 0 :break ;case 1 :break ;switch (OFF) {case 0 :break ;case 1 :if (((s32)DST < 0 ) == ((s32)IMM < 0 ))else break ;switch (IMM) {case 16 :break ;case 32 :break ;case 64 :break ;switch (IMM) {case 16 :break ;case 32 :break ;case 64 :break ;switch (IMM) {case 16 :break ;case 32 :break ;case 64 :break ;/* CALL */ /* Function call scratches BPF_R1-BPF_R5 registers, * preserves BPF_R6 - BPF_R9 , and stores return value * into BPF_R0 . 1 );struct bpf_map *map = (struct bpf_map *) (unsigned long ) BPF_R2;struct bpf_array *array = container_of(map, struct bpf_array, map);struct bpf_prog *prog;if (unlikely(index >= array->map.max_entries))goto out;if (unlikely(tail_call_cnt >= MAX_TAIL_CALL_CNT))goto out;if (!prog)goto out;/* ARG1 at this point is guaranteed to point to CTX from * the verifier side due to the fact that the tail call is * handled like a helper , that is , bpf_tail_call_proto , * where arg1_type is ARG_PTR_TO_CTX . goto select_insn;return BPF_R0;/* JMP */ #define COND_JMP(SIGN, OPCODE, CMP_OP) \## OPCODE## _X: \if ((SIGN## 64 ) DST CMP_OP (SIGN## 64 ) SRC) { \## OPCODE## _X: \if ((SIGN## 32 ) DST CMP_OP (SIGN## 32 ) SRC) { \## OPCODE## _K: \if ((SIGN## 64 ) DST CMP_OP (SIGN## 64 ) IMM) { \## OPCODE## _K: \if ((SIGN## 32 ) DST CMP_OP (SIGN## 32 ) IMM) { \#undef COND_JMP/* ST, STX and LDX*/ /* Speculation barrier for mitigating Speculative Store Bypass, * Bounds - Check Bypass and Type Confusion . In case of arm64 , we * rely on the firmware mitigation as controlled via the ssbd * kernel parameter . Whenever the mitigation is enabled , it * works for all of the kernel code with no need to provide any * additional instructions here . In case of x86 , we use ' lfence ' * insn for mitigation . We reuse preexisting logic from Spectre * v1 mitigation that happens to produce the required code on * x86 for v4 as well . #define LDST(SIZEOP, SIZE) \## SIZEOP: \unsigned long ) (DST + insn->off) = SRC; \## SIZEOP: \unsigned long ) (DST + insn->off) = IMM; \## SIZEOP: \unsigned long ) (SRC + insn->off); \## SIZEOP: \sizeof (SIZE), \const void *)(long ) (SRC + insn->off)); \#undef LDST#define LDSX(SIZEOP, SIZE) \## SIZEOP: \unsigned long ) (SRC + insn->off); \## SIZEOP: \sizeof (SIZE), \const void *)(long ) (SRC + insn->off)); \#undef LDSX#define ATOMIC_ALU_OP(BOP, KOP) \case BOP: \if (BPF_SIZE(insn->code) == BPF_W) \## KOP((u32) SRC, (atomic_t *)(unsigned long ) \else if (BPF_SIZE(insn->code) == BPF_DW) \## KOP((u64) SRC, (atomic64_t *)(unsigned long ) \else \goto default_label; \break ; \case BOP | BPF_FETCH: \if (BPF_SIZE(insn->code) == BPF_W) \## KOP( \unsigned long ) (DST + insn->off)); \else if (BPF_SIZE(insn->code) == BPF_DW) \## KOP( \unsigned long ) (DST + insn->off)); \else \goto default_label; \break ;switch (IMM) {/* Atomic read-modify-write instructions support only W and DW * size modifiers . and )or )xor )#undef ATOMIC_ALU_OPcase BPF_XCHG:if (BPF_SIZE(insn->code) == BPF_W)unsigned long ) (DST + insn->off),else if (BPF_SIZE(insn->code) == BPF_DW)unsigned long ) (DST + insn->off),else goto default_label;break ;case BPF_CMPXCHG:if (BPF_SIZE(insn->code) == BPF_W)unsigned long ) (DST + insn->off),else if (BPF_SIZE(insn->code) == BPF_DW)unsigned long ) (DST + insn->off),else goto default_label;break ;/* Atomic load and store instructions support all size * modifiers . case BPF_LOAD_ACQ:switch (BPF_SIZE(insn->code)) {#define LOAD_ACQUIRE(SIZEOP, SIZE) \case BPF_## SIZEOP: \unsigned long )(SRC + insn->off)); \break ;#ifdef CONFIG_64BIT#endif #undef LOAD_ACQUIREdefault :goto default_label;break ;case BPF_STORE_REL:switch (BPF_SIZE(insn->code)) {#define STORE_RELEASE(SIZEOP, SIZE) \case BPF_## SIZEOP: \unsigned long )(DST + insn->off), (SIZE)SRC); \break ;#ifdef CONFIG_64BIT#endif #undef STORE_RELEASEdefault :goto default_label;break ;default :goto default_label;/* If we ever reach this, we have a bug somewhere. Die hard here * instead of just returning 0 ; we could be somewhere in a subprog , * so execution could continue otherwise which we do / not / want . * * Note , verifier whitelists all opcodes in bpf_opcode_in_insntable ( ) . "BPF interpreter: unknown opcode %02x (imm: 0x%x)\n" ,1 );return 0 ;#define PROG_NAME(stack_size) __bpf_prog_run## stack_size#define DEFINE_BPF_PROG_RUN(stack_size) \static unsigned int PROG_NAME(stack_size)(const void *ctx, const struct bpf_insn *insn) \sizeof (u64)]; \sizeof (stack)); \unsigned long ) &stack[ARRAY_SIZE(stack)]; \unsigned long ) ctx; \return ___bpf_prog_run(regs, insn); \#define PROG_NAME_ARGS(stack_size) __bpf_prog_run_args## stack_size#define DEFINE_BPF_PROG_RUN_ARGS(stack_size) \static u64 PROG_NAME_ARGS(stack_size)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5, \const struct bpf_insn *insn) \sizeof (u64)]; \sizeof (stack)); \unsigned long ) &stack[ARRAY_SIZE(stack)]; \return ___bpf_prog_run(regs, insn); \#define EVAL1(FN, X) FN(X)#define EVAL2(FN, X, Y...) FN(X) EVAL1(FN, Y)#define EVAL3(FN, X, Y...) FN(X) EVAL2(FN, Y)#define EVAL4(FN, X, Y...) FN(X) EVAL3(FN, Y)#define EVAL5(FN, X, Y...) FN(X) EVAL4(FN, Y)#define EVAL6(FN, X, Y...) FN(X) EVAL5(FN, Y)32 , 64 , 96 , 128 , 160 , 192 );224 , 256 , 288 , 320 , 352 , 384 );416 , 448 , 480 , 512 );32 , 64 , 96 , 128 , 160 , 192 );224 , 256 , 288 , 320 , 352 , 384 );416 , 448 , 480 , 512 );#define PROG_NAME_LIST(stack_size) PROG_NAME(stack_size),static unsigned int (*interpreters[])(const void *ctx,const struct bpf_insn *insn) = {32 , 64 , 96 , 128 , 160 , 192 )224 , 256 , 288 , 320 , 352 , 384 )416 , 448 , 480 , 512 )#undef PROG_NAME_LIST#define PROG_NAME_LIST(stack_size) PROG_NAME_ARGS(stack_size),static __maybe_unusedconst struct bpf_insn *insn) = {32 , 64 , 96 , 128 , 160 , 192 )224 , 256 , 288 , 320 , 352 , 384 )416 , 448 , 480 , 512 )#undef PROG_NAME_LIST#ifdef CONFIG_BPF_SYSCALLvoid bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth)1 );32 ) / 32 ) - 1 ] -#endif #endif static unsigned int __bpf_prog_ret0_warn(const void *ctx,const struct bpf_insn *insn)/* If this handler ever gets executed, then BPF_JIT_ALWAYS_ON * is not working properly , so warn about it ! 1 );return 0 ;static bool __bpf_prog_map_compatible(struct bpf_map *map,const struct bpf_prog *fp)enum bpf_prog_type prog_type = resolve_prog_type(fp);struct bpf_prog_aux *aux = fp->aux;enum bpf_cgroup_storage_type i;bool ret = false ;if (fp->kprobe_override)return ret;/* There's no owner yet where we could check for compatibility. */ if (!map->owner) {if (!map->owner)goto err;0 ;true ;else {if (ret &&false ;if (!ret)break ;0 ;if (ret &&switch (prog_type) {case BPF_PROG_TYPE_TRACING:case BPF_PROG_TYPE_LSM:case BPF_PROG_TYPE_EXT:case BPF_PROG_TYPE_STRUCT_OPS:false ;break ;default :break ;return ret;bool bpf_prog_map_compatible(struct bpf_map *map, const struct bpf_prog *fp)/* XDP programs inserted into maps are not guaranteed to run on * a particular netdev ( and can run outside driver context entirely * in the case of devmap and cpumap ) . Until device checks * are implemented , prohibit adding dev - bound programs to program maps . if (bpf_prog_is_dev_bound(fp->aux))return false ;return __bpf_prog_map_compatible(map, fp);static int bpf_check_tail_call(const struct bpf_prog *fp)struct bpf_prog_aux *aux = fp->aux;int i, ret = 0 ;for (i = 0 ; i < aux->used_map_cnt; i++) {struct bpf_map *map = aux->used_maps[i];if (!map_type_contains_progs(map))continue ;if (!__bpf_prog_map_compatible(map, fp)) {goto out;return ret;static bool bpf_prog_select_interpreter(struct bpf_prog *fp)bool select_interpreter = false ;#ifndef CONFIG_BPF_JIT_ALWAYS_ON1 );32 ) / 32 ) - 1 ;/* may_goto may cause stack size > 512, leading to idx out-of-bounds. * But for non - JITed programs , we don ' t need bpf_func , so no bounds * check needed . if (idx < ARRAY_SIZE(interpreters)) {true ;else {#else #endif return select_interpreter;/** * bpf_prog_select_runtime - select exec runtime for BPF program * @ fp : bpf_prog populated with BPF program * @ err : pointer to error variable * * Try to JIT eBPF program , if JIT is not available , use interpreter . * The BPF program will be executed via bpf_prog_run ( ) function . * * Return : the & fp argument along with & err set to 0 for success or * a negative errno code on failure struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)/* In case of BPF to BPF calls, verifier did all the prep * work with regards to JITing , etc . bool jit_needed = false ;if (fp->bpf_func)goto finalize;if (IS_ENABLED(CONFIG_BPF_JIT_ALWAYS_ON) ||true ;if (!bpf_prog_select_interpreter(fp))true ;/* eBPF JITs can rewrite the program in case constant * blinding is active . However , in case of error during * blinding , bpf_int_jit_compile ( ) must always return a * valid program , which in this case would simply not * be JITed , but falls back to the interpreter . if (!bpf_prog_is_offloaded(fp->aux)) {if (*err)return fp;if (!fp->jited && jit_needed) {return fp;else {if (*err)return fp;if (*err)return fp;/* The tail call compatibility check can only be done at * this late stage as we need to determine , if we deal * with JITed or non JITed program concatenations and not * all eBPF JITs might immediately support all features . return fp;static unsigned int __bpf_prog_ret1(const void *ctx,const struct bpf_insn *insn)return 1 ;static struct bpf_prog_dummy {struct bpf_prog prog;struct bpf_empty_prog_array bpf_empty_prog_array = {struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags)struct bpf_prog_array *p;if (prog_cnt)1 ), flags);else return p;void bpf_prog_array_free(struct bpf_prog_array *progs)if (!progs || progs == &bpf_empty_prog_array.hdr)return ;static void __bpf_prog_array_free_sleepable_cb(struct rcu_head *rcu)struct bpf_prog_array *progs;/* If RCU Tasks Trace grace period implies RCU grace period, there is * no need to call kfree_rcu ( ) , just call kfree ( ) directly . struct bpf_prog_array, rcu);if (rcu_trace_implies_rcu_gp())else void bpf_prog_array_free_sleepable(struct bpf_prog_array *progs)if (!progs || progs == &bpf_empty_prog_array.hdr)return ;int bpf_prog_array_length(struct bpf_prog_array *array)struct bpf_prog_array_item *item;0 ;for (item = array->items; item->prog; item++)if (item->prog != &dummy_bpf_prog.prog)return cnt;bool bpf_prog_array_is_empty(struct bpf_prog_array *array)struct bpf_prog_array_item *item;for (item = array->items; item->prog; item++)if (item->prog != &dummy_bpf_prog.prog)return false ;return true ;static bool bpf_prog_array_copy_core(struct bpf_prog_array *array,struct bpf_prog_array_item *item;int i = 0 ;for (item = array->items; item->prog; item++) {if (item->prog == &dummy_bpf_prog.prog)continue ;if (++i == request_cnt) {break ;return !!(item->prog);int bpf_prog_array_copy_to_user(struct bpf_prog_array *array,unsigned long err = 0 ;bool nospc;/* users of this function are doing: * cnt = bpf_prog_array_length ( ) ; * if ( cnt > 0 ) * bpf_prog_array_copy_to_user ( . . . , cnt ) ; * so below kcalloc doesn ' t need extra cnt > 0 check . sizeof (u32), GFP_USER | __GFP_NOWARN);if (!ids)return -ENOMEM;sizeof (u32));if (err)return -EFAULT;if (nospc)return -ENOSPC;return 0 ;void bpf_prog_array_delete_safe(struct bpf_prog_array *array,struct bpf_prog *old_prog)struct bpf_prog_array_item *item;for (item = array->items; item->prog; item++)if (item->prog == old_prog) {break ;/** * bpf_prog_array_delete_safe_at ( ) - Replaces the program at the given * index into the program array with * a dummy no - op program . * @ array : a bpf_prog_array * @ index : the index of the program to replace * * Skips over dummy programs , by not counting them , when calculating * the position of the program to replace . * * Return : * * 0 - Success * * - EINVAL - Invalid index value . Must be a non - negative integer . * * - ENOENT - Index out of range int bpf_prog_array_delete_safe_at(struct bpf_prog_array *array, int index)return bpf_prog_array_update_at(array, index, &dummy_bpf_prog.prog);/** * bpf_prog_array_update_at ( ) - Updates the program at the given index * into the program array . * @ array : a bpf_prog_array * @ index : the index of the program to update * @ prog : the program to insert into the array * * Skips over dummy programs , by not counting them , when calculating * the position of the program to update . * * Return : * * 0 - Success * * - EINVAL - Invalid index value . Must be a non - negative integer . * * - ENOENT - Index out of range int bpf_prog_array_update_at(struct bpf_prog_array *array, int index,struct bpf_prog *prog)struct bpf_prog_array_item *item;if (unlikely(index < 0 ))return -EINVAL;for (item = array->items; item->prog; item++) {if (item->prog == &dummy_bpf_prog.prog)continue ;if (!index) {return 0 ;return -ENOENT;int bpf_prog_array_copy(struct bpf_prog_array *old_array,struct bpf_prog *exclude_prog,struct bpf_prog *include_prog,struct bpf_prog_array **new_array)int new_prog_cnt, carry_prog_cnt = 0 ;struct bpf_prog_array_item *existing, *new ;struct bpf_prog_array *array;bool found_exclude = false ;/* Figure out how many existing progs we need to carry over to * the new array . if (old_array) {for (; existing->prog; existing++) {if (existing->prog == exclude_prog) {true ;continue ;if (existing->prog != &dummy_bpf_prog.prog)if (existing->prog == include_prog)return -EEXIST;if (exclude_prog && !found_exclude)return -ENOENT;/* How many progs (not NULL) will be in the new array? */ if (include_prog)1 ;/* Do we have any prog (not NULL) in the new array? */ if (!new_prog_cnt) {return 0 ;/* +1 as the end of prog_array is marked with NULL */ 1 , GFP_KERNEL);if (!array)return -ENOMEM;new = array->items;/* Fill in the new prog array */ if (carry_prog_cnt) {for (; existing->prog; existing++) {if (existing->prog == exclude_prog ||continue ;new ->prog = existing->prog;new ->bpf_cookie = existing->bpf_cookie;new ++;if (include_prog) {new ->prog = include_prog;new ->bpf_cookie = bpf_cookie;new ++;new ->prog = NULL;return 0 ;int bpf_prog_array_copy_info(struct bpf_prog_array *array,0 ;if (array)/* return early if user requested only program count or nothing to copy */ if (!request_cnt || !cnt)return 0 ;/* this function is called under trace/bpf_trace.c: bpf_event_mutex */ return bpf_prog_array_copy_core(array, prog_ids, request_cnt) ? -ENOSPC0 ;void __bpf_free_used_maps(struct bpf_prog_aux *aux,struct bpf_map **used_maps, u32 len)struct bpf_map *map;bool sleepable;for (i = 0 ; i < len; i++) {if (map->ops->map_poke_untrack)if (sleepable)static void bpf_free_used_maps(struct bpf_prog_aux *aux)void __bpf_free_used_btfs(struct btf_mod_pair *used_btfs, u32 len)#ifdef CONFIG_BPF_SYSCALLstruct btf_mod_pair *btf_mod;for (i = 0 ; i < len; i++) {if (btf_mod->module)#endif static void bpf_free_used_btfs(struct bpf_prog_aux *aux)static void bpf_prog_free_deferred(struct work_struct *work)struct bpf_prog_aux *aux;int i;struct bpf_prog_aux, work);#ifdef CONFIG_BPF_SYSCALL#endif #ifdef CONFIG_CGROUP_BPFif (aux->cgroup_atype != CGROUP_BPF_ATTACH_TYPE_INVALID)#endif if (bpf_prog_is_dev_bound(aux))#ifdef CONFIG_PERF_EVENTSif (aux->prog->has_callchain_buf)#endif if (aux->dst_trampoline)for (i = 0 ; i < aux->real_func_cnt; i++) {/* We can just unlink the subprog poke descriptor table as * it was originally linked to the main program and is also * released along with it . if (aux->real_func_cnt) {else {void bpf_prog_free(struct bpf_prog *fp)struct bpf_prog_aux *aux = fp->aux;if (aux->dst_prog)/* RNG for unprivileged user space with separated state from prandom_u32(). */ static DEFINE_PER_CPU(struct rnd_state, bpf_user_rnd_state);void bpf_user_rnd_init_once(void )/* Should someone ever have the rather unwise idea to use some * of the registers passed into this function , then note that * this function is called from native eBPF and classic - to - eBPF * transformations . Register assignments from both sides are * different , f . e . classic always sets fn ( ctx , A , X ) here . struct rnd_state *state;return res;return raw_smp_processor_id();/* Weak definitions of helper functions in case we don't have bpf syscall. */ const struct bpf_func_proto bpf_map_lookup_elem_proto __weak;const struct bpf_func_proto bpf_map_update_elem_proto __weak;const struct bpf_func_proto bpf_map_delete_elem_proto __weak;const struct bpf_func_proto bpf_map_push_elem_proto __weak;const struct bpf_func_proto bpf_map_pop_elem_proto __weak;const struct bpf_func_proto bpf_map_peek_elem_proto __weak;const struct bpf_func_proto bpf_map_lookup_percpu_elem_proto __weak;const struct bpf_func_proto bpf_spin_lock_proto __weak;const struct bpf_func_proto bpf_spin_unlock_proto __weak;const struct bpf_func_proto bpf_jiffies64_proto __weak;const struct bpf_func_proto bpf_get_prandom_u32_proto __weak;const struct bpf_func_proto bpf_get_smp_processor_id_proto __weak;const struct bpf_func_proto bpf_get_numa_node_id_proto __weak;const struct bpf_func_proto bpf_ktime_get_ns_proto __weak;const struct bpf_func_proto bpf_ktime_get_boot_ns_proto __weak;const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto __weak;const struct bpf_func_proto bpf_ktime_get_tai_ns_proto __weak;const struct bpf_func_proto bpf_get_current_pid_tgid_proto __weak;const struct bpf_func_proto bpf_get_current_uid_gid_proto __weak;const struct bpf_func_proto bpf_get_current_comm_proto __weak;const struct bpf_func_proto bpf_get_current_cgroup_id_proto __weak;const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto __weak;const struct bpf_func_proto bpf_get_local_storage_proto __weak;const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto __weak;const struct bpf_func_proto bpf_snprintf_btf_proto __weak;const struct bpf_func_proto bpf_seq_printf_btf_proto __weak;const struct bpf_func_proto bpf_set_retval_proto __weak;const struct bpf_func_proto bpf_get_retval_proto __weak;const struct bpf_func_proto * __weak bpf_get_trace_printk_proto(void )return NULL;const struct bpf_func_proto * __weak bpf_get_trace_vprintk_proto(void )return NULL;const struct bpf_func_proto * __weak bpf_get_perf_event_read_value_proto(void )return NULL;struct bpf_map *map, u64 flags, void *meta, u64 meta_size,void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy)return -ENOTSUPP;/* Always built-in helper functions. */ const struct bpf_func_proto bpf_tail_call_proto = {/* func is unused for tail_call, we set it to pass the * get_helper_proto check false ,/* Stub for JITs that only support cBPF. eBPF programs are interpreted. * It is encouraged to implement bpf_int_jit_compile ( ) instead , so that * eBPF and implicitly also cBPF can get JITed ! struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog)return prog;/* Stub for JITs that support eBPF. All cBPF code gets transformed into * eBPF by the kernel and is later compiled by bpf_int_jit_compile ( ) . void __weak bpf_jit_compile(struct bpf_prog *prog)bool __weak bpf_helper_changes_pkt_data(enum bpf_func_id func_id)return false ;/* Return TRUE if the JIT backend wants verifier to enable sub-register usage * analysis code and wants explicit zero extension inserted by verifier . * Otherwise , return FALSE . * * The verifier inserts an explicit zero extension after BPF_CMPXCHGs even if * you don ' t override this . JITs that don ' t want these extra insns can detect * them using insn_is_zext . bool __weak bpf_jit_needs_zext(void )return false ;/* By default, enable the verifier's mitigations against Spectre v1 and v4 for * all archs . The value returned must not change at runtime as there is * currently no support for reloading programs that were loaded without * mitigations . bool __weak bpf_jit_bypass_spec_v1(void )return false ;bool __weak bpf_jit_bypass_spec_v4(void )return false ;/* Return true if the JIT inlines the call to the helper corresponding to * the imm . * * The verifier will not patch the insn - > imm for the call to the helper if * this returns true . bool __weak bpf_jit_inlines_helper_call(s32 imm)return false ;/* Return TRUE if the JIT backend supports mixing bpf2bpf and tailcalls. */ bool __weak bpf_jit_supports_subprog_tailcalls(void )return false ;bool __weak bpf_jit_supports_percpu_insn(void )return false ;bool __weak bpf_jit_supports_kfunc_call(void )return false ;bool __weak bpf_jit_supports_far_kfunc_call(void )return false ;bool __weak bpf_jit_supports_arena(void )return false ;bool __weak bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)return false ;void )#if defined (CONFIG_64BIT) && defined (CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE)return TASK_SIZE;#else return 0 ;#endif /* Return TRUE if the JIT backend satisfies the following two conditions: * 1 ) JIT backend supports atomic_xchg ( ) on pointer - sized words . * 2 ) Under the specific arch , the implementation of xchg ( ) is the same * as atomic_xchg ( ) on pointer - sized words . bool __weak bpf_jit_supports_ptr_xchg(void )return false ;/* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call * skb_copy_bits ( ) , so provide a weak definition of it for NET - less config . int __weak skb_copy_bits(const struct sk_buff *skb, int offset, void *to,int len)return -EFAULT;int __weak bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,void *addr1, void *addr2)return -ENOTSUPP;void * __weak bpf_arch_text_copy(void *dst, void *src, size_t len)return ERR_PTR(-ENOTSUPP);int __weak bpf_arch_text_invalidate(void *dst, size_t len)return -ENOTSUPP;bool __weak bpf_jit_supports_exceptions(void )return false ;bool __weak bpf_jit_supports_private_stack(void )return false ;void __weak arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie)bool __weak bpf_jit_supports_timed_may_goto(void )return false ;void )return 0 ;static noinline void bpf_prog_report_may_goto_violation(void )#ifdef CONFIG_BPF_SYSCALLstruct bpf_stream_stage ss;struct bpf_prog *prog;if (!prog)return ;"ERROR: Timeout detected for may_goto instruction\n" );#endif struct bpf_timed_may_goto *p)/* Populate the timestamp for this stack frame, and refresh count. */ if (!p->timestamp) {return BPF_MAX_TIMED_LOOPS;/* Check if we've exhausted our time slice, and zero count. */ if (unlikely(time - p->timestamp >= (NSEC_PER_SEC / 4 ))) {return 0 ;/* Refresh the count for the stack frame. */ return BPF_MAX_TIMED_LOOPS;/* for configs without MMU or 32-bit */ const struct bpf_map_ops arena_map_ops;struct bpf_arena *arena)return 0 ;struct bpf_arena *arena)return 0 ;#ifdef CONFIG_BPF_SYSCALLstatic int __init bpf_global_ma_init(void )int ret;0 , false );return ret;#endif /* All definitions of tracepoints related to BPF. */ #define CREATE_TRACE_POINTS#include <linux/bpf_trace.h>#ifdef CONFIG_BPF_SYSCALLint bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char **filep,const char **linep, int *nump)int idx = -1 , insn_start, insn_end, len;struct bpf_line_info *linfo;void **jited_linfo;struct btf *btf;int nr_linfo;if (!btf || !linfo || !jited_linfo)return -EINVAL;0 ].insn_off;for (int i = 0 ; i < nr_linfo &&if (jited_linfo[i] >= (void *)ip)break ;if (idx == -1 )return -ENOENT;/* Get base component of the file path. */ /* Obtain the source line, and strip whitespace in prefix. */ while (isspace(**linep))1 ;return 0 ;struct walk_stack_ctx {struct bpf_prog *prog;static bool find_from_stack_cb(void *cookie, u64 ip, u64 sp, u64 bp)struct walk_stack_ctx *ctxp = cookie;struct bpf_prog *prog;/* * The RCU read lock is held to safely traverse the latch tree , but we * don ' t need its protection when accessing the prog , since it has an * active stack frame on the current stack trace , and won ' t disappear . if (!prog)return true ;if (bpf_is_subprog(prog))return true ;return false ;struct bpf_prog *bpf_prog_find_from_stack(void )struct walk_stack_ctx ctx = {};return ctx.prog;#endif
Messung V0.5 in Prozent C=97 H=94 G=95
¤ Dauer der Verarbeitung: 0.64 Sekunden
(vorverarbeitet am 2026-09-28)
¤ *© Formatika GbR, Deutschland
