15 #define BPF_JMP_UNC(ins)	((ins) += (ins)->off)  argument
17 #define BPF_JMP_CND_REG(reg, ins, op, type)	\  argument
18 	((ins) += \
19 		((type)(reg)[(ins)->dst_reg] op (type)(reg)[(ins)->src_reg]) ? \
20 		(ins)->off : 0)
22 #define BPF_JMP_CND_IMM(reg, ins, op, type)	\  argument
23 	((ins) += \
24 		((type)(reg)[(ins)->dst_reg] op (type)(ins)->imm) ? \
25 		(ins)->off : 0)
27 #define BPF_NEG_ALU(reg, ins, type)	\  argument
28 	((reg)[(ins)->dst_reg] = (type)(-(reg)[(ins)->dst_reg]))
30 #define EBPF_MOV_ALU_REG(reg, ins, type)	\  argument
31 	((reg)[(ins)->dst_reg] = (type)(reg)[(ins)->src_reg])
33 #define BPF_OP_ALU_REG(reg, ins, op, type)	\  argument
34 	((reg)[(ins)->dst_reg] = \
35 		(type)(reg)[(ins)->dst_reg] op (type)(reg)[(ins)->src_reg])
37 #define EBPF_MOV_ALU_IMM(reg, ins, type)	\  argument
38 	((reg)[(ins)->dst_reg] = (type)(ins)->imm)
40 #define BPF_OP_ALU_IMM(reg, ins, op, type)	\  argument
41 	((reg)[(ins)->dst_reg] = \
42 		(type)(reg)[(ins)->dst_reg] op (type)(ins)->imm)
44 #define BPF_DIV_ZERO_CHECK(bpf, reg, ins, type) do { \  argument
45 	if ((type)(reg)[(ins)->src_reg] == 0) { \
49 			(uintptr_t)(ins) - (uintptr_t)(bpf)->prm.ins); \
54 #define BPF_LD_REG(reg, ins, type)	\  argument
55 	((reg)[(ins)->dst_reg] = \
56 		*(type *)(uintptr_t)((reg)[(ins)->src_reg] + (ins)->off))
58 #define BPF_ST_IMM(reg, ins, type)	\  argument
59 	(*(type *)(uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off) = \
60 		(type)(ins)->imm)
62 #define BPF_ST_REG(reg, ins, type)	\  argument
63 	(*(type *)(uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off) = \
64 		(type)(reg)[(ins)->src_reg])
66 #define BPF_ST_XADD_REG(reg, ins, tp)	\  argument
68 		(uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off), \
69 		reg[ins->src_reg]))
75 #define BPF_LD_ABS(bpf, reg, ins, type, op) do { \  argument
76 	const type *p = bpf_ld_mbuf(bpf, reg, ins, (ins)->imm, sizeof(type)); \
82 #define BPF_LD_IND(bpf, reg, ins, type, op) do { \  argument
83 	uint32_t ofs = reg[ins->src_reg] + (ins)->imm; \
84 	const type *p = bpf_ld_mbuf(bpf, reg, ins, ofs, sizeof(type)); \
92 bpf_alu_be(uint64_t reg[EBPF_REG_NUM], const struct ebpf_insn *ins)  in bpf_alu_be()  argument
96 	v = reg + ins->dst_reg;  in bpf_alu_be()
97 	switch (ins->imm) {  in bpf_alu_be()
111 bpf_alu_le(uint64_t reg[EBPF_REG_NUM], const struct ebpf_insn *ins)  in bpf_alu_le()  argument
115 	v = reg + ins->dst_reg;  in bpf_alu_le()
116 	switch (ins->imm) {  in bpf_alu_le()
131 	const struct ebpf_insn *ins, uint32_t off, uint32_t len)  in bpf_ld_mbuf()  argument
142 			(uintptr_t)(ins) - (uintptr_t)(bpf)->prm.ins);  in bpf_ld_mbuf()
149 	const struct ebpf_insn *ins;  in bpf_exec()  local
151 	for (ins = bpf->prm.ins; ; ins++) {  in bpf_exec()
152 		switch (ins->code) {  in bpf_exec()
155 			BPF_OP_ALU_IMM(reg, ins, +, uint32_t);  in bpf_exec()
158 			BPF_OP_ALU_IMM(reg, ins, -, uint32_t);  in bpf_exec()
161 			BPF_OP_ALU_IMM(reg, ins, &, uint32_t);  in bpf_exec()
164 			BPF_OP_ALU_IMM(reg, ins, |, uint32_t);  in bpf_exec()
167 			BPF_OP_ALU_IMM(reg, ins, <<, uint32_t);  in bpf_exec()
170 			BPF_OP_ALU_IMM(reg, ins, >>, uint32_t);  in bpf_exec()
173 			BPF_OP_ALU_IMM(reg, ins, ^, uint32_t);  in bpf_exec()
176 			BPF_OP_ALU_IMM(reg, ins, *, uint32_t);  in bpf_exec()
179 			BPF_OP_ALU_IMM(reg, ins, /, uint32_t);  in bpf_exec()
182 			BPF_OP_ALU_IMM(reg, ins, %, uint32_t);  in bpf_exec()
185 			EBPF_MOV_ALU_IMM(reg, ins, uint32_t);  in bpf_exec()
189 			BPF_OP_ALU_REG(reg, ins, +, uint32_t);  in bpf_exec()
192 			BPF_OP_ALU_REG(reg, ins, -, uint32_t);  in bpf_exec()
195 			BPF_OP_ALU_REG(reg, ins, &, uint32_t);  in bpf_exec()
198 			BPF_OP_ALU_REG(reg, ins, |, uint32_t);  in bpf_exec()
201 			BPF_OP_ALU_REG(reg, ins, <<, uint32_t);  in bpf_exec()
204 			BPF_OP_ALU_REG(reg, ins, >>, uint32_t);  in bpf_exec()
207 			BPF_OP_ALU_REG(reg, ins, ^, uint32_t);  in bpf_exec()
210 			BPF_OP_ALU_REG(reg, ins, *, uint32_t);  in bpf_exec()
213 			BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint32_t);  in bpf_exec()
214 			BPF_OP_ALU_REG(reg, ins, /, uint32_t);  in bpf_exec()
217 			BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint32_t);  in bpf_exec()
218 			BPF_OP_ALU_REG(reg, ins, %, uint32_t);  in bpf_exec()
221 			EBPF_MOV_ALU_REG(reg, ins, uint32_t);  in bpf_exec()
224 			BPF_NEG_ALU(reg, ins, uint32_t);  in bpf_exec()
227 			bpf_alu_be(reg, ins);  in bpf_exec()
230 			bpf_alu_le(reg, ins);  in bpf_exec()
234 			BPF_OP_ALU_IMM(reg, ins, +, uint64_t);  in bpf_exec()
237 			BPF_OP_ALU_IMM(reg, ins, -, uint64_t);  in bpf_exec()
240 			BPF_OP_ALU_IMM(reg, ins, &, uint64_t);  in bpf_exec()
243 			BPF_OP_ALU_IMM(reg, ins, |, uint64_t);  in bpf_exec()
246 			BPF_OP_ALU_IMM(reg, ins, <<, uint64_t);  in bpf_exec()
249 			BPF_OP_ALU_IMM(reg, ins, >>, uint64_t);  in bpf_exec()
252 			BPF_OP_ALU_IMM(reg, ins, >>, int64_t);  in bpf_exec()
255 			BPF_OP_ALU_IMM(reg, ins, ^, uint64_t);  in bpf_exec()
258 			BPF_OP_ALU_IMM(reg, ins, *, uint64_t);  in bpf_exec()
261 			BPF_OP_ALU_IMM(reg, ins, /, uint64_t);  in bpf_exec()
264 			BPF_OP_ALU_IMM(reg, ins, %, uint64_t);  in bpf_exec()
267 			EBPF_MOV_ALU_IMM(reg, ins, uint64_t);  in bpf_exec()
271 			BPF_OP_ALU_REG(reg, ins, +, uint64_t);  in bpf_exec()
274 			BPF_OP_ALU_REG(reg, ins, -, uint64_t);  in bpf_exec()
277 			BPF_OP_ALU_REG(reg, ins, &, uint64_t);  in bpf_exec()
280 			BPF_OP_ALU_REG(reg, ins, |, uint64_t);  in bpf_exec()
283 			BPF_OP_ALU_REG(reg, ins, <<, uint64_t);  in bpf_exec()
286 			BPF_OP_ALU_REG(reg, ins, >>, uint64_t);  in bpf_exec()
289 			BPF_OP_ALU_REG(reg, ins, >>, int64_t);  in bpf_exec()
292 			BPF_OP_ALU_REG(reg, ins, ^, uint64_t);  in bpf_exec()
295 			BPF_OP_ALU_REG(reg, ins, *, uint64_t);  in bpf_exec()
298 			BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint64_t);  in bpf_exec()
299 			BPF_OP_ALU_REG(reg, ins, /, uint64_t);  in bpf_exec()
302 			BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint64_t);  in bpf_exec()
303 			BPF_OP_ALU_REG(reg, ins, %, uint64_t);  in bpf_exec()
306 			EBPF_MOV_ALU_REG(reg, ins, uint64_t);  in bpf_exec()
309 			BPF_NEG_ALU(reg, ins, uint64_t);  in bpf_exec()
313 			BPF_LD_REG(reg, ins, uint8_t);  in bpf_exec()
316 			BPF_LD_REG(reg, ins, uint16_t);  in bpf_exec()
319 			BPF_LD_REG(reg, ins, uint32_t);  in bpf_exec()
322 			BPF_LD_REG(reg, ins, uint64_t);  in bpf_exec()
326 			reg[ins->dst_reg] = (uint32_t)ins[0].imm |  in bpf_exec()
327 				(uint64_t)(uint32_t)ins[1].imm << 32;  in bpf_exec()
328 			ins++;  in bpf_exec()
332 			BPF_LD_ABS(bpf, reg, ins, uint8_t, NOP);  in bpf_exec()
335 			BPF_LD_ABS(bpf, reg, ins, uint16_t, rte_be_to_cpu_16);  in bpf_exec()
338 			BPF_LD_ABS(bpf, reg, ins, uint32_t, rte_be_to_cpu_32);  in bpf_exec()
342 			BPF_LD_IND(bpf, reg, ins, uint8_t, NOP);  in bpf_exec()
345 			BPF_LD_IND(bpf, reg, ins, uint16_t, rte_be_to_cpu_16);  in bpf_exec()
348 			BPF_LD_IND(bpf, reg, ins, uint32_t, rte_be_to_cpu_32);  in bpf_exec()
352 			BPF_ST_REG(reg, ins, uint8_t);  in bpf_exec()
355 			BPF_ST_REG(reg, ins, uint16_t);  in bpf_exec()
358 			BPF_ST_REG(reg, ins, uint32_t);  in bpf_exec()
361 			BPF_ST_REG(reg, ins, uint64_t);  in bpf_exec()
364 			BPF_ST_IMM(reg, ins, uint8_t);  in bpf_exec()
367 			BPF_ST_IMM(reg, ins, uint16_t);  in bpf_exec()
370 			BPF_ST_IMM(reg, ins, uint32_t);  in bpf_exec()
373 			BPF_ST_IMM(reg, ins, uint64_t);  in bpf_exec()
377 			BPF_ST_XADD_REG(reg, ins, 32);  in bpf_exec()
380 			BPF_ST_XADD_REG(reg, ins, 64);  in bpf_exec()
384 			BPF_JMP_UNC(ins);  in bpf_exec()
388 			BPF_JMP_CND_IMM(reg, ins, ==, uint64_t);  in bpf_exec()
391 			BPF_JMP_CND_IMM(reg, ins, !=, uint64_t);  in bpf_exec()
394 			BPF_JMP_CND_IMM(reg, ins, >, uint64_t);  in bpf_exec()
397 			BPF_JMP_CND_IMM(reg, ins, <, uint64_t);  in bpf_exec()
400 			BPF_JMP_CND_IMM(reg, ins, >=, uint64_t);  in bpf_exec()
403 			BPF_JMP_CND_IMM(reg, ins, <=, uint64_t);  in bpf_exec()
406 			BPF_JMP_CND_IMM(reg, ins, >, int64_t);  in bpf_exec()
409 			BPF_JMP_CND_IMM(reg, ins, <, int64_t);  in bpf_exec()
412 			BPF_JMP_CND_IMM(reg, ins, >=, int64_t);  in bpf_exec()
415 			BPF_JMP_CND_IMM(reg, ins, <=, int64_t);  in bpf_exec()
418 			BPF_JMP_CND_IMM(reg, ins, &, uint64_t);  in bpf_exec()
422 			BPF_JMP_CND_REG(reg, ins, ==, uint64_t);  in bpf_exec()
425 			BPF_JMP_CND_REG(reg, ins, !=, uint64_t);  in bpf_exec()
428 			BPF_JMP_CND_REG(reg, ins, >, uint64_t);  in bpf_exec()
431 			BPF_JMP_CND_REG(reg, ins, <, uint64_t);  in bpf_exec()
434 			BPF_JMP_CND_REG(reg, ins, >=, uint64_t);  in bpf_exec()
437 			BPF_JMP_CND_REG(reg, ins, <=, uint64_t);  in bpf_exec()
440 			BPF_JMP_CND_REG(reg, ins, >, int64_t);  in bpf_exec()
443 			BPF_JMP_CND_REG(reg, ins, <, int64_t);  in bpf_exec()
446 			BPF_JMP_CND_REG(reg, ins, >=, int64_t);  in bpf_exec()
449 			BPF_JMP_CND_REG(reg, ins, <=, int64_t);  in bpf_exec()
452 			BPF_JMP_CND_REG(reg, ins, &, uint64_t);  in bpf_exec()
456 			reg[EBPF_REG_0] = bpf->prm.xsym[ins->imm].func.val(  in bpf_exec()
467 				__func__, bpf, ins->code,  in bpf_exec()
468 				(uintptr_t)ins - (uintptr_t)bpf->prm.ins);  in bpf_exec()