bpf: allow extended BPF programs access skb fields

introduce user accessible mirror of in-kernel 'struct sk_buff': struct __sk_buff { __u32 len; __u32 pkt_type; __u32 mark; __u32 queue_mapping; }; bpf programs can do: int bpf_prog(struct __sk_buff *skb) { __u32 var = skb->pkt_type; which will be compiled to bpf assembler as: dst_reg = *(u32 *)(src_reg + 4) // 4 == offsetof(struct __sk_buff, pkt_type) bpf verifier will check validity of access and will convert it to: dst_reg = *(u8 *)(src_reg + offsetof(struct sk_buff, __pkt_type_offset)) dst_reg &= 7 since skb->pkt_type is a bitfield.

Alexei Starovoitov committed 11 years ago

Commit 9bac3d61 parent a498cfe5 files · +234 −35Browse files

Unified · Split

5 files

include/linux/bpf.h +4 −1
include/uapi/linux/bpf.h +10 −0
kernel/bpf/syscall.c +1 −1
kernel/bpf/verifier.c +136 −16
net/core/filter.c +83 −17

include/linux/bpf.h+4−1

 	 * with 'type' (read or write) is allowed
 	 */
 	bool (*is_valid_access)(int off, int size, enum bpf_access_type type);
+	u32 (*convert_ctx_access)(int dst_reg, int src_reg, int ctx_off,
+				  struct bpf_insn *insn);
 };
 struct bpf_prog_type_list {
 void bpf_map_put(struct bpf_map *map);
 /* verify correctness of eBPF program */
-int bpf_check(struct bpf_prog *fp, union bpf_attr *attr);
+int bpf_check(struct bpf_prog **fp, union bpf_attr *attr);
 #else
 static inline void bpf_register_prog_type(struct bpf_prog_type_list *tl)
 {

include/uapi/linux/bpf.h+10−0

@@ -170,4 +170,14 @@ enum bpf_func_id { · 169 unmodified lines
170	__BPF_FUNC_MAX_ID,	170	__BPF_FUNC_MAX_ID,
171	};	171	};
172		172
		173	/* user accessible mirror of in-kernel sk_buff.
		174	* new fields can only be added to the end of this structure
		175	*/
		176	struct __sk_buff {
		177	__u32 len;
		178	__u32 pkt_type;
		179	__u32 mark;
		180	__u32 queue_mapping;
		181	};
		182
173	#endif /* _UAPI__LINUX_BPF_H__ */	183	#endif /* _UAPI__LINUX_BPF_H__ */

kernel/bpf/syscall.c+1−1

@@ -519,7 +519,7 @@ static int bpf_prog_load(union bpf_attr *attr) · 518 unmodified lines
519	goto free_prog;	519	goto free_prog;
520		520
521	/* run eBPF verifier */	521	/* run eBPF verifier */
522	err = bpf_check(prog, attr);	522	err = bpf_check(&prog, attr);
523	if (err < 0)	523	if (err < 0)
524	goto free_used_maps;	524	goto free_used_maps;
525		525

kernel/bpf/verifier.c+136−16

@@ -1620,11 +1620,10 @@ static int do_check(struct verifier_env *env) · 1619 unmodified lines
1620	return err;	1620	return err;
1621		1621
1622	} else if (class == BPF_LDX) {	1622	} else if (class == BPF_LDX) {
1623	if (BPF_MODE(insn->code) != BPF_MEM \|\|	1623	enum bpf_reg_type src_reg_type;
1624	insn->imm != 0) {	1624
1625	verbose("BPF_LDX uses reserved fields\n");	1625	/* check for reserved fields is already done */
1626	return -EINVAL;	1626
1627	}
1628	/* check src operand */	1627	/* check src operand */
1629	err = check_reg_arg(regs, insn->src_reg, SRC_OP);	1628	err = check_reg_arg(regs, insn->src_reg, SRC_OP);
1630	if (err)	1629	if (err)
@@ -1643,6 +1642,29 @@ static int do_check(struct verifier_env *env) · 12 unmodified lines
1643	if (err)	1642	if (err)
1644	return err;	1643	return err;
1645		1644
		1645	src_reg_type = regs[insn->src_reg].type;
		1646
		1647	if (insn->imm == 0 && BPF_SIZE(insn->code) == BPF_W) {
		1648	/* saw a valid insn
		1649	* dst_reg = (u32 )(src_reg + off)
		1650	* use reserved 'imm' field to mark this insn
		1651	*/
		1652	insn->imm = src_reg_type;
		1653
		1654	} else if (src_reg_type != insn->imm &&
		1655	(src_reg_type == PTR_TO_CTX \|\|
		1656	insn->imm == PTR_TO_CTX)) {
		1657	/* ABuser program is trying to use the same insn
		1658	* dst_reg = (u32) (src_reg + off)
		1659	* with different pointer types:
		1660	* src_reg == ctx in one branch and
		1661	* src_reg == stack\|map in some other branch.
		1662	* Reject it.
		1663	*/
		1664	verbose("same insn cannot be used with different pointers\n");
		1665	return -EINVAL;
		1666	}
		1667
1646	} else if (class == BPF_STX) {	1668	} else if (class == BPF_STX) {
1647	if (BPF_MODE(insn->code) == BPF_XADD) {	1669	if (BPF_MODE(insn->code) == BPF_XADD) {
1648	err = check_xadd(env, insn);	1670	err = check_xadd(env, insn);
@@ -1790,6 +1812,13 @@ static int replace_map_fd_with_map_ptr(struct verifier_env *env) · 141 unmodified lines
1790	int i, j;	1812	int i, j;
1791		1813
1792	for (i = 0; i < insn_cnt; i++, insn++) {	1814	for (i = 0; i < insn_cnt; i++, insn++) {
		1815	if (BPF_CLASS(insn->code) == BPF_LDX &&
		1816	(BPF_MODE(insn->code) != BPF_MEM \|\|
		1817	insn->imm != 0)) {
		1818	verbose("BPF_LDX uses reserved fields\n");
		1819	return -EINVAL;
		1820	}
		1821
1793	if (insn[0].code == (BPF_LD \| BPF_IMM \| BPF_DW)) {	1822	if (insn[0].code == (BPF_LD \| BPF_IMM \| BPF_DW)) {
1794	struct bpf_map *map;	1823	struct bpf_map *map;
1795	struct fd f;	1824	struct fd f;
@@ -1881,6 +1910,92 @@ static void convert_pseudo_ld_imm64(struct verifier_env *env) · 85 unmodified lines
1881	insn->src_reg = 0;	1910	insn->src_reg = 0;
1882	}	1911	}
1883		1912
		1913	static void adjust_branches(struct bpf_prog *prog, int pos, int delta)
		1914	{
		1915	struct bpf_insn *insn = prog->insnsi;
		1916	int insn_cnt = prog->len;
		1917	int i;
		1918
		1919	for (i = 0; i < insn_cnt; i++, insn++) {
		1920	if (BPF_CLASS(insn->code) != BPF_JMP \|\|
		1921	BPF_OP(insn->code) == BPF_CALL \|\|
		1922	BPF_OP(insn->code) == BPF_EXIT)
		1923	continue;
		1924
		1925	/* adjust offset of jmps if necessary */
		1926	if (i < pos && i + insn->off + 1 > pos)
		1927	insn->off += delta;
		1928	else if (i > pos && i + insn->off + 1 < pos)
		1929	insn->off -= delta;
		1930	}
		1931	}
		1932
		1933	/* convert load instructions that access fields of 'struct __sk_buff'
		1934	* into sequence of instructions that access fields of 'struct sk_buff'
		1935	*/
		1936	static int convert_ctx_accesses(struct verifier_env *env)
		1937	{
		1938	struct bpf_insn *insn = env->prog->insnsi;
		1939	int insn_cnt = env->prog->len;
		1940	struct bpf_insn insn_buf[16];
		1941	struct bpf_prog *new_prog;
		1942	u32 cnt;
		1943	int i;
		1944
		1945	if (!env->prog->aux->ops->convert_ctx_access)
		1946	return 0;
		1947
		1948	for (i = 0; i < insn_cnt; i++, insn++) {
		1949	if (insn->code != (BPF_LDX \| BPF_MEM \| BPF_W))
		1950	continue;
		1951
		1952	if (insn->imm != PTR_TO_CTX) {
		1953	/* clear internal mark */
		1954	insn->imm = 0;
		1955	continue;
		1956	}
		1957
		1958	cnt = env->prog->aux->ops->
		1959	convert_ctx_access(insn->dst_reg, insn->src_reg,
		1960	insn->off, insn_buf);
		1961	if (cnt == 0 \|\| cnt >= ARRAY_SIZE(insn_buf)) {
		1962	verbose("bpf verifier is misconfigured\n");
		1963	return -EINVAL;
		1964	}
		1965
		1966	if (cnt == 1) {
		1967	memcpy(insn, insn_buf, sizeof(*insn));
		1968	continue;
		1969	}
		1970
		1971	/* several new insns need to be inserted. Make room for them */
		1972	insn_cnt += cnt - 1;
		1973	new_prog = bpf_prog_realloc(env->prog,
		1974	bpf_prog_size(insn_cnt),
		1975	GFP_USER);
		1976	if (!new_prog)
		1977	return -ENOMEM;
		1978
		1979	new_prog->len = insn_cnt;
		1980
		1981	memmove(new_prog->insnsi + i + cnt, new_prog->insns + i + 1,
		1982	sizeof(insn) (insn_cnt - i - cnt));
		1983
		1984	/* copy substitute insns in place of load instruction */
		1985	memcpy(new_prog->insnsi + i, insn_buf, sizeof(insn) cnt);
		1986
		1987	/* adjust branches in the whole program */
		1988	adjust_branches(new_prog, i, cnt - 1);
		1989
		1990	/* keep walking new program and skip insns we just inserted */
		1991	env->prog = new_prog;
		1992	insn = new_prog->insnsi + i + cnt - 1;
		1993	i += cnt - 1;
		1994	}
		1995
		1996	return 0;
		1997	}
		1998
1884	static void free_states(struct verifier_env *env)	1999	static void free_states(struct verifier_env *env)
1885	{	2000	{
1886	struct verifier_state_list sl, sln;	2001	struct verifier_state_list sl, sln;
@@ -1903,13 +2018,13 @@ static void free_states(struct verifier_env *env) · 16 unmodified lines
1903	kfree(env->explored_states);	2018	kfree(env->explored_states);
1904	}	2019	}
1905		2020
1906	int bpf_check(struct bpf_prog prog, union bpf_attr attr)	2021	int bpf_check(struct bpf_prog *prog, union bpf_attr attr)
1907	{	2022	{
1908	char __user *log_ubuf = NULL;	2023	char __user *log_ubuf = NULL;
1909	struct verifier_env *env;	2024	struct verifier_env *env;
1910	int ret = -EINVAL;	2025	int ret = -EINVAL;
1911		2026
1912	if (prog->len <= 0 \|\| prog->len > BPF_MAXINSNS)	2027	if ((prog)->len <= 0 \|\| (prog)->len > BPF_MAXINSNS)
1913	return -E2BIG;	2028	return -E2BIG;
1914		2029
1915	/* 'struct verifier_env' can be global, but since it's not small,	2030	/* 'struct verifier_env' can be global, but since it's not small,
@@ -1919,7 +2034,7 @@ int bpf_check(struct bpf_prog prog, union bpf_attr attr) · 3 unmodified lines
1919	if (!env)	2034	if (!env)
1920	return -ENOMEM;	2035	return -ENOMEM;
1921		2036
1922	env->prog = prog;	2037	env->prog = *prog;
1923		2038
1924	/* grab the mutex to protect few globals used by verifier */	2039	/* grab the mutex to protect few globals used by verifier */
1925	mutex_lock(&bpf_verifier_lock);	2040	mutex_lock(&bpf_verifier_lock);
@@ -1951,7 +2066,7 @@ int bpf_check(struct bpf_prog prog, union bpf_attr attr) · 25 unmodified lines
1951	if (ret < 0)	2066	if (ret < 0)
1952	goto skip_full_check;	2067	goto skip_full_check;
1953		2068
1954	env->explored_states = kcalloc(prog->len,	2069	env->explored_states = kcalloc(env->prog->len,
1955	sizeof(struct verifier_state_list *),	2070	sizeof(struct verifier_state_list *),
1956	GFP_USER);	2071	GFP_USER);
1957	ret = -ENOMEM;	2072	ret = -ENOMEM;
@@ -1968,6 +2083,10 @@ skip_full_check: · 10 unmodified lines
1968	while (pop_stack(env, NULL) >= 0);	2083	while (pop_stack(env, NULL) >= 0);
1969	free_states(env);	2084	free_states(env);
1970		2085
		2086	if (ret == 0)
		2087	/* program is valid, convert (u32)(ctx + off) accesses */
		2088	ret = convert_ctx_accesses(env);
		2089
1971	if (log_level && log_len >= log_size - 1) {	2090	if (log_level && log_len >= log_size - 1) {
1972	BUG_ON(log_len >= log_size);	2091	BUG_ON(log_len >= log_size);
1973	/* verifier log exceeded user supplied buffer */	2092	/* verifier log exceeded user supplied buffer */
@@ -1983,18 +2102,18 @@ skip_full_check: · 9 unmodified lines
1983		2102
1984	if (ret == 0 && env->used_map_cnt) {	2103	if (ret == 0 && env->used_map_cnt) {
1985	/* if program passed verifier, update used_maps in bpf_prog_info */	2104	/* if program passed verifier, update used_maps in bpf_prog_info */
1986	prog->aux->used_maps = kmalloc_array(env->used_map_cnt,	2105	env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt,
1987	sizeof(env->used_maps[0]),	2106	sizeof(env->used_maps[0]),
1988	GFP_KERNEL);	2107	GFP_KERNEL);
1989		2108
1990	if (!prog->aux->used_maps) {	2109	if (!env->prog->aux->used_maps) {
1991	ret = -ENOMEM;	2110	ret = -ENOMEM;
1992	goto free_log_buf;	2111	goto free_log_buf;
1993	}	2112	}
1994		2113
1995	memcpy(prog->aux->used_maps, env->used_maps,	2114	memcpy(env->prog->aux->used_maps, env->used_maps,
1996	sizeof(env->used_maps[0]) * env->used_map_cnt);	2115	sizeof(env->used_maps[0]) * env->used_map_cnt);
1997	prog->aux->used_map_cnt = env->used_map_cnt;	2116	env->prog->aux->used_map_cnt = env->used_map_cnt;
1998		2117
1999	/* program is valid. Convert pseudo bpf_ld_imm64 into generic	2118	/* program is valid. Convert pseudo bpf_ld_imm64 into generic
2000	* bpf_ld_imm64 instructions	2119	* bpf_ld_imm64 instructions
@@ -2006,11 +2125,12 @@ free_log_buf: · 5 unmodified lines
2006	if (log_level)	2125	if (log_level)
2007	vfree(log_buf);	2126	vfree(log_buf);
2008	free_env:	2127	free_env:
2009	if (!prog->aux->used_maps)	2128	if (!env->prog->aux->used_maps)
2010	/* if we didn't copy map pointers into bpf_prog_info, release	2129	/* if we didn't copy map pointers into bpf_prog_info, release
2011	* them now. Otherwise free_bpf_prog_info() will release them.	2130	* them now. Otherwise free_bpf_prog_info() will release them.
2012	*/	2131	*/
2013	release_maps(env);	2132	release_maps(env);
		2133	*prog = env->prog;
2014	kfree(env);	2134	kfree(env);
2015	mutex_unlock(&bpf_verifier_lock);	2135	mutex_unlock(&bpf_verifier_lock);
2016	return ret;	2136	return ret;

net/core/filter.c+83−17

@@ -150,10 +150,43 @@ static u64 __get_random_u32(u64 ctx, u64 a, u64 x, u64 r4, u64 r5) · 149 unmodified lines
150	return prandom_u32();	150	return prandom_u32();
151	}	151	}
152		152
		153	static u32 convert_skb_access(int skb_field, int dst_reg, int src_reg,
		154	struct bpf_insn *insn_buf)
		155	{
		156	struct bpf_insn *insn = insn_buf;
		157
		158	switch (skb_field) {
		159	case SKF_AD_MARK:
		160	BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4);
		161
		162	*insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg,
		163	offsetof(struct sk_buff, mark));
		164	break;
		165
		166	case SKF_AD_PKTTYPE:
		167	*insn++ = BPF_LDX_MEM(BPF_B, dst_reg, src_reg, PKT_TYPE_OFFSET());
		168	*insn++ = BPF_ALU32_IMM(BPF_AND, dst_reg, PKT_TYPE_MAX);
		169	#ifdef __BIG_ENDIAN_BITFIELD
		170	*insn++ = BPF_ALU32_IMM(BPF_RSH, dst_reg, 5);
		171	#endif
		172	break;
		173
		174	case SKF_AD_QUEUE:
		175	BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, queue_mapping) != 2);
		176
		177	*insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg,
		178	offsetof(struct sk_buff, queue_mapping));
		179	break;
		180	}
		181
		182	return insn - insn_buf;
		183	}
		184
153	static bool convert_bpf_extensions(struct sock_filter *fp,	185	static bool convert_bpf_extensions(struct sock_filter *fp,
154	struct bpf_insn **insnp)	186	struct bpf_insn **insnp)
155	{	187	{
156	struct bpf_insn insn = insnp;	188	struct bpf_insn insn = insnp;
		189	u32 cnt;
157		190
158	switch (fp->k) {	191	switch (fp->k) {
159	case SKF_AD_OFF + SKF_AD_PROTOCOL:	192	case SKF_AD_OFF + SKF_AD_PROTOCOL:
@@ -167,13 +200,8 @@ static bool convert_bpf_extensions(struct sock_filter *fp, · 7 unmodified lines
167	break;	200	break;
168		201
169	case SKF_AD_OFF + SKF_AD_PKTTYPE:	202	case SKF_AD_OFF + SKF_AD_PKTTYPE:
170	*insn++ = BPF_LDX_MEM(BPF_B, BPF_REG_A, BPF_REG_CTX,	203	cnt = convert_skb_access(SKF_AD_PKTTYPE, BPF_REG_A, BPF_REG_CTX, insn);
171	PKT_TYPE_OFFSET());	204	insn += cnt - 1;
172	*insn = BPF_ALU32_IMM(BPF_AND, BPF_REG_A, PKT_TYPE_MAX);
173	#ifdef __BIG_ENDIAN_BITFIELD
174	insn++;
175	*insn = BPF_ALU32_IMM(BPF_RSH, BPF_REG_A, 5);
176	#endif
177	break;	205	break;
178		206
179	case SKF_AD_OFF + SKF_AD_IFINDEX:	207	case SKF_AD_OFF + SKF_AD_IFINDEX:
@@ -197,10 +225,8 @@ static bool convert_bpf_extensions(struct sock_filter *fp, · 17 unmodified lines
197	break;	225	break;
198		226
199	case SKF_AD_OFF + SKF_AD_MARK:	227	case SKF_AD_OFF + SKF_AD_MARK:
200	BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4);	228	cnt = convert_skb_access(SKF_AD_MARK, BPF_REG_A, BPF_REG_CTX, insn);
201		229	insn += cnt - 1;
202	*insn = BPF_LDX_MEM(BPF_W, BPF_REG_A, BPF_REG_CTX,
203	offsetof(struct sk_buff, mark));
204	break;	230	break;
205		231
206	case SKF_AD_OFF + SKF_AD_RXHASH:	232	case SKF_AD_OFF + SKF_AD_RXHASH:
@@ -211,10 +237,8 @@ static bool convert_bpf_extensions(struct sock_filter *fp, · 4 unmodified lines
211	break;	237	break;
212		238
213	case SKF_AD_OFF + SKF_AD_QUEUE:	239	case SKF_AD_OFF + SKF_AD_QUEUE:
214	BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, queue_mapping) != 2);	240	cnt = convert_skb_access(SKF_AD_QUEUE, BPF_REG_A, BPF_REG_CTX, insn);
215		241	insn += cnt - 1;
216	*insn = BPF_LDX_MEM(BPF_H, BPF_REG_A, BPF_REG_CTX,
217	offsetof(struct sk_buff, queue_mapping));
218	break;	242	break;
219		243
220	case SKF_AD_OFF + SKF_AD_VLAN_TAG:	244	case SKF_AD_OFF + SKF_AD_VLAN_TAG:
@@ -1151,13 +1175,55 @@ sk_filter_func_proto(enum bpf_func_id func_id) · 930 unmodified lines
1151	static bool sk_filter_is_valid_access(int off, int size,	1175	static bool sk_filter_is_valid_access(int off, int size,
1152	enum bpf_access_type type)	1176	enum bpf_access_type type)
1153	{	1177	{
1154	/* skb fields cannot be accessed yet */	1178	/* only read is allowed */
1155	return false;	1179	if (type != BPF_READ)
		1180	return false;
		1181
		1182	/* check bounds */
		1183	if (off < 0 \|\| off >= sizeof(struct __sk_buff))
		1184	return false;
		1185
		1186	/* disallow misaligned access */
		1187	if (off % size != 0)
		1188	return false;
		1189
		1190	/* all __sk_buff fields are __u32 */
		1191	if (size != 4)
		1192	return false;
		1193
		1194	return true;
		1195	}
		1196
		1197	static u32 sk_filter_convert_ctx_access(int dst_reg, int src_reg, int ctx_off,
		1198	struct bpf_insn *insn_buf)
		1199	{
		1200	struct bpf_insn *insn = insn_buf;
		1201
		1202	switch (ctx_off) {
		1203	case offsetof(struct __sk_buff, len):
		1204	BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4);
		1205
		1206	*insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg,
		1207	offsetof(struct sk_buff, len));
		1208	break;
		1209
		1210	case offsetof(struct __sk_buff, mark):
		1211	return convert_skb_access(SKF_AD_MARK, dst_reg, src_reg, insn);
		1212
		1213	case offsetof(struct __sk_buff, pkt_type):
		1214	return convert_skb_access(SKF_AD_PKTTYPE, dst_reg, src_reg, insn);
		1215
		1216	case offsetof(struct __sk_buff, queue_mapping):
		1217	return convert_skb_access(SKF_AD_QUEUE, dst_reg, src_reg, insn);
		1218	}
		1219
		1220	return insn - insn_buf;
1156	}	1221	}
1157		1222
1158	static const struct bpf_verifier_ops sk_filter_ops = {	1223	static const struct bpf_verifier_ops sk_filter_ops = {
1159	.get_func_proto = sk_filter_func_proto,	1224	.get_func_proto = sk_filter_func_proto,
1160	.is_valid_access = sk_filter_is_valid_access,	1225	.is_valid_access = sk_filter_is_valid_access,
		1226	.convert_ctx_access = sk_filter_convert_ctx_access,
1161	};	1227	};
1162		1228
1163	static struct bpf_prog_type_list sk_filter_type __read_mostly = {	1229	static struct bpf_prog_type_list sk_filter_type __read_mostly = {

bpf: allow extended BPF programs access skb fields

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code