History

bpf: allow extended BPF programs access skb fields

introduce user accessible mirror of in-kernel 'struct sk_buff': struct __sk_buff { __u32 len; __u32 pkt_type; __u32 mark; __u32 queue_mapping; }; bpf programs can do: int bpf_prog(struct __sk_buff *skb) { __u32 var = skb->pkt_type; which will be compiled to bpf assembler as: dst_reg = *(u32 *)(src_reg + 4) // 4 == offsetof(struct __sk_buff, pkt_type) bpf verifier will check validity of access and will convert it to: dst_reg = *(u8 *)(src_reg + offsetof(struct sk_buff, __pkt_type_offset)) dst_reg &= 7 since skb->pkt_type is a bitfield.

Alexei Starovoitov committed 11 years ago

Commit 9bac3d61 parent a498cfe5 files · +234 −35Browse files

Unified · Split

5 files
  1. include/linux/bpf.h +4 −1
  2. include/uapi/linux/bpf.h +10 −0
  3. kernel/bpf/syscall.c +1 −1
  4. kernel/bpf/verifier.c +136 −16
  5. net/core/filter.c +83 −17
include/linux/bpf.h+4−1
@@ -103,6 +103,9 @@ struct bpf_verifier_ops { · 102 unmodified lines
103103 * with 'type' (read or write) is allowed
104104 */
105105 bool (*is_valid_access)(int off, int size, enum bpf_access_type type);
106+
107+ u32 (*convert_ctx_access)(int dst_reg, int src_reg, int ctx_off,
108+ struct bpf_insn *insn);
106109 };
107110
108111 struct bpf_prog_type_list {
@@ -133,7 +136,7 @@ struct bpf_map *bpf_map_get(struct fd f); · 24 unmodified lines
133136 void bpf_map_put(struct bpf_map *map);
134137
135138 /* verify correctness of eBPF program */
136−int bpf_check(struct bpf_prog *fp, union bpf_attr *attr);
139+int bpf_check(struct bpf_prog **fp, union bpf_attr *attr);
137140 #else
138141 static inline void bpf_register_prog_type(struct bpf_prog_type_list *tl)
139142 {
include/uapi/linux/bpf.h+10−0
@@ -170,4 +170,14 @@ enum bpf_func_id { · 169 unmodified lines
170170 __BPF_FUNC_MAX_ID,
171171 };
172172
173+/* user accessible mirror of in-kernel sk_buff.
174+ * new fields can only be added to the end of this structure
175+ */
176+struct __sk_buff {
177+ __u32 len;
178+ __u32 pkt_type;
179+ __u32 mark;
180+ __u32 queue_mapping;
181+};
182+
173183 #endif /* _UAPI__LINUX_BPF_H__ */
kernel/bpf/syscall.c+1−1
@@ -519,7 +519,7 @@ static int bpf_prog_load(union bpf_attr *attr) · 518 unmodified lines
519519 goto free_prog;
520520
521521 /* run eBPF verifier */
522− err = bpf_check(prog, attr);
522+ err = bpf_check(&prog, attr);
523523 if (err < 0)
524524 goto free_used_maps;
525525
kernel/bpf/verifier.c+136−16
@@ -1620,11 +1620,10 @@ static int do_check(struct verifier_env *env) · 1619 unmodified lines
16201620 return err;
16211621
16221622 } else if (class == BPF_LDX) {
1623− if (BPF_MODE(insn->code) != BPF_MEM ||
1624− insn->imm != 0) {
1625− verbose("BPF_LDX uses reserved fields\n");
1626− return -EINVAL;
1627− }
1623+ enum bpf_reg_type src_reg_type;
1624+
1625+ /* check for reserved fields is already done */
1626+
16281627 /* check src operand */
16291628 err = check_reg_arg(regs, insn->src_reg, SRC_OP);
16301629 if (err)
@@ -1643,6 +1642,29 @@ static int do_check(struct verifier_env *env) · 12 unmodified lines
16431642 if (err)
16441643 return err;
16451644
1645+ src_reg_type = regs[insn->src_reg].type;
1646+
1647+ if (insn->imm == 0 && BPF_SIZE(insn->code) == BPF_W) {
1648+ /* saw a valid insn
1649+ * dst_reg = *(u32 *)(src_reg + off)
1650+ * use reserved 'imm' field to mark this insn
1651+ */
1652+ insn->imm = src_reg_type;
1653+
1654+ } else if (src_reg_type != insn->imm &&
1655+ (src_reg_type == PTR_TO_CTX ||
1656+ insn->imm == PTR_TO_CTX)) {
1657+ /* ABuser program is trying to use the same insn
1658+ * dst_reg = *(u32*) (src_reg + off)
1659+ * with different pointer types:
1660+ * src_reg == ctx in one branch and
1661+ * src_reg == stack|map in some other branch.
1662+ * Reject it.
1663+ */
1664+ verbose("same insn cannot be used with different pointers\n");
1665+ return -EINVAL;
1666+ }
1667+
16461668 } else if (class == BPF_STX) {
16471669 if (BPF_MODE(insn->code) == BPF_XADD) {
16481670 err = check_xadd(env, insn);
@@ -1790,6 +1812,13 @@ static int replace_map_fd_with_map_ptr(struct verifier_env *env) · 141 unmodified lines
17901812 int i, j;
17911813
17921814 for (i = 0; i < insn_cnt; i++, insn++) {
1815+ if (BPF_CLASS(insn->code) == BPF_LDX &&
1816+ (BPF_MODE(insn->code) != BPF_MEM ||
1817+ insn->imm != 0)) {
1818+ verbose("BPF_LDX uses reserved fields\n");
1819+ return -EINVAL;
1820+ }
1821+
17931822 if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) {
17941823 struct bpf_map *map;
17951824 struct fd f;
@@ -1881,6 +1910,92 @@ static void convert_pseudo_ld_imm64(struct verifier_env *env) · 85 unmodified lines
18811910 insn->src_reg = 0;
18821911 }
18831912
1913+static void adjust_branches(struct bpf_prog *prog, int pos, int delta)
1914+{
1915+ struct bpf_insn *insn = prog->insnsi;
1916+ int insn_cnt = prog->len;
1917+ int i;
1918+
1919+ for (i = 0; i < insn_cnt; i++, insn++) {
1920+ if (BPF_CLASS(insn->code) != BPF_JMP ||
1921+ BPF_OP(insn->code) == BPF_CALL ||
1922+ BPF_OP(insn->code) == BPF_EXIT)
1923+ continue;
1924+
1925+ /* adjust offset of jmps if necessary */
1926+ if (i < pos && i + insn->off + 1 > pos)
1927+ insn->off += delta;
1928+ else if (i > pos && i + insn->off + 1 < pos)
1929+ insn->off -= delta;
1930+ }
1931+}
1932+
1933+/* convert load instructions that access fields of 'struct __sk_buff'
1934+ * into sequence of instructions that access fields of 'struct sk_buff'
1935+ */
1936+static int convert_ctx_accesses(struct verifier_env *env)
1937+{
1938+ struct bpf_insn *insn = env->prog->insnsi;
1939+ int insn_cnt = env->prog->len;
1940+ struct bpf_insn insn_buf[16];
1941+ struct bpf_prog *new_prog;
1942+ u32 cnt;
1943+ int i;
1944+
1945+ if (!env->prog->aux->ops->convert_ctx_access)
1946+ return 0;
1947+
1948+ for (i = 0; i < insn_cnt; i++, insn++) {
1949+ if (insn->code != (BPF_LDX | BPF_MEM | BPF_W))
1950+ continue;
1951+
1952+ if (insn->imm != PTR_TO_CTX) {
1953+ /* clear internal mark */
1954+ insn->imm = 0;
1955+ continue;
1956+ }
1957+
1958+ cnt = env->prog->aux->ops->
1959+ convert_ctx_access(insn->dst_reg, insn->src_reg,
1960+ insn->off, insn_buf);
1961+ if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) {
1962+ verbose("bpf verifier is misconfigured\n");
1963+ return -EINVAL;
1964+ }
1965+
1966+ if (cnt == 1) {
1967+ memcpy(insn, insn_buf, sizeof(*insn));
1968+ continue;
1969+ }
1970+
1971+ /* several new insns need to be inserted. Make room for them */
1972+ insn_cnt += cnt - 1;
1973+ new_prog = bpf_prog_realloc(env->prog,
1974+ bpf_prog_size(insn_cnt),
1975+ GFP_USER);
1976+ if (!new_prog)
1977+ return -ENOMEM;
1978+
1979+ new_prog->len = insn_cnt;
1980+
1981+ memmove(new_prog->insnsi + i + cnt, new_prog->insns + i + 1,
1982+ sizeof(*insn) * (insn_cnt - i - cnt));
1983+
1984+ /* copy substitute insns in place of load instruction */
1985+ memcpy(new_prog->insnsi + i, insn_buf, sizeof(*insn) * cnt);
1986+
1987+ /* adjust branches in the whole program */
1988+ adjust_branches(new_prog, i, cnt - 1);
1989+
1990+ /* keep walking new program and skip insns we just inserted */
1991+ env->prog = new_prog;
1992+ insn = new_prog->insnsi + i + cnt - 1;
1993+ i += cnt - 1;
1994+ }
1995+
1996+ return 0;
1997+}
1998+
18841999 static void free_states(struct verifier_env *env)
18852000 {
18862001 struct verifier_state_list *sl, *sln;
@@ -1903,13 +2018,13 @@ static void free_states(struct verifier_env *env) · 16 unmodified lines
19032018 kfree(env->explored_states);
19042019 }
19052020
1906−int bpf_check(struct bpf_prog *prog, union bpf_attr *attr)
2021+int bpf_check(struct bpf_prog **prog, union bpf_attr *attr)
19072022 {
19082023 char __user *log_ubuf = NULL;
19092024 struct verifier_env *env;
19102025 int ret = -EINVAL;
19112026
1912− if (prog->len <= 0 || prog->len > BPF_MAXINSNS)
2027+ if ((*prog)->len <= 0 || (*prog)->len > BPF_MAXINSNS)
19132028 return -E2BIG;
19142029
19152030 /* 'struct verifier_env' can be global, but since it's not small,
@@ -1919,7 +2034,7 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr *attr) · 3 unmodified lines
19192034 if (!env)
19202035 return -ENOMEM;
19212036
1922− env->prog = prog;
2037+ env->prog = *prog;
19232038
19242039 /* grab the mutex to protect few globals used by verifier */
19252040 mutex_lock(&bpf_verifier_lock);
@@ -1951,7 +2066,7 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr *attr) · 25 unmodified lines
19512066 if (ret < 0)
19522067 goto skip_full_check;
19532068
1954− env->explored_states = kcalloc(prog->len,
2069+ env->explored_states = kcalloc(env->prog->len,
19552070 sizeof(struct verifier_state_list *),
19562071 GFP_USER);
19572072 ret = -ENOMEM;
@@ -1968,6 +2083,10 @@ skip_full_check: · 10 unmodified lines
19682083 while (pop_stack(env, NULL) >= 0);
19692084 free_states(env);
19702085
2086+ if (ret == 0)
2087+ /* program is valid, convert *(u32*)(ctx + off) accesses */
2088+ ret = convert_ctx_accesses(env);
2089+
19712090 if (log_level && log_len >= log_size - 1) {
19722091 BUG_ON(log_len >= log_size);
19732092 /* verifier log exceeded user supplied buffer */
@@ -1983,18 +2102,18 @@ skip_full_check: · 9 unmodified lines
19832102
19842103 if (ret == 0 && env->used_map_cnt) {
19852104 /* if program passed verifier, update used_maps in bpf_prog_info */
1986− prog->aux->used_maps = kmalloc_array(env->used_map_cnt,
1987− sizeof(env->used_maps[0]),
1988− GFP_KERNEL);
2105+ env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt,
2106+ sizeof(env->used_maps[0]),
2107+ GFP_KERNEL);
19892108
1990− if (!prog->aux->used_maps) {
2109+ if (!env->prog->aux->used_maps) {
19912110 ret = -ENOMEM;
19922111 goto free_log_buf;
19932112 }
19942113
1995− memcpy(prog->aux->used_maps, env->used_maps,
2114+ memcpy(env->prog->aux->used_maps, env->used_maps,
19962115 sizeof(env->used_maps[0]) * env->used_map_cnt);
1997− prog->aux->used_map_cnt = env->used_map_cnt;
2116+ env->prog->aux->used_map_cnt = env->used_map_cnt;
19982117
19992118 /* program is valid. Convert pseudo bpf_ld_imm64 into generic
20002119 * bpf_ld_imm64 instructions
@@ -2006,11 +2125,12 @@ free_log_buf: · 5 unmodified lines
20062125 if (log_level)
20072126 vfree(log_buf);
20082127 free_env:
2009− if (!prog->aux->used_maps)
2128+ if (!env->prog->aux->used_maps)
20102129 /* if we didn't copy map pointers into bpf_prog_info, release
20112130 * them now. Otherwise free_bpf_prog_info() will release them.
20122131 */
20132132 release_maps(env);
2133+ *prog = env->prog;
20142134 kfree(env);
20152135 mutex_unlock(&bpf_verifier_lock);
20162136 return ret;
net/core/filter.c+83−17
@@ -150,10 +150,43 @@ static u64 __get_random_u32(u64 ctx, u64 a, u64 x, u64 r4, u64 r5) · 149 unmodified lines
150150 return prandom_u32();
151151 }
152152
153+static u32 convert_skb_access(int skb_field, int dst_reg, int src_reg,
154+ struct bpf_insn *insn_buf)
155+{
156+ struct bpf_insn *insn = insn_buf;
157+
158+ switch (skb_field) {
159+ case SKF_AD_MARK:
160+ BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4);
161+
162+ *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg,
163+ offsetof(struct sk_buff, mark));
164+ break;
165+
166+ case SKF_AD_PKTTYPE:
167+ *insn++ = BPF_LDX_MEM(BPF_B, dst_reg, src_reg, PKT_TYPE_OFFSET());
168+ *insn++ = BPF_ALU32_IMM(BPF_AND, dst_reg, PKT_TYPE_MAX);
169+#ifdef __BIG_ENDIAN_BITFIELD
170+ *insn++ = BPF_ALU32_IMM(BPF_RSH, dst_reg, 5);
171+#endif
172+ break;
173+
174+ case SKF_AD_QUEUE:
175+ BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, queue_mapping) != 2);
176+
177+ *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg,
178+ offsetof(struct sk_buff, queue_mapping));
179+ break;
180+ }
181+
182+ return insn - insn_buf;
183+}
184+
153185 static bool convert_bpf_extensions(struct sock_filter *fp,
154186 struct bpf_insn **insnp)
155187 {
156188 struct bpf_insn *insn = *insnp;
189+ u32 cnt;
157190
158191 switch (fp->k) {
159192 case SKF_AD_OFF + SKF_AD_PROTOCOL:
@@ -167,13 +200,8 @@ static bool convert_bpf_extensions(struct sock_filter *fp, · 7 unmodified lines
167200 break;
168201
169202 case SKF_AD_OFF + SKF_AD_PKTTYPE:
170− *insn++ = BPF_LDX_MEM(BPF_B, BPF_REG_A, BPF_REG_CTX,
171− PKT_TYPE_OFFSET());
172− *insn = BPF_ALU32_IMM(BPF_AND, BPF_REG_A, PKT_TYPE_MAX);
173−#ifdef __BIG_ENDIAN_BITFIELD
174− insn++;
175− *insn = BPF_ALU32_IMM(BPF_RSH, BPF_REG_A, 5);
176−#endif
203+ cnt = convert_skb_access(SKF_AD_PKTTYPE, BPF_REG_A, BPF_REG_CTX, insn);
204+ insn += cnt - 1;
177205 break;
178206
179207 case SKF_AD_OFF + SKF_AD_IFINDEX:
@@ -197,10 +225,8 @@ static bool convert_bpf_extensions(struct sock_filter *fp, · 17 unmodified lines
197225 break;
198226
199227 case SKF_AD_OFF + SKF_AD_MARK:
200− BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4);
201−
202− *insn = BPF_LDX_MEM(BPF_W, BPF_REG_A, BPF_REG_CTX,
203− offsetof(struct sk_buff, mark));
228+ cnt = convert_skb_access(SKF_AD_MARK, BPF_REG_A, BPF_REG_CTX, insn);
229+ insn += cnt - 1;
204230 break;
205231
206232 case SKF_AD_OFF + SKF_AD_RXHASH:
@@ -211,10 +237,8 @@ static bool convert_bpf_extensions(struct sock_filter *fp, · 4 unmodified lines
211237 break;
212238
213239 case SKF_AD_OFF + SKF_AD_QUEUE:
214− BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, queue_mapping) != 2);
215−
216− *insn = BPF_LDX_MEM(BPF_H, BPF_REG_A, BPF_REG_CTX,
217− offsetof(struct sk_buff, queue_mapping));
240+ cnt = convert_skb_access(SKF_AD_QUEUE, BPF_REG_A, BPF_REG_CTX, insn);
241+ insn += cnt - 1;
218242 break;
219243
220244 case SKF_AD_OFF + SKF_AD_VLAN_TAG:
@@ -1151,13 +1175,55 @@ sk_filter_func_proto(enum bpf_func_id func_id) · 930 unmodified lines
11511175 static bool sk_filter_is_valid_access(int off, int size,
11521176 enum bpf_access_type type)
11531177 {
1154− /* skb fields cannot be accessed yet */
1155− return false;
1178+ /* only read is allowed */
1179+ if (type != BPF_READ)
1180+ return false;
1181+
1182+ /* check bounds */
1183+ if (off < 0 || off >= sizeof(struct __sk_buff))
1184+ return false;
1185+
1186+ /* disallow misaligned access */
1187+ if (off % size != 0)
1188+ return false;
1189+
1190+ /* all __sk_buff fields are __u32 */
1191+ if (size != 4)
1192+ return false;
1193+
1194+ return true;
1195+}
1196+
1197+static u32 sk_filter_convert_ctx_access(int dst_reg, int src_reg, int ctx_off,
1198+ struct bpf_insn *insn_buf)
1199+{
1200+ struct bpf_insn *insn = insn_buf;
1201+
1202+ switch (ctx_off) {
1203+ case offsetof(struct __sk_buff, len):
1204+ BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4);
1205+
1206+ *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg,
1207+ offsetof(struct sk_buff, len));
1208+ break;
1209+
1210+ case offsetof(struct __sk_buff, mark):
1211+ return convert_skb_access(SKF_AD_MARK, dst_reg, src_reg, insn);
1212+
1213+ case offsetof(struct __sk_buff, pkt_type):
1214+ return convert_skb_access(SKF_AD_PKTTYPE, dst_reg, src_reg, insn);
1215+
1216+ case offsetof(struct __sk_buff, queue_mapping):
1217+ return convert_skb_access(SKF_AD_QUEUE, dst_reg, src_reg, insn);
1218+ }
1219+
1220+ return insn - insn_buf;
11561221 }
11571222
11581223 static const struct bpf_verifier_ops sk_filter_ops = {
11591224 .get_func_proto = sk_filter_func_proto,
11601225 .is_valid_access = sk_filter_is_valid_access,
1226+ .convert_ctx_access = sk_filter_convert_ctx_access,
11611227 };
11621228
11631229 static struct bpf_prog_type_list sk_filter_type __read_mostly = {

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t