History

bpf: Move getsockopt retval to struct bpf_cg_run_ctx

The retval value is moved to struct bpf_cg_run_ctx for ease of access in different prog types with different context structs layouts. The helper implementation (to be added in a later patch in the series) can simply perform a container_of from current->bpf_ctx to retrieve bpf_cg_run_ctx. Unfortunately, there is no easy way to access the current task_struct via the verifier BPF bytecode rewrite, aside from possibly calling a helper, so a pointer to current task is added to struct bpf_sockopt_kern so that the rewritten BPF bytecode can access struct bpf_cg_run_ctx with an indirection. For backward compatibility, if a getsockopt program rejects a syscall by returning 0, an -EPERM will be generated, by having the BPF_PROG_RUN_ARRAY_CG family macros automatically set the retval to -EPERM. Unlike prior to this patch, this -EPERM will be visible to ctx->retval for any other hooks down the line in the prog array. Additionally, the restriction that getsockopt filters can only set the retval to 0 is removed, considering that certain getsockopt implementations may return optlen. Filters are now able to set the value arbitrarily.

YiFei Zhu committed 4 years ago

Commit c4dcfdd1 parent f10d0593 files · +63 −44Browse files

Unified · Split

3 files
  1. include/linux/bpf.h +11 −9
  2. include/linux/filter.h +4 −1
  3. kernel/bpf/cgroup.c +48 −34
include/linux/bpf.h+11−9
@@ -1245,6 +1245,7 @@ struct bpf_run_ctx {}; · 1244 unmodified lines
12451245 struct bpf_cg_run_ctx {
12461246 struct bpf_run_ctx run_ctx;
12471247 const struct bpf_prog_array_item *prog_item;
1248+ int retval;
12481249 };
12491250
12501251 struct bpf_trace_run_ctx {
@@ -1280,16 +1281,16 @@ typedef u32 (*bpf_prog_run_fn)(const struct bpf_prog *prog, const void *ctx); · 29 unmodified lines
12801281 static __always_inline int
12811282 BPF_PROG_RUN_ARRAY_CG_FLAGS(const struct bpf_prog_array __rcu *array_rcu,
12821283 const void *ctx, bpf_prog_run_fn run_prog,
1283− u32 *ret_flags)
1284+ int retval, u32 *ret_flags)
12841285 {
12851286 const struct bpf_prog_array_item *item;
12861287 const struct bpf_prog *prog;
12871288 const struct bpf_prog_array *array;
12881289 struct bpf_run_ctx *old_run_ctx;
12891290 struct bpf_cg_run_ctx run_ctx;
1290− int ret = 0;
12911291 u32 func_ret;
12921292
1293+ run_ctx.retval = retval;
12931294 migrate_disable();
12941295 rcu_read_lock();
12951296 array = rcu_dereference(array_rcu);
@@ -1299,27 +1300,28 @@ BPF_PROG_RUN_ARRAY_CG_FLAGS(const struct bpf_prog_array __rcu *array_rcu, · 3 unmodified lines
12991300 run_ctx.prog_item = item;
13001301 func_ret = run_prog(prog, ctx);
13011302 if (!(func_ret & 1))
1302− ret = -EPERM;
1303+ run_ctx.retval = -EPERM;
13031304 *(ret_flags) |= (func_ret >> 1);
13041305 item++;
13051306 }
13061307 bpf_reset_run_ctx(old_run_ctx);
13071308 rcu_read_unlock();
13081309 migrate_enable();
1309− return ret;
1310+ return run_ctx.retval;
13101311 }
13111312
13121313 static __always_inline int
13131314 BPF_PROG_RUN_ARRAY_CG(const struct bpf_prog_array __rcu *array_rcu,
1314− const void *ctx, bpf_prog_run_fn run_prog)
1315+ const void *ctx, bpf_prog_run_fn run_prog,
1316+ int retval)
13151317 {
13161318 const struct bpf_prog_array_item *item;
13171319 const struct bpf_prog *prog;
13181320 const struct bpf_prog_array *array;
13191321 struct bpf_run_ctx *old_run_ctx;
13201322 struct bpf_cg_run_ctx run_ctx;
1321− int ret = 0;
13221323
1324+ run_ctx.retval = retval;
13231325 migrate_disable();
13241326 rcu_read_lock();
13251327 array = rcu_dereference(array_rcu);
@@ -1328,13 +1330,13 @@ BPF_PROG_RUN_ARRAY_CG(const struct bpf_prog_array __rcu *array_rcu, · 2 unmodified lines
13281330 while ((prog = READ_ONCE(item->prog))) {
13291331 run_ctx.prog_item = item;
13301332 if (!run_prog(prog, ctx))
1331− ret = -EPERM;
1333+ run_ctx.retval = -EPERM;
13321334 item++;
13331335 }
13341336 bpf_reset_run_ctx(old_run_ctx);
13351337 rcu_read_unlock();
13361338 migrate_enable();
1337− return ret;
1339+ return run_ctx.retval;
13381340 }
13391341
13401342 static __always_inline u32
@@ -1394,7 +1396,7 @@ out: · 53 unmodified lines
13941396 u32 _flags = 0; \
13951397 bool _cn; \
13961398 u32 _ret; \
1397− _ret = BPF_PROG_RUN_ARRAY_CG_FLAGS(array, ctx, func, &_flags); \
1399+ _ret = BPF_PROG_RUN_ARRAY_CG_FLAGS(array, ctx, func, 0, &_flags); \
13981400 _cn = _flags & BPF_RET_SET_CN; \
13991401 if (!_ret) \
14001402 _ret = (_cn ? NET_XMIT_CN : NET_XMIT_SUCCESS); \
include/linux/filter.h+4−1
@@ -1356,7 +1356,10 @@ struct bpf_sockopt_kern { · 1355 unmodified lines
13561356 s32 level;
13571357 s32 optname;
13581358 s32 optlen;
1359− s32 retval;
1359+ /* for retval in struct bpf_cg_run_ctx */
1360+ struct task_struct *current_task;
1361+ /* Temporary "register" for indirect stores to ppos. */
1362+ u64 tmp_reg;
13601363 };
13611364
13621365 int copy_bpf_fprog_from_user(struct sock_fprog *dst, sockptr_t src, int len);
kernel/bpf/cgroup.c+48−34
@@ -1079,7 +1079,7 @@ int __cgroup_bpf_run_filter_skb(struct sock *sk, · 1078 unmodified lines
10791079 cgrp->bpf.effective[atype], skb, __bpf_prog_run_save_cb);
10801080 } else {
10811081 ret = BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[atype], skb,
1082− __bpf_prog_run_save_cb);
1082+ __bpf_prog_run_save_cb, 0);
10831083 }
10841084 bpf_restore_data_end(skb, saved_data_end);
10851085 __skb_pull(skb, offset);
@@ -1108,7 +1108,7 @@ int __cgroup_bpf_run_filter_sk(struct sock *sk, · 22 unmodified lines
11081108 struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
11091109
11101110 return BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[atype], sk,
1111− bpf_prog_run);
1111+ bpf_prog_run, 0);
11121112 }
11131113 EXPORT_SYMBOL(__cgroup_bpf_run_filter_sk);
11141114
@@ -1154,7 +1154,7 @@ int __cgroup_bpf_run_filter_sock_addr(struct sock *sk, · 39 unmodified lines
11541154
11551155 cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
11561156 return BPF_PROG_RUN_ARRAY_CG_FLAGS(cgrp->bpf.effective[atype], &ctx,
1157− bpf_prog_run, flags);
1157+ bpf_prog_run, 0, flags);
11581158 }
11591159 EXPORT_SYMBOL(__cgroup_bpf_run_filter_sock_addr);
11601160
@@ -1181,7 +1181,7 @@ int __cgroup_bpf_run_filter_sock_ops(struct sock *sk, · 20 unmodified lines
11811181 struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
11821182
11831183 return BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[atype], sock_ops,
1184− bpf_prog_run);
1184+ bpf_prog_run, 0);
11851185 }
11861186 EXPORT_SYMBOL(__cgroup_bpf_run_filter_sock_ops);
11871187
@@ -1199,7 +1199,7 @@ int __cgroup_bpf_check_dev_permission(short dev_type, u32 major, u32 minor, · 11 unmodified lines
11991199 rcu_read_lock();
12001200 cgrp = task_dfl_cgroup(current);
12011201 ret = BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[atype], &ctx,
1202− bpf_prog_run);
1202+ bpf_prog_run, 0);
12031203 rcu_read_unlock();
12041204
12051205 return ret;
@@ -1330,7 +1330,8 @@ int __cgroup_bpf_run_filter_sysctl(struct ctl_table_header *head, · 124 unmodified lines
13301330
13311331 rcu_read_lock();
13321332 cgrp = task_dfl_cgroup(current);
1333− ret = BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[atype], &ctx, bpf_prog_run);
1333+ ret = BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[atype], &ctx,
1334+ bpf_prog_run, 0);
13341335 rcu_read_unlock();
13351336
13361337 kfree(ctx.cur_val);
@@ -1445,7 +1446,7 @@ int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level, · 108 unmodified lines
14451446
14461447 lock_sock(sk);
14471448 ret = BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[CGROUP_SETSOCKOPT],
1448− &ctx, bpf_prog_run);
1449+ &ctx, bpf_prog_run, 0);
14491450 release_sock(sk);
14501451
14511452 if (ret)
@@ -1509,7 +1510,7 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, · 57 unmodified lines
15091510 .sk = sk,
15101511 .level = level,
15111512 .optname = optname,
1512− .retval = retval,
1513+ .current_task = current,
15131514 };
15141515 int ret;
15151516
@@ -1553,10 +1554,10 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, · 37 unmodified lines
15531554
15541555 lock_sock(sk);
15551556 ret = BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[CGROUP_GETSOCKOPT],
1556− &ctx, bpf_prog_run);
1557+ &ctx, bpf_prog_run, retval);
15571558 release_sock(sk);
15581559
1559− if (ret)
1560+ if (ret < 0)
15601561 goto out;
15611562
15621563 if (ctx.optlen > max_optlen || ctx.optlen < 0) {
@@ -1564,14 +1565,6 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, · 1 unmodified line
15641565 goto out;
15651566 }
15661567
1567− /* BPF programs only allowed to set retval to 0, not some
1568− * arbitrary value.
1569− */
1570− if (ctx.retval != 0 && ctx.retval != retval) {
1571− ret = -EFAULT;
1572− goto out;
1573− }
1574−
15751568 if (ctx.optlen != 0) {
15761569 if (copy_to_user(optval, ctx.optval, ctx.optlen) ||
15771570 put_user(ctx.optlen, optlen)) {
@@ -1580,8 +1573,6 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, · 2 unmodified lines
15801573 }
15811574 }
15821575
1583− ret = ctx.retval;
1584−
15851576 out:
15861577 sockopt_free_buf(&ctx, &buf);
15871578 return ret;
@@ -1596,10 +1587,10 @@ int __cgroup_bpf_run_filter_getsockopt_kern(struct sock *sk, int level, · 8 unmodified lines
15961587 .sk = sk,
15971588 .level = level,
15981589 .optname = optname,
1599− .retval = retval,
16001590 .optlen = *optlen,
16011591 .optval = optval,
16021592 .optval_end = optval + *optlen,
1593+ .current_task = current,
16031594 };
16041595 int ret;
16051596
@@ -1612,25 +1603,19 @@ int __cgroup_bpf_run_filter_getsockopt_kern(struct sock *sk, int level, · 6 unmodified lines
16121603 */
16131604
16141605 ret = BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[CGROUP_GETSOCKOPT],
1615− &ctx, bpf_prog_run);
1616− if (ret)
1606+ &ctx, bpf_prog_run, retval);
1607+ if (ret < 0)
16171608 return ret;
16181609
16191610 if (ctx.optlen > *optlen)
16201611 return -EFAULT;
16211612
1622− /* BPF programs only allowed to set retval to 0, not some
1623− * arbitrary value.
1624− */
1625− if (ctx.retval != 0 && ctx.retval != retval)
1626− return -EFAULT;
1627−
16281613 /* BPF programs can shrink the buffer, export the modifications.
16291614 */
16301615 if (ctx.optlen != 0)
16311616 *optlen = ctx.optlen;
16321617
1633− return ctx.retval;
1618+ return ret;
16341619 }
16351620 #endif
16361621
@@ -2046,10 +2031,39 @@ static u32 cg_sockopt_convert_ctx_access(enum bpf_access_type type, · 409 unmodified lines
20462031 *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, optlen);
20472032 break;
20482033 case offsetof(struct bpf_sockopt, retval):
2049− if (type == BPF_WRITE)
2050− *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_STX_MEM, retval);
2051− else
2052− *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, retval);
2034+ BUILD_BUG_ON(offsetof(struct bpf_cg_run_ctx, run_ctx) != 0);
2035+
2036+ if (type == BPF_WRITE) {
2037+ int treg = BPF_REG_9;
2038+
2039+ if (si->src_reg == treg || si->dst_reg == treg)
2040+ --treg;
2041+ if (si->src_reg == treg || si->dst_reg == treg)
2042+ --treg;
2043+ *insn++ = BPF_STX_MEM(BPF_DW, si->dst_reg, treg,
2044+ offsetof(struct bpf_sockopt_kern, tmp_reg));
2045+ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sockopt_kern, current_task),
2046+ treg, si->dst_reg,
2047+ offsetof(struct bpf_sockopt_kern, current_task));
2048+ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct task_struct, bpf_ctx),
2049+ treg, treg,
2050+ offsetof(struct task_struct, bpf_ctx));
2051+ *insn++ = BPF_STX_MEM(BPF_FIELD_SIZEOF(struct bpf_cg_run_ctx, retval),
2052+ treg, si->src_reg,
2053+ offsetof(struct bpf_cg_run_ctx, retval));
2054+ *insn++ = BPF_LDX_MEM(BPF_DW, treg, si->dst_reg,
2055+ offsetof(struct bpf_sockopt_kern, tmp_reg));
2056+ } else {
2057+ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sockopt_kern, current_task),
2058+ si->dst_reg, si->src_reg,
2059+ offsetof(struct bpf_sockopt_kern, current_task));
2060+ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct task_struct, bpf_ctx),
2061+ si->dst_reg, si->dst_reg,
2062+ offsetof(struct task_struct, bpf_ctx));
2063+ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_cg_run_ctx, retval),
2064+ si->dst_reg, si->dst_reg,
2065+ offsetof(struct bpf_cg_run_ctx, retval));
2066+ }
20532067 break;
20542068 case offsetof(struct bpf_sockopt, optval):
20552069 *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, optval);