History

bpf: Move getsockopt retval to struct bpf_cg_run_ctx

The retval value is moved to struct bpf_cg_run_ctx for ease of access in different prog types with different context structs layouts. The helper implementation (to be added in a later patch in the series) can simply perform a container_of from current->bpf_ctx to retrieve bpf_cg_run_ctx. Unfortunately, there is no easy way to access the current task_struct via the verifier BPF bytecode rewrite, aside from possibly calling a helper, so a pointer to current task is added to struct bpf_sockopt_kern so that the rewritten BPF bytecode can access struct bpf_cg_run_ctx with an indirection. For backward compatibility, if a getsockopt program rejects a syscall by returning 0, an -EPERM will be generated, by having the BPF_PROG_RUN_ARRAY_CG family macros automatically set the retval to -EPERM. Unlike prior to this patch, this -EPERM will be visible to ctx->retval for any other hooks down the line in the prog array. Additionally, the restriction that getsockopt filters can only set the retval to 0 is removed, considering that certain getsockopt implementations may return optlen. Filters are now able to set the value arbitrarily.

YiFei Zhu committed 4 years ago

Commit c4dcfdd1 parent f10d0593 files · +63 −44Browse files

Unified · Split

3 files
  1. include/linux/bpf.h +11 −9
  2. include/linux/filter.h +4 −1
  3. kernel/bpf/cgroup.c +48 −34
include/linux/bpf.h+11−9
@@ -1245,6 +1245,7 @@ struct bpf_run_ctx {}; · 1244 unmodified lines
1245struct bpf_cg_run_ctx {1245struct bpf_cg_run_ctx {
1246 struct bpf_run_ctx run_ctx;1246 struct bpf_run_ctx run_ctx;
1247 const struct bpf_prog_array_item *prog_item;1247 const struct bpf_prog_array_item *prog_item;
1248 int retval;
1248};1249};
1249 1250
1250struct bpf_trace_run_ctx {1251struct bpf_trace_run_ctx {
@@ -1280,16 +1281,16 @@ typedef u32 (*bpf_prog_run_fn)(const struct bpf_prog *prog, const void *ctx); · 29 unmodified lines
1280static __always_inline int1281static __always_inline int
1281BPF_PROG_RUN_ARRAY_CG_FLAGS(const struct bpf_prog_array __rcu *array_rcu,1282BPF_PROG_RUN_ARRAY_CG_FLAGS(const struct bpf_prog_array __rcu *array_rcu,
1282 const void *ctx, bpf_prog_run_fn run_prog,1283 const void *ctx, bpf_prog_run_fn run_prog,
1283 u32 *ret_flags)1284 int retval, u32 *ret_flags)
1284{1285{
1285 const struct bpf_prog_array_item *item;1286 const struct bpf_prog_array_item *item;
1286 const struct bpf_prog *prog;1287 const struct bpf_prog *prog;
1287 const struct bpf_prog_array *array;1288 const struct bpf_prog_array *array;
1288 struct bpf_run_ctx *old_run_ctx;1289 struct bpf_run_ctx *old_run_ctx;
1289 struct bpf_cg_run_ctx run_ctx;1290 struct bpf_cg_run_ctx run_ctx;
1290 int ret = 0;
1291 u32 func_ret;1291 u32 func_ret;
1292 1292
1293 run_ctx.retval = retval;
1293 migrate_disable();1294 migrate_disable();
1294 rcu_read_lock();1295 rcu_read_lock();
1295 array = rcu_dereference(array_rcu);1296 array = rcu_dereference(array_rcu);
@@ -1299,27 +1300,28 @@ BPF_PROG_RUN_ARRAY_CG_FLAGS(const struct bpf_prog_array __rcu *array_rcu, · 3 unmodified lines
1299 run_ctx.prog_item = item;1300 run_ctx.prog_item = item;
1300 func_ret = run_prog(prog, ctx);1301 func_ret = run_prog(prog, ctx);
1301 if (!(func_ret & 1))1302 if (!(func_ret & 1))
1302 ret = -EPERM;1303 run_ctx.retval = -EPERM;
1303 *(ret_flags) |= (func_ret >> 1);1304 *(ret_flags) |= (func_ret >> 1);
1304 item++;1305 item++;
1305 }1306 }
1306 bpf_reset_run_ctx(old_run_ctx);1307 bpf_reset_run_ctx(old_run_ctx);
1307 rcu_read_unlock();1308 rcu_read_unlock();
1308 migrate_enable();1309 migrate_enable();
1309 return ret;1310 return run_ctx.retval;
1310}1311}
1311 1312
1312static __always_inline int1313static __always_inline int
1313BPF_PROG_RUN_ARRAY_CG(const struct bpf_prog_array __rcu *array_rcu,1314BPF_PROG_RUN_ARRAY_CG(const struct bpf_prog_array __rcu *array_rcu,
1314 const void *ctx, bpf_prog_run_fn run_prog)1315 const void *ctx, bpf_prog_run_fn run_prog,
1316 int retval)
1315{1317{
1316 const struct bpf_prog_array_item *item;1318 const struct bpf_prog_array_item *item;
1317 const struct bpf_prog *prog;1319 const struct bpf_prog *prog;
1318 const struct bpf_prog_array *array;1320 const struct bpf_prog_array *array;
1319 struct bpf_run_ctx *old_run_ctx;1321 struct bpf_run_ctx *old_run_ctx;
1320 struct bpf_cg_run_ctx run_ctx;1322 struct bpf_cg_run_ctx run_ctx;
1321 int ret = 0;
1322 1323
1324 run_ctx.retval = retval;
1323 migrate_disable();1325 migrate_disable();
1324 rcu_read_lock();1326 rcu_read_lock();
1325 array = rcu_dereference(array_rcu);1327 array = rcu_dereference(array_rcu);
@@ -1328,13 +1330,13 @@ BPF_PROG_RUN_ARRAY_CG(const struct bpf_prog_array __rcu *array_rcu, · 2 unmodified lines
1328 while ((prog = READ_ONCE(item->prog))) {1330 while ((prog = READ_ONCE(item->prog))) {
1329 run_ctx.prog_item = item;1331 run_ctx.prog_item = item;
1330 if (!run_prog(prog, ctx))1332 if (!run_prog(prog, ctx))
1331 ret = -EPERM;1333 run_ctx.retval = -EPERM;
1332 item++;1334 item++;
1333 }1335 }
1334 bpf_reset_run_ctx(old_run_ctx);1336 bpf_reset_run_ctx(old_run_ctx);
1335 rcu_read_unlock();1337 rcu_read_unlock();
1336 migrate_enable();1338 migrate_enable();
1337 return ret;1339 return run_ctx.retval;
1338}1340}
1339 1341
1340static __always_inline u321342static __always_inline u32
@@ -1394,7 +1396,7 @@ out: · 53 unmodified lines
1394 u32 _flags = 0; \1396 u32 _flags = 0; \
1395 bool _cn; \1397 bool _cn; \
1396 u32 _ret; \1398 u32 _ret; \
1397 _ret = BPF_PROG_RUN_ARRAY_CG_FLAGS(array, ctx, func, &_flags); \1399 _ret = BPF_PROG_RUN_ARRAY_CG_FLAGS(array, ctx, func, 0, &_flags); \
1398 _cn = _flags & BPF_RET_SET_CN; \1400 _cn = _flags & BPF_RET_SET_CN; \
1399 if (!_ret) \1401 if (!_ret) \
1400 _ret = (_cn ? NET_XMIT_CN : NET_XMIT_SUCCESS); \1402 _ret = (_cn ? NET_XMIT_CN : NET_XMIT_SUCCESS); \
include/linux/filter.h+4−1
@@ -1356,7 +1356,10 @@ struct bpf_sockopt_kern { · 1355 unmodified lines
1356 s32 level;1356 s32 level;
1357 s32 optname;1357 s32 optname;
1358 s32 optlen;1358 s32 optlen;
1359 s32 retval;1359 /* for retval in struct bpf_cg_run_ctx */
1360 struct task_struct *current_task;
1361 /* Temporary "register" for indirect stores to ppos. */
1362 u64 tmp_reg;
1360};1363};
1361 1364
1362int copy_bpf_fprog_from_user(struct sock_fprog *dst, sockptr_t src, int len);1365int copy_bpf_fprog_from_user(struct sock_fprog *dst, sockptr_t src, int len);
kernel/bpf/cgroup.c+48−34
@@ -1079,7 +1079,7 @@ int __cgroup_bpf_run_filter_skb(struct sock *sk, · 1078 unmodified lines
1079 cgrp->bpf.effective[atype], skb, __bpf_prog_run_save_cb);1079 cgrp->bpf.effective[atype], skb, __bpf_prog_run_save_cb);
1080 } else {1080 } else {
1081 ret = BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[atype], skb,1081 ret = BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[atype], skb,
1082 __bpf_prog_run_save_cb);1082 __bpf_prog_run_save_cb, 0);
1083 }1083 }
1084 bpf_restore_data_end(skb, saved_data_end);1084 bpf_restore_data_end(skb, saved_data_end);
1085 __skb_pull(skb, offset);1085 __skb_pull(skb, offset);
@@ -1108,7 +1108,7 @@ int __cgroup_bpf_run_filter_sk(struct sock *sk, · 22 unmodified lines
1108 struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);1108 struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
1109 1109
1110 return BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[atype], sk,1110 return BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[atype], sk,
1111 bpf_prog_run);1111 bpf_prog_run, 0);
1112}1112}
1113EXPORT_SYMBOL(__cgroup_bpf_run_filter_sk);1113EXPORT_SYMBOL(__cgroup_bpf_run_filter_sk);
1114 1114
@@ -1154,7 +1154,7 @@ int __cgroup_bpf_run_filter_sock_addr(struct sock *sk, · 39 unmodified lines
1154 1154
1155 cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);1155 cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
1156 return BPF_PROG_RUN_ARRAY_CG_FLAGS(cgrp->bpf.effective[atype], &ctx,1156 return BPF_PROG_RUN_ARRAY_CG_FLAGS(cgrp->bpf.effective[atype], &ctx,
1157 bpf_prog_run, flags);1157 bpf_prog_run, 0, flags);
1158}1158}
1159EXPORT_SYMBOL(__cgroup_bpf_run_filter_sock_addr);1159EXPORT_SYMBOL(__cgroup_bpf_run_filter_sock_addr);
1160 1160
@@ -1181,7 +1181,7 @@ int __cgroup_bpf_run_filter_sock_ops(struct sock *sk, · 20 unmodified lines
1181 struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);1181 struct cgroup *cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
1182 1182
1183 return BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[atype], sock_ops,1183 return BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[atype], sock_ops,
1184 bpf_prog_run);1184 bpf_prog_run, 0);
1185}1185}
1186EXPORT_SYMBOL(__cgroup_bpf_run_filter_sock_ops);1186EXPORT_SYMBOL(__cgroup_bpf_run_filter_sock_ops);
1187 1187
@@ -1199,7 +1199,7 @@ int __cgroup_bpf_check_dev_permission(short dev_type, u32 major, u32 minor, · 11 unmodified lines
1199 rcu_read_lock();1199 rcu_read_lock();
1200 cgrp = task_dfl_cgroup(current);1200 cgrp = task_dfl_cgroup(current);
1201 ret = BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[atype], &ctx,1201 ret = BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[atype], &ctx,
1202 bpf_prog_run);1202 bpf_prog_run, 0);
1203 rcu_read_unlock();1203 rcu_read_unlock();
1204 1204
1205 return ret;1205 return ret;
@@ -1330,7 +1330,8 @@ int __cgroup_bpf_run_filter_sysctl(struct ctl_table_header *head, · 124 unmodified lines
1330 1330
1331 rcu_read_lock();1331 rcu_read_lock();
1332 cgrp = task_dfl_cgroup(current);1332 cgrp = task_dfl_cgroup(current);
1333 ret = BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[atype], &ctx, bpf_prog_run);1333 ret = BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[atype], &ctx,
1334 bpf_prog_run, 0);
1334 rcu_read_unlock();1335 rcu_read_unlock();
1335 1336
1336 kfree(ctx.cur_val);1337 kfree(ctx.cur_val);
@@ -1445,7 +1446,7 @@ int __cgroup_bpf_run_filter_setsockopt(struct sock *sk, int *level, · 108 unmodified lines
1445 1446
1446 lock_sock(sk);1447 lock_sock(sk);
1447 ret = BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[CGROUP_SETSOCKOPT],1448 ret = BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[CGROUP_SETSOCKOPT],
1448 &ctx, bpf_prog_run);1449 &ctx, bpf_prog_run, 0);
1449 release_sock(sk);1450 release_sock(sk);
1450 1451
1451 if (ret)1452 if (ret)
@@ -1509,7 +1510,7 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, · 57 unmodified lines
1509 .sk = sk,1510 .sk = sk,
1510 .level = level,1511 .level = level,
1511 .optname = optname,1512 .optname = optname,
1512 .retval = retval,1513 .current_task = current,
1513 };1514 };
1514 int ret;1515 int ret;
1515 1516
@@ -1553,10 +1554,10 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, · 37 unmodified lines
1553 1554
1554 lock_sock(sk);1555 lock_sock(sk);
1555 ret = BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[CGROUP_GETSOCKOPT],1556 ret = BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[CGROUP_GETSOCKOPT],
1556 &ctx, bpf_prog_run);1557 &ctx, bpf_prog_run, retval);
1557 release_sock(sk);1558 release_sock(sk);
1558 1559
1559 if (ret)1560 if (ret < 0)
1560 goto out;1561 goto out;
1561 1562
1562 if (ctx.optlen > max_optlen || ctx.optlen < 0) {1563 if (ctx.optlen > max_optlen || ctx.optlen < 0) {
@@ -1564,14 +1565,6 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, · 1 unmodified line
1564 goto out;1565 goto out;
1565 }1566 }
1566 1567
1567 /* BPF programs only allowed to set retval to 0, not some
1568 * arbitrary value.
1569 */
1570 if (ctx.retval != 0 && ctx.retval != retval) {
1571 ret = -EFAULT;
1572 goto out;
1573 }
1574
1575 if (ctx.optlen != 0) {1568 if (ctx.optlen != 0) {
1576 if (copy_to_user(optval, ctx.optval, ctx.optlen) ||1569 if (copy_to_user(optval, ctx.optval, ctx.optlen) ||
1577 put_user(ctx.optlen, optlen)) {1570 put_user(ctx.optlen, optlen)) {
@@ -1580,8 +1573,6 @@ int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level, · 2 unmodified lines
1580 }1573 }
1581 }1574 }
1582 1575
1583 ret = ctx.retval;
1584
1585out:1576out:
1586 sockopt_free_buf(&ctx, &buf);1577 sockopt_free_buf(&ctx, &buf);
1587 return ret;1578 return ret;
@@ -1596,10 +1587,10 @@ int __cgroup_bpf_run_filter_getsockopt_kern(struct sock *sk, int level, · 8 unmodified lines
1596 .sk = sk,1587 .sk = sk,
1597 .level = level,1588 .level = level,
1598 .optname = optname,1589 .optname = optname,
1599 .retval = retval,
1600 .optlen = *optlen,1590 .optlen = *optlen,
1601 .optval = optval,1591 .optval = optval,
1602 .optval_end = optval + *optlen,1592 .optval_end = optval + *optlen,
1593 .current_task = current,
1603 };1594 };
1604 int ret;1595 int ret;
1605 1596
@@ -1612,25 +1603,19 @@ int __cgroup_bpf_run_filter_getsockopt_kern(struct sock *sk, int level, · 6 unmodified lines
1612 */1603 */
1613 1604
1614 ret = BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[CGROUP_GETSOCKOPT],1605 ret = BPF_PROG_RUN_ARRAY_CG(cgrp->bpf.effective[CGROUP_GETSOCKOPT],
1615 &ctx, bpf_prog_run);1606 &ctx, bpf_prog_run, retval);
1616 if (ret)1607 if (ret < 0)
1617 return ret;1608 return ret;
1618 1609
1619 if (ctx.optlen > *optlen)1610 if (ctx.optlen > *optlen)
1620 return -EFAULT;1611 return -EFAULT;
1621 1612
1622 /* BPF programs only allowed to set retval to 0, not some
1623 * arbitrary value.
1624 */
1625 if (ctx.retval != 0 && ctx.retval != retval)
1626 return -EFAULT;
1627
1628 /* BPF programs can shrink the buffer, export the modifications.1613 /* BPF programs can shrink the buffer, export the modifications.
1629 */1614 */
1630 if (ctx.optlen != 0)1615 if (ctx.optlen != 0)
1631 *optlen = ctx.optlen;1616 *optlen = ctx.optlen;
1632 1617
1633 return ctx.retval;1618 return ret;
1634}1619}
1635#endif1620#endif
1636 1621
@@ -2046,10 +2031,39 @@ static u32 cg_sockopt_convert_ctx_access(enum bpf_access_type type, · 409 unmodified lines
2046 *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, optlen);2031 *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, optlen);
2047 break;2032 break;
2048 case offsetof(struct bpf_sockopt, retval):2033 case offsetof(struct bpf_sockopt, retval):
2049 if (type == BPF_WRITE)2034 BUILD_BUG_ON(offsetof(struct bpf_cg_run_ctx, run_ctx) != 0);
2050 *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_STX_MEM, retval);2035
2051 else2036 if (type == BPF_WRITE) {
2052 *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, retval);2037 int treg = BPF_REG_9;
2038
2039 if (si->src_reg == treg || si->dst_reg == treg)
2040 --treg;
2041 if (si->src_reg == treg || si->dst_reg == treg)
2042 --treg;
2043 *insn++ = BPF_STX_MEM(BPF_DW, si->dst_reg, treg,
2044 offsetof(struct bpf_sockopt_kern, tmp_reg));
2045 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sockopt_kern, current_task),
2046 treg, si->dst_reg,
2047 offsetof(struct bpf_sockopt_kern, current_task));
2048 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct task_struct, bpf_ctx),
2049 treg, treg,
2050 offsetof(struct task_struct, bpf_ctx));
2051 *insn++ = BPF_STX_MEM(BPF_FIELD_SIZEOF(struct bpf_cg_run_ctx, retval),
2052 treg, si->src_reg,
2053 offsetof(struct bpf_cg_run_ctx, retval));
2054 *insn++ = BPF_LDX_MEM(BPF_DW, treg, si->dst_reg,
2055 offsetof(struct bpf_sockopt_kern, tmp_reg));
2056 } else {
2057 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sockopt_kern, current_task),
2058 si->dst_reg, si->src_reg,
2059 offsetof(struct bpf_sockopt_kern, current_task));
2060 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct task_struct, bpf_ctx),
2061 si->dst_reg, si->dst_reg,
2062 offsetof(struct task_struct, bpf_ctx));
2063 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_cg_run_ctx, retval),
2064 si->dst_reg, si->dst_reg,
2065 offsetof(struct bpf_cg_run_ctx, retval));
2066 }
2053 break;2067 break;
2054 case offsetof(struct bpf_sockopt, optval):2068 case offsetof(struct bpf_sockopt, optval):
2055 *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, optval);2069 *insn++ = CG_SOCKOPT_ACCESS_FIELD(BPF_LDX_MEM, optval);