The retval value is moved to struct bpf_cg_run_ctx for ease of access in different prog types with different context structs layouts. The helper implementation (to be added in a later patch in the series) can simply perform a container_of from current->bpf_ctx to retrieve bpf_cg_run_ctx. Unfortunately, there is no easy way to access the current task_struct via the verifier BPF bytecode rewrite, aside from possibly calling a helper, so a pointer to current task is added to struct bpf_sockopt_kern so that the rewritten BPF bytecode can access struct bpf_cg_run_ctx with an indirection. For backward compatibility, if a getsockopt program rejects a syscall by returning 0, an -EPERM will be generated, by having the BPF_PROG_RUN_ARRAY_CG family macros automatically set the retval to -EPERM. Unlike prior to this patch, this -EPERM will be visible to ctx->retval for any other hooks down the line in the prog array. Additionally, the restriction that getsockopt filters can only set the retval to 0 is removed, considering that certain getsockopt implementations may return optlen. Filters are now able to set the value arbitrarily.
Showing the changes to include (2 of 3 files). Show all
Unified · Split
| @@ -1245,6 +1245,7 @@ struct bpf_run_ctx {}; · 1244 unmodified lines | ||
| 1245 | 1245 | struct bpf_cg_run_ctx { |
| 1246 | 1246 | struct bpf_run_ctx run_ctx; |
| 1247 | 1247 | const struct bpf_prog_array_item *prog_item; |
| 1248 | + int retval; | |
| 1248 | 1249 | }; |
| 1249 | 1250 | |
| 1250 | 1251 | struct bpf_trace_run_ctx { |
| @@ -1280,16 +1281,16 @@ typedef u32 (*bpf_prog_run_fn)(const struct bpf_prog *prog, const void *ctx); · 29 unmodified lines | ||
| 1280 | 1281 | static __always_inline int |
| 1281 | 1282 | BPF_PROG_RUN_ARRAY_CG_FLAGS(const struct bpf_prog_array __rcu *array_rcu, |
| 1282 | 1283 | const void *ctx, bpf_prog_run_fn run_prog, |
| 1283 | − u32 *ret_flags) | |
| 1284 | + int retval, u32 *ret_flags) | |
| 1284 | 1285 | { |
| 1285 | 1286 | const struct bpf_prog_array_item *item; |
| 1286 | 1287 | const struct bpf_prog *prog; |
| 1287 | 1288 | const struct bpf_prog_array *array; |
| 1288 | 1289 | struct bpf_run_ctx *old_run_ctx; |
| 1289 | 1290 | struct bpf_cg_run_ctx run_ctx; |
| 1290 | − int ret = 0; | |
| 1291 | 1291 | u32 func_ret; |
| 1292 | 1292 | |
| 1293 | + run_ctx.retval = retval; | |
| 1293 | 1294 | migrate_disable(); |
| 1294 | 1295 | rcu_read_lock(); |
| 1295 | 1296 | array = rcu_dereference(array_rcu); |
| @@ -1299,27 +1300,28 @@ BPF_PROG_RUN_ARRAY_CG_FLAGS(const struct bpf_prog_array __rcu *array_rcu, · 3 unmodified lines | ||
| 1299 | 1300 | run_ctx.prog_item = item; |
| 1300 | 1301 | func_ret = run_prog(prog, ctx); |
| 1301 | 1302 | if (!(func_ret & 1)) |
| 1302 | − ret = -EPERM; | |
| 1303 | + run_ctx.retval = -EPERM; | |
| 1303 | 1304 | *(ret_flags) |= (func_ret >> 1); |
| 1304 | 1305 | item++; |
| 1305 | 1306 | } |
| 1306 | 1307 | bpf_reset_run_ctx(old_run_ctx); |
| 1307 | 1308 | rcu_read_unlock(); |
| 1308 | 1309 | migrate_enable(); |
| 1309 | − return ret; | |
| 1310 | + return run_ctx.retval; | |
| 1310 | 1311 | } |
| 1311 | 1312 | |
| 1312 | 1313 | static __always_inline int |
| 1313 | 1314 | BPF_PROG_RUN_ARRAY_CG(const struct bpf_prog_array __rcu *array_rcu, |
| 1314 | − const void *ctx, bpf_prog_run_fn run_prog) | |
| 1315 | + const void *ctx, bpf_prog_run_fn run_prog, | |
| 1316 | + int retval) | |
| 1315 | 1317 | { |
| 1316 | 1318 | const struct bpf_prog_array_item *item; |
| 1317 | 1319 | const struct bpf_prog *prog; |
| 1318 | 1320 | const struct bpf_prog_array *array; |
| 1319 | 1321 | struct bpf_run_ctx *old_run_ctx; |
| 1320 | 1322 | struct bpf_cg_run_ctx run_ctx; |
| 1321 | − int ret = 0; | |
| 1322 | 1323 | |
| 1324 | + run_ctx.retval = retval; | |
| 1323 | 1325 | migrate_disable(); |
| 1324 | 1326 | rcu_read_lock(); |
| 1325 | 1327 | array = rcu_dereference(array_rcu); |
| @@ -1328,13 +1330,13 @@ BPF_PROG_RUN_ARRAY_CG(const struct bpf_prog_array __rcu *array_rcu, · 2 unmodified lines | ||
| 1328 | 1330 | while ((prog = READ_ONCE(item->prog))) { |
| 1329 | 1331 | run_ctx.prog_item = item; |
| 1330 | 1332 | if (!run_prog(prog, ctx)) |
| 1331 | − ret = -EPERM; | |
| 1333 | + run_ctx.retval = -EPERM; | |
| 1332 | 1334 | item++; |
| 1333 | 1335 | } |
| 1334 | 1336 | bpf_reset_run_ctx(old_run_ctx); |
| 1335 | 1337 | rcu_read_unlock(); |
| 1336 | 1338 | migrate_enable(); |
| 1337 | − return ret; | |
| 1339 | + return run_ctx.retval; | |
| 1338 | 1340 | } |
| 1339 | 1341 | |
| 1340 | 1342 | static __always_inline u32 |
| @@ -1394,7 +1396,7 @@ out: · 53 unmodified lines | ||
| 1394 | 1396 | u32 _flags = 0; \ |
| 1395 | 1397 | bool _cn; \ |
| 1396 | 1398 | u32 _ret; \ |
| 1397 | − _ret = BPF_PROG_RUN_ARRAY_CG_FLAGS(array, ctx, func, &_flags); \ | |
| 1399 | + _ret = BPF_PROG_RUN_ARRAY_CG_FLAGS(array, ctx, func, 0, &_flags); \ | |
| 1398 | 1400 | _cn = _flags & BPF_RET_SET_CN; \ |
| 1399 | 1401 | if (!_ret) \ |
| 1400 | 1402 | _ret = (_cn ? NET_XMIT_CN : NET_XMIT_SUCCESS); \ |
| @@ -1356,7 +1356,10 @@ struct bpf_sockopt_kern { · 1355 unmodified lines | ||
| 1356 | 1356 | s32 level; |
| 1357 | 1357 | s32 optname; |
| 1358 | 1358 | s32 optlen; |
| 1359 | − s32 retval; | |
| 1359 | + /* for retval in struct bpf_cg_run_ctx */ | |
| 1360 | + struct task_struct *current_task; | |
| 1361 | + /* Temporary "register" for indirect stores to ppos. */ | |
| 1362 | + u64 tmp_reg; | |
| 1360 | 1363 | }; |
| 1361 | 1364 | |
| 1362 | 1365 | int copy_bpf_fprog_from_user(struct sock_fprog *dst, sockptr_t src, int len); |