Since both sleepable and non-sleepable programs execute under migrate_disable add recursion prevention mechanism to both types of programs when they're executed via bpf trampoline.
Showing the changes to kernel (2 of 7 files). Show all
Unified · Split
| @@ -91,6 +91,12 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag · 90 unmodified lines | ||
| 91 | 91 | vfree(fp); |
| 92 | 92 | return NULL; |
| 93 | 93 | } |
| 94 | + fp->active = alloc_percpu_gfp(int, GFP_KERNEL_ACCOUNT | gfp_extra_flags); | |
| 95 | + if (!fp->active) { | |
| 96 | + vfree(fp); | |
| 97 | + kfree(aux); | |
| 98 | + return NULL; | |
| 99 | + } | |
| 94 | 100 | |
| 95 | 101 | fp->pages = size / PAGE_SIZE; |
| 96 | 102 | fp->aux = aux; |
| @@ -116,6 +122,7 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) · 19 unmodified lines | ||
| 116 | 122 | |
| 117 | 123 | prog->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags); |
| 118 | 124 | if (!prog->stats) { |
| 125 | + free_percpu(prog->active); | |
| 119 | 126 | kfree(prog->aux); |
| 120 | 127 | vfree(prog); |
| 121 | 128 | return NULL; |
| @@ -253,6 +260,7 @@ void __bpf_prog_free(struct bpf_prog *fp) · 131 unmodified lines | ||
| 253 | 260 | kfree(fp->aux); |
| 254 | 261 | } |
| 255 | 262 | free_percpu(fp->stats); |
| 263 | + free_percpu(fp->active); | |
| 256 | 264 | vfree(fp); |
| 257 | 265 | } |
| 258 | 266 | |
| @@ -381,13 +381,16 @@ out: · 380 unmodified lines | ||
| 381 | 381 | mutex_unlock(&trampoline_mutex); |
| 382 | 382 | } |
| 383 | 383 | |
| 384 | −#define NO_START_TIME 0 | |
| 384 | +#define NO_START_TIME 1 | |
| 385 | 385 | static u64 notrace bpf_prog_start_time(void) |
| 386 | 386 | { |
| 387 | 387 | u64 start = NO_START_TIME; |
| 388 | 388 | |
| 389 | − if (static_branch_unlikely(&bpf_stats_enabled_key)) | |
| 389 | + if (static_branch_unlikely(&bpf_stats_enabled_key)) { | |
| 390 | 390 | start = sched_clock(); |
| 391 | + if (unlikely(!start)) | |
| 392 | + start = NO_START_TIME; | |
| 393 | + } | |
| 391 | 394 | return start; |
| 392 | 395 | } |
| 393 | 396 | |
| @@ -397,12 +400,20 @@ static u64 notrace bpf_prog_start_time(void) · 3 unmodified lines | ||
| 397 | 400 | * call __bpf_prog_enter |
| 398 | 401 | * call prog->bpf_func |
| 399 | 402 | * call __bpf_prog_exit |
| 403 | + * | |
| 404 | + * __bpf_prog_enter returns: | |
| 405 | + * 0 - skip execution of the bpf prog | |
| 406 | + * 1 - execute bpf prog | |
| 407 | + * [2..MAX_U64] - excute bpf prog and record execution time. | |
| 408 | + * This is start time. | |
| 400 | 409 | */ |
| 401 | −u64 notrace __bpf_prog_enter(void) | |
| 410 | +u64 notrace __bpf_prog_enter(struct bpf_prog *prog) | |
| 402 | 411 | __acquires(RCU) |
| 403 | 412 | { |
| 404 | 413 | rcu_read_lock(); |
| 405 | 414 | migrate_disable(); |
| 415 | + if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) | |
| 416 | + return 0; | |
| 406 | 417 | return bpf_prog_start_time(); |
| 407 | 418 | } |
| 408 | 419 | |
| @@ -430,21 +441,25 @@ void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start) · 21 unmodified lines | ||
| 430 | 441 | __releases(RCU) |
| 431 | 442 | { |
| 432 | 443 | update_prog_stats(prog, start); |
| 444 | + __this_cpu_dec(*(prog->active)); | |
| 433 | 445 | migrate_enable(); |
| 434 | 446 | rcu_read_unlock(); |
| 435 | 447 | } |
| 436 | 448 | |
| 437 | −u64 notrace __bpf_prog_enter_sleepable(void) | |
| 449 | +u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog) | |
| 438 | 450 | { |
| 439 | 451 | rcu_read_lock_trace(); |
| 440 | 452 | migrate_disable(); |
| 441 | 453 | might_fault(); |
| 454 | + if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) | |
| 455 | + return 0; | |
| 442 | 456 | return bpf_prog_start_time(); |
| 443 | 457 | } |
| 444 | 458 | |
| 445 | 459 | void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start) |
| 446 | 460 | { |
| 447 | 461 | update_prog_stats(prog, start); |
| 462 | + __this_cpu_dec(*(prog->active)); | |
| 448 | 463 | migrate_enable(); |
| 449 | 464 | rcu_read_unlock_trace(); |
| 450 | 465 | } |