Since both sleepable and non-sleepable programs execute under migrate_disable add recursion prevention mechanism to both types of programs when they're executed via bpf trampoline.
Showing the changes to kernel (2 of 7 files). Show all
Unified · Split
| @@ -91,6 +91,12 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag · 90 unmodified lines | |||
| 91 | vfree(fp); | 91 | vfree(fp); |
| 92 | return NULL; | 92 | return NULL; |
| 93 | } | 93 | } |
| 94 | fp->active = alloc_percpu_gfp(int, GFP_KERNEL_ACCOUNT | gfp_extra_flags); | ||
| 95 | if (!fp->active) { | ||
| 96 | vfree(fp); | ||
| 97 | kfree(aux); | ||
| 98 | return NULL; | ||
| 99 | } | ||
| 94 | 100 | ||
| 95 | fp->pages = size / PAGE_SIZE; | 101 | fp->pages = size / PAGE_SIZE; |
| 96 | fp->aux = aux; | 102 | fp->aux = aux; |
| @@ -116,6 +122,7 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) · 19 unmodified lines | |||
| 116 | 122 | ||
| 117 | prog->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags); | 123 | prog->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags); |
| 118 | if (!prog->stats) { | 124 | if (!prog->stats) { |
| 125 | free_percpu(prog->active); | ||
| 119 | kfree(prog->aux); | 126 | kfree(prog->aux); |
| 120 | vfree(prog); | 127 | vfree(prog); |
| 121 | return NULL; | 128 | return NULL; |
| @@ -253,6 +260,7 @@ void __bpf_prog_free(struct bpf_prog *fp) · 131 unmodified lines | |||
| 253 | kfree(fp->aux); | 260 | kfree(fp->aux); |
| 254 | } | 261 | } |
| 255 | free_percpu(fp->stats); | 262 | free_percpu(fp->stats); |
| 263 | free_percpu(fp->active); | ||
| 256 | vfree(fp); | 264 | vfree(fp); |
| 257 | } | 265 | } |
| 258 | 266 | ||
| @@ -381,13 +381,16 @@ out: · 380 unmodified lines | |||
| 381 | mutex_unlock(&trampoline_mutex); | 381 | mutex_unlock(&trampoline_mutex); |
| 382 | } | 382 | } |
| 383 | 383 | ||
| 384 | #define NO_START_TIME 0 | 384 | #define NO_START_TIME 1 |
| 385 | static u64 notrace bpf_prog_start_time(void) | 385 | static u64 notrace bpf_prog_start_time(void) |
| 386 | { | 386 | { |
| 387 | u64 start = NO_START_TIME; | 387 | u64 start = NO_START_TIME; |
| 388 | 388 | ||
| 389 | if (static_branch_unlikely(&bpf_stats_enabled_key)) | 389 | if (static_branch_unlikely(&bpf_stats_enabled_key)) { |
| 390 | start = sched_clock(); | 390 | start = sched_clock(); |
| 391 | if (unlikely(!start)) | ||
| 392 | start = NO_START_TIME; | ||
| 393 | } | ||
| 391 | return start; | 394 | return start; |
| 392 | } | 395 | } |
| 393 | 396 | ||
| @@ -397,12 +400,20 @@ static u64 notrace bpf_prog_start_time(void) · 3 unmodified lines | |||
| 397 | * call __bpf_prog_enter | 400 | * call __bpf_prog_enter |
| 398 | * call prog->bpf_func | 401 | * call prog->bpf_func |
| 399 | * call __bpf_prog_exit | 402 | * call __bpf_prog_exit |
| 403 | * | ||
| 404 | * __bpf_prog_enter returns: | ||
| 405 | * 0 - skip execution of the bpf prog | ||
| 406 | * 1 - execute bpf prog | ||
| 407 | * [2..MAX_U64] - excute bpf prog and record execution time. | ||
| 408 | * This is start time. | ||
| 400 | */ | 409 | */ |
| 401 | u64 notrace __bpf_prog_enter(void) | 410 | u64 notrace __bpf_prog_enter(struct bpf_prog *prog) |
| 402 | __acquires(RCU) | 411 | __acquires(RCU) |
| 403 | { | 412 | { |
| 404 | rcu_read_lock(); | 413 | rcu_read_lock(); |
| 405 | migrate_disable(); | 414 | migrate_disable(); |
| 415 | if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) | ||
| 416 | return 0; | ||
| 406 | return bpf_prog_start_time(); | 417 | return bpf_prog_start_time(); |
| 407 | } | 418 | } |
| 408 | 419 | ||
| @@ -430,21 +441,25 @@ void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start) · 21 unmodified lines | |||
| 430 | __releases(RCU) | 441 | __releases(RCU) |
| 431 | { | 442 | { |
| 432 | update_prog_stats(prog, start); | 443 | update_prog_stats(prog, start); |
| 444 | __this_cpu_dec(*(prog->active)); | ||
| 433 | migrate_enable(); | 445 | migrate_enable(); |
| 434 | rcu_read_unlock(); | 446 | rcu_read_unlock(); |
| 435 | } | 447 | } |
| 436 | 448 | ||
| 437 | u64 notrace __bpf_prog_enter_sleepable(void) | 449 | u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog) |
| 438 | { | 450 | { |
| 439 | rcu_read_lock_trace(); | 451 | rcu_read_lock_trace(); |
| 440 | migrate_disable(); | 452 | migrate_disable(); |
| 441 | might_fault(); | 453 | might_fault(); |
| 454 | if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) | ||
| 455 | return 0; | ||
| 442 | return bpf_prog_start_time(); | 456 | return bpf_prog_start_time(); |
| 443 | } | 457 | } |
| 444 | 458 | ||
| 445 | void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start) | 459 | void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start) |
| 446 | { | 460 | { |
| 447 | update_prog_stats(prog, start); | 461 | update_prog_stats(prog, start); |
| 462 | __this_cpu_dec(*(prog->active)); | ||
| 448 | migrate_enable(); | 463 | migrate_enable(); |
| 449 | rcu_read_unlock_trace(); | 464 | rcu_read_unlock_trace(); |
| 450 | } | 465 | } |