Move three global variables protected by bpf_verifier_lock into 'struct bpf_verifier_env' to allow parallel verification.
Unified · Split
| @@ -295,6 +295,11 @@ struct bpf_verifier_env { · 294 unmodified lines | ||
| 295 | 295 | const struct bpf_line_info *prev_linfo; |
| 296 | 296 | struct bpf_verifier_log log; |
| 297 | 297 | struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 1]; |
| 298 | + struct { | |
| 299 | + int *insn_state; | |
| 300 | + int *insn_stack; | |
| 301 | + int cur_stack; | |
| 302 | + } cfg; | |
| 298 | 303 | u32 subprog_cnt; |
| 299 | 304 | /* number of instructions analyzed by the verifier */ |
| 300 | 305 | u32 insn_processed; |
| @@ -5369,10 +5369,6 @@ enum { · 5368 unmodified lines | ||
| 5369 | 5369 | |
| 5370 | 5370 | #define STATE_LIST_MARK ((struct bpf_verifier_state_list *) -1L) |
| 5371 | 5371 | |
| 5372 | −static int *insn_stack; /* stack of insns to process */ | |
| 5373 | −static int cur_stack; /* current stack index */ | |
| 5374 | −static int *insn_state; | |
| 5375 | − | |
| 5376 | 5372 | /* t, w, e - match pseudo-code above: |
| 5377 | 5373 | * t - index of current instruction |
| 5378 | 5374 | * w - next instruction |
| @@ -5380,6 +5376,9 @@ static int *insn_state; · 1 unmodified line | ||
| 5380 | 5376 | */ |
| 5381 | 5377 | static int push_insn(int t, int w, int e, struct bpf_verifier_env *env) |
| 5382 | 5378 | { |
| 5379 | + int *insn_stack = env->cfg.insn_stack; | |
| 5380 | + int *insn_state = env->cfg.insn_state; | |
| 5381 | + | |
| 5383 | 5382 | if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH)) |
| 5384 | 5383 | return 0; |
| 5385 | 5384 | |
| @@ -5400,9 +5399,9 @@ static int push_insn(int t, int w, int e, struct bpf_verifier_env *env) · 14 unmodified lines | ||
| 5400 | 5399 | /* tree-edge */ |
| 5401 | 5400 | insn_state[t] = DISCOVERED | e; |
| 5402 | 5401 | insn_state[w] = DISCOVERED; |
| 5403 | − if (cur_stack >= env->prog->len) | |
| 5402 | + if (env->cfg.cur_stack >= env->prog->len) | |
| 5404 | 5403 | return -E2BIG; |
| 5405 | − insn_stack[cur_stack++] = w; | |
| 5404 | + insn_stack[env->cfg.cur_stack++] = w; | |
| 5406 | 5405 | return 1; |
| 5407 | 5406 | } else if ((insn_state[w] & 0xF0) == DISCOVERED) { |
| 5408 | 5407 | verbose_linfo(env, t, "%d: ", t); |
| @@ -5426,14 +5425,15 @@ static int check_cfg(struct bpf_verifier_env *env) · 17 unmodified lines | ||
| 5426 | 5425 | { |
| 5427 | 5426 | struct bpf_insn *insns = env->prog->insnsi; |
| 5428 | 5427 | int insn_cnt = env->prog->len; |
| 5428 | + int *insn_stack, *insn_state; | |
| 5429 | 5429 | int ret = 0; |
| 5430 | 5430 | int i, t; |
| 5431 | 5431 | |
| 5432 | − insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); | |
| 5432 | + insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); | |
| 5433 | 5433 | if (!insn_state) |
| 5434 | 5434 | return -ENOMEM; |
| 5435 | 5435 | |
| 5436 | − insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); | |
| 5436 | + insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); | |
| 5437 | 5437 | if (!insn_stack) { |
| 5438 | 5438 | kvfree(insn_state); |
| 5439 | 5439 | return -ENOMEM; |
| @@ -5441,12 +5441,12 @@ static int check_cfg(struct bpf_verifier_env *env) · 1 unmodified line | ||
| 5441 | 5441 | |
| 5442 | 5442 | insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */ |
| 5443 | 5443 | insn_stack[0] = 0; /* 0 is the first instruction */ |
| 5444 | − cur_stack = 1; | |
| 5444 | + env->cfg.cur_stack = 1; | |
| 5445 | 5445 | |
| 5446 | 5446 | peek_stack: |
| 5447 | − if (cur_stack == 0) | |
| 5447 | + if (env->cfg.cur_stack == 0) | |
| 5448 | 5448 | goto check_state; |
| 5449 | − t = insn_stack[cur_stack - 1]; | |
| 5449 | + t = insn_stack[env->cfg.cur_stack - 1]; | |
| 5450 | 5450 | |
| 5451 | 5451 | if (BPF_CLASS(insns[t].code) == BPF_JMP || |
| 5452 | 5452 | BPF_CLASS(insns[t].code) == BPF_JMP32) { |
| @@ -5515,7 +5515,7 @@ peek_stack: · 62 unmodified lines | ||
| 5515 | 5515 | |
| 5516 | 5516 | mark_explored: |
| 5517 | 5517 | insn_state[t] = EXPLORED; |
| 5518 | − if (cur_stack-- <= 0) { | |
| 5518 | + if (env->cfg.cur_stack-- <= 0) { | |
| 5519 | 5519 | verbose(env, "pop stack internal bug\n"); |
| 5520 | 5520 | ret = -EFAULT; |
| 5521 | 5521 | goto err_free; |
| @@ -5535,6 +5535,7 @@ check_state: · 13 unmodified lines | ||
| 5535 | 5535 | err_free: |
| 5536 | 5536 | kvfree(insn_state); |
| 5537 | 5537 | kvfree(insn_stack); |
| 5538 | + env->cfg.insn_state = env->cfg.insn_stack = NULL; | |
| 5538 | 5539 | return ret; |
| 5539 | 5540 | } |
| 5540 | 5541 | |