Move three global variables protected by bpf_verifier_lock into 'struct bpf_verifier_env' to allow parallel verification.
Unified · Split
| @@ -295,6 +295,11 @@ struct bpf_verifier_env { · 294 unmodified lines | |||
| 295 | const struct bpf_line_info *prev_linfo; | 295 | const struct bpf_line_info *prev_linfo; |
| 296 | struct bpf_verifier_log log; | 296 | struct bpf_verifier_log log; |
| 297 | struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 1]; | 297 | struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 1]; |
| 298 | struct { | ||
| 299 | int *insn_state; | ||
| 300 | int *insn_stack; | ||
| 301 | int cur_stack; | ||
| 302 | } cfg; | ||
| 298 | u32 subprog_cnt; | 303 | u32 subprog_cnt; |
| 299 | /* number of instructions analyzed by the verifier */ | 304 | /* number of instructions analyzed by the verifier */ |
| 300 | u32 insn_processed; | 305 | u32 insn_processed; |
| @@ -5369,10 +5369,6 @@ enum { · 5368 unmodified lines | |||
| 5369 | 5369 | ||
| 5370 | #define STATE_LIST_MARK ((struct bpf_verifier_state_list *) -1L) | 5370 | #define STATE_LIST_MARK ((struct bpf_verifier_state_list *) -1L) |
| 5371 | 5371 | ||
| 5372 | static int *insn_stack; /* stack of insns to process */ | ||
| 5373 | static int cur_stack; /* current stack index */ | ||
| 5374 | static int *insn_state; | ||
| 5375 | |||
| 5376 | /* t, w, e - match pseudo-code above: | 5372 | /* t, w, e - match pseudo-code above: |
| 5377 | * t - index of current instruction | 5373 | * t - index of current instruction |
| 5378 | * w - next instruction | 5374 | * w - next instruction |
| @@ -5380,6 +5376,9 @@ static int *insn_state; · 1 unmodified line | |||
| 5380 | */ | 5376 | */ |
| 5381 | static int push_insn(int t, int w, int e, struct bpf_verifier_env *env) | 5377 | static int push_insn(int t, int w, int e, struct bpf_verifier_env *env) |
| 5382 | { | 5378 | { |
| 5379 | int *insn_stack = env->cfg.insn_stack; | ||
| 5380 | int *insn_state = env->cfg.insn_state; | ||
| 5381 | |||
| 5383 | if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH)) | 5382 | if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH)) |
| 5384 | return 0; | 5383 | return 0; |
| 5385 | 5384 | ||
| @@ -5400,9 +5399,9 @@ static int push_insn(int t, int w, int e, struct bpf_verifier_env *env) · 14 unmodified lines | |||
| 5400 | /* tree-edge */ | 5399 | /* tree-edge */ |
| 5401 | insn_state[t] = DISCOVERED | e; | 5400 | insn_state[t] = DISCOVERED | e; |
| 5402 | insn_state[w] = DISCOVERED; | 5401 | insn_state[w] = DISCOVERED; |
| 5403 | if (cur_stack >= env->prog->len) | 5402 | if (env->cfg.cur_stack >= env->prog->len) |
| 5404 | return -E2BIG; | 5403 | return -E2BIG; |
| 5405 | insn_stack[cur_stack++] = w; | 5404 | insn_stack[env->cfg.cur_stack++] = w; |
| 5406 | return 1; | 5405 | return 1; |
| 5407 | } else if ((insn_state[w] & 0xF0) == DISCOVERED) { | 5406 | } else if ((insn_state[w] & 0xF0) == DISCOVERED) { |
| 5408 | verbose_linfo(env, t, "%d: ", t); | 5407 | verbose_linfo(env, t, "%d: ", t); |
| @@ -5426,14 +5425,15 @@ static int check_cfg(struct bpf_verifier_env *env) · 17 unmodified lines | |||
| 5426 | { | 5425 | { |
| 5427 | struct bpf_insn *insns = env->prog->insnsi; | 5426 | struct bpf_insn *insns = env->prog->insnsi; |
| 5428 | int insn_cnt = env->prog->len; | 5427 | int insn_cnt = env->prog->len; |
| 5428 | int *insn_stack, *insn_state; | ||
| 5429 | int ret = 0; | 5429 | int ret = 0; |
| 5430 | int i, t; | 5430 | int i, t; |
| 5431 | 5431 | ||
| 5432 | insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); | 5432 | insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); |
| 5433 | if (!insn_state) | 5433 | if (!insn_state) |
| 5434 | return -ENOMEM; | 5434 | return -ENOMEM; |
| 5435 | 5435 | ||
| 5436 | insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); | 5436 | insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); |
| 5437 | if (!insn_stack) { | 5437 | if (!insn_stack) { |
| 5438 | kvfree(insn_state); | 5438 | kvfree(insn_state); |
| 5439 | return -ENOMEM; | 5439 | return -ENOMEM; |
| @@ -5441,12 +5441,12 @@ static int check_cfg(struct bpf_verifier_env *env) · 1 unmodified line | |||
| 5441 | 5441 | ||
| 5442 | insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */ | 5442 | insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */ |
| 5443 | insn_stack[0] = 0; /* 0 is the first instruction */ | 5443 | insn_stack[0] = 0; /* 0 is the first instruction */ |
| 5444 | cur_stack = 1; | 5444 | env->cfg.cur_stack = 1; |
| 5445 | 5445 | ||
| 5446 | peek_stack: | 5446 | peek_stack: |
| 5447 | if (cur_stack == 0) | 5447 | if (env->cfg.cur_stack == 0) |
| 5448 | goto check_state; | 5448 | goto check_state; |
| 5449 | t = insn_stack[cur_stack - 1]; | 5449 | t = insn_stack[env->cfg.cur_stack - 1]; |
| 5450 | 5450 | ||
| 5451 | if (BPF_CLASS(insns[t].code) == BPF_JMP || | 5451 | if (BPF_CLASS(insns[t].code) == BPF_JMP || |
| 5452 | BPF_CLASS(insns[t].code) == BPF_JMP32) { | 5452 | BPF_CLASS(insns[t].code) == BPF_JMP32) { |
| @@ -5515,7 +5515,7 @@ peek_stack: · 62 unmodified lines | |||
| 5515 | 5515 | ||
| 5516 | mark_explored: | 5516 | mark_explored: |
| 5517 | insn_state[t] = EXPLORED; | 5517 | insn_state[t] = EXPLORED; |
| 5518 | if (cur_stack-- <= 0) { | 5518 | if (env->cfg.cur_stack-- <= 0) { |
| 5519 | verbose(env, "pop stack internal bug\n"); | 5519 | verbose(env, "pop stack internal bug\n"); |
| 5520 | ret = -EFAULT; | 5520 | ret = -EFAULT; |
| 5521 | goto err_free; | 5521 | goto err_free; |
| @@ -5535,6 +5535,7 @@ check_state: · 13 unmodified lines | |||
| 5535 | err_free: | 5535 | err_free: |
| 5536 | kvfree(insn_state); | 5536 | kvfree(insn_state); |
| 5537 | kvfree(insn_stack); | 5537 | kvfree(insn_stack); |
| 5538 | env->cfg.insn_state = env->cfg.insn_stack = NULL; | ||
| 5538 | return ret; | 5539 | return ret; |
| 5539 | } | 5540 | } |
| 5540 | 5541 | ||