Only update psock->saved_* if psock->sk_proto has not been initialized yet. This allows us to get rid of tcp_bpf_reinit_sk_prot.
Unified · Split
| @@ -347,11 +347,23 @@ static inline void sk_psock_update_proto(struct sock *sk, · 346 unmodified lines | |||
| 347 | struct sk_psock *psock, | 347 | struct sk_psock *psock, |
| 348 | struct proto *ops) | 348 | struct proto *ops) |
| 349 | { | 349 | { |
| 350 | psock->saved_unhash = sk->sk_prot->unhash; | 350 | /* Initialize saved callbacks and original proto only once, since this |
| 351 | psock->saved_close = sk->sk_prot->close; | 351 | * function may be called multiple times for a psock, e.g. when |
| 352 | psock->saved_write_space = sk->sk_write_space; | 352 | * psock->progs.msg_parser is updated. |
| 353 | * | ||
| 354 | * Since we've not installed the new proto, psock is not yet in use and | ||
| 355 | * we can initialize it without synchronization. | ||
| 356 | */ | ||
| 357 | if (!psock->sk_proto) { | ||
| 358 | struct proto *orig = READ_ONCE(sk->sk_prot); | ||
| 359 | |||
| 360 | psock->saved_unhash = orig->unhash; | ||
| 361 | psock->saved_close = orig->close; | ||
| 362 | psock->saved_write_space = sk->sk_write_space; | ||
| 363 | |||
| 364 | psock->sk_proto = orig; | ||
| 365 | } | ||
| 353 | 366 | ||
| 354 | psock->sk_proto = sk->sk_prot; | ||
| 355 | /* Pairs with lockless read in sk_clone_lock() */ | 367 | /* Pairs with lockless read in sk_clone_lock() */ |
| 356 | WRITE_ONCE(sk->sk_prot, ops); | 368 | WRITE_ONCE(sk->sk_prot, ops); |
| 357 | } | 369 | } |
| @@ -637,20 +637,6 @@ static void tcp_bpf_update_sk_prot(struct sock *sk, struct sk_psock *psock) · 636 unmodified lines | |||
| 637 | sk_psock_update_proto(sk, psock, &tcp_bpf_prots[family][config]); | 637 | sk_psock_update_proto(sk, psock, &tcp_bpf_prots[family][config]); |
| 638 | } | 638 | } |
| 639 | 639 | ||
| 640 | static void tcp_bpf_reinit_sk_prot(struct sock *sk, struct sk_psock *psock) | ||
| 641 | { | ||
| 642 | int family = sk->sk_family == AF_INET6 ? TCP_BPF_IPV6 : TCP_BPF_IPV4; | ||
| 643 | int config = psock->progs.msg_parser ? TCP_BPF_TX : TCP_BPF_BASE; | ||
| 644 | |||
| 645 | /* Reinit occurs when program types change e.g. TCP_BPF_TX is removed | ||
| 646 | * or added requiring sk_prot hook updates. We keep original saved | ||
| 647 | * hooks in this case. | ||
| 648 | * | ||
| 649 | * Pairs with lockless read in sk_clone_lock(). | ||
| 650 | */ | ||
| 651 | WRITE_ONCE(sk->sk_prot, &tcp_bpf_prots[family][config]); | ||
| 652 | } | ||
| 653 | |||
| 654 | static int tcp_bpf_assert_proto_ops(struct proto *ops) | 640 | static int tcp_bpf_assert_proto_ops(struct proto *ops) |
| 655 | { | 641 | { |
| 656 | /* In order to avoid retpoline, we make assumptions when we call | 642 | /* In order to avoid retpoline, we make assumptions when we call |
| @@ -670,7 +656,7 @@ void tcp_bpf_reinit(struct sock *sk) · 13 unmodified lines | |||
| 670 | 656 | ||
| 671 | rcu_read_lock(); | 657 | rcu_read_lock(); |
| 672 | psock = sk_psock(sk); | 658 | psock = sk_psock(sk); |
| 673 | tcp_bpf_reinit_sk_prot(sk, psock); | 659 | tcp_bpf_update_sk_prot(sk, psock); |
| 674 | rcu_read_unlock(); | 660 | rcu_read_unlock(); |
| 675 | } | 661 | } |
| 676 | 662 | ||