Only update psock->saved_* if psock->sk_proto has not been initialized yet. This allows us to get rid of tcp_bpf_reinit_sk_prot.
Unified · Split
| @@ -347,11 +347,23 @@ static inline void sk_psock_update_proto(struct sock *sk, · 346 unmodified lines | ||
| 347 | 347 | struct sk_psock *psock, |
| 348 | 348 | struct proto *ops) |
| 349 | 349 | { |
| 350 | − psock->saved_unhash = sk->sk_prot->unhash; | |
| 351 | − psock->saved_close = sk->sk_prot->close; | |
| 352 | − psock->saved_write_space = sk->sk_write_space; | |
| 350 | + /* Initialize saved callbacks and original proto only once, since this | |
| 351 | + * function may be called multiple times for a psock, e.g. when | |
| 352 | + * psock->progs.msg_parser is updated. | |
| 353 | + * | |
| 354 | + * Since we've not installed the new proto, psock is not yet in use and | |
| 355 | + * we can initialize it without synchronization. | |
| 356 | + */ | |
| 357 | + if (!psock->sk_proto) { | |
| 358 | + struct proto *orig = READ_ONCE(sk->sk_prot); | |
| 359 | + | |
| 360 | + psock->saved_unhash = orig->unhash; | |
| 361 | + psock->saved_close = orig->close; | |
| 362 | + psock->saved_write_space = sk->sk_write_space; | |
| 363 | + | |
| 364 | + psock->sk_proto = orig; | |
| 365 | + } | |
| 353 | 366 | |
| 354 | − psock->sk_proto = sk->sk_prot; | |
| 355 | 367 | /* Pairs with lockless read in sk_clone_lock() */ |
| 356 | 368 | WRITE_ONCE(sk->sk_prot, ops); |
| 357 | 369 | } |
| @@ -637,20 +637,6 @@ static void tcp_bpf_update_sk_prot(struct sock *sk, struct sk_psock *psock) · 636 unmodified lines | ||
| 637 | 637 | sk_psock_update_proto(sk, psock, &tcp_bpf_prots[family][config]); |
| 638 | 638 | } |
| 639 | 639 | |
| 640 | −static void tcp_bpf_reinit_sk_prot(struct sock *sk, struct sk_psock *psock) | |
| 641 | −{ | |
| 642 | − int family = sk->sk_family == AF_INET6 ? TCP_BPF_IPV6 : TCP_BPF_IPV4; | |
| 643 | − int config = psock->progs.msg_parser ? TCP_BPF_TX : TCP_BPF_BASE; | |
| 644 | − | |
| 645 | − /* Reinit occurs when program types change e.g. TCP_BPF_TX is removed | |
| 646 | − * or added requiring sk_prot hook updates. We keep original saved | |
| 647 | − * hooks in this case. | |
| 648 | − * | |
| 649 | − * Pairs with lockless read in sk_clone_lock(). | |
| 650 | − */ | |
| 651 | − WRITE_ONCE(sk->sk_prot, &tcp_bpf_prots[family][config]); | |
| 652 | −} | |
| 653 | − | |
| 654 | 640 | static int tcp_bpf_assert_proto_ops(struct proto *ops) |
| 655 | 641 | { |
| 656 | 642 | /* In order to avoid retpoline, we make assumptions when we call |
| @@ -670,7 +656,7 @@ void tcp_bpf_reinit(struct sock *sk) · 13 unmodified lines | ||
| 670 | 656 | |
| 671 | 657 | rcu_read_lock(); |
| 672 | 658 | psock = sk_psock(sk); |
| 673 | − tcp_bpf_reinit_sk_prot(sk, psock); | |
| 659 | + tcp_bpf_update_sk_prot(sk, psock); | |
| 674 | 660 | rcu_read_unlock(); |
| 675 | 661 | } |
| 676 | 662 | |