There is a need for a polling interface for SRCU grace periods. This polling needs to initiate an SRCU grace period without having to queue (and manage) a callback. This commit therefore splits the Tree SRCU __call_srcu() function into callback-initialization and queuing/start-grace-period portions, with the latter in a new function named srcu_gp_start_if_needed(). This function may be passed a NULL callback pointer, in which case it will refrain from queuing anything. Why have the new function mess with queuing? Locking considerations, of course!
Showing the changes to kernel (1 of 1 file). Show all
Unified · Split
| @@ -807,6 +807,42 @@ static void srcu_leak_callback(struct rcu_head *rhp) · 806 unmodified lines | |||
| 807 | { | 807 | { |
| 808 | } | 808 | } |
| 809 | 809 | ||
| 810 | /* | ||
| 811 | * Start an SRCU grace period, and also queue the callback if non-NULL. | ||
| 812 | */ | ||
| 813 | static void srcu_gp_start_if_needed(struct srcu_struct *ssp, struct rcu_head *rhp, bool do_norm) | ||
| 814 | { | ||
| 815 | unsigned long flags; | ||
| 816 | int idx; | ||
| 817 | bool needexp = false; | ||
| 818 | bool needgp = false; | ||
| 819 | unsigned long s; | ||
| 820 | struct srcu_data *sdp; | ||
| 821 | |||
| 822 | idx = srcu_read_lock(ssp); | ||
| 823 | sdp = raw_cpu_ptr(ssp->sda); | ||
| 824 | spin_lock_irqsave_rcu_node(sdp, flags); | ||
| 825 | rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp); | ||
| 826 | rcu_segcblist_advance(&sdp->srcu_cblist, | ||
| 827 | rcu_seq_current(&ssp->srcu_gp_seq)); | ||
| 828 | s = rcu_seq_snap(&ssp->srcu_gp_seq); | ||
| 829 | (void)rcu_segcblist_accelerate(&sdp->srcu_cblist, s); | ||
| 830 | if (ULONG_CMP_LT(sdp->srcu_gp_seq_needed, s)) { | ||
| 831 | sdp->srcu_gp_seq_needed = s; | ||
| 832 | needgp = true; | ||
| 833 | } | ||
| 834 | if (!do_norm && ULONG_CMP_LT(sdp->srcu_gp_seq_needed_exp, s)) { | ||
| 835 | sdp->srcu_gp_seq_needed_exp = s; | ||
| 836 | needexp = true; | ||
| 837 | } | ||
| 838 | spin_unlock_irqrestore_rcu_node(sdp, flags); | ||
| 839 | if (needgp) | ||
| 840 | srcu_funnel_gp_start(ssp, sdp, s, do_norm); | ||
| 841 | else if (needexp) | ||
| 842 | srcu_funnel_exp_start(ssp, sdp->mynode, s); | ||
| 843 | srcu_read_unlock(ssp, idx); | ||
| 844 | } | ||
| 845 | |||
| 810 | /* | 846 | /* |
| 811 | * Enqueue an SRCU callback on the srcu_data structure associated with | 847 | * Enqueue an SRCU callback on the srcu_data structure associated with |
| 812 | * the current CPU and the specified srcu_struct structure, initiating | 848 | * the current CPU and the specified srcu_struct structure, initiating |
| @@ -838,13 +874,6 @@ static void srcu_leak_callback(struct rcu_head *rhp) · 25 unmodified lines | |||
| 838 | static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, | 874 | static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, |
| 839 | rcu_callback_t func, bool do_norm) | 875 | rcu_callback_t func, bool do_norm) |
| 840 | { | 876 | { |
| 841 | unsigned long flags; | ||
| 842 | int idx; | ||
| 843 | bool needexp = false; | ||
| 844 | bool needgp = false; | ||
| 845 | unsigned long s; | ||
| 846 | struct srcu_data *sdp; | ||
| 847 | |||
| 848 | check_init_srcu_struct(ssp); | 877 | check_init_srcu_struct(ssp); |
| 849 | if (debug_rcu_head_queue(rhp)) { | 878 | if (debug_rcu_head_queue(rhp)) { |
| 850 | /* Probable double call_srcu(), so leak the callback. */ | 879 | /* Probable double call_srcu(), so leak the callback. */ |
| @@ -853,28 +882,7 @@ static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, · 2 unmodified lines | |||
| 853 | return; | 882 | return; |
| 854 | } | 883 | } |
| 855 | rhp->func = func; | 884 | rhp->func = func; |
| 856 | idx = srcu_read_lock(ssp); | 885 | srcu_gp_start_if_needed(ssp, rhp, do_norm); |
| 857 | sdp = raw_cpu_ptr(ssp->sda); | ||
| 858 | spin_lock_irqsave_rcu_node(sdp, flags); | ||
| 859 | rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp); | ||
| 860 | rcu_segcblist_advance(&sdp->srcu_cblist, | ||
| 861 | rcu_seq_current(&ssp->srcu_gp_seq)); | ||
| 862 | s = rcu_seq_snap(&ssp->srcu_gp_seq); | ||
| 863 | (void)rcu_segcblist_accelerate(&sdp->srcu_cblist, s); | ||
| 864 | if (ULONG_CMP_LT(sdp->srcu_gp_seq_needed, s)) { | ||
| 865 | sdp->srcu_gp_seq_needed = s; | ||
| 866 | needgp = true; | ||
| 867 | } | ||
| 868 | if (!do_norm && ULONG_CMP_LT(sdp->srcu_gp_seq_needed_exp, s)) { | ||
| 869 | sdp->srcu_gp_seq_needed_exp = s; | ||
| 870 | needexp = true; | ||
| 871 | } | ||
| 872 | spin_unlock_irqrestore_rcu_node(sdp, flags); | ||
| 873 | if (needgp) | ||
| 874 | srcu_funnel_gp_start(ssp, sdp, s, do_norm); | ||
| 875 | else if (needexp) | ||
| 876 | srcu_funnel_exp_start(ssp, sdp->mynode, s); | ||
| 877 | srcu_read_unlock(ssp, idx); | ||
| 878 | } | 886 | } |
| 879 | 887 | ||
| 880 | /** | 888 | /** |