There is a need for a polling interface for SRCU grace periods. This polling needs to initiate an SRCU grace period without having to queue (and manage) a callback. This commit therefore splits the Tree SRCU __call_srcu() function into callback-initialization and queuing/start-grace-period portions, with the latter in a new function named srcu_gp_start_if_needed(). This function may be passed a NULL callback pointer, in which case it will refrain from queuing anything. Why have the new function mess with queuing? Locking considerations, of course!
Unified · Split
| @@ -807,6 +807,42 @@ static void srcu_leak_callback(struct rcu_head *rhp) · 806 unmodified lines | |||
| 807 | { | 807 | { |
| 808 | } | 808 | } |
| 809 | 809 | ||
| 810 | /* | ||
| 811 | * Start an SRCU grace period, and also queue the callback if non-NULL. | ||
| 812 | */ | ||
| 813 | static void srcu_gp_start_if_needed(struct srcu_struct *ssp, struct rcu_head *rhp, bool do_norm) | ||
| 814 | { | ||
| 815 | unsigned long flags; | ||
| 816 | int idx; | ||
| 817 | bool needexp = false; | ||
| 818 | bool needgp = false; | ||
| 819 | unsigned long s; | ||
| 820 | struct srcu_data *sdp; | ||
| 821 | |||
| 822 | idx = srcu_read_lock(ssp); | ||
| 823 | sdp = raw_cpu_ptr(ssp->sda); | ||
| 824 | spin_lock_irqsave_rcu_node(sdp, flags); | ||
| 825 | rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp); | ||
| 826 | rcu_segcblist_advance(&sdp->srcu_cblist, | ||
| 827 | rcu_seq_current(&ssp->srcu_gp_seq)); | ||
| 828 | s = rcu_seq_snap(&ssp->srcu_gp_seq); | ||
| 829 | (void)rcu_segcblist_accelerate(&sdp->srcu_cblist, s); | ||
| 830 | if (ULONG_CMP_LT(sdp->srcu_gp_seq_needed, s)) { | ||
| 831 | sdp->srcu_gp_seq_needed = s; | ||
| 832 | needgp = true; | ||
| 833 | } | ||
| 834 | if (!do_norm && ULONG_CMP_LT(sdp->srcu_gp_seq_needed_exp, s)) { | ||
| 835 | sdp->srcu_gp_seq_needed_exp = s; | ||
| 836 | needexp = true; | ||
| 837 | } | ||
| 838 | spin_unlock_irqrestore_rcu_node(sdp, flags); | ||
| 839 | if (needgp) | ||
| 840 | srcu_funnel_gp_start(ssp, sdp, s, do_norm); | ||
| 841 | else if (needexp) | ||
| 842 | srcu_funnel_exp_start(ssp, sdp->mynode, s); | ||
| 843 | srcu_read_unlock(ssp, idx); | ||
| 844 | } | ||
| 845 | |||
| 810 | /* | 846 | /* |
| 811 | * Enqueue an SRCU callback on the srcu_data structure associated with | 847 | * Enqueue an SRCU callback on the srcu_data structure associated with |
| 812 | * the current CPU and the specified srcu_struct structure, initiating | 848 | * the current CPU and the specified srcu_struct structure, initiating |
| @@ -838,13 +874,6 @@ static void srcu_leak_callback(struct rcu_head *rhp) · 25 unmodified lines | |||
| 838 | static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, | 874 | static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, |
| 839 | rcu_callback_t func, bool do_norm) | 875 | rcu_callback_t func, bool do_norm) |
| 840 | { | 876 | { |
| 841 | unsigned long flags; | ||
| 842 | int idx; | ||
| 843 | bool needexp = false; | ||
| 844 | bool needgp = false; | ||
| 845 | unsigned long s; | ||
| 846 | struct srcu_data *sdp; | ||
| 847 | |||
| 848 | check_init_srcu_struct(ssp); | 877 | check_init_srcu_struct(ssp); |
| 849 | if (debug_rcu_head_queue(rhp)) { | 878 | if (debug_rcu_head_queue(rhp)) { |
| 850 | /* Probable double call_srcu(), so leak the callback. */ | 879 | /* Probable double call_srcu(), so leak the callback. */ |
| @@ -853,28 +882,7 @@ static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, · 2 unmodified lines | |||
| 853 | return; | 882 | return; |
| 854 | } | 883 | } |
| 855 | rhp->func = func; | 884 | rhp->func = func; |
| 856 | idx = srcu_read_lock(ssp); | 885 | srcu_gp_start_if_needed(ssp, rhp, do_norm); |
| 857 | sdp = raw_cpu_ptr(ssp->sda); | ||
| 858 | spin_lock_irqsave_rcu_node(sdp, flags); | ||
| 859 | rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp); | ||
| 860 | rcu_segcblist_advance(&sdp->srcu_cblist, | ||
| 861 | rcu_seq_current(&ssp->srcu_gp_seq)); | ||
| 862 | s = rcu_seq_snap(&ssp->srcu_gp_seq); | ||
| 863 | (void)rcu_segcblist_accelerate(&sdp->srcu_cblist, s); | ||
| 864 | if (ULONG_CMP_LT(sdp->srcu_gp_seq_needed, s)) { | ||
| 865 | sdp->srcu_gp_seq_needed = s; | ||
| 866 | needgp = true; | ||
| 867 | } | ||
| 868 | if (!do_norm && ULONG_CMP_LT(sdp->srcu_gp_seq_needed_exp, s)) { | ||
| 869 | sdp->srcu_gp_seq_needed_exp = s; | ||
| 870 | needexp = true; | ||
| 871 | } | ||
| 872 | spin_unlock_irqrestore_rcu_node(sdp, flags); | ||
| 873 | if (needgp) | ||
| 874 | srcu_funnel_gp_start(ssp, sdp, s, do_norm); | ||
| 875 | else if (needexp) | ||
| 876 | srcu_funnel_exp_start(ssp, sdp->mynode, s); | ||
| 877 | srcu_read_unlock(ssp, idx); | ||
| 878 | } | 886 | } |
| 879 | 887 | ||
| 880 | /** | 888 | /** |