There is a need for a polling interface for SRCU grace periods. This polling needs to initiate an SRCU grace period without having to queue (and manage) a callback. This commit therefore splits the Tree SRCU __call_srcu() function into callback-initialization and queuing/start-grace-period portions, with the latter in a new function named srcu_gp_start_if_needed(). This function may be passed a NULL callback pointer, in which case it will refrain from queuing anything. Why have the new function mess with queuing? Locking considerations, of course!
Unified · Split
| @@ -807,6 +807,42 @@ static void srcu_leak_callback(struct rcu_head *rhp) · 806 unmodified lines | ||
| 807 | 807 | { |
| 808 | 808 | } |
| 809 | 809 | |
| 810 | +/* | |
| 811 | + * Start an SRCU grace period, and also queue the callback if non-NULL. | |
| 812 | + */ | |
| 813 | +static void srcu_gp_start_if_needed(struct srcu_struct *ssp, struct rcu_head *rhp, bool do_norm) | |
| 814 | +{ | |
| 815 | + unsigned long flags; | |
| 816 | + int idx; | |
| 817 | + bool needexp = false; | |
| 818 | + bool needgp = false; | |
| 819 | + unsigned long s; | |
| 820 | + struct srcu_data *sdp; | |
| 821 | + | |
| 822 | + idx = srcu_read_lock(ssp); | |
| 823 | + sdp = raw_cpu_ptr(ssp->sda); | |
| 824 | + spin_lock_irqsave_rcu_node(sdp, flags); | |
| 825 | + rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp); | |
| 826 | + rcu_segcblist_advance(&sdp->srcu_cblist, | |
| 827 | + rcu_seq_current(&ssp->srcu_gp_seq)); | |
| 828 | + s = rcu_seq_snap(&ssp->srcu_gp_seq); | |
| 829 | + (void)rcu_segcblist_accelerate(&sdp->srcu_cblist, s); | |
| 830 | + if (ULONG_CMP_LT(sdp->srcu_gp_seq_needed, s)) { | |
| 831 | + sdp->srcu_gp_seq_needed = s; | |
| 832 | + needgp = true; | |
| 833 | + } | |
| 834 | + if (!do_norm && ULONG_CMP_LT(sdp->srcu_gp_seq_needed_exp, s)) { | |
| 835 | + sdp->srcu_gp_seq_needed_exp = s; | |
| 836 | + needexp = true; | |
| 837 | + } | |
| 838 | + spin_unlock_irqrestore_rcu_node(sdp, flags); | |
| 839 | + if (needgp) | |
| 840 | + srcu_funnel_gp_start(ssp, sdp, s, do_norm); | |
| 841 | + else if (needexp) | |
| 842 | + srcu_funnel_exp_start(ssp, sdp->mynode, s); | |
| 843 | + srcu_read_unlock(ssp, idx); | |
| 844 | +} | |
| 845 | + | |
| 810 | 846 | /* |
| 811 | 847 | * Enqueue an SRCU callback on the srcu_data structure associated with |
| 812 | 848 | * the current CPU and the specified srcu_struct structure, initiating |
| @@ -838,13 +874,6 @@ static void srcu_leak_callback(struct rcu_head *rhp) · 25 unmodified lines | ||
| 838 | 874 | static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, |
| 839 | 875 | rcu_callback_t func, bool do_norm) |
| 840 | 876 | { |
| 841 | − unsigned long flags; | |
| 842 | − int idx; | |
| 843 | − bool needexp = false; | |
| 844 | − bool needgp = false; | |
| 845 | − unsigned long s; | |
| 846 | − struct srcu_data *sdp; | |
| 847 | − | |
| 848 | 877 | check_init_srcu_struct(ssp); |
| 849 | 878 | if (debug_rcu_head_queue(rhp)) { |
| 850 | 879 | /* Probable double call_srcu(), so leak the callback. */ |
| @@ -853,28 +882,7 @@ static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, · 2 unmodified lines | ||
| 853 | 882 | return; |
| 854 | 883 | } |
| 855 | 884 | rhp->func = func; |
| 856 | − idx = srcu_read_lock(ssp); | |
| 857 | − sdp = raw_cpu_ptr(ssp->sda); | |
| 858 | − spin_lock_irqsave_rcu_node(sdp, flags); | |
| 859 | − rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp); | |
| 860 | − rcu_segcblist_advance(&sdp->srcu_cblist, | |
| 861 | − rcu_seq_current(&ssp->srcu_gp_seq)); | |
| 862 | − s = rcu_seq_snap(&ssp->srcu_gp_seq); | |
| 863 | − (void)rcu_segcblist_accelerate(&sdp->srcu_cblist, s); | |
| 864 | − if (ULONG_CMP_LT(sdp->srcu_gp_seq_needed, s)) { | |
| 865 | − sdp->srcu_gp_seq_needed = s; | |
| 866 | − needgp = true; | |
| 867 | − } | |
| 868 | − if (!do_norm && ULONG_CMP_LT(sdp->srcu_gp_seq_needed_exp, s)) { | |
| 869 | − sdp->srcu_gp_seq_needed_exp = s; | |
| 870 | − needexp = true; | |
| 871 | − } | |
| 872 | − spin_unlock_irqrestore_rcu_node(sdp, flags); | |
| 873 | − if (needgp) | |
| 874 | − srcu_funnel_gp_start(ssp, sdp, s, do_norm); | |
| 875 | − else if (needexp) | |
| 876 | − srcu_funnel_exp_start(ssp, sdp->mynode, s); | |
| 877 | − srcu_read_unlock(ssp, idx); | |
| 885 | + srcu_gp_start_if_needed(ssp, rhp, do_norm); | |
| 878 | 886 | } |
| 879 | 887 | |
| 880 | 888 | /** |