History

rcu: Expedite first two FQS scans under callback-overload conditions

Even if some CPUs have excessive numbers of callbacks, RCU's grace-period kthread will still wait normally between successive force-quiescent-state scans. The first two are the most important, as they are the ones that enlist aid from the scheduler when overloaded. This commit therefore omits the wait before the first and the second force-quiescent-state scan under callback-overload conditions. This approach was inspired by a discussion with Jeff Roberson.

Paul E. McKenney committed 6 years ago

Commit 1fca4d11 parent 53965db2 files · +16 −4Browse files

Showing the changes to kernel (2 of 2 files). Show all

Unified · Split

2 files
  1. kernel/rcu/tree.c +15 −4
  2. kernel/rcu/tree.h +1 −0
kernel/rcu/tree.c+15−4
@@ -1624,12 +1624,16 @@ static bool rcu_gp_fqs_check_wake(int *gfp) · 1623 unmodified lines
16241624 {
16251625 struct rcu_node *rnp = rcu_get_root();
16261626
1627− /* Someone like call_rcu() requested a force-quiescent-state scan. */
1627+ // If under overload conditions, force an immediate FQS scan.
1628+ if (*gfp & RCU_GP_FLAG_OVLD)
1629+ return true;
1630+
1631+ // Someone like call_rcu() requested a force-quiescent-state scan.
16281632 *gfp = READ_ONCE(rcu_state.gp_flags);
16291633 if (*gfp & RCU_GP_FLAG_FQS)
16301634 return true;
16311635
1632− /* The current grace period has completed. */
1636+ // The current grace period has completed.
16331637 if (!READ_ONCE(rnp->qsmask) && !rcu_preempt_blocked_readers_cgp(rnp))
16341638 return true;
16351639
@@ -1667,13 +1671,15 @@ static void rcu_gp_fqs(bool first_time) · 31 unmodified lines
16671671 static void rcu_gp_fqs_loop(void)
16681672 {
16691673 bool first_gp_fqs;
1670− int gf;
1674+ int gf = 0;
16711675 unsigned long j;
16721676 int ret;
16731677 struct rcu_node *rnp = rcu_get_root();
16741678
16751679 first_gp_fqs = true;
16761680 j = READ_ONCE(jiffies_till_first_fqs);
1681+ if (rcu_state.cbovld)
1682+ gf = RCU_GP_FLAG_OVLD;
16771683 ret = 0;
16781684 for (;;) {
16791685 if (!ret) {
@@ -1698,7 +1704,11 @@ static void rcu_gp_fqs_loop(void) · 18 unmodified lines
16981704 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
16991705 TPS("fqsstart"));
17001706 rcu_gp_fqs(first_gp_fqs);
1701− first_gp_fqs = false;
1707+ gf = 0;
1708+ if (first_gp_fqs) {
1709+ first_gp_fqs = false;
1710+ gf = rcu_state.cbovld ? RCU_GP_FLAG_OVLD : 0;
1711+ }
17021712 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
17031713 TPS("fqsend"));
17041714 cond_resched_tasks_rcu_qs();
@@ -1718,6 +1728,7 @@ static void rcu_gp_fqs_loop(void) · 13 unmodified lines
17181728 j = 1;
17191729 else
17201730 j = rcu_state.jiffies_force_qs - j;
1731+ gf = 0;
17211732 }
17221733 }
17231734 }
kernel/rcu/tree.h+1−0
@@ -359,6 +359,7 @@ struct rcu_state { · 358 unmodified lines
359359 /* Values for rcu_state structure's gp_flags field. */
360360 #define RCU_GP_FLAG_INIT 0x1 /* Need grace-period initialization. */
361361 #define RCU_GP_FLAG_FQS 0x2 /* Need grace-period quiescent-state forcing. */
362+#define RCU_GP_FLAG_OVLD 0x4 /* Experiencing callback overload. */
362363
363364 /* Values for rcu_state structure's gp_state field. */
364365 #define RCU_GP_IDLE 0 /* Initial state and no GP in progress. */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t