History

rcu: Expedite first two FQS scans under callback-overload conditions

Even if some CPUs have excessive numbers of callbacks, RCU's grace-period kthread will still wait normally between successive force-quiescent-state scans. The first two are the most important, as they are the ones that enlist aid from the scheduler when overloaded. This commit therefore omits the wait before the first and the second force-quiescent-state scan under callback-overload conditions. This approach was inspired by a discussion with Jeff Roberson.

Paul E. McKenney committed 6 years ago

Commit 1fca4d11 parent 53965db2 files · +16 −4Browse files

Showing the changes to kernel (2 of 2 files). Show all

Unified · Split

2 files
  1. kernel/rcu/tree.c +15 −4
  2. kernel/rcu/tree.h +1 −0
kernel/rcu/tree.c+15−4
@@ -1624,12 +1624,16 @@ static bool rcu_gp_fqs_check_wake(int *gfp) · 1623 unmodified lines
1624{1624{
1625 struct rcu_node *rnp = rcu_get_root();1625 struct rcu_node *rnp = rcu_get_root();
1626 1626
1627 /* Someone like call_rcu() requested a force-quiescent-state scan. */1627 // If under overload conditions, force an immediate FQS scan.
1628 if (*gfp & RCU_GP_FLAG_OVLD)
1629 return true;
1630
1631 // Someone like call_rcu() requested a force-quiescent-state scan.
1628 *gfp = READ_ONCE(rcu_state.gp_flags);1632 *gfp = READ_ONCE(rcu_state.gp_flags);
1629 if (*gfp & RCU_GP_FLAG_FQS)1633 if (*gfp & RCU_GP_FLAG_FQS)
1630 return true;1634 return true;
1631 1635
1632 /* The current grace period has completed. */1636 // The current grace period has completed.
1633 if (!READ_ONCE(rnp->qsmask) && !rcu_preempt_blocked_readers_cgp(rnp))1637 if (!READ_ONCE(rnp->qsmask) && !rcu_preempt_blocked_readers_cgp(rnp))
1634 return true;1638 return true;
1635 1639
@@ -1667,13 +1671,15 @@ static void rcu_gp_fqs(bool first_time) · 31 unmodified lines
1667static void rcu_gp_fqs_loop(void)1671static void rcu_gp_fqs_loop(void)
1668{1672{
1669 bool first_gp_fqs;1673 bool first_gp_fqs;
1670 int gf;1674 int gf = 0;
1671 unsigned long j;1675 unsigned long j;
1672 int ret;1676 int ret;
1673 struct rcu_node *rnp = rcu_get_root();1677 struct rcu_node *rnp = rcu_get_root();
1674 1678
1675 first_gp_fqs = true;1679 first_gp_fqs = true;
1676 j = READ_ONCE(jiffies_till_first_fqs);1680 j = READ_ONCE(jiffies_till_first_fqs);
1681 if (rcu_state.cbovld)
1682 gf = RCU_GP_FLAG_OVLD;
1677 ret = 0;1683 ret = 0;
1678 for (;;) {1684 for (;;) {
1679 if (!ret) {1685 if (!ret) {
@@ -1698,7 +1704,11 @@ static void rcu_gp_fqs_loop(void) · 18 unmodified lines
1698 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,1704 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
1699 TPS("fqsstart"));1705 TPS("fqsstart"));
1700 rcu_gp_fqs(first_gp_fqs);1706 rcu_gp_fqs(first_gp_fqs);
1701 first_gp_fqs = false;1707 gf = 0;
1708 if (first_gp_fqs) {
1709 first_gp_fqs = false;
1710 gf = rcu_state.cbovld ? RCU_GP_FLAG_OVLD : 0;
1711 }
1702 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,1712 trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
1703 TPS("fqsend"));1713 TPS("fqsend"));
1704 cond_resched_tasks_rcu_qs();1714 cond_resched_tasks_rcu_qs();
@@ -1718,6 +1728,7 @@ static void rcu_gp_fqs_loop(void) · 13 unmodified lines
1718 j = 1;1728 j = 1;
1719 else1729 else
1720 j = rcu_state.jiffies_force_qs - j;1730 j = rcu_state.jiffies_force_qs - j;
1731 gf = 0;
1721 }1732 }
1722 }1733 }
1723}1734}
kernel/rcu/tree.h+1−0
@@ -359,6 +359,7 @@ struct rcu_state { · 358 unmodified lines
359/* Values for rcu_state structure's gp_flags field. */359/* Values for rcu_state structure's gp_flags field. */
360#define RCU_GP_FLAG_INIT 0x1 /* Need grace-period initialization. */360#define RCU_GP_FLAG_INIT 0x1 /* Need grace-period initialization. */
361#define RCU_GP_FLAG_FQS 0x2 /* Need grace-period quiescent-state forcing. */361#define RCU_GP_FLAG_FQS 0x2 /* Need grace-period quiescent-state forcing. */
362#define RCU_GP_FLAG_OVLD 0x4 /* Experiencing callback overload. */
362 363
363/* Values for rcu_state structure's gp_state field. */364/* Values for rcu_state structure's gp_state field. */
364#define RCU_GP_IDLE 0 /* Initial state and no GP in progress. */365#define RCU_GP_IDLE 0 /* Initial state and no GP in progress. */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t