History

rcu: Allow rcu_do_batch() to dynamically adjust batch sizes

Bimodal behavior of rcu_do_batch() is not really suited to Google applications like gfe servers. When a process with millions of sockets exits, closing all files queues two rcu callbacks per socket. This eventually reaches the point where RCU enters an emergency mode, where rcu_do_batch() do not return until whole queue is flushed. Each rcu callback lasts at least 70 nsec, so with millions of elements, we easily spend more than 100 msec without rescheduling. Goal of this patch is to avoid the infamous message like following "need_resched set for > 51999388 ns (52 ticks) without schedule" We dynamically adjust the number of elements we process, instead of 10 / INFINITE choices, we use a floor of ~1 % of current entries. If the number is above 1000, we switch to a time based limit of 3 msec per batch, adjustable with /sys/module/rcutree/parameters/rcu_resched_ns

Eric Dumazet committed 7 years ago

Commit cfcdef51 parent f48fe4c1 file · +19 −1Browse files

Unified · Split

kernel/rcu/tree.c+19−1
@@ -56,6 +56,7 @@ · 55 unmodified lines
5656 #include <linux/smpboot.h>
5757 #include <linux/jiffies.h>
5858 #include <linux/sched/isolation.h>
59+#include <linux/sched/clock.h>
5960 #include "../time/tick-internal.h"
6061
6162 #include "tree.h"
@@ -416,6 +417,12 @@ module_param(qlowmark, long, 0444); · 354 unmodified lines
416417 static ulong jiffies_till_first_fqs = ULONG_MAX;
417418 static ulong jiffies_till_next_fqs = ULONG_MAX;
418419 static bool rcu_kick_kthreads;
420+static int rcu_divisor = 7;
421+module_param(rcu_divisor, int, 0644);
422+
423+/* Force an exit from rcu_do_batch() after 3 milliseconds. */
424+static long rcu_resched_ns = 3 * NSEC_PER_MSEC;
425+module_param(rcu_resched_ns, long, 0644);
419426
420427 /*
421428 * How long the grace period must be before we start recruiting
@@ -2109,6 +2116,7 @@ static void rcu_do_batch(struct rcu_data *rdp) · 1687 unmodified lines
21092116 struct rcu_head *rhp;
21102117 struct rcu_cblist rcl = RCU_CBLIST_INITIALIZER(rcl);
21112118 long bl, count;
2119+ long pending, tlimit = 0;
21122120
21132121 /* If no callbacks are ready, just return. */
21142122 if (!rcu_segcblist_ready_cbs(&rdp->cblist)) {
@@ -2130,7 +2138,10 @@ static void rcu_do_batch(struct rcu_data *rdp) · 15 unmodified lines
21302138 local_irq_save(flags);
21312139 rcu_nocb_lock(rdp);
21322140 WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));
2133− bl = rdp->blimit;
2141+ pending = rcu_segcblist_n_cbs(&rdp->cblist);
2142+ bl = max(rdp->blimit, pending >> rcu_divisor);
2143+ if (unlikely(bl > 100))
2144+ tlimit = local_clock() + rcu_resched_ns;
21342145 trace_rcu_batch_start(rcu_state.name,
21352146 rcu_segcblist_n_lazy_cbs(&rdp->cblist),
21362147 rcu_segcblist_n_cbs(&rdp->cblist), bl);
@@ -2153,6 +2164,13 @@ static void rcu_do_batch(struct rcu_data *rdp) · 16 unmodified lines
21532164 (need_resched() ||
21542165 (!is_idle_task(current) && !rcu_is_callbacks_kthread())))
21552166 break;
2167+ if (unlikely(tlimit)) {
2168+ /* only call local_clock() every 32 callbacks */
2169+ if (likely((-rcl.len & 31) || local_clock() < tlimit))
2170+ continue;
2171+ /* Exceeded the time limit, so leave. */
2172+ break;
2173+ }
21562174 if (offloaded) {
21572175 WARN_ON_ONCE(in_serving_softirq());
21582176 local_bh_enable();

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t