This commit causes all the readers to start running unmeasured load until all readers have done at least one such run (thus having warmed up), then run the measured load, and then run unmeasured load until all readers have completed their measured load. This approach avoids any thread running measured load while other readers are idle.
Unified · Split
| @@ -100,6 +100,8 @@ static atomic_t nreaders_exp; · 99 unmodified lines | ||
| 100 | 100 | // Use to wait for all threads to start. |
| 101 | 101 | static atomic_t n_init; |
| 102 | 102 | static atomic_t n_started; |
| 103 | +static atomic_t n_warmedup; | |
| 104 | +static atomic_t n_cooleddown; | |
| 103 | 105 | |
| 104 | 106 | // Track which experiment is currently running. |
| 105 | 107 | static int exp_idx; |
| @@ -260,8 +262,15 @@ repeat: · 154 unmodified lines | ||
| 260 | 262 | |
| 261 | 263 | VERBOSE_PERFOUT("ref_perf_reader %ld: experiment %d started", me, exp_idx); |
| 262 | 264 | |
| 263 | − // To prevent noise, keep interrupts disabled. This also has the | |
| 264 | − // effect of preventing entries into slow path for rcu_read_unlock(). | |
| 265 | + | |
| 266 | + // To reduce noise, do an initial cache-warming invocation, check | |
| 267 | + // in, and then keep warming until everyone has checked in. | |
| 268 | + cur_ops->readsection(loops); | |
| 269 | + if (!atomic_dec_return(&n_warmedup)) | |
| 270 | + while (atomic_read_acquire(&n_warmedup)) | |
| 271 | + cur_ops->readsection(loops); | |
| 272 | + // Also keep interrupts disabled. This also has the effect | |
| 273 | + // of preventing entries into slow path for rcu_read_unlock(). | |
| 265 | 274 | local_irq_save(flags); |
| 266 | 275 | start = ktime_get_mono_fast_ns(); |
| 267 | 276 | |
| @@ -271,6 +280,11 @@ repeat: · 3 unmodified lines | ||
| 271 | 280 | local_irq_restore(flags); |
| 272 | 281 | |
| 273 | 282 | rt->last_duration_ns = WARN_ON_ONCE(duration < 0) ? 0 : duration; |
| 283 | + // To reduce runtime-skew noise, do maintain-load invocations until | |
| 284 | + // everyone is done. | |
| 285 | + if (!atomic_dec_return(&n_cooleddown)) | |
| 286 | + while (atomic_read_acquire(&n_cooleddown)) | |
| 287 | + cur_ops->readsection(loops); | |
| 274 | 288 | |
| 275 | 289 | if (atomic_dec_and_test(&nreaders_exp)) |
| 276 | 290 | wake_up(&main_wq); |
| @@ -372,6 +386,8 @@ static int main_func(void *arg) · 95 unmodified lines | ||
| 372 | 386 | reset_readers(); |
| 373 | 387 | atomic_set(&nreaders_exp, nreaders); |
| 374 | 388 | atomic_set(&n_started, nreaders); |
| 389 | + atomic_set(&n_warmedup, nreaders); | |
| 390 | + atomic_set(&n_cooleddown, nreaders); | |
| 375 | 391 | |
| 376 | 392 | exp_idx = exp; |
| 377 | 393 | |