History

mm, mempolicy: simplify rebinding mempolicies when updating cpusets

Commit c0ff7453bb5c ("cpuset,mm: fix no node to alloc memory when changing cpuset's mems") has introduced a two-step protocol when rebinding task's mempolicy due to cpuset update, in order to avoid a parallel allocation seeing an empty effective nodemask and failing. Later, commit cc9a6c877661 ("cpuset: mm: reduce large amounts of memory barrier related damage v3") introduced a seqlock protection and removed the synchronization point between the two update steps. At that point (or perhaps later), the two-step rebinding became unnecessary. Currently it only makes sure that the update first adds new nodes in step 1 and then removes nodes in step 2. Without memory barriers the effects are questionable, and even then this cannot prevent a parallel zonelist iteration checking the nodemask at each step to observe all nodes as unusable for allocation. We now fully rely on the seqlock to prevent premature OOMs and allocation failures. We can thus remove the two-step update parts and simplify the code.

Vlastimil Babka committed 9 years ago

Commit 213980c1 parent 04ec6264 files · +21 −99Browse files

Unified · Split

4 files
  1. include/linux/mempolicy.h +2 −4
  2. include/uapi/linux/mempolicy.h +0 −8
  3. kernel/cgroup/cpuset.c +1 −3
  4. mm/mempolicy.c +18 −84
include/linux/mempolicy.h+2−4
@@ -142,8 +142,7 @@ bool vma_policy_mof(struct vm_area_struct *vma); · 141 unmodified lines
142 142
143extern void numa_default_policy(void);143extern void numa_default_policy(void);
144extern void numa_policy_init(void);144extern void numa_policy_init(void);
145extern void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new,145extern void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new);
146 enum mpol_rebind_step step);
147extern void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new);146extern void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new);
148 147
149extern int huge_node(struct vm_area_struct *vma,148extern int huge_node(struct vm_area_struct *vma,
@@ -260,8 +259,7 @@ static inline void numa_default_policy(void) · 110 unmodified lines
260}259}
261 260
262static inline void mpol_rebind_task(struct task_struct *tsk,261static inline void mpol_rebind_task(struct task_struct *tsk,
263 const nodemask_t *new,262 const nodemask_t *new)
264 enum mpol_rebind_step step)
265{263{
266}264}
267 265
include/uapi/linux/mempolicy.h+0−8
@@ -24,13 +24,6 @@ enum { · 23 unmodified lines
24 MPOL_MAX, /* always last member of enum */24 MPOL_MAX, /* always last member of enum */
25};25};
26 26
27enum mpol_rebind_step {
28 MPOL_REBIND_ONCE, /* do rebind work at once(not by two step) */
29 MPOL_REBIND_STEP1, /* first step(set all the newly nodes) */
30 MPOL_REBIND_STEP2, /* second step(clean all the disallowed nodes)*/
31 MPOL_REBIND_NSTEP,
32};
33
34/* Flags for set_mempolicy */27/* Flags for set_mempolicy */
35#define MPOL_F_STATIC_NODES (1 << 15)28#define MPOL_F_STATIC_NODES (1 << 15)
36#define MPOL_F_RELATIVE_NODES (1 << 14)29#define MPOL_F_RELATIVE_NODES (1 << 14)
@@ -65,7 +58,6 @@ enum mpol_rebind_step { · 28 unmodified lines
65 */58 */
66#define MPOL_F_SHARED (1 << 0) /* identify shared policies */59#define MPOL_F_SHARED (1 << 0) /* identify shared policies */
67#define MPOL_F_LOCAL (1 << 1) /* preferred local allocation */60#define MPOL_F_LOCAL (1 << 1) /* preferred local allocation */
68#define MPOL_F_REBINDING (1 << 2) /* identify policies in rebinding */
69#define MPOL_F_MOF (1 << 3) /* this policy wants migrate on fault */61#define MPOL_F_MOF (1 << 3) /* this policy wants migrate on fault */
70#define MPOL_F_MORON (1 << 4) /* Migrate On protnone Reference On Node */62#define MPOL_F_MORON (1 << 4) /* Migrate On protnone Reference On Node */
71 63
kernel/cgroup/cpuset.c+1−3
@@ -1063,9 +1063,7 @@ static void cpuset_change_task_nodemask(struct task_struct *tsk, · 1062 unmodified lines
1063 }1063 }
1064 1064
1065 nodes_or(tsk->mems_allowed, tsk->mems_allowed, *newmems);1065 nodes_or(tsk->mems_allowed, tsk->mems_allowed, *newmems);
1066 mpol_rebind_task(tsk, newmems, MPOL_REBIND_STEP1);1066 mpol_rebind_task(tsk, newmems);
1067
1068 mpol_rebind_task(tsk, newmems, MPOL_REBIND_STEP2);
1069 tsk->mems_allowed = *newmems;1067 tsk->mems_allowed = *newmems;
1070 1068
1071 if (need_loop) {1069 if (need_loop) {
mm/mempolicy.c+18−84
@@ -146,22 +146,7 @@ struct mempolicy *get_task_policy(struct task_struct *p) · 145 unmodified lines
146 146
147static const struct mempolicy_operations {147static const struct mempolicy_operations {
148 int (*create)(struct mempolicy *pol, const nodemask_t *nodes);148 int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
149 /*149 void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes);
150 * If read-side task has no lock to protect task->mempolicy, write-side
151 * task will rebind the task->mempolicy by two step. The first step is
152 * setting all the newly nodes, and the second step is cleaning all the
153 * disallowed nodes. In this way, we can avoid finding no node to alloc
154 * page.
155 * If we have a lock to protect task->mempolicy in read-side, we do
156 * rebind directly.
157 *
158 * step:
159 * MPOL_REBIND_ONCE - do rebind work at once
160 * MPOL_REBIND_STEP1 - set all the newly nodes
161 * MPOL_REBIND_STEP2 - clean all the disallowed nodes
162 */
163 void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes,
164 enum mpol_rebind_step step);
165} mpol_ops[MPOL_MAX];150} mpol_ops[MPOL_MAX];
166 151
167static inline int mpol_store_user_nodemask(const struct mempolicy *pol)152static inline int mpol_store_user_nodemask(const struct mempolicy *pol)
@@ -304,19 +289,11 @@ void __mpol_put(struct mempolicy *p) · 136 unmodified lines
304 kmem_cache_free(policy_cache, p);289 kmem_cache_free(policy_cache, p);
305}290}
306 291
307static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes,292static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes)
308 enum mpol_rebind_step step)
309{293{
310}294}
311 295
312/*296static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes)
313 * step:
314 * MPOL_REBIND_ONCE - do rebind work at once
315 * MPOL_REBIND_STEP1 - set all the newly nodes
316 * MPOL_REBIND_STEP2 - clean all the disallowed nodes
317 */
318static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes,
319 enum mpol_rebind_step step)
320{297{
321 nodemask_t tmp;298 nodemask_t tmp;
322 299
@@ -325,35 +302,19 @@ static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes, · 2 unmodified lines
325 else if (pol->flags & MPOL_F_RELATIVE_NODES)302 else if (pol->flags & MPOL_F_RELATIVE_NODES)
326 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);303 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
327 else {304 else {
328 /*305 nodes_remap(tmp, pol->v.nodes,pol->w.cpuset_mems_allowed,
329 * if step == 1, we use ->w.cpuset_mems_allowed to cache the306 *nodes);
330 * result307 pol->w.cpuset_mems_allowed = tmp;
331 */
332 if (step == MPOL_REBIND_ONCE || step == MPOL_REBIND_STEP1) {
333 nodes_remap(tmp, pol->v.nodes,
334 pol->w.cpuset_mems_allowed, *nodes);
335 pol->w.cpuset_mems_allowed = step ? tmp : *nodes;
336 } else if (step == MPOL_REBIND_STEP2) {
337 tmp = pol->w.cpuset_mems_allowed;
338 pol->w.cpuset_mems_allowed = *nodes;
339 } else
340 BUG();
341 }308 }
342 309
343 if (nodes_empty(tmp))310 if (nodes_empty(tmp))
344 tmp = *nodes;311 tmp = *nodes;
345 312
346 if (step == MPOL_REBIND_STEP1)313 pol->v.nodes = tmp;
347 nodes_or(pol->v.nodes, pol->v.nodes, tmp);
348 else if (step == MPOL_REBIND_ONCE || step == MPOL_REBIND_STEP2)
349 pol->v.nodes = tmp;
350 else
351 BUG();
352}314}
353 315
354static void mpol_rebind_preferred(struct mempolicy *pol,316static void mpol_rebind_preferred(struct mempolicy *pol,
355 const nodemask_t *nodes,317 const nodemask_t *nodes)
356 enum mpol_rebind_step step)
357{318{
358 nodemask_t tmp;319 nodemask_t tmp;
359 320
@@ -379,42 +340,19 @@ static void mpol_rebind_preferred(struct mempolicy *pol, · 19 unmodified lines
379/*340/*
380 * mpol_rebind_policy - Migrate a policy to a different set of nodes341 * mpol_rebind_policy - Migrate a policy to a different set of nodes
381 *342 *
382 * If read-side task has no lock to protect task->mempolicy, write-side343 * Per-vma policies are protected by mmap_sem. Allocations using per-task
383 * task will rebind the task->mempolicy by two step. The first step is344 * policies are protected by task->mems_allowed_seq to prevent a premature
384 * setting all the newly nodes, and the second step is cleaning all the345 * OOM/allocation failure due to parallel nodemask modification.
385 * disallowed nodes. In this way, we can avoid finding no node to alloc
386 * page.
387 * If we have a lock to protect task->mempolicy in read-side, we do
388 * rebind directly.
389 *
390 * step:
391 * MPOL_REBIND_ONCE - do rebind work at once
392 * MPOL_REBIND_STEP1 - set all the newly nodes
393 * MPOL_REBIND_STEP2 - clean all the disallowed nodes
394 */346 */
395static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask,347static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask)
396 enum mpol_rebind_step step)
397{348{
398 if (!pol)349 if (!pol)
399 return;350 return;
400 if (!mpol_store_user_nodemask(pol) && step == MPOL_REBIND_ONCE &&351 if (!mpol_store_user_nodemask(pol) &&
401 nodes_equal(pol->w.cpuset_mems_allowed, *newmask))352 nodes_equal(pol->w.cpuset_mems_allowed, *newmask))
402 return;353 return;
403 354
404 if (step == MPOL_REBIND_STEP1 && (pol->flags & MPOL_F_REBINDING))355 mpol_ops[pol->mode].rebind(pol, newmask);
405 return;
406
407 if (step == MPOL_REBIND_STEP2 && !(pol->flags & MPOL_F_REBINDING))
408 BUG();
409
410 if (step == MPOL_REBIND_STEP1)
411 pol->flags |= MPOL_F_REBINDING;
412 else if (step == MPOL_REBIND_STEP2)
413 pol->flags &= ~MPOL_F_REBINDING;
414 else if (step >= MPOL_REBIND_NSTEP)
415 BUG();
416
417 mpol_ops[pol->mode].rebind(pol, newmask, step);
418}356}
419 357
420/*358/*
@@ -424,10 +362,9 @@ static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask, · 3 unmodified lines
424 * Called with task's alloc_lock held.362 * Called with task's alloc_lock held.
425 */363 */
426 364
427void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new,365void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new)
428 enum mpol_rebind_step step)
429{366{
430 mpol_rebind_policy(tsk->mempolicy, new, step);367 mpol_rebind_policy(tsk->mempolicy, new);
431}368}
432 369
433/*370/*
@@ -442,7 +379,7 @@ void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new) · 8 unmodified lines
442 379
443 down_write(&mm->mmap_sem);380 down_write(&mm->mmap_sem);
444 for (vma = mm->mmap; vma; vma = vma->vm_next)381 for (vma = mm->mmap; vma; vma = vma->vm_next)
445 mpol_rebind_policy(vma->vm_policy, new, MPOL_REBIND_ONCE);382 mpol_rebind_policy(vma->vm_policy, new);
446 up_write(&mm->mmap_sem);383 up_write(&mm->mmap_sem);
447}384}
448 385
@@ -2101,10 +2038,7 @@ struct mempolicy *__mpol_dup(struct mempolicy *old) · 1652 unmodified lines
2101 2038
2102 if (current_cpuset_is_being_rebound()) {2039 if (current_cpuset_is_being_rebound()) {
2103 nodemask_t mems = cpuset_mems_allowed(current);2040 nodemask_t mems = cpuset_mems_allowed(current);
2104 if (new->flags & MPOL_F_REBINDING)2041 mpol_rebind_policy(new, &mems);
2105 mpol_rebind_policy(new, &mems, MPOL_REBIND_STEP2);
2106 else
2107 mpol_rebind_policy(new, &mems, MPOL_REBIND_ONCE);
2108 }2042 }
2109 atomic_set(&new->refcnt, 1);2043 atomic_set(&new->refcnt, 1);
2110 return new;2044 return new;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t