History

mm, mempolicy: simplify rebinding mempolicies when updating cpusets

Commit c0ff7453bb5c ("cpuset,mm: fix no node to alloc memory when changing cpuset's mems") has introduced a two-step protocol when rebinding task's mempolicy due to cpuset update, in order to avoid a parallel allocation seeing an empty effective nodemask and failing. Later, commit cc9a6c877661 ("cpuset: mm: reduce large amounts of memory barrier related damage v3") introduced a seqlock protection and removed the synchronization point between the two update steps. At that point (or perhaps later), the two-step rebinding became unnecessary. Currently it only makes sure that the update first adds new nodes in step 1 and then removes nodes in step 2. Without memory barriers the effects are questionable, and even then this cannot prevent a parallel zonelist iteration checking the nodemask at each step to observe all nodes as unusable for allocation. We now fully rely on the seqlock to prevent premature OOMs and allocation failures. We can thus remove the two-step update parts and simplify the code.

Vlastimil Babka committed 9 years ago

Commit 213980c1 parent 04ec6264 files · +21 −99Browse files

Unified · Split

4 files
  1. include/linux/mempolicy.h +2 −4
  2. include/uapi/linux/mempolicy.h +0 −8
  3. kernel/cgroup/cpuset.c +1 −3
  4. mm/mempolicy.c +18 −84
include/linux/mempolicy.h+2−4
@@ -142,8 +142,7 @@ bool vma_policy_mof(struct vm_area_struct *vma); · 141 unmodified lines
142142
143143 extern void numa_default_policy(void);
144144 extern void numa_policy_init(void);
145−extern void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new,
146− enum mpol_rebind_step step);
145+extern void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new);
147146 extern void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new);
148147
149148 extern int huge_node(struct vm_area_struct *vma,
@@ -260,8 +259,7 @@ static inline void numa_default_policy(void) · 110 unmodified lines
260259 }
261260
262261 static inline void mpol_rebind_task(struct task_struct *tsk,
263− const nodemask_t *new,
264− enum mpol_rebind_step step)
262+ const nodemask_t *new)
265263 {
266264 }
267265
include/uapi/linux/mempolicy.h+0−8
@@ -24,13 +24,6 @@ enum { · 23 unmodified lines
2424 MPOL_MAX, /* always last member of enum */
2525 };
2626
27−enum mpol_rebind_step {
28− MPOL_REBIND_ONCE, /* do rebind work at once(not by two step) */
29− MPOL_REBIND_STEP1, /* first step(set all the newly nodes) */
30− MPOL_REBIND_STEP2, /* second step(clean all the disallowed nodes)*/
31− MPOL_REBIND_NSTEP,
32−};
33−
3427 /* Flags for set_mempolicy */
3528 #define MPOL_F_STATIC_NODES (1 << 15)
3629 #define MPOL_F_RELATIVE_NODES (1 << 14)
@@ -65,7 +58,6 @@ enum mpol_rebind_step { · 28 unmodified lines
6558 */
6659 #define MPOL_F_SHARED (1 << 0) /* identify shared policies */
6760 #define MPOL_F_LOCAL (1 << 1) /* preferred local allocation */
68−#define MPOL_F_REBINDING (1 << 2) /* identify policies in rebinding */
6961 #define MPOL_F_MOF (1 << 3) /* this policy wants migrate on fault */
7062 #define MPOL_F_MORON (1 << 4) /* Migrate On protnone Reference On Node */
7163
kernel/cgroup/cpuset.c+1−3
@@ -1063,9 +1063,7 @@ static void cpuset_change_task_nodemask(struct task_struct *tsk, · 1062 unmodified lines
10631063 }
10641064
10651065 nodes_or(tsk->mems_allowed, tsk->mems_allowed, *newmems);
1066− mpol_rebind_task(tsk, newmems, MPOL_REBIND_STEP1);
1067−
1068− mpol_rebind_task(tsk, newmems, MPOL_REBIND_STEP2);
1066+ mpol_rebind_task(tsk, newmems);
10691067 tsk->mems_allowed = *newmems;
10701068
10711069 if (need_loop) {
mm/mempolicy.c+18−84
@@ -146,22 +146,7 @@ struct mempolicy *get_task_policy(struct task_struct *p) · 145 unmodified lines
146146
147147 static const struct mempolicy_operations {
148148 int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
149− /*
150− * If read-side task has no lock to protect task->mempolicy, write-side
151− * task will rebind the task->mempolicy by two step. The first step is
152− * setting all the newly nodes, and the second step is cleaning all the
153− * disallowed nodes. In this way, we can avoid finding no node to alloc
154− * page.
155− * If we have a lock to protect task->mempolicy in read-side, we do
156− * rebind directly.
157− *
158− * step:
159− * MPOL_REBIND_ONCE - do rebind work at once
160− * MPOL_REBIND_STEP1 - set all the newly nodes
161− * MPOL_REBIND_STEP2 - clean all the disallowed nodes
162− */
163− void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes,
164− enum mpol_rebind_step step);
149+ void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes);
165150 } mpol_ops[MPOL_MAX];
166151
167152 static inline int mpol_store_user_nodemask(const struct mempolicy *pol)
@@ -304,19 +289,11 @@ void __mpol_put(struct mempolicy *p) · 136 unmodified lines
304289 kmem_cache_free(policy_cache, p);
305290 }
306291
307−static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes,
308− enum mpol_rebind_step step)
292+static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes)
309293 {
310294 }
311295
312−/*
313− * step:
314− * MPOL_REBIND_ONCE - do rebind work at once
315− * MPOL_REBIND_STEP1 - set all the newly nodes
316− * MPOL_REBIND_STEP2 - clean all the disallowed nodes
317− */
318−static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes,
319− enum mpol_rebind_step step)
296+static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes)
320297 {
321298 nodemask_t tmp;
322299
@@ -325,35 +302,19 @@ static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes, · 2 unmodified lines
325302 else if (pol->flags & MPOL_F_RELATIVE_NODES)
326303 mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
327304 else {
328− /*
329− * if step == 1, we use ->w.cpuset_mems_allowed to cache the
330− * result
331− */
332− if (step == MPOL_REBIND_ONCE || step == MPOL_REBIND_STEP1) {
333− nodes_remap(tmp, pol->v.nodes,
334− pol->w.cpuset_mems_allowed, *nodes);
335− pol->w.cpuset_mems_allowed = step ? tmp : *nodes;
336− } else if (step == MPOL_REBIND_STEP2) {
337− tmp = pol->w.cpuset_mems_allowed;
338− pol->w.cpuset_mems_allowed = *nodes;
339− } else
340− BUG();
305+ nodes_remap(tmp, pol->v.nodes,pol->w.cpuset_mems_allowed,
306+ *nodes);
307+ pol->w.cpuset_mems_allowed = tmp;
341308 }
342309
343310 if (nodes_empty(tmp))
344311 tmp = *nodes;
345312
346− if (step == MPOL_REBIND_STEP1)
347− nodes_or(pol->v.nodes, pol->v.nodes, tmp);
348− else if (step == MPOL_REBIND_ONCE || step == MPOL_REBIND_STEP2)
349− pol->v.nodes = tmp;
350− else
351− BUG();
313+ pol->v.nodes = tmp;
352314 }
353315
354316 static void mpol_rebind_preferred(struct mempolicy *pol,
355− const nodemask_t *nodes,
356− enum mpol_rebind_step step)
317+ const nodemask_t *nodes)
357318 {
358319 nodemask_t tmp;
359320
@@ -379,42 +340,19 @@ static void mpol_rebind_preferred(struct mempolicy *pol, · 19 unmodified lines
379340 /*
380341 * mpol_rebind_policy - Migrate a policy to a different set of nodes
381342 *
382− * If read-side task has no lock to protect task->mempolicy, write-side
383− * task will rebind the task->mempolicy by two step. The first step is
384− * setting all the newly nodes, and the second step is cleaning all the
385− * disallowed nodes. In this way, we can avoid finding no node to alloc
386− * page.
387− * If we have a lock to protect task->mempolicy in read-side, we do
388− * rebind directly.
389− *
390− * step:
391− * MPOL_REBIND_ONCE - do rebind work at once
392− * MPOL_REBIND_STEP1 - set all the newly nodes
393− * MPOL_REBIND_STEP2 - clean all the disallowed nodes
343+ * Per-vma policies are protected by mmap_sem. Allocations using per-task
344+ * policies are protected by task->mems_allowed_seq to prevent a premature
345+ * OOM/allocation failure due to parallel nodemask modification.
394346 */
395−static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask,
396− enum mpol_rebind_step step)
347+static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask)
397348 {
398349 if (!pol)
399350 return;
400− if (!mpol_store_user_nodemask(pol) && step == MPOL_REBIND_ONCE &&
351+ if (!mpol_store_user_nodemask(pol) &&
401352 nodes_equal(pol->w.cpuset_mems_allowed, *newmask))
402353 return;
403354
404− if (step == MPOL_REBIND_STEP1 && (pol->flags & MPOL_F_REBINDING))
405− return;
406−
407− if (step == MPOL_REBIND_STEP2 && !(pol->flags & MPOL_F_REBINDING))
408− BUG();
409−
410− if (step == MPOL_REBIND_STEP1)
411− pol->flags |= MPOL_F_REBINDING;
412− else if (step == MPOL_REBIND_STEP2)
413− pol->flags &= ~MPOL_F_REBINDING;
414− else if (step >= MPOL_REBIND_NSTEP)
415− BUG();
416−
417− mpol_ops[pol->mode].rebind(pol, newmask, step);
355+ mpol_ops[pol->mode].rebind(pol, newmask);
418356 }
419357
420358 /*
@@ -424,10 +362,9 @@ static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask, · 3 unmodified lines
424362 * Called with task's alloc_lock held.
425363 */
426364
427−void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new,
428− enum mpol_rebind_step step)
365+void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new)
429366 {
430− mpol_rebind_policy(tsk->mempolicy, new, step);
367+ mpol_rebind_policy(tsk->mempolicy, new);
431368 }
432369
433370 /*
@@ -442,7 +379,7 @@ void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new) · 8 unmodified lines
442379
443380 down_write(&mm->mmap_sem);
444381 for (vma = mm->mmap; vma; vma = vma->vm_next)
445− mpol_rebind_policy(vma->vm_policy, new, MPOL_REBIND_ONCE);
382+ mpol_rebind_policy(vma->vm_policy, new);
446383 up_write(&mm->mmap_sem);
447384 }
448385
@@ -2101,10 +2038,7 @@ struct mempolicy *__mpol_dup(struct mempolicy *old) · 1652 unmodified lines
21012038
21022039 if (current_cpuset_is_being_rebound()) {
21032040 nodemask_t mems = cpuset_mems_allowed(current);
2104− if (new->flags & MPOL_F_REBINDING)
2105− mpol_rebind_policy(new, &mems, MPOL_REBIND_STEP2);
2106− else
2107− mpol_rebind_policy(new, &mems, MPOL_REBIND_ONCE);
2041+ mpol_rebind_policy(new, &mems);
21082042 }
21092043 atomic_set(&new->refcnt, 1);
21102044 return new;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t