| @@ -146,22 +146,7 @@ struct mempolicy *get_task_policy(struct task_struct *p) · 145 unmodified lines |
| 146 | 146 | |
| 147 | 147 | static const struct mempolicy_operations { |
| 148 | 148 | int (*create)(struct mempolicy *pol, const nodemask_t *nodes); |
| 149 | | − /* |
| 150 | | − * If read-side task has no lock to protect task->mempolicy, write-side |
| 151 | | − * task will rebind the task->mempolicy by two step. The first step is |
| 152 | | − * setting all the newly nodes, and the second step is cleaning all the |
| 153 | | − * disallowed nodes. In this way, we can avoid finding no node to alloc |
| 154 | | − * page. |
| 155 | | − * If we have a lock to protect task->mempolicy in read-side, we do |
| 156 | | − * rebind directly. |
| 157 | | − * |
| 158 | | − * step: |
| 159 | | − * MPOL_REBIND_ONCE - do rebind work at once |
| 160 | | − * MPOL_REBIND_STEP1 - set all the newly nodes |
| 161 | | − * MPOL_REBIND_STEP2 - clean all the disallowed nodes |
| 162 | | − */ |
| 163 | | − void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes, |
| 164 | | − enum mpol_rebind_step step); |
| 149 | + void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes); |
| 165 | 150 | } mpol_ops[MPOL_MAX]; |
| 166 | 151 | |
| 167 | 152 | static inline int mpol_store_user_nodemask(const struct mempolicy *pol) |
| @@ -304,19 +289,11 @@ void __mpol_put(struct mempolicy *p) · 136 unmodified lines |
| 304 | 289 | kmem_cache_free(policy_cache, p); |
| 305 | 290 | } |
| 306 | 291 | |
| 307 | | −static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes, |
| 308 | | − enum mpol_rebind_step step) |
| 292 | +static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes) |
| 309 | 293 | { |
| 310 | 294 | } |
| 311 | 295 | |
| 312 | | −/* |
| 313 | | − * step: |
| 314 | | − * MPOL_REBIND_ONCE - do rebind work at once |
| 315 | | − * MPOL_REBIND_STEP1 - set all the newly nodes |
| 316 | | − * MPOL_REBIND_STEP2 - clean all the disallowed nodes |
| 317 | | − */ |
| 318 | | −static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes, |
| 319 | | − enum mpol_rebind_step step) |
| 296 | +static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes) |
| 320 | 297 | { |
| 321 | 298 | nodemask_t tmp; |
| 322 | 299 | |
| @@ -325,35 +302,19 @@ static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes, · 2 unmodified lines |
| 325 | 302 | else if (pol->flags & MPOL_F_RELATIVE_NODES) |
| 326 | 303 | mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes); |
| 327 | 304 | else { |
| 328 | | − /* |
| 329 | | − * if step == 1, we use ->w.cpuset_mems_allowed to cache the |
| 330 | | − * result |
| 331 | | − */ |
| 332 | | − if (step == MPOL_REBIND_ONCE || step == MPOL_REBIND_STEP1) { |
| 333 | | − nodes_remap(tmp, pol->v.nodes, |
| 334 | | − pol->w.cpuset_mems_allowed, *nodes); |
| 335 | | − pol->w.cpuset_mems_allowed = step ? tmp : *nodes; |
| 336 | | − } else if (step == MPOL_REBIND_STEP2) { |
| 337 | | − tmp = pol->w.cpuset_mems_allowed; |
| 338 | | − pol->w.cpuset_mems_allowed = *nodes; |
| 339 | | − } else |
| 340 | | − BUG(); |
| 305 | + nodes_remap(tmp, pol->v.nodes,pol->w.cpuset_mems_allowed, |
| 306 | + *nodes); |
| 307 | + pol->w.cpuset_mems_allowed = tmp; |
| 341 | 308 | } |
| 342 | 309 | |
| 343 | 310 | if (nodes_empty(tmp)) |
| 344 | 311 | tmp = *nodes; |
| 345 | 312 | |
| 346 | | − if (step == MPOL_REBIND_STEP1) |
| 347 | | − nodes_or(pol->v.nodes, pol->v.nodes, tmp); |
| 348 | | − else if (step == MPOL_REBIND_ONCE || step == MPOL_REBIND_STEP2) |
| 349 | | − pol->v.nodes = tmp; |
| 350 | | − else |
| 351 | | − BUG(); |
| 313 | + pol->v.nodes = tmp; |
| 352 | 314 | } |
| 353 | 315 | |
| 354 | 316 | static void mpol_rebind_preferred(struct mempolicy *pol, |
| 355 | | − const nodemask_t *nodes, |
| 356 | | − enum mpol_rebind_step step) |
| 317 | + const nodemask_t *nodes) |
| 357 | 318 | { |
| 358 | 319 | nodemask_t tmp; |
| 359 | 320 | |
| @@ -379,42 +340,19 @@ static void mpol_rebind_preferred(struct mempolicy *pol, · 19 unmodified lines |
| 379 | 340 | /* |
| 380 | 341 | * mpol_rebind_policy - Migrate a policy to a different set of nodes |
| 381 | 342 | * |
| 382 | | − * If read-side task has no lock to protect task->mempolicy, write-side |
| 383 | | − * task will rebind the task->mempolicy by two step. The first step is |
| 384 | | − * setting all the newly nodes, and the second step is cleaning all the |
| 385 | | − * disallowed nodes. In this way, we can avoid finding no node to alloc |
| 386 | | − * page. |
| 387 | | − * If we have a lock to protect task->mempolicy in read-side, we do |
| 388 | | − * rebind directly. |
| 389 | | − * |
| 390 | | − * step: |
| 391 | | − * MPOL_REBIND_ONCE - do rebind work at once |
| 392 | | − * MPOL_REBIND_STEP1 - set all the newly nodes |
| 393 | | − * MPOL_REBIND_STEP2 - clean all the disallowed nodes |
| 343 | + * Per-vma policies are protected by mmap_sem. Allocations using per-task |
| 344 | + * policies are protected by task->mems_allowed_seq to prevent a premature |
| 345 | + * OOM/allocation failure due to parallel nodemask modification. |
| 394 | 346 | */ |
| 395 | | −static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask, |
| 396 | | − enum mpol_rebind_step step) |
| 347 | +static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask) |
| 397 | 348 | { |
| 398 | 349 | if (!pol) |
| 399 | 350 | return; |
| 400 | | − if (!mpol_store_user_nodemask(pol) && step == MPOL_REBIND_ONCE && |
| 351 | + if (!mpol_store_user_nodemask(pol) && |
| 401 | 352 | nodes_equal(pol->w.cpuset_mems_allowed, *newmask)) |
| 402 | 353 | return; |
| 403 | 354 | |
| 404 | | − if (step == MPOL_REBIND_STEP1 && (pol->flags & MPOL_F_REBINDING)) |
| 405 | | − return; |
| 406 | | − |
| 407 | | − if (step == MPOL_REBIND_STEP2 && !(pol->flags & MPOL_F_REBINDING)) |
| 408 | | − BUG(); |
| 409 | | − |
| 410 | | − if (step == MPOL_REBIND_STEP1) |
| 411 | | − pol->flags |= MPOL_F_REBINDING; |
| 412 | | − else if (step == MPOL_REBIND_STEP2) |
| 413 | | − pol->flags &= ~MPOL_F_REBINDING; |
| 414 | | − else if (step >= MPOL_REBIND_NSTEP) |
| 415 | | − BUG(); |
| 416 | | − |
| 417 | | − mpol_ops[pol->mode].rebind(pol, newmask, step); |
| 355 | + mpol_ops[pol->mode].rebind(pol, newmask); |
| 418 | 356 | } |
| 419 | 357 | |
| 420 | 358 | /* |
| @@ -424,10 +362,9 @@ static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask, · 3 unmodified lines |
| 424 | 362 | * Called with task's alloc_lock held. |
| 425 | 363 | */ |
| 426 | 364 | |
| 427 | | −void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new, |
| 428 | | − enum mpol_rebind_step step) |
| 365 | +void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new) |
| 429 | 366 | { |
| 430 | | − mpol_rebind_policy(tsk->mempolicy, new, step); |
| 367 | + mpol_rebind_policy(tsk->mempolicy, new); |
| 431 | 368 | } |
| 432 | 369 | |
| 433 | 370 | /* |
| @@ -442,7 +379,7 @@ void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new) · 8 unmodified lines |
| 442 | 379 | |
| 443 | 380 | down_write(&mm->mmap_sem); |
| 444 | 381 | for (vma = mm->mmap; vma; vma = vma->vm_next) |
| 445 | | − mpol_rebind_policy(vma->vm_policy, new, MPOL_REBIND_ONCE); |
| 382 | + mpol_rebind_policy(vma->vm_policy, new); |
| 446 | 383 | up_write(&mm->mmap_sem); |
| 447 | 384 | } |
| 448 | 385 | |
| @@ -2101,10 +2038,7 @@ struct mempolicy *__mpol_dup(struct mempolicy *old) · 1652 unmodified lines |
| 2101 | 2038 | |
| 2102 | 2039 | if (current_cpuset_is_being_rebound()) { |
| 2103 | 2040 | nodemask_t mems = cpuset_mems_allowed(current); |
| 2104 | | − if (new->flags & MPOL_F_REBINDING) |
| 2105 | | − mpol_rebind_policy(new, &mems, MPOL_REBIND_STEP2); |
| 2106 | | − else |
| 2107 | | − mpol_rebind_policy(new, &mems, MPOL_REBIND_ONCE); |
| 2041 | + mpol_rebind_policy(new, &mems); |
| 2108 | 2042 | } |
| 2109 | 2043 | atomic_set(&new->refcnt, 1); |
| 2110 | 2044 | return new; |