| @@ -146,22 +146,7 @@ struct mempolicy *get_task_policy(struct task_struct *p) · 145 unmodified lines |
| 146 | | 146 | |
| 147 | static const struct mempolicy_operations { | 147 | static const struct mempolicy_operations { |
| 148 | int (*create)(struct mempolicy *pol, const nodemask_t *nodes); | 148 | int (*create)(struct mempolicy *pol, const nodemask_t *nodes); |
| 149 | /* | 149 | void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes); |
| 150 | * If read-side task has no lock to protect task->mempolicy, write-side | | |
| 151 | * task will rebind the task->mempolicy by two step. The first step is | | |
| 152 | * setting all the newly nodes, and the second step is cleaning all the | | |
| 153 | * disallowed nodes. In this way, we can avoid finding no node to alloc | | |
| 154 | * page. | | |
| 155 | * If we have a lock to protect task->mempolicy in read-side, we do | | |
| 156 | * rebind directly. | | |
| 157 | * | | |
| 158 | * step: | | |
| 159 | * MPOL_REBIND_ONCE - do rebind work at once | | |
| 160 | * MPOL_REBIND_STEP1 - set all the newly nodes | | |
| 161 | * MPOL_REBIND_STEP2 - clean all the disallowed nodes | | |
| 162 | */ | | |
| 163 | void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes, | | |
| 164 | enum mpol_rebind_step step); | | |
| 165 | } mpol_ops[MPOL_MAX]; | 150 | } mpol_ops[MPOL_MAX]; |
| 166 | | 151 | |
| 167 | static inline int mpol_store_user_nodemask(const struct mempolicy *pol) | 152 | static inline int mpol_store_user_nodemask(const struct mempolicy *pol) |
| @@ -304,19 +289,11 @@ void __mpol_put(struct mempolicy *p) · 136 unmodified lines |
| 304 | kmem_cache_free(policy_cache, p); | 289 | kmem_cache_free(policy_cache, p); |
| 305 | } | 290 | } |
| 306 | | 291 | |
| 307 | static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes, | 292 | static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes) |
| 308 | enum mpol_rebind_step step) | | |
| 309 | { | 293 | { |
| 310 | } | 294 | } |
| 311 | | 295 | |
| 312 | /* | 296 | static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes) |
| 313 | * step: | | |
| 314 | * MPOL_REBIND_ONCE - do rebind work at once | | |
| 315 | * MPOL_REBIND_STEP1 - set all the newly nodes | | |
| 316 | * MPOL_REBIND_STEP2 - clean all the disallowed nodes | | |
| 317 | */ | | |
| 318 | static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes, | | |
| 319 | enum mpol_rebind_step step) | | |
| 320 | { | 297 | { |
| 321 | nodemask_t tmp; | 298 | nodemask_t tmp; |
| 322 | | 299 | |
| @@ -325,35 +302,19 @@ static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes, · 2 unmodified lines |
| 325 | else if (pol->flags & MPOL_F_RELATIVE_NODES) | 302 | else if (pol->flags & MPOL_F_RELATIVE_NODES) |
| 326 | mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes); | 303 | mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes); |
| 327 | else { | 304 | else { |
| 328 | /* | 305 | nodes_remap(tmp, pol->v.nodes,pol->w.cpuset_mems_allowed, |
| 329 | * if step == 1, we use ->w.cpuset_mems_allowed to cache the | 306 | *nodes); |
| 330 | * result | 307 | pol->w.cpuset_mems_allowed = tmp; |
| 331 | */ | | |
| 332 | if (step == MPOL_REBIND_ONCE || step == MPOL_REBIND_STEP1) { | | |
| 333 | nodes_remap(tmp, pol->v.nodes, | | |
| 334 | pol->w.cpuset_mems_allowed, *nodes); | | |
| 335 | pol->w.cpuset_mems_allowed = step ? tmp : *nodes; | | |
| 336 | } else if (step == MPOL_REBIND_STEP2) { | | |
| 337 | tmp = pol->w.cpuset_mems_allowed; | | |
| 338 | pol->w.cpuset_mems_allowed = *nodes; | | |
| 339 | } else | | |
| 340 | BUG(); | | |
| 341 | } | 308 | } |
| 342 | | 309 | |
| 343 | if (nodes_empty(tmp)) | 310 | if (nodes_empty(tmp)) |
| 344 | tmp = *nodes; | 311 | tmp = *nodes; |
| 345 | | 312 | |
| 346 | if (step == MPOL_REBIND_STEP1) | 313 | pol->v.nodes = tmp; |
| 347 | nodes_or(pol->v.nodes, pol->v.nodes, tmp); | | |
| 348 | else if (step == MPOL_REBIND_ONCE || step == MPOL_REBIND_STEP2) | | |
| 349 | pol->v.nodes = tmp; | | |
| 350 | else | | |
| 351 | BUG(); | | |
| 352 | } | 314 | } |
| 353 | | 315 | |
| 354 | static void mpol_rebind_preferred(struct mempolicy *pol, | 316 | static void mpol_rebind_preferred(struct mempolicy *pol, |
| 355 | const nodemask_t *nodes, | 317 | const nodemask_t *nodes) |
| 356 | enum mpol_rebind_step step) | | |
| 357 | { | 318 | { |
| 358 | nodemask_t tmp; | 319 | nodemask_t tmp; |
| 359 | | 320 | |
| @@ -379,42 +340,19 @@ static void mpol_rebind_preferred(struct mempolicy *pol, · 19 unmodified lines |
| 379 | /* | 340 | /* |
| 380 | * mpol_rebind_policy - Migrate a policy to a different set of nodes | 341 | * mpol_rebind_policy - Migrate a policy to a different set of nodes |
| 381 | * | 342 | * |
| 382 | * If read-side task has no lock to protect task->mempolicy, write-side | 343 | * Per-vma policies are protected by mmap_sem. Allocations using per-task |
| 383 | * task will rebind the task->mempolicy by two step. The first step is | 344 | * policies are protected by task->mems_allowed_seq to prevent a premature |
| 384 | * setting all the newly nodes, and the second step is cleaning all the | 345 | * OOM/allocation failure due to parallel nodemask modification. |
| 385 | * disallowed nodes. In this way, we can avoid finding no node to alloc | | |
| 386 | * page. | | |
| 387 | * If we have a lock to protect task->mempolicy in read-side, we do | | |
| 388 | * rebind directly. | | |
| 389 | * | | |
| 390 | * step: | | |
| 391 | * MPOL_REBIND_ONCE - do rebind work at once | | |
| 392 | * MPOL_REBIND_STEP1 - set all the newly nodes | | |
| 393 | * MPOL_REBIND_STEP2 - clean all the disallowed nodes | | |
| 394 | */ | 346 | */ |
| 395 | static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask, | 347 | static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask) |
| 396 | enum mpol_rebind_step step) | | |
| 397 | { | 348 | { |
| 398 | if (!pol) | 349 | if (!pol) |
| 399 | return; | 350 | return; |
| 400 | if (!mpol_store_user_nodemask(pol) && step == MPOL_REBIND_ONCE && | 351 | if (!mpol_store_user_nodemask(pol) && |
| 401 | nodes_equal(pol->w.cpuset_mems_allowed, *newmask)) | 352 | nodes_equal(pol->w.cpuset_mems_allowed, *newmask)) |
| 402 | return; | 353 | return; |
| 403 | | 354 | |
| 404 | if (step == MPOL_REBIND_STEP1 && (pol->flags & MPOL_F_REBINDING)) | 355 | mpol_ops[pol->mode].rebind(pol, newmask); |
| 405 | return; | | |
| 406 | | | |
| 407 | if (step == MPOL_REBIND_STEP2 && !(pol->flags & MPOL_F_REBINDING)) | | |
| 408 | BUG(); | | |
| 409 | | | |
| 410 | if (step == MPOL_REBIND_STEP1) | | |
| 411 | pol->flags |= MPOL_F_REBINDING; | | |
| 412 | else if (step == MPOL_REBIND_STEP2) | | |
| 413 | pol->flags &= ~MPOL_F_REBINDING; | | |
| 414 | else if (step >= MPOL_REBIND_NSTEP) | | |
| 415 | BUG(); | | |
| 416 | | | |
| 417 | mpol_ops[pol->mode].rebind(pol, newmask, step); | | |
| 418 | } | 356 | } |
| 419 | | 357 | |
| 420 | /* | 358 | /* |
| @@ -424,10 +362,9 @@ static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask, · 3 unmodified lines |
| 424 | * Called with task's alloc_lock held. | 362 | * Called with task's alloc_lock held. |
| 425 | */ | 363 | */ |
| 426 | | 364 | |
| 427 | void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new, | 365 | void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new) |
| 428 | enum mpol_rebind_step step) | | |
| 429 | { | 366 | { |
| 430 | mpol_rebind_policy(tsk->mempolicy, new, step); | 367 | mpol_rebind_policy(tsk->mempolicy, new); |
| 431 | } | 368 | } |
| 432 | | 369 | |
| 433 | /* | 370 | /* |
| @@ -442,7 +379,7 @@ void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new) · 8 unmodified lines |
| 442 | | 379 | |
| 443 | down_write(&mm->mmap_sem); | 380 | down_write(&mm->mmap_sem); |
| 444 | for (vma = mm->mmap; vma; vma = vma->vm_next) | 381 | for (vma = mm->mmap; vma; vma = vma->vm_next) |
| 445 | mpol_rebind_policy(vma->vm_policy, new, MPOL_REBIND_ONCE); | 382 | mpol_rebind_policy(vma->vm_policy, new); |
| 446 | up_write(&mm->mmap_sem); | 383 | up_write(&mm->mmap_sem); |
| 447 | } | 384 | } |
| 448 | | 385 | |
| @@ -2101,10 +2038,7 @@ struct mempolicy *__mpol_dup(struct mempolicy *old) · 1652 unmodified lines |
| 2101 | | 2038 | |
| 2102 | if (current_cpuset_is_being_rebound()) { | 2039 | if (current_cpuset_is_being_rebound()) { |
| 2103 | nodemask_t mems = cpuset_mems_allowed(current); | 2040 | nodemask_t mems = cpuset_mems_allowed(current); |
| 2104 | if (new->flags & MPOL_F_REBINDING) | 2041 | mpol_rebind_policy(new, &mems); |
| 2105 | mpol_rebind_policy(new, &mems, MPOL_REBIND_STEP2); | | |
| 2106 | else | | |
| 2107 | mpol_rebind_policy(new, &mems, MPOL_REBIND_ONCE); | | |
| 2108 | } | 2042 | } |
| 2109 | atomic_set(&new->refcnt, 1); | 2043 | atomic_set(&new->refcnt, 1); |
| 2110 | return new; | 2044 | return new; |