Add commentry to document the chain walk and the protection mechanisms and their scope.
Unified · Split
| @@ -337,6 +337,48 @@ static inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p) · 336 unmodified lines | ||
| 337 | 337 | * @top_task: the current top waiter |
| 338 | 338 | * |
| 339 | 339 | * Returns 0 or -EDEADLK. |
| 340 | + * | |
| 341 | + * Chain walk basics and protection scope | |
| 342 | + * | |
| 343 | + * [R] refcount on task | |
| 344 | + * [P] task->pi_lock held | |
| 345 | + * [L] rtmutex->wait_lock held | |
| 346 | + * | |
| 347 | + * Step Description Protected by | |
| 348 | + * function arguments: | |
| 349 | + * @task [R] | |
| 350 | + * @orig_lock if != NULL @top_task is blocked on it | |
| 351 | + * @next_lock Unprotected. Cannot be | |
| 352 | + * dereferenced. Only used for | |
| 353 | + * comparison. | |
| 354 | + * @orig_waiter if != NULL @top_task is blocked on it | |
| 355 | + * @top_task current, or in case of proxy | |
| 356 | + * locking protected by calling | |
| 357 | + * code | |
| 358 | + * again: | |
| 359 | + * loop_sanity_check(); | |
| 360 | + * retry: | |
| 361 | + * [1] lock(task->pi_lock); [R] acquire [P] | |
| 362 | + * [2] waiter = task->pi_blocked_on; [P] | |
| 363 | + * [3] check_exit_conditions_1(); [P] | |
| 364 | + * [4] lock = waiter->lock; [P] | |
| 365 | + * [5] if (!try_lock(lock->wait_lock)) { [P] try to acquire [L] | |
| 366 | + * unlock(task->pi_lock); release [P] | |
| 367 | + * goto retry; | |
| 368 | + * } | |
| 369 | + * [6] check_exit_conditions_2(); [P] + [L] | |
| 370 | + * [7] requeue_lock_waiter(lock, waiter); [P] + [L] | |
| 371 | + * [8] unlock(task->pi_lock); release [P] | |
| 372 | + * put_task_struct(task); release [R] | |
| 373 | + * [9] check_exit_conditions_3(); [L] | |
| 374 | + * [10] task = owner(lock); [L] | |
| 375 | + * get_task_struct(task); [L] acquire [R] | |
| 376 | + * lock(task->pi_lock); [L] acquire [P] | |
| 377 | + * [11] requeue_pi_waiter(tsk, waiters(lock));[P] + [L] | |
| 378 | + * [12] check_exit_conditions_4(); [P] + [L] | |
| 379 | + * [13] unlock(task->pi_lock); release [P] | |
| 380 | + * unlock(lock->wait_lock); release [L] | |
| 381 | + * goto again; | |
| 340 | 382 | */ |
| 341 | 383 | static int rt_mutex_adjust_prio_chain(struct task_struct *task, |
| 342 | 384 | int deadlock_detect, |
| @@ -361,6 +403,9 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 18 unmodified lines | ||
| 361 | 403 | * carefully whether things change under us. |
| 362 | 404 | */ |
| 363 | 405 | again: |
| 406 | + /* | |
| 407 | + * We limit the lock chain length for each invocation. | |
| 408 | + */ | |
| 364 | 409 | if (++depth > max_lock_depth) { |
| 365 | 410 | static int prev_max; |
| 366 | 411 | |
| @@ -378,13 +423,28 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 11 unmodified lines | ||
| 378 | 423 | |
| 379 | 424 | return -EDEADLK; |
| 380 | 425 | } |
| 426 | + | |
| 427 | + /* | |
| 428 | + * We are fully preemptible here and only hold the refcount on | |
| 429 | + * @task. So everything can have changed under us since the | |
| 430 | + * caller or our own code below (goto retry/again) dropped all | |
| 431 | + * locks. | |
| 432 | + */ | |
| 381 | 433 | retry: |
| 382 | 434 | /* |
| 383 | − * Task can not go away as we did a get_task() before ! | |
| 435 | + * [1] Task cannot go away as we did a get_task() before ! | |
| 384 | 436 | */ |
| 385 | 437 | raw_spin_lock_irqsave(&task->pi_lock, flags); |
| 386 | 438 | |
| 439 | + /* | |
| 440 | + * [2] Get the waiter on which @task is blocked on. | |
| 441 | + */ | |
| 387 | 442 | waiter = task->pi_blocked_on; |
| 443 | + | |
| 444 | + /* | |
| 445 | + * [3] check_exit_conditions_1() protected by task->pi_lock. | |
| 446 | + */ | |
| 447 | + | |
| 388 | 448 | /* |
| 389 | 449 | * Check whether the end of the boosting chain has been |
| 390 | 450 | * reached or the state of the chain has changed while we |
| @@ -435,7 +495,15 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 44 unmodified lines | ||
| 435 | 495 | if (!detect_deadlock && waiter->prio == task->prio) |
| 436 | 496 | goto out_unlock_pi; |
| 437 | 497 | |
| 498 | + /* | |
| 499 | + * [4] Get the next lock | |
| 500 | + */ | |
| 438 | 501 | lock = waiter->lock; |
| 502 | + /* | |
| 503 | + * [5] We need to trylock here as we are holding task->pi_lock, | |
| 504 | + * which is the reverse lock order versus the other rtmutex | |
| 505 | + * operations. | |
| 506 | + */ | |
| 439 | 507 | if (!raw_spin_trylock(&lock->wait_lock)) { |
| 440 | 508 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
| 441 | 509 | cpu_relax(); |
| @@ -443,6 +511,9 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 1 unmodified line | ||
| 443 | 511 | } |
| 444 | 512 | |
| 445 | 513 | /* |
| 514 | + * [6] check_exit_conditions_2() protected by task->pi_lock and | |
| 515 | + * lock->wait_lock. | |
| 516 | + * | |
| 446 | 517 | * Deadlock detection. If the lock is the same as the original |
| 447 | 518 | * lock which caused us to walk the lock chain or if the |
| 448 | 519 | * current lock is owned by the task which initiated the chain |
| @@ -462,24 +533,27 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 13 unmodified lines | ||
| 462 | 533 | */ |
| 463 | 534 | prerequeue_top_waiter = rt_mutex_top_waiter(lock); |
| 464 | 535 | |
| 465 | − /* Requeue the waiter in the lock waiter list. */ | |
| 536 | + /* [7] Requeue the waiter in the lock waiter list. */ | |
| 466 | 537 | rt_mutex_dequeue(lock, waiter); |
| 467 | 538 | waiter->prio = task->prio; |
| 468 | 539 | rt_mutex_enqueue(lock, waiter); |
| 469 | 540 | |
| 470 | − /* Release the task */ | |
| 541 | + /* [8] Release the task */ | |
| 471 | 542 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
| 472 | 543 | put_task_struct(task); |
| 473 | 544 | |
| 474 | 545 | /* |
| 546 | + * [9] check_exit_conditions_3 protected by lock->wait_lock. | |
| 547 | + * | |
| 475 | 548 | * We must abort the chain walk if there is no lock owner even |
| 476 | 549 | * in the dead lock detection case, as we have nothing to |
| 477 | 550 | * follow here. This is the end of the chain we are walking. |
| 478 | 551 | */ |
| 479 | 552 | if (!rt_mutex_owner(lock)) { |
| 480 | 553 | /* |
| 481 | − * If the requeue above changed the top waiter, then we need | |
| 482 | − * to wake the new top waiter up to try to get the lock. | |
| 554 | + * If the requeue [7] above changed the top waiter, | |
| 555 | + * then we need to wake the new top waiter up to try | |
| 556 | + * to get the lock. | |
| 483 | 557 | */ |
| 484 | 558 | if (prerequeue_top_waiter != rt_mutex_top_waiter(lock)) |
| 485 | 559 | wake_up_process(rt_mutex_top_waiter(lock)->task); |
| @@ -487,11 +561,12 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 1 unmodified line | ||
| 487 | 561 | return 0; |
| 488 | 562 | } |
| 489 | 563 | |
| 490 | − /* Grab the next task, i.e. the owner of @lock */ | |
| 564 | + /* [10] Grab the next task, i.e. the owner of @lock */ | |
| 491 | 565 | task = rt_mutex_owner(lock); |
| 492 | 566 | get_task_struct(task); |
| 493 | 567 | raw_spin_lock_irqsave(&task->pi_lock, flags); |
| 494 | 568 | |
| 569 | + /* [11] requeue the pi waiters if necessary */ | |
| 495 | 570 | if (waiter == rt_mutex_top_waiter(lock)) { |
| 496 | 571 | /* |
| 497 | 572 | * The waiter became the new top (highest priority) |
| @@ -526,23 +601,30 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 28 unmodified lines | ||
| 526 | 601 | } |
| 527 | 602 | |
| 528 | 603 | /* |
| 604 | + * [12] check_exit_conditions_4() protected by task->pi_lock | |
| 605 | + * and lock->wait_lock. The actual decisions are made after we | |
| 606 | + * dropped the locks. | |
| 607 | + * | |
| 529 | 608 | * Check whether the task which owns the current lock is pi |
| 530 | 609 | * blocked itself. If yes we store a pointer to the lock for |
| 531 | 610 | * the lock chain change detection above. After we dropped |
| 532 | 611 | * task->pi_lock next_lock cannot be dereferenced anymore. |
| 533 | 612 | */ |
| 534 | 613 | next_lock = task_blocked_on_lock(task); |
| 535 | − | |
| 536 | − raw_spin_unlock_irqrestore(&task->pi_lock, flags); | |
| 537 | − | |
| 538 | 614 | /* |
| 539 | 615 | * Store the top waiter of @lock for the end of chain walk |
| 540 | 616 | * decision below. |
| 541 | 617 | */ |
| 542 | 618 | top_waiter = rt_mutex_top_waiter(lock); |
| 619 | + | |
| 620 | + /* [13] Drop the locks */ | |
| 621 | + raw_spin_unlock_irqrestore(&task->pi_lock, flags); | |
| 543 | 622 | raw_spin_unlock(&lock->wait_lock); |
| 544 | 623 | |
| 545 | 624 | /* |
| 625 | + * Make the actual exit decisions [12], based on the stored | |
| 626 | + * values. | |
| 627 | + * | |
| 546 | 628 | * We reached the end of the lock chain. Stop right here. No |
| 547 | 629 | * point to go back just to figure that out. |
| 548 | 630 | */ |