| @@ -83,6 +83,47 @@ static inline void mark_rt_mutex_waiters(struct rt_mutex *lock) · 82 unmodified lines |
| 83 | owner = *p; | 83 | owner = *p; |
| 84 | } while (cmpxchg(p, owner, owner | RT_MUTEX_HAS_WAITERS) != owner); | 84 | } while (cmpxchg(p, owner, owner | RT_MUTEX_HAS_WAITERS) != owner); |
| 85 | } | 85 | } |
| | 86 | |
| | 87 | /* |
| | 88 | * Safe fastpath aware unlock: |
| | 89 | * 1) Clear the waiters bit |
| | 90 | * 2) Drop lock->wait_lock |
| | 91 | * 3) Try to unlock the lock with cmpxchg |
| | 92 | */ |
| | 93 | static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock) |
| | 94 | __releases(lock->wait_lock) |
| | 95 | { |
| | 96 | struct task_struct *owner = rt_mutex_owner(lock); |
| | 97 | |
| | 98 | clear_rt_mutex_waiters(lock); |
| | 99 | raw_spin_unlock(&lock->wait_lock); |
| | 100 | /* |
| | 101 | * If a new waiter comes in between the unlock and the cmpxchg |
| | 102 | * we have two situations: |
| | 103 | * |
| | 104 | * unlock(wait_lock); |
| | 105 | * lock(wait_lock); |
| | 106 | * cmpxchg(p, owner, 0) == owner |
| | 107 | * mark_rt_mutex_waiters(lock); |
| | 108 | * acquire(lock); |
| | 109 | * or: |
| | 110 | * |
| | 111 | * unlock(wait_lock); |
| | 112 | * lock(wait_lock); |
| | 113 | * mark_rt_mutex_waiters(lock); |
| | 114 | * |
| | 115 | * cmpxchg(p, owner, 0) != owner |
| | 116 | * enqueue_waiter(); |
| | 117 | * unlock(wait_lock); |
| | 118 | * lock(wait_lock); |
| | 119 | * wake waiter(); |
| | 120 | * unlock(wait_lock); |
| | 121 | * lock(wait_lock); |
| | 122 | * acquire(lock); |
| | 123 | */ |
| | 124 | return rt_mutex_cmpxchg(lock, owner, NULL); |
| | 125 | } |
| | 126 | |
| 86 | #else | 127 | #else |
| 87 | # define rt_mutex_cmpxchg(l,c,n) (0) | 128 | # define rt_mutex_cmpxchg(l,c,n) (0) |
| 88 | static inline void mark_rt_mutex_waiters(struct rt_mutex *lock) | 129 | static inline void mark_rt_mutex_waiters(struct rt_mutex *lock) |
| @@ -90,6 +131,17 @@ static inline void mark_rt_mutex_waiters(struct rt_mutex *lock) · 1 unmodified line |
| 90 | lock->owner = (struct task_struct *) | 131 | lock->owner = (struct task_struct *) |
| 91 | ((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS); | 132 | ((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS); |
| 92 | } | 133 | } |
| | 134 | |
| | 135 | /* |
| | 136 | * Simple slow path only version: lock->owner is protected by lock->wait_lock. |
| | 137 | */ |
| | 138 | static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock) |
| | 139 | __releases(lock->wait_lock) |
| | 140 | { |
| | 141 | lock->owner = NULL; |
| | 142 | raw_spin_unlock(&lock->wait_lock); |
| | 143 | return true; |
| | 144 | } |
| 93 | #endif | 145 | #endif |
| 94 | | 146 | |
| 95 | static inline int | 147 | static inline int |
| @@ -260,27 +312,36 @@ static void rt_mutex_adjust_prio(struct task_struct *task) · 164 unmodified lines |
| 260 | */ | 312 | */ |
| 261 | int max_lock_depth = 1024; | 313 | int max_lock_depth = 1024; |
| 262 | | 314 | |
| | 315 | static inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p) |
| | 316 | { |
| | 317 | return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL; |
| | 318 | } |
| | 319 | |
| 263 | /* | 320 | /* |
| 264 | * Adjust the priority chain. Also used for deadlock detection. | 321 | * Adjust the priority chain. Also used for deadlock detection. |
| 265 | * Decreases task's usage by one - may thus free the task. | 322 | * Decreases task's usage by one - may thus free the task. |
| 266 | * | 323 | * |
| 267 | * @task: the task owning the mutex (owner) for which a chain walk is probably | 324 | * @task: the task owning the mutex (owner) for which a chain walk is |
| 268 | * needed | 325 | * probably needed |
| 269 | * @deadlock_detect: do we have to carry out deadlock detection? | 326 | * @deadlock_detect: do we have to carry out deadlock detection? |
| 270 | * @orig_lock: the mutex (can be NULL if we are walking the chain to recheck | 327 | * @orig_lock: the mutex (can be NULL if we are walking the chain to recheck |
| 271 | * things for a task that has just got its priority adjusted, and | 328 | * things for a task that has just got its priority adjusted, and |
| 272 | * is waiting on a mutex) | 329 | * is waiting on a mutex) |
| | 330 | * @next_lock: the mutex on which the owner of @orig_lock was blocked before |
| | 331 | * we dropped its pi_lock. Is never dereferenced, only used for |
| | 332 | * comparison to detect lock chain changes. |
| 273 | * @orig_waiter: rt_mutex_waiter struct for the task that has just donated | 333 | * @orig_waiter: rt_mutex_waiter struct for the task that has just donated |
| 274 | * its priority to the mutex owner (can be NULL in the case | 334 | * its priority to the mutex owner (can be NULL in the case |
| 275 | * depicted above or if the top waiter is gone away and we are | 335 | * depicted above or if the top waiter is gone away and we are |
| 276 | * actually deboosting the owner) | 336 | * actually deboosting the owner) |
| 277 | * @top_task: the current top waiter | 337 | * @top_task: the current top waiter |
| 278 | * | 338 | * |
| 279 | * Returns 0 or -EDEADLK. | 339 | * Returns 0 or -EDEADLK. |
| 280 | */ | 340 | */ |
| 281 | static int rt_mutex_adjust_prio_chain(struct task_struct *task, | 341 | static int rt_mutex_adjust_prio_chain(struct task_struct *task, |
| 282 | int deadlock_detect, | 342 | int deadlock_detect, |
| 283 | struct rt_mutex *orig_lock, | 343 | struct rt_mutex *orig_lock, |
| | 344 | struct rt_mutex *next_lock, |
| 284 | struct rt_mutex_waiter *orig_waiter, | 345 | struct rt_mutex_waiter *orig_waiter, |
| 285 | struct task_struct *top_task) | 346 | struct task_struct *top_task) |
| 286 | { | 347 | { |
| @@ -314,7 +375,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 27 unmodified lines |
| 314 | } | 375 | } |
| 315 | put_task_struct(task); | 376 | put_task_struct(task); |
| 316 | | 377 | |
| 317 | return deadlock_detect ? -EDEADLK : 0; | 378 | return -EDEADLK; |
| 318 | } | 379 | } |
| 319 | retry: | 380 | retry: |
| 320 | /* | 381 | /* |
| @@ -338,6 +399,18 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 17 unmodified lines |
| 338 | if (orig_waiter && !rt_mutex_owner(orig_lock)) | 399 | if (orig_waiter && !rt_mutex_owner(orig_lock)) |
| 339 | goto out_unlock_pi; | 400 | goto out_unlock_pi; |
| 340 | | 401 | |
| | 402 | /* |
| | 403 | * We dropped all locks after taking a refcount on @task, so |
| | 404 | * the task might have moved on in the lock chain or even left |
| | 405 | * the chain completely and blocks now on an unrelated lock or |
| | 406 | * on @orig_lock. |
| | 407 | * |
| | 408 | * We stored the lock on which @task was blocked in @next_lock, |
| | 409 | * so we can detect the chain change. |
| | 410 | */ |
| | 411 | if (next_lock != waiter->lock) |
| | 412 | goto out_unlock_pi; |
| | 413 | |
| 341 | /* | 414 | /* |
| 342 | * Drop out, when the task has no waiters. Note, | 415 | * Drop out, when the task has no waiters. Note, |
| 343 | * top_waiter can be NULL, when we are in the deboosting | 416 | * top_waiter can be NULL, when we are in the deboosting |
| @@ -377,7 +450,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 33 unmodified lines |
| 377 | if (lock == orig_lock || rt_mutex_owner(lock) == top_task) { | 450 | if (lock == orig_lock || rt_mutex_owner(lock) == top_task) { |
| 378 | debug_rt_mutex_deadlock(deadlock_detect, orig_waiter, lock); | 451 | debug_rt_mutex_deadlock(deadlock_detect, orig_waiter, lock); |
| 379 | raw_spin_unlock(&lock->wait_lock); | 452 | raw_spin_unlock(&lock->wait_lock); |
| 380 | ret = deadlock_detect ? -EDEADLK : 0; | 453 | ret = -EDEADLK; |
| 381 | goto out_unlock_pi; | 454 | goto out_unlock_pi; |
| 382 | } | 455 | } |
| 383 | | 456 | |
| @@ -422,11 +495,26 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 38 unmodified lines |
| 422 | __rt_mutex_adjust_prio(task); | 495 | __rt_mutex_adjust_prio(task); |
| 423 | } | 496 | } |
| 424 | | 497 | |
| | 498 | /* |
| | 499 | * Check whether the task which owns the current lock is pi |
| | 500 | * blocked itself. If yes we store a pointer to the lock for |
| | 501 | * the lock chain change detection above. After we dropped |
| | 502 | * task->pi_lock next_lock cannot be dereferenced anymore. |
| | 503 | */ |
| | 504 | next_lock = task_blocked_on_lock(task); |
| | 505 | |
| 425 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); | 506 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
| 426 | | 507 | |
| 427 | top_waiter = rt_mutex_top_waiter(lock); | 508 | top_waiter = rt_mutex_top_waiter(lock); |
| 428 | raw_spin_unlock(&lock->wait_lock); | 509 | raw_spin_unlock(&lock->wait_lock); |
| 429 | | 510 | |
| | 511 | /* |
| | 512 | * We reached the end of the lock chain. Stop right here. No |
| | 513 | * point to go back just to figure that out. |
| | 514 | */ |
| | 515 | if (!next_lock) |
| | 516 | goto out_put_task; |
| | 517 | |
| 430 | if (!detect_deadlock && waiter != top_waiter) | 518 | if (!detect_deadlock && waiter != top_waiter) |
| 431 | goto out_put_task; | 519 | goto out_put_task; |
| 432 | | 520 | |
| @@ -536,8 +624,9 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, · 103 unmodified lines |
| 536 | { | 624 | { |
| 537 | struct task_struct *owner = rt_mutex_owner(lock); | 625 | struct task_struct *owner = rt_mutex_owner(lock); |
| 538 | struct rt_mutex_waiter *top_waiter = waiter; | 626 | struct rt_mutex_waiter *top_waiter = waiter; |
| 539 | unsigned long flags; | 627 | struct rt_mutex *next_lock; |
| 540 | int chain_walk = 0, res; | 628 | int chain_walk = 0, res; |
| | 629 | unsigned long flags; |
| 541 | | 630 | |
| 542 | /* | 631 | /* |
| 543 | * Early deadlock detection. We really don't want the task to | 632 | * Early deadlock detection. We really don't want the task to |
| @@ -548,7 +637,7 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, · 4 unmodified lines |
| 548 | * which is wrong, as the other waiter is not in a deadlock | 637 | * which is wrong, as the other waiter is not in a deadlock |
| 549 | * situation. | 638 | * situation. |
| 550 | */ | 639 | */ |
| 551 | if (detect_deadlock && owner == task) | 640 | if (owner == task) |
| 552 | return -EDEADLK; | 641 | return -EDEADLK; |
| 553 | | 642 | |
| 554 | raw_spin_lock_irqsave(&task->pi_lock, flags); | 643 | raw_spin_lock_irqsave(&task->pi_lock, flags); |
| @@ -569,20 +658,28 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, · 14 unmodified lines |
| 569 | if (!owner) | 658 | if (!owner) |
| 570 | return 0; | 659 | return 0; |
| 571 | | 660 | |
| | 661 | raw_spin_lock_irqsave(&owner->pi_lock, flags); |
| 572 | if (waiter == rt_mutex_top_waiter(lock)) { | 662 | if (waiter == rt_mutex_top_waiter(lock)) { |
| 573 | raw_spin_lock_irqsave(&owner->pi_lock, flags); | | |
| 574 | rt_mutex_dequeue_pi(owner, top_waiter); | 663 | rt_mutex_dequeue_pi(owner, top_waiter); |
| 575 | rt_mutex_enqueue_pi(owner, waiter); | 664 | rt_mutex_enqueue_pi(owner, waiter); |
| 576 | | 665 | |
| 577 | __rt_mutex_adjust_prio(owner); | 666 | __rt_mutex_adjust_prio(owner); |
| 578 | if (owner->pi_blocked_on) | 667 | if (owner->pi_blocked_on) |
| 579 | chain_walk = 1; | 668 | chain_walk = 1; |
| 580 | raw_spin_unlock_irqrestore(&owner->pi_lock, flags); | 669 | } else if (debug_rt_mutex_detect_deadlock(waiter, detect_deadlock)) { |
| 581 | } | | |
| 582 | else if (debug_rt_mutex_detect_deadlock(waiter, detect_deadlock)) | | |
| 583 | chain_walk = 1; | 670 | chain_walk = 1; |
| | 671 | } |
| 584 | | 672 | |
| 585 | if (!chain_walk) | 673 | /* Store the lock on which owner is blocked or NULL */ |
| | 674 | next_lock = task_blocked_on_lock(owner); |
| | 675 | |
| | 676 | raw_spin_unlock_irqrestore(&owner->pi_lock, flags); |
| | 677 | /* |
| | 678 | * Even if full deadlock detection is on, if the owner is not |
| | 679 | * blocked itself, we can avoid finding this out in the chain |
| | 680 | * walk. |
| | 681 | */ |
| | 682 | if (!chain_walk || !next_lock) |
| 586 | return 0; | 683 | return 0; |
| 587 | | 684 | |
| 588 | /* | 685 | /* |
| @@ -594,8 +691,8 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, · 5 unmodified lines |
| 594 | | 691 | |
| 595 | raw_spin_unlock(&lock->wait_lock); | 692 | raw_spin_unlock(&lock->wait_lock); |
| 596 | | 693 | |
| 597 | res = rt_mutex_adjust_prio_chain(owner, detect_deadlock, lock, waiter, | 694 | res = rt_mutex_adjust_prio_chain(owner, detect_deadlock, lock, |
| 598 | task); | 695 | next_lock, waiter, task); |
| 599 | | 696 | |
| 600 | raw_spin_lock(&lock->wait_lock); | 697 | raw_spin_lock(&lock->wait_lock); |
| 601 | | 698 | |
| @@ -605,7 +702,8 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, · 3 unmodified lines |
| 605 | /* | 702 | /* |
| 606 | * Wake up the next waiter on the lock. | 703 | * Wake up the next waiter on the lock. |
| 607 | * | 704 | * |
| 608 | * Remove the top waiter from the current tasks waiter list and wake it up. | 705 | * Remove the top waiter from the current tasks pi waiter list and |
| | 706 | * wake it up. |
| 609 | * | 707 | * |
| 610 | * Called with lock->wait_lock held. | 708 | * Called with lock->wait_lock held. |
| 611 | */ | 709 | */ |
| @@ -626,10 +724,23 @@ static void wakeup_next_waiter(struct rt_mutex *lock) · 14 unmodified lines |
| 626 | */ | 724 | */ |
| 627 | rt_mutex_dequeue_pi(current, waiter); | 725 | rt_mutex_dequeue_pi(current, waiter); |
| 628 | | 726 | |
| 629 | rt_mutex_set_owner(lock, NULL); | 727 | /* |
| | 728 | * As we are waking up the top waiter, and the waiter stays |
| | 729 | * queued on the lock until it gets the lock, this lock |
| | 730 | * obviously has waiters. Just set the bit here and this has |
| | 731 | * the added benefit of forcing all new tasks into the |
| | 732 | * slow path making sure no task of lower priority than |
| | 733 | * the top waiter can steal this lock. |
| | 734 | */ |
| | 735 | lock->owner = (void *) RT_MUTEX_HAS_WAITERS; |
| 630 | | 736 | |
| 631 | raw_spin_unlock_irqrestore(¤t->pi_lock, flags); | 737 | raw_spin_unlock_irqrestore(¤t->pi_lock, flags); |
| 632 | | 738 | |
| | 739 | /* |
| | 740 | * It's safe to dereference waiter as it cannot go away as |
| | 741 | * long as we hold lock->wait_lock. The waiter task needs to |
| | 742 | * acquire it in order to dequeue the waiter. |
| | 743 | */ |
| 633 | wake_up_process(waiter->task); | 744 | wake_up_process(waiter->task); |
| 634 | } | 745 | } |
| 635 | | 746 | |
| @@ -644,8 +755,8 @@ static void remove_waiter(struct rt_mutex *lock, · 8 unmodified lines |
| 644 | { | 755 | { |
| 645 | int first = (waiter == rt_mutex_top_waiter(lock)); | 756 | int first = (waiter == rt_mutex_top_waiter(lock)); |
| 646 | struct task_struct *owner = rt_mutex_owner(lock); | 757 | struct task_struct *owner = rt_mutex_owner(lock); |
| | 758 | struct rt_mutex *next_lock = NULL; |
| 647 | unsigned long flags; | 759 | unsigned long flags; |
| 648 | int chain_walk = 0; | | |
| 649 | | 760 | |
| 650 | raw_spin_lock_irqsave(¤t->pi_lock, flags); | 761 | raw_spin_lock_irqsave(¤t->pi_lock, flags); |
| 651 | rt_mutex_dequeue(lock, waiter); | 762 | rt_mutex_dequeue(lock, waiter); |
| @@ -669,13 +780,13 @@ static void remove_waiter(struct rt_mutex *lock, · 17 unmodified lines |
| 669 | } | 780 | } |
| 670 | __rt_mutex_adjust_prio(owner); | 781 | __rt_mutex_adjust_prio(owner); |
| 671 | | 782 | |
| 672 | if (owner->pi_blocked_on) | 783 | /* Store the lock on which owner is blocked or NULL */ |
| 673 | chain_walk = 1; | 784 | next_lock = task_blocked_on_lock(owner); |
| 674 | | 785 | |
| 675 | raw_spin_unlock_irqrestore(&owner->pi_lock, flags); | 786 | raw_spin_unlock_irqrestore(&owner->pi_lock, flags); |
| 676 | } | 787 | } |
| 677 | | 788 | |
| 678 | if (!chain_walk) | 789 | if (!next_lock) |
| 679 | return; | 790 | return; |
| 680 | | 791 | |
| 681 | /* gets dropped in rt_mutex_adjust_prio_chain()! */ | 792 | /* gets dropped in rt_mutex_adjust_prio_chain()! */ |
| @@ -683,7 +794,7 @@ static void remove_waiter(struct rt_mutex *lock, · 1 unmodified line |
| 683 | | 794 | |
| 684 | raw_spin_unlock(&lock->wait_lock); | 795 | raw_spin_unlock(&lock->wait_lock); |
| 685 | | 796 | |
| 686 | rt_mutex_adjust_prio_chain(owner, 0, lock, NULL, current); | 797 | rt_mutex_adjust_prio_chain(owner, 0, lock, next_lock, NULL, current); |
| 687 | | 798 | |
| 688 | raw_spin_lock(&lock->wait_lock); | 799 | raw_spin_lock(&lock->wait_lock); |
| 689 | } | 800 | } |
| @@ -696,6 +807,7 @@ static void remove_waiter(struct rt_mutex *lock, · 6 unmodified lines |
| 696 | void rt_mutex_adjust_pi(struct task_struct *task) | 807 | void rt_mutex_adjust_pi(struct task_struct *task) |
| 697 | { | 808 | { |
| 698 | struct rt_mutex_waiter *waiter; | 809 | struct rt_mutex_waiter *waiter; |
| | 810 | struct rt_mutex *next_lock; |
| 699 | unsigned long flags; | 811 | unsigned long flags; |
| 700 | | 812 | |
| 701 | raw_spin_lock_irqsave(&task->pi_lock, flags); | 813 | raw_spin_lock_irqsave(&task->pi_lock, flags); |
| @@ -706,12 +818,13 @@ void rt_mutex_adjust_pi(struct task_struct *task) · 4 unmodified lines |
| 706 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); | 818 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
| 707 | return; | 819 | return; |
| 708 | } | 820 | } |
| 709 | | 821 | next_lock = waiter->lock; |
| 710 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); | 822 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
| 711 | | 823 | |
| 712 | /* gets dropped in rt_mutex_adjust_prio_chain()! */ | 824 | /* gets dropped in rt_mutex_adjust_prio_chain()! */ |
| 713 | get_task_struct(task); | 825 | get_task_struct(task); |
| 714 | rt_mutex_adjust_prio_chain(task, 0, NULL, NULL, task); | 826 | |
| | 827 | rt_mutex_adjust_prio_chain(task, 0, NULL, next_lock, NULL, task); |
| 715 | } | 828 | } |
| 716 | | 829 | |
| 717 | /** | 830 | /** |
| @@ -763,6 +876,26 @@ __rt_mutex_slowlock(struct rt_mutex *lock, int state, · 45 unmodified lines |
| 763 | return ret; | 876 | return ret; |
| 764 | } | 877 | } |
| 765 | | 878 | |
| | 879 | static void rt_mutex_handle_deadlock(int res, int detect_deadlock, |
| | 880 | struct rt_mutex_waiter *w) |
| | 881 | { |
| | 882 | /* |
| | 883 | * If the result is not -EDEADLOCK or the caller requested |
| | 884 | * deadlock detection, nothing to do here. |
| | 885 | */ |
| | 886 | if (res != -EDEADLOCK || detect_deadlock) |
| | 887 | return; |
| | 888 | |
| | 889 | /* |
| | 890 | * Yell lowdly and stop the task right here. |
| | 891 | */ |
| | 892 | rt_mutex_print_deadlock(w); |
| | 893 | while (1) { |
| | 894 | set_current_state(TASK_INTERRUPTIBLE); |
| | 895 | schedule(); |
| | 896 | } |
| | 897 | } |
| | 898 | |
| 766 | /* | 899 | /* |
| 767 | * Slow path lock function: | 900 | * Slow path lock function: |
| 768 | */ | 901 | */ |
| @@ -802,8 +935,10 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state, · 33 unmodified lines |
| 802 | | 935 | |
| 803 | set_current_state(TASK_RUNNING); | 936 | set_current_state(TASK_RUNNING); |
| 804 | | 937 | |
| 805 | if (unlikely(ret)) | 938 | if (unlikely(ret)) { |
| 806 | remove_waiter(lock, &waiter); | 939 | remove_waiter(lock, &waiter); |
| | 940 | rt_mutex_handle_deadlock(ret, detect_deadlock, &waiter); |
| | 941 | } |
| 807 | | 942 | |
| 808 | /* | 943 | /* |
| 809 | * try_to_take_rt_mutex() sets the waiter bit | 944 | * try_to_take_rt_mutex() sets the waiter bit |
| @@ -859,12 +994,49 @@ rt_mutex_slowunlock(struct rt_mutex *lock) · 49 unmodified lines |
| 859 | | 994 | |
| 860 | rt_mutex_deadlock_account_unlock(current); | 995 | rt_mutex_deadlock_account_unlock(current); |
| 861 | | 996 | |
| 862 | if (!rt_mutex_has_waiters(lock)) { | 997 | /* |
| 863 | lock->owner = NULL; | 998 | * We must be careful here if the fast path is enabled. If we |
| 864 | raw_spin_unlock(&lock->wait_lock); | 999 | * have no waiters queued we cannot set owner to NULL here |
| 865 | return; | 1000 | * because of: |
| | 1001 | * |
| | 1002 | * foo->lock->owner = NULL; |
| | 1003 | * rtmutex_lock(foo->lock); <- fast path |
| | 1004 | * free = atomic_dec_and_test(foo->refcnt); |
| | 1005 | * rtmutex_unlock(foo->lock); <- fast path |
| | 1006 | * if (free) |
| | 1007 | * kfree(foo); |
| | 1008 | * raw_spin_unlock(foo->lock->wait_lock); |
| | 1009 | * |
| | 1010 | * So for the fastpath enabled kernel: |
| | 1011 | * |
| | 1012 | * Nothing can set the waiters bit as long as we hold |
| | 1013 | * lock->wait_lock. So we do the following sequence: |
| | 1014 | * |
| | 1015 | * owner = rt_mutex_owner(lock); |
| | 1016 | * clear_rt_mutex_waiters(lock); |
| | 1017 | * raw_spin_unlock(&lock->wait_lock); |
| | 1018 | * if (cmpxchg(&lock->owner, owner, 0) == owner) |
| | 1019 | * return; |
| | 1020 | * goto retry; |
| | 1021 | * |
| | 1022 | * The fastpath disabled variant is simple as all access to |
| | 1023 | * lock->owner is serialized by lock->wait_lock: |
| | 1024 | * |
| | 1025 | * lock->owner = NULL; |
| | 1026 | * raw_spin_unlock(&lock->wait_lock); |
| | 1027 | */ |
| | 1028 | while (!rt_mutex_has_waiters(lock)) { |
| | 1029 | /* Drops lock->wait_lock ! */ |
| | 1030 | if (unlock_rt_mutex_safe(lock) == true) |
| | 1031 | return; |
| | 1032 | /* Relock the rtmutex and try again */ |
| | 1033 | raw_spin_lock(&lock->wait_lock); |
| 866 | } | 1034 | } |
| 867 | | 1035 | |
| | 1036 | /* |
| | 1037 | * The wakeup next waiter path does not suffer from the above |
| | 1038 | * race. See the comments there. |
| | 1039 | */ |
| 868 | wakeup_next_waiter(lock); | 1040 | wakeup_next_waiter(lock); |
| 869 | | 1041 | |
| 870 | raw_spin_unlock(&lock->wait_lock); | 1042 | raw_spin_unlock(&lock->wait_lock); |
| @@ -1112,7 +1284,8 @@ int rt_mutex_start_proxy_lock(struct rt_mutex *lock, · 241 unmodified lines |
| 1112 | return 1; | 1284 | return 1; |
| 1113 | } | 1285 | } |
| 1114 | | 1286 | |
| 1115 | ret = task_blocks_on_rt_mutex(lock, waiter, task, detect_deadlock); | 1287 | /* We enforce deadlock detection for futexes */ |
| | 1288 | ret = task_blocks_on_rt_mutex(lock, waiter, task, 1); |
| 1116 | | 1289 | |
| 1117 | if (ret && !rt_mutex_owner(lock)) { | 1290 | if (ret && !rt_mutex_owner(lock)) { |
| 1118 | /* | 1291 | /* |