| @@ -2099,20 +2099,7 @@ queue_unlock(struct futex_hash_bucket *hb) · 2098 unmodified lines |
| 2099 | 2099 | hb_waiters_dec(hb); |
| 2100 | 2100 | } |
| 2101 | 2101 | |
| 2102 | | −/** |
| 2103 | | − * queue_me() - Enqueue the futex_q on the futex_hash_bucket |
| 2104 | | − * @q: The futex_q to enqueue |
| 2105 | | − * @hb: The destination hash bucket |
| 2106 | | − * |
| 2107 | | − * The hb->lock must be held by the caller, and is released here. A call to |
| 2108 | | − * queue_me() is typically paired with exactly one call to unqueue_me(). The |
| 2109 | | − * exceptions involve the PI related operations, which may use unqueue_me_pi() |
| 2110 | | − * or nothing if the unqueue is done as part of the wake process and the unqueue |
| 2111 | | − * state is implicit in the state of woken task (see futex_wait_requeue_pi() for |
| 2112 | | − * an example). |
| 2113 | | − */ |
| 2114 | | −static inline void queue_me(struct futex_q *q, struct futex_hash_bucket *hb) |
| 2115 | | − __releases(&hb->lock) |
| 2102 | +static inline void __queue_me(struct futex_q *q, struct futex_hash_bucket *hb) |
| 2116 | 2103 | { |
| 2117 | 2104 | int prio; |
| 2118 | 2105 | |
| @@ -2129,6 +2116,24 @@ static inline void queue_me(struct futex_q *q, struct futex_hash_bucket *hb) · 10 unmodified lines |
| 2129 | 2116 | plist_node_init(&q->list, prio); |
| 2130 | 2117 | plist_add(&q->list, &hb->chain); |
| 2131 | 2118 | q->task = current; |
| 2119 | +} |
| 2120 | + |
| 2121 | +/** |
| 2122 | + * queue_me() - Enqueue the futex_q on the futex_hash_bucket |
| 2123 | + * @q: The futex_q to enqueue |
| 2124 | + * @hb: The destination hash bucket |
| 2125 | + * |
| 2126 | + * The hb->lock must be held by the caller, and is released here. A call to |
| 2127 | + * queue_me() is typically paired with exactly one call to unqueue_me(). The |
| 2128 | + * exceptions involve the PI related operations, which may use unqueue_me_pi() |
| 2129 | + * or nothing if the unqueue is done as part of the wake process and the unqueue |
| 2130 | + * state is implicit in the state of woken task (see futex_wait_requeue_pi() for |
| 2131 | + * an example). |
| 2132 | + */ |
| 2133 | +static inline void queue_me(struct futex_q *q, struct futex_hash_bucket *hb) |
| 2134 | + __releases(&hb->lock) |
| 2135 | +{ |
| 2136 | + __queue_me(q, hb); |
| 2132 | 2137 | spin_unlock(&hb->lock); |
| 2133 | 2138 | } |
| 2134 | 2139 | |
| @@ -2587,6 +2592,7 @@ static int futex_lock_pi(u32 __user *uaddr, unsigned int flags, · 452 unmodified lines |
| 2587 | 2592 | { |
| 2588 | 2593 | struct hrtimer_sleeper timeout, *to = NULL; |
| 2589 | 2594 | struct futex_pi_state *pi_state = NULL; |
| 2595 | + struct rt_mutex_waiter rt_waiter; |
| 2590 | 2596 | struct futex_hash_bucket *hb; |
| 2591 | 2597 | struct futex_q q = futex_q_init; |
| 2592 | 2598 | int res, ret; |
| @@ -2639,24 +2645,51 @@ retry_private: · 46 unmodified lines |
| 2639 | 2645 | } |
| 2640 | 2646 | } |
| 2641 | 2647 | |
| 2648 | + WARN_ON(!q.pi_state); |
| 2649 | + |
| 2642 | 2650 | /* |
| 2643 | 2651 | * Only actually queue now that the atomic ops are done: |
| 2644 | 2652 | */ |
| 2645 | | − queue_me(&q, hb); |
| 2653 | + __queue_me(&q, hb); |
| 2646 | 2654 | |
| 2647 | | − WARN_ON(!q.pi_state); |
| 2648 | | − /* |
| 2649 | | − * Block on the PI mutex: |
| 2650 | | − */ |
| 2651 | | − if (!trylock) { |
| 2652 | | − ret = rt_mutex_timed_futex_lock(&q.pi_state->pi_mutex, to); |
| 2653 | | − } else { |
| 2655 | + if (trylock) { |
| 2654 | 2656 | ret = rt_mutex_futex_trylock(&q.pi_state->pi_mutex); |
| 2655 | 2657 | /* Fixup the trylock return value: */ |
| 2656 | 2658 | ret = ret ? 0 : -EWOULDBLOCK; |
| 2659 | + goto no_block; |
| 2660 | + } |
| 2661 | + |
| 2662 | + /* |
| 2663 | + * We must add ourselves to the rt_mutex waitlist while holding hb->lock |
| 2664 | + * such that the hb and rt_mutex wait lists match. |
| 2665 | + */ |
| 2666 | + rt_mutex_init_waiter(&rt_waiter); |
| 2667 | + ret = rt_mutex_start_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter, current); |
| 2668 | + if (ret) { |
| 2669 | + if (ret == 1) |
| 2670 | + ret = 0; |
| 2671 | + |
| 2672 | + goto no_block; |
| 2657 | 2673 | } |
| 2658 | 2674 | |
| 2675 | + spin_unlock(q.lock_ptr); |
| 2676 | + |
| 2677 | + if (unlikely(to)) |
| 2678 | + hrtimer_start_expires(&to->timer, HRTIMER_MODE_ABS); |
| 2679 | + |
| 2680 | + ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter); |
| 2681 | + |
| 2659 | 2682 | spin_lock(q.lock_ptr); |
| 2683 | + /* |
| 2684 | + * If we failed to acquire the lock (signal/timeout), we must |
| 2685 | + * first acquire the hb->lock before removing the lock from the |
| 2686 | + * rt_mutex waitqueue, such that we can keep the hb and rt_mutex |
| 2687 | + * wait lists consistent. |
| 2688 | + */ |
| 2689 | + if (ret && !rt_mutex_cleanup_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter)) |
| 2690 | + ret = 0; |
| 2691 | + |
| 2692 | +no_block: |
| 2660 | 2693 | /* |
| 2661 | 2694 | * Fixup the pi_state owner and possibly acquire the lock if we |
| 2662 | 2695 | * haven't already. |