By moving the initial blkg lookup into blkg_tryget_closest we get a nicely self contained routines that does all the RCU locking.
Unified · Split
| @@ -1716,19 +1716,20 @@ void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta) · 1715 unmodified lines | ||
| 1716 | 1716 | |
| 1717 | 1717 | /** |
| 1718 | 1718 | * blkg_tryget_closest - try and get a blkg ref on the closet blkg |
| 1719 | − * @blkg: blkg to get | |
| 1719 | + * @bio: target bio | |
| 1720 | + * @css: target css | |
| 1720 | 1721 | * |
| 1721 | − * This needs to be called rcu protected. As the failure mode here is to walk | |
| 1722 | − * up the blkg tree, this ensure that the blkg->parent pointers are always | |
| 1723 | − * valid. This returns the blkg that it ended up taking a reference on or %NULL | |
| 1724 | − * if no reference was taken. | |
| 1722 | + * As the failure mode here is to walk up the blkg tree, this ensure that the | |
| 1723 | + * blkg->parent pointers are always valid. This returns the blkg that it ended | |
| 1724 | + * up taking a reference on or %NULL if no reference was taken. | |
| 1725 | 1725 | */ |
| 1726 | −static inline struct blkcg_gq *blkg_tryget_closest(struct blkcg_gq *blkg) | |
| 1726 | +static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio, | |
| 1727 | + struct cgroup_subsys_state *css) | |
| 1727 | 1728 | { |
| 1728 | − struct blkcg_gq *ret_blkg = NULL; | |
| 1729 | − | |
| 1730 | − WARN_ON_ONCE(!rcu_read_lock_held()); | |
| 1729 | + struct blkcg_gq *blkg, *ret_blkg = NULL; | |
| 1731 | 1730 | |
| 1731 | + rcu_read_lock(); | |
| 1732 | + blkg = blkg_lookup_create(css_to_blkcg(css), bio->bi_disk->queue); | |
| 1732 | 1733 | while (blkg) { |
| 1733 | 1734 | if (blkg_tryget(blkg)) { |
| 1734 | 1735 | ret_blkg = blkg; |
| @@ -1736,6 +1737,7 @@ static inline struct blkcg_gq *blkg_tryget_closest(struct blkcg_gq *blkg) · 1 unmodified line | ||
| 1736 | 1737 | } |
| 1737 | 1738 | blkg = blkg->parent; |
| 1738 | 1739 | } |
| 1740 | + rcu_read_unlock(); | |
| 1739 | 1741 | |
| 1740 | 1742 | return ret_blkg; |
| 1741 | 1743 | } |
| @@ -1757,21 +1759,14 @@ static inline struct blkcg_gq *blkg_tryget_closest(struct blkcg_gq *blkg) · 15 unmodified lines | ||
| 1757 | 1759 | void bio_associate_blkg_from_css(struct bio *bio, |
| 1758 | 1760 | struct cgroup_subsys_state *css) |
| 1759 | 1761 | { |
| 1760 | − struct request_queue *q = bio->bi_disk->queue; | |
| 1761 | − | |
| 1762 | 1762 | if (bio->bi_blkg) |
| 1763 | 1763 | blkg_put(bio->bi_blkg); |
| 1764 | 1764 | |
| 1765 | 1765 | if (css && css->parent) { |
| 1766 | − struct blkcg_gq *blkg; | |
| 1767 | − | |
| 1768 | − rcu_read_lock(); | |
| 1769 | − blkg = blkg_lookup_create(css_to_blkcg(css), q); | |
| 1770 | − bio->bi_blkg = blkg_tryget_closest(blkg); | |
| 1771 | − rcu_read_unlock(); | |
| 1766 | + bio->bi_blkg = blkg_tryget_closest(bio, css); | |
| 1772 | 1767 | } else { |
| 1773 | − blkg_get(q->root_blkg); | |
| 1774 | − bio->bi_blkg = q->root_blkg; | |
| 1768 | + blkg_get(bio->bi_disk->queue->root_blkg); | |
| 1769 | + bio->bi_blkg = bio->bi_disk->queue->root_blkg; | |
| 1775 | 1770 | } |
| 1776 | 1771 | } |
| 1777 | 1772 | EXPORT_SYMBOL_GPL(bio_associate_blkg_from_css); |