When a shared kthread needs to issue a bio for a cgroup, doing so synchronously can lead to priority inversions as the kthread can be trapped waiting for that cgroup. This patch implements REQ_CGROUP_PUNT flag which makes submit_bio() punt the actual issuing to a dedicated per-blkcg work item to avoid such priority inversions. This will be used to fix priority inversions in btrfs compression and should be generally useful as we grow filesystem support for comprehensive IO control.
Showing the changes to include/linux (4 of 6 files). Show all
Unified · Split
| @@ -48,6 +48,7 @@ extern spinlock_t bdi_lock; · 47 unmodified lines | |||
| 48 | extern struct list_head bdi_list; | 48 | extern struct list_head bdi_list; |
| 49 | 49 | ||
| 50 | extern struct workqueue_struct *bdi_wq; | 50 | extern struct workqueue_struct *bdi_wq; |
| 51 | extern struct workqueue_struct *bdi_async_bio_wq; | ||
| 51 | 52 | ||
| 52 | static inline bool wb_has_dirty_io(struct bdi_writeback *wb) | 53 | static inline bool wb_has_dirty_io(struct bdi_writeback *wb) |
| 53 | { | 54 | { |
| @@ -132,13 +132,17 @@ struct blkcg_gq { · 131 unmodified lines | |||
| 132 | 132 | ||
| 133 | struct blkg_policy_data *pd[BLKCG_MAX_POLS]; | 133 | struct blkg_policy_data *pd[BLKCG_MAX_POLS]; |
| 134 | 134 | ||
| 135 | struct rcu_head rcu_head; | 135 | spinlock_t async_bio_lock; |
| 136 | struct bio_list async_bios; | ||
| 137 | struct work_struct async_bio_work; | ||
| 136 | 138 | ||
| 137 | atomic_t use_delay; | 139 | atomic_t use_delay; |
| 138 | atomic64_t delay_nsec; | 140 | atomic64_t delay_nsec; |
| 139 | atomic64_t delay_start; | 141 | atomic64_t delay_start; |
| 140 | u64 last_delay; | 142 | u64 last_delay; |
| 141 | int last_use; | 143 | int last_use; |
| 144 | |||
| 145 | struct rcu_head rcu_head; | ||
| 142 | }; | 146 | }; |
| 143 | 147 | ||
| 144 | typedef struct blkcg_policy_data *(blkcg_pol_alloc_cpd_fn)(gfp_t gfp); | 148 | typedef struct blkcg_policy_data *(blkcg_pol_alloc_cpd_fn)(gfp_t gfp); |
| @@ -701,6 +705,15 @@ static inline bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg · 556 unmodified lines | |||
| 701 | struct bio *bio) { return false; } | 705 | struct bio *bio) { return false; } |
| 702 | #endif | 706 | #endif |
| 703 | 707 | ||
| 708 | bool __blkcg_punt_bio_submit(struct bio *bio); | ||
| 709 | |||
| 710 | static inline bool blkcg_punt_bio_submit(struct bio *bio) | ||
| 711 | { | ||
| 712 | if (bio->bi_opf & REQ_CGROUP_PUNT) | ||
| 713 | return __blkcg_punt_bio_submit(bio); | ||
| 714 | else | ||
| 715 | return false; | ||
| 716 | } | ||
| 704 | 717 | ||
| 705 | static inline void blkcg_bio_issue_init(struct bio *bio) | 718 | static inline void blkcg_bio_issue_init(struct bio *bio) |
| 706 | { | 719 | { |
| @@ -848,6 +861,7 @@ static inline char *blkg_path(struct blkcg_gq *blkg) { return NULL; } · 141 unmodified lines | |||
| 848 | static inline void blkg_get(struct blkcg_gq *blkg) { } | 861 | static inline void blkg_get(struct blkcg_gq *blkg) { } |
| 849 | static inline void blkg_put(struct blkcg_gq *blkg) { } | 862 | static inline void blkg_put(struct blkcg_gq *blkg) { } |
| 850 | 863 | ||
| 864 | static inline bool blkcg_punt_bio_submit(struct bio *bio) { return false; } | ||
| 851 | static inline void blkcg_bio_issue_init(struct bio *bio) { } | 865 | static inline void blkcg_bio_issue_init(struct bio *bio) { } |
| 852 | static inline bool blkcg_bio_issue_check(struct request_queue *q, | 866 | static inline bool blkcg_bio_issue_check(struct request_queue *q, |
| 853 | struct bio *bio) { return true; } | 867 | struct bio *bio) { return true; } |
| @@ -311,6 +311,14 @@ enum req_flag_bits { · 310 unmodified lines | |||
| 311 | __REQ_RAHEAD, /* read ahead, can fail anytime */ | 311 | __REQ_RAHEAD, /* read ahead, can fail anytime */ |
| 312 | __REQ_BACKGROUND, /* background IO */ | 312 | __REQ_BACKGROUND, /* background IO */ |
| 313 | __REQ_NOWAIT, /* Don't wait if request will block */ | 313 | __REQ_NOWAIT, /* Don't wait if request will block */ |
| 314 | /* | ||
| 315 | * When a shared kthread needs to issue a bio for a cgroup, doing | ||
| 316 | * so synchronously can lead to priority inversions as the kthread | ||
| 317 | * can be trapped waiting for that cgroup. CGROUP_PUNT flag makes | ||
| 318 | * submit_bio() punt the actual issuing to a dedicated per-blkcg | ||
| 319 | * work item to avoid such priority inversions. | ||
| 320 | */ | ||
| 321 | __REQ_CGROUP_PUNT, | ||
| 314 | 322 | ||
| 315 | /* command specific flags for REQ_OP_WRITE_ZEROES: */ | 323 | /* command specific flags for REQ_OP_WRITE_ZEROES: */ |
| 316 | __REQ_NOUNMAP, /* do not free blocks when zeroing */ | 324 | __REQ_NOUNMAP, /* do not free blocks when zeroing */ |
| @@ -337,6 +345,8 @@ enum req_flag_bits { · 20 unmodified lines | |||
| 337 | #define REQ_RAHEAD (1ULL << __REQ_RAHEAD) | 345 | #define REQ_RAHEAD (1ULL << __REQ_RAHEAD) |
| 338 | #define REQ_BACKGROUND (1ULL << __REQ_BACKGROUND) | 346 | #define REQ_BACKGROUND (1ULL << __REQ_BACKGROUND) |
| 339 | #define REQ_NOWAIT (1ULL << __REQ_NOWAIT) | 347 | #define REQ_NOWAIT (1ULL << __REQ_NOWAIT) |
| 348 | #define REQ_CGROUP_PUNT (1ULL << __REQ_CGROUP_PUNT) | ||
| 349 | |||
| 340 | #define REQ_NOUNMAP (1ULL << __REQ_NOUNMAP) | 350 | #define REQ_NOUNMAP (1ULL << __REQ_NOUNMAP) |
| 341 | #define REQ_HIPRI (1ULL << __REQ_HIPRI) | 351 | #define REQ_HIPRI (1ULL << __REQ_HIPRI) |
| 342 | 352 | ||
| @@ -78,6 +78,8 @@ struct writeback_control { · 77 unmodified lines | |||
| 78 | */ | 78 | */ |
| 79 | unsigned no_cgroup_owner:1; | 79 | unsigned no_cgroup_owner:1; |
| 80 | 80 | ||
| 81 | unsigned punt_to_cgroup:1; /* cgrp punting, see __REQ_CGROUP_PUNT */ | ||
| 82 | |||
| 81 | #ifdef CONFIG_CGROUP_WRITEBACK | 83 | #ifdef CONFIG_CGROUP_WRITEBACK |
| 82 | struct bdi_writeback *wb; /* wb this writeback is issued under */ | 84 | struct bdi_writeback *wb; /* wb this writeback is issued under */ |
| 83 | struct inode *inode; /* inode being written out */ | 85 | struct inode *inode; /* inode being written out */ |
| @@ -94,12 +96,17 @@ struct writeback_control { · 10 unmodified lines | |||
| 94 | 96 | ||
| 95 | static inline int wbc_to_write_flags(struct writeback_control *wbc) | 97 | static inline int wbc_to_write_flags(struct writeback_control *wbc) |
| 96 | { | 98 | { |
| 99 | int flags = 0; | ||
| 100 | |||
| 101 | if (wbc->punt_to_cgroup) | ||
| 102 | flags = REQ_CGROUP_PUNT; | ||
| 103 | |||
| 97 | if (wbc->sync_mode == WB_SYNC_ALL) | 104 | if (wbc->sync_mode == WB_SYNC_ALL) |
| 98 | return REQ_SYNC; | 105 | flags |= REQ_SYNC; |
| 99 | else if (wbc->for_kupdate || wbc->for_background) | 106 | else if (wbc->for_kupdate || wbc->for_background) |
| 100 | return REQ_BACKGROUND; | 107 | flags |= REQ_BACKGROUND; |
| 101 | 108 | ||
| 102 | return 0; | 109 | return flags; |
| 103 | } | 110 | } |
| 104 | 111 | ||
| 105 | static inline struct cgroup_subsys_state * | 112 | static inline struct cgroup_subsys_state * |