History

Merge tag 'for-6.11/block-20240722' of git://git.kernel.dk/linux

Pull more block updates from Jens Axboe: - MD fixes via Song: - md-cluster fixes (Heming Zhao) - raid1 fix (Mateusz Jończyk) - s390/dasd module description (Jeff) - Series cleaning up and hardening the blk-mq debugfs flag handling (John, Christoph) - blk-cgroup cleanup (Xiu) - Error polled IO attempts if backend doesn't support it (hexue) - Fix for an sbitmap hang (Yang) * tag 'for-6.11/block-20240722' of git://git.kernel.dk/linux: (23 commits) blk-cgroup: move congestion_count to struct blkcg sbitmap: fix io hung due to race on sbitmap_word::cleared block: avoid polling configuration errors block: Catch possible entries missing from rqf_name[] block: Simplify definition of RQF_NAME() block: Use enum to define RQF_x bit indexes block: Catch possible entries missing from cmd_flag_name[] block: Catch possible entries missing from alloc_policy_name[] block: Catch possible entries missing from hctx_flag_name[] block: Catch possible entries missing from hctx_state_name[] block: Catch possible entries missing from blk_queue_flag_name[] block: Make QUEUE_FLAG_x as an enum block: Relocate BLK_MQ_MAX_DEPTH block: Relocate BLK_MQ_CPU_WORK_BATCH block: remove QUEUE_FLAG_STOPPED block: Add missing entry to hctx_flag_name[] block: Add zone write plugging entry to rqf_name[] block: Add missing entries from cmd_flag_name[] s390/dasd: fix error checks in dasd_copy_pair_store() s390/dasd: add missing MODULE_DESCRIPTION() macros ...

Linus Torvalds committed 2 years ago

Commit 7d080fa2 parents 0256994 89ed6c919 files · +233 −102Browse files

Unified · Split

19 files
  1. block/blk-cgroup.c +4 −3
  2. block/blk-cgroup.h +6 −4
  3. block/blk-core.c +4 −1
  4. block/blk-mq-debugfs.c +21 −5
  5. block/blk-mq.h +2 −0
  6. drivers/md/md-cluster.c +39 −10
  7. drivers/md/md-cluster.h +2 −0
  8. drivers/md/md.c +14 −3
  9. drivers/md/raid1.c +1 −0
  10. drivers/s390/block/dasd_devmap.c +8 −2
  11. drivers/s390/block/dasd_diag.c +1 −0
  12. drivers/s390/block/dasd_eckd.c +1 −0
  13. drivers/s390/block/dasd_fba.c +1 −0
  14. include/linux/blk-mq.h +78 −49
  15. include/linux/blk_types.h +1 −0
  16. include/linux/blkdev.h +16 −15
  17. include/linux/cgroup-defs.h +0 −3
  18. include/linux/sbitmap.h +5 −0
  19. lib/sbitmap.c +29 −7
block/blk-cgroup.c+4−3
@@ -2182,12 +2182,13 @@ void blk_cgroup_bio_start(struct bio *bio) · 2181 unmodified lines
2182 2182
2183bool blk_cgroup_congested(void)2183bool blk_cgroup_congested(void)
2184{2184{
2185 struct cgroup_subsys_state *css;2185 struct blkcg *blkcg;
2186 bool ret = false;2186 bool ret = false;
2187 2187
2188 rcu_read_lock();2188 rcu_read_lock();
2189 for (css = blkcg_css(); css; css = css->parent) {2189 for (blkcg = css_to_blkcg(blkcg_css()); blkcg;
2190 if (atomic_read(&css->cgroup->congestion_count)) {2190 blkcg = blkcg_parent(blkcg)) {
2191 if (atomic_read(&blkcg->congestion_count)) {
2191 ret = true;2192 ret = true;
2192 break;2193 break;
2193 }2194 }
block/blk-cgroup.h+6−4
@@ -95,6 +95,8 @@ struct blkcg { · 94 unmodified lines
95 struct cgroup_subsys_state css;95 struct cgroup_subsys_state css;
96 spinlock_t lock;96 spinlock_t lock;
97 refcount_t online_pin;97 refcount_t online_pin;
98 /* If there is block congestion on this cgroup. */
99 atomic_t congestion_count;
98 100
99 struct radix_tree_root blkg_tree;101 struct radix_tree_root blkg_tree;
100 struct blkcg_gq __rcu *blkg_hint;102 struct blkcg_gq __rcu *blkg_hint;
@@ -374,7 +376,7 @@ static inline void blkcg_use_delay(struct blkcg_gq *blkg) · 273 unmodified lines
374 if (WARN_ON_ONCE(atomic_read(&blkg->use_delay) < 0))376 if (WARN_ON_ONCE(atomic_read(&blkg->use_delay) < 0))
375 return;377 return;
376 if (atomic_add_return(1, &blkg->use_delay) == 1)378 if (atomic_add_return(1, &blkg->use_delay) == 1)
377 atomic_inc(&blkg->blkcg->css.cgroup->congestion_count);379 atomic_inc(&blkg->blkcg->congestion_count);
378}380}
379 381
380static inline int blkcg_unuse_delay(struct blkcg_gq *blkg)382static inline int blkcg_unuse_delay(struct blkcg_gq *blkg)
@@ -399,7 +401,7 @@ static inline int blkcg_unuse_delay(struct blkcg_gq *blkg) · 18 unmodified lines
399 if (old == 0)401 if (old == 0)
400 return 0;402 return 0;
401 if (old == 1)403 if (old == 1)
402 atomic_dec(&blkg->blkcg->css.cgroup->congestion_count);404 atomic_dec(&blkg->blkcg->congestion_count);
403 return 1;405 return 1;
404}406}
405 407
@@ -418,7 +420,7 @@ static inline void blkcg_set_delay(struct blkcg_gq *blkg, u64 delay) · 12 unmodified lines
418 420
419 /* We only want 1 person setting the congestion count for this blkg. */421 /* We only want 1 person setting the congestion count for this blkg. */
420 if (!old && atomic_try_cmpxchg(&blkg->use_delay, &old, -1))422 if (!old && atomic_try_cmpxchg(&blkg->use_delay, &old, -1))
421 atomic_inc(&blkg->blkcg->css.cgroup->congestion_count);423 atomic_inc(&blkg->blkcg->congestion_count);
422 424
423 atomic64_set(&blkg->delay_nsec, delay);425 atomic64_set(&blkg->delay_nsec, delay);
424}426}
@@ -435,7 +437,7 @@ static inline void blkcg_clear_delay(struct blkcg_gq *blkg) · 10 unmodified lines
435 437
436 /* We only want 1 person clearing the congestion count for this blkg. */438 /* We only want 1 person clearing the congestion count for this blkg. */
437 if (old && atomic_try_cmpxchg(&blkg->use_delay, &old, 0))439 if (old && atomic_try_cmpxchg(&blkg->use_delay, &old, 0))
438 atomic_dec(&blkg->blkcg->css.cgroup->congestion_count);440 atomic_dec(&blkg->blkcg->congestion_count);
439}441}
440 442
441/**443/**
block/blk-core.c+4−1
@@ -791,8 +791,11 @@ void submit_bio_noacct(struct bio *bio) · 790 unmodified lines
791 }791 }
792 }792 }
793 793
794 if (!(q->limits.features & BLK_FEAT_POLL))794 if (!(q->limits.features & BLK_FEAT_POLL) &&
795 (bio->bi_opf & REQ_POLLED)) {
795 bio_clear_polled(bio);796 bio_clear_polled(bio);
797 goto not_supported;
798 }
796 799
797 switch (bio_op(bio)) {800 switch (bio_op(bio)) {
798 case REQ_OP_READ:801 case REQ_OP_READ:
block/blk-mq-debugfs.c+21−5
@@ -5,6 +5,7 @@ · 4 unmodified lines
5 5
6#include <linux/kernel.h>6#include <linux/kernel.h>
7#include <linux/blkdev.h>7#include <linux/blkdev.h>
8#include <linux/build_bug.h>
8#include <linux/debugfs.h>9#include <linux/debugfs.h>
9 10
10#include "blk.h"11#include "blk.h"
@@ -79,7 +80,6 @@ static int queue_pm_only_show(void *data, struct seq_file *m) · 68 unmodified lines
79 80
80#define QUEUE_FLAG_NAME(name) [QUEUE_FLAG_##name] = #name81#define QUEUE_FLAG_NAME(name) [QUEUE_FLAG_##name] = #name
81static const char *const blk_queue_flag_name[] = {82static const char *const blk_queue_flag_name[] = {
82 QUEUE_FLAG_NAME(STOPPED),
83 QUEUE_FLAG_NAME(DYING),83 QUEUE_FLAG_NAME(DYING),
84 QUEUE_FLAG_NAME(NOMERGES),84 QUEUE_FLAG_NAME(NOMERGES),
85 QUEUE_FLAG_NAME(SAME_COMP),85 QUEUE_FLAG_NAME(SAME_COMP),
@@ -100,6 +100,7 @@ static int queue_state_show(void *data, struct seq_file *m) · 14 unmodified lines
100{100{
101 struct request_queue *q = data;101 struct request_queue *q = data;
102 102
103 BUILD_BUG_ON(ARRAY_SIZE(blk_queue_flag_name) != QUEUE_FLAG_MAX);
103 blk_flags_show(m, q->queue_flags, blk_queue_flag_name,104 blk_flags_show(m, q->queue_flags, blk_queue_flag_name,
104 ARRAY_SIZE(blk_queue_flag_name));105 ARRAY_SIZE(blk_queue_flag_name));
105 seq_puts(m, "\n");106 seq_puts(m, "\n");
@@ -164,6 +165,7 @@ static int hctx_state_show(void *data, struct seq_file *m) · 58 unmodified lines
164{165{
165 struct blk_mq_hw_ctx *hctx = data;166 struct blk_mq_hw_ctx *hctx = data;
166 167
168 BUILD_BUG_ON(ARRAY_SIZE(hctx_state_name) != BLK_MQ_S_MAX);
167 blk_flags_show(m, hctx->state, hctx_state_name,169 blk_flags_show(m, hctx->state, hctx_state_name,
168 ARRAY_SIZE(hctx_state_name));170 ARRAY_SIZE(hctx_state_name));
169 seq_puts(m, "\n");171 seq_puts(m, "\n");
@@ -181,10 +183,11 @@ static const char *const alloc_policy_name[] = { · 11 unmodified lines
181static const char *const hctx_flag_name[] = {183static const char *const hctx_flag_name[] = {
182 HCTX_FLAG_NAME(SHOULD_MERGE),184 HCTX_FLAG_NAME(SHOULD_MERGE),
183 HCTX_FLAG_NAME(TAG_QUEUE_SHARED),185 HCTX_FLAG_NAME(TAG_QUEUE_SHARED),
184 HCTX_FLAG_NAME(BLOCKING),
185 HCTX_FLAG_NAME(NO_SCHED),
186 HCTX_FLAG_NAME(STACKING),186 HCTX_FLAG_NAME(STACKING),
187 HCTX_FLAG_NAME(TAG_HCTX_SHARED),187 HCTX_FLAG_NAME(TAG_HCTX_SHARED),
188 HCTX_FLAG_NAME(BLOCKING),
189 HCTX_FLAG_NAME(NO_SCHED),
190 HCTX_FLAG_NAME(NO_SCHED_BY_DEFAULT),
188};191};
189#undef HCTX_FLAG_NAME192#undef HCTX_FLAG_NAME
190 193
@@ -193,6 +196,10 @@ static int hctx_flags_show(void *data, struct seq_file *m) · 2 unmodified lines
193 struct blk_mq_hw_ctx *hctx = data;196 struct blk_mq_hw_ctx *hctx = data;
194 const int alloc_policy = BLK_MQ_FLAG_TO_ALLOC_POLICY(hctx->flags);197 const int alloc_policy = BLK_MQ_FLAG_TO_ALLOC_POLICY(hctx->flags);
195 198
199 BUILD_BUG_ON(ARRAY_SIZE(hctx_flag_name) !=
200 BLK_MQ_F_ALLOC_POLICY_START_BIT);
201 BUILD_BUG_ON(ARRAY_SIZE(alloc_policy_name) != BLK_TAG_ALLOC_MAX);
202
196 seq_puts(m, "alloc_policy=");203 seq_puts(m, "alloc_policy=");
197 if (alloc_policy < ARRAY_SIZE(alloc_policy_name) &&204 if (alloc_policy < ARRAY_SIZE(alloc_policy_name) &&
198 alloc_policy_name[alloc_policy])205 alloc_policy_name[alloc_policy])
@@ -223,12 +230,17 @@ static const char *const cmd_flag_name[] = { · 24 unmodified lines
223 CMD_FLAG_NAME(RAHEAD),230 CMD_FLAG_NAME(RAHEAD),
224 CMD_FLAG_NAME(BACKGROUND),231 CMD_FLAG_NAME(BACKGROUND),
225 CMD_FLAG_NAME(NOWAIT),232 CMD_FLAG_NAME(NOWAIT),
226 CMD_FLAG_NAME(NOUNMAP),
227 CMD_FLAG_NAME(POLLED),233 CMD_FLAG_NAME(POLLED),
234 CMD_FLAG_NAME(ALLOC_CACHE),
235 CMD_FLAG_NAME(SWAP),
236 CMD_FLAG_NAME(DRV),
237 CMD_FLAG_NAME(FS_PRIVATE),
238 CMD_FLAG_NAME(ATOMIC),
239 CMD_FLAG_NAME(NOUNMAP),
228};240};
229#undef CMD_FLAG_NAME241#undef CMD_FLAG_NAME
230 242
231#define RQF_NAME(name) [ilog2((__force u32)RQF_##name)] = #name243#define RQF_NAME(name) [__RQF_##name] = #name
232static const char *const rqf_name[] = {244static const char *const rqf_name[] = {
233 RQF_NAME(STARTED),245 RQF_NAME(STARTED),
234 RQF_NAME(FLUSH_SEQ),246 RQF_NAME(FLUSH_SEQ),
@@ -243,6 +255,7 @@ static const char *const rqf_name[] = { · 8 unmodified lines
243 RQF_NAME(HASHED),255 RQF_NAME(HASHED),
244 RQF_NAME(STATS),256 RQF_NAME(STATS),
245 RQF_NAME(SPECIAL_PAYLOAD),257 RQF_NAME(SPECIAL_PAYLOAD),
258 RQF_NAME(ZONE_WRITE_PLUGGING),
246 RQF_NAME(TIMED_OUT),259 RQF_NAME(TIMED_OUT),
247 RQF_NAME(RESV),260 RQF_NAME(RESV),
248};261};
@@ -268,6 +281,9 @@ int __blk_mq_debugfs_rq_show(struct seq_file *m, struct request *rq) · 19 unmodified lines
268 const enum req_op op = req_op(rq);281 const enum req_op op = req_op(rq);
269 const char *op_str = blk_op_str(op);282 const char *op_str = blk_op_str(op);
270 283
284 BUILD_BUG_ON(ARRAY_SIZE(cmd_flag_name) != __REQ_NR_BITS);
285 BUILD_BUG_ON(ARRAY_SIZE(rqf_name) != __RQF_BITS);
286
271 seq_printf(m, "%p {.op=", rq);287 seq_printf(m, "%p {.op=", rq);
272 if (strcmp(op_str, "UNKNOWN") == 0)288 if (strcmp(op_str, "UNKNOWN") == 0)
273 seq_printf(m, "%u", op);289 seq_printf(m, "%u", op);
block/blk-mq.h+2−0
@@ -36,6 +36,8 @@ enum { · 35 unmodified lines
36 BLK_MQ_TAG_MAX = BLK_MQ_NO_TAG - 1,36 BLK_MQ_TAG_MAX = BLK_MQ_NO_TAG - 1,
37};37};
38 38
39#define BLK_MQ_CPU_WORK_BATCH (8)
40
39typedef unsigned int __bitwise blk_insert_t;41typedef unsigned int __bitwise blk_insert_t;
40#define BLK_MQ_INSERT_AT_HEAD ((__force blk_insert_t)0x01)42#define BLK_MQ_INSERT_AT_HEAD ((__force blk_insert_t)0x01)
41 43
drivers/md/md-cluster.c+39−10
@@ -15,6 +15,7 @@ · 14 unmodified lines
15 15
16#define LVB_SIZE 6416#define LVB_SIZE 64
17#define NEW_DEV_TIMEOUT 500017#define NEW_DEV_TIMEOUT 5000
18#define WAIT_DLM_LOCK_TIMEOUT (30 * HZ)
18 19
19struct dlm_lock_resource {20struct dlm_lock_resource {
20 dlm_lockspace_t *ls;21 dlm_lockspace_t *ls;
@@ -56,6 +57,7 @@ struct resync_info { · 35 unmodified lines
56#define MD_CLUSTER_ALREADY_IN_CLUSTER 657#define MD_CLUSTER_ALREADY_IN_CLUSTER 6
57#define MD_CLUSTER_PENDING_RECV_EVENT 758#define MD_CLUSTER_PENDING_RECV_EVENT 7
58#define MD_CLUSTER_HOLDING_MUTEX_FOR_RECVD 859#define MD_CLUSTER_HOLDING_MUTEX_FOR_RECVD 8
60#define MD_CLUSTER_WAITING_FOR_SYNC 9
59 61
60struct md_cluster_info {62struct md_cluster_info {
61 struct mddev *mddev; /* the md device which md_cluster_info belongs to */63 struct mddev *mddev; /* the md device which md_cluster_info belongs to */
@@ -91,6 +93,7 @@ struct md_cluster_info { · 29 unmodified lines
91 sector_t sync_hi;93 sector_t sync_hi;
92};94};
93 95
96/* For compatibility, add the new msg_type at the end. */
94enum msg_type {97enum msg_type {
95 METADATA_UPDATED = 0,98 METADATA_UPDATED = 0,
96 RESYNCING,99 RESYNCING,
@@ -100,6 +103,7 @@ enum msg_type { · 3 unmodified lines
100 BITMAP_NEEDS_SYNC,103 BITMAP_NEEDS_SYNC,
101 CHANGE_CAPACITY,104 CHANGE_CAPACITY,
102 BITMAP_RESIZE,105 BITMAP_RESIZE,
106 RESYNCING_START,
103};107};
104 108
105struct cluster_msg {109struct cluster_msg {
@@ -130,8 +134,13 @@ static int dlm_lock_sync(struct dlm_lock_resource *res, int mode) · 24 unmodified lines
130 0, sync_ast, res, res->bast);134 0, sync_ast, res, res->bast);
131 if (ret)135 if (ret)
132 return ret;136 return ret;
133 wait_event(res->sync_locking, res->sync_locking_done);137 ret = wait_event_timeout(res->sync_locking, res->sync_locking_done,
138 WAIT_DLM_LOCK_TIMEOUT);
134 res->sync_locking_done = false;139 res->sync_locking_done = false;
140 if (!ret) {
141 pr_err("locking DLM '%s' timeout!\n", res->name);
142 return -EBUSY;
143 }
135 if (res->lksb.sb_status == 0)144 if (res->lksb.sb_status == 0)
136 res->mode = mode;145 res->mode = mode;
137 return res->lksb.sb_status;146 return res->lksb.sb_status;
@@ -455,6 +464,7 @@ static void process_suspend_info(struct mddev *mddev, · 317 unmodified lines
455 clear_bit(MD_RESYNCING_REMOTE, &mddev->recovery);464 clear_bit(MD_RESYNCING_REMOTE, &mddev->recovery);
456 remove_suspend_info(mddev, slot);465 remove_suspend_info(mddev, slot);
457 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);466 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
467 clear_bit(MD_CLUSTER_WAITING_FOR_SYNC, &cinfo->state);
458 md_wakeup_thread(mddev->thread);468 md_wakeup_thread(mddev->thread);
459 return;469 return;
460 }470 }
@@ -525,6 +535,7 @@ static int process_add_new_disk(struct mddev *mddev, struct cluster_msg *cmsg) · 64 unmodified lines
525 res = -1;535 res = -1;
526 }536 }
527 clear_bit(MD_CLUSTER_WAITING_FOR_NEWDISK, &cinfo->state);537 clear_bit(MD_CLUSTER_WAITING_FOR_NEWDISK, &cinfo->state);
538 set_bit(MD_CLUSTER_WAITING_FOR_SYNC, &cinfo->state);
528 return res;539 return res;
529}540}
530 541
@@ -593,6 +604,9 @@ static int process_recvd_msg(struct mddev *mddev, struct cluster_msg *msg) · 62 unmodified lines
593 case CHANGE_CAPACITY:604 case CHANGE_CAPACITY:
594 set_capacity_and_notify(mddev->gendisk, mddev->array_sectors);605 set_capacity_and_notify(mddev->gendisk, mddev->array_sectors);
595 break;606 break;
607 case RESYNCING_START:
608 clear_bit(MD_CLUSTER_WAITING_FOR_SYNC, &mddev->cluster_info->state);
609 break;
596 case RESYNCING:610 case RESYNCING:
597 set_bit(MD_RESYNCING_REMOTE, &mddev->recovery);611 set_bit(MD_RESYNCING_REMOTE, &mddev->recovery);
598 process_suspend_info(mddev, le32_to_cpu(msg->slot),612 process_suspend_info(mddev, le32_to_cpu(msg->slot),
@@ -743,7 +757,7 @@ static void unlock_comm(struct md_cluster_info *cinfo) · 144 unmodified lines
743 */757 */
744static int __sendmsg(struct md_cluster_info *cinfo, struct cluster_msg *cmsg)758static int __sendmsg(struct md_cluster_info *cinfo, struct cluster_msg *cmsg)
745{759{
746 int error;760 int error, unlock_error;
747 int slot = cinfo->slot_number - 1;761 int slot = cinfo->slot_number - 1;
748 762
749 cmsg->slot = cpu_to_le32(slot);763 cmsg->slot = cpu_to_le32(slot);
@@ -751,7 +765,7 @@ static int __sendmsg(struct md_cluster_info *cinfo, struct cluster_msg *cmsg) · 1 unmodified line
751 error = dlm_lock_sync(cinfo->message_lockres, DLM_LOCK_EX);765 error = dlm_lock_sync(cinfo->message_lockres, DLM_LOCK_EX);
752 if (error) {766 if (error) {
753 pr_err("md-cluster: failed to get EX on MESSAGE (%d)\n", error);767 pr_err("md-cluster: failed to get EX on MESSAGE (%d)\n", error);
754 goto failed_message;768 return error;
755 }769 }
756 770
757 memcpy(cinfo->message_lockres->lksb.sb_lvbptr, (void *)cmsg,771 memcpy(cinfo->message_lockres->lksb.sb_lvbptr, (void *)cmsg,
@@ -781,14 +795,10 @@ static int __sendmsg(struct md_cluster_info *cinfo, struct cluster_msg *cmsg) · 23 unmodified lines
781 }795 }
782 796
783failed_ack:797failed_ack:
784 error = dlm_unlock_sync(cinfo->message_lockres);798 while ((unlock_error = dlm_unlock_sync(cinfo->message_lockres)))
785 if (unlikely(error != 0)) {
786 pr_err("md-cluster: failed convert to NL on MESSAGE(%d)\n",799 pr_err("md-cluster: failed convert to NL on MESSAGE(%d)\n",
787 error);800 unlock_error);
788 /* in case the message can't be released due to some reason */801
789 goto failed_ack;
790 }
791failed_message:
792 return error;802 return error;
793}803}
794 804
@@ -1343,6 +1353,23 @@ static void resync_info_get(struct mddev *mddev, sector_t *lo, sector_t *hi) · 548 unmodified lines
1343 spin_unlock_irq(&cinfo->suspend_lock);1353 spin_unlock_irq(&cinfo->suspend_lock);
1344}1354}
1345 1355
1356static int resync_status_get(struct mddev *mddev)
1357{
1358 struct md_cluster_info *cinfo = mddev->cluster_info;
1359
1360 return test_bit(MD_CLUSTER_WAITING_FOR_SYNC, &cinfo->state);
1361}
1362
1363static int resync_start_notify(struct mddev *mddev)
1364{
1365 struct md_cluster_info *cinfo = mddev->cluster_info;
1366 struct cluster_msg cmsg = {0};
1367
1368 cmsg.type = cpu_to_le32(RESYNCING_START);
1369
1370 return sendmsg(cinfo, &cmsg, 0);
1371}
1372
1346static int resync_info_update(struct mddev *mddev, sector_t lo, sector_t hi)1373static int resync_info_update(struct mddev *mddev, sector_t lo, sector_t hi)
1347{1374{
1348 struct md_cluster_info *cinfo = mddev->cluster_info;1375 struct md_cluster_info *cinfo = mddev->cluster_info;
@@ -1577,6 +1604,8 @@ static const struct md_cluster_operations cluster_ops = { · 228 unmodified lines
1577 .resync_start = resync_start,1604 .resync_start = resync_start,
1578 .resync_finish = resync_finish,1605 .resync_finish = resync_finish,
1579 .resync_info_update = resync_info_update,1606 .resync_info_update = resync_info_update,
1607 .resync_start_notify = resync_start_notify,
1608 .resync_status_get = resync_status_get,
1580 .resync_info_get = resync_info_get,1609 .resync_info_get = resync_info_get,
1581 .metadata_update_start = metadata_update_start,1610 .metadata_update_start = metadata_update_start,
1582 .metadata_update_finish = metadata_update_finish,1611 .metadata_update_finish = metadata_update_finish,
drivers/md/md-cluster.h+2−0
@@ -14,6 +14,8 @@ struct md_cluster_operations { · 13 unmodified lines
14 int (*leave)(struct mddev *mddev);14 int (*leave)(struct mddev *mddev);
15 int (*slot_number)(struct mddev *mddev);15 int (*slot_number)(struct mddev *mddev);
16 int (*resync_info_update)(struct mddev *mddev, sector_t lo, sector_t hi);16 int (*resync_info_update)(struct mddev *mddev, sector_t lo, sector_t hi);
17 int (*resync_start_notify)(struct mddev *mddev);
18 int (*resync_status_get)(struct mddev *mddev);
17 void (*resync_info_get)(struct mddev *mddev, sector_t *lo, sector_t *hi);19 void (*resync_info_get)(struct mddev *mddev, sector_t *lo, sector_t *hi);
18 int (*metadata_update_start)(struct mddev *mddev);20 int (*metadata_update_start)(struct mddev *mddev);
19 int (*metadata_update_finish)(struct mddev *mddev);21 int (*metadata_update_finish)(struct mddev *mddev);
drivers/md/md.c+14−3
@@ -8978,7 +8978,8 @@ void md_do_sync(struct md_thread *thread) · 8977 unmodified lines
8978 * This will mean we have to start checking from the beginning again.8978 * This will mean we have to start checking from the beginning again.
8979 *8979 *
8980 */8980 */
8981 8981 if (mddev_is_clustered(mddev))
8982 md_cluster_ops->resync_start_notify(mddev);
8982 do {8983 do {
8983 int mddev2_minor = -1;8984 int mddev2_minor = -1;
8984 mddev->curr_resync = MD_RESYNC_DELAYED;8985 mddev->curr_resync = MD_RESYNC_DELAYED;
@@ -9992,8 +9993,18 @@ static void check_sb_changes(struct mddev *mddev, struct md_rdev *rdev) · 1007 unmodified lines
9992 */9993 */
9993 if (rdev2->raid_disk == -1 && role != MD_DISK_ROLE_SPARE &&9994 if (rdev2->raid_disk == -1 && role != MD_DISK_ROLE_SPARE &&
9994 !(le32_to_cpu(sb->feature_map) &9995 !(le32_to_cpu(sb->feature_map) &
9995 MD_FEATURE_RESHAPE_ACTIVE)) {9996 MD_FEATURE_RESHAPE_ACTIVE) &&
9996 rdev2->saved_raid_disk = role;9997 !md_cluster_ops->resync_status_get(mddev)) {
9998 /*
9999 * -1 to make raid1_add_disk() set conf->fullsync
10000 * to 1. This could avoid skipping sync when the
10001 * remote node is down during resyncing.
10002 */
10003 if ((le32_to_cpu(sb->feature_map)
10004 & MD_FEATURE_RECOVERY_OFFSET))
10005 rdev2->saved_raid_disk = -1;
10006 else
10007 rdev2->saved_raid_disk = role;
9997 ret = remove_and_add_spares(mddev, rdev2);10008 ret = remove_and_add_spares(mddev, rdev2);
9998 pr_info("Activated spare: %pg\n",10009 pr_info("Activated spare: %pg\n",
9999 rdev2->bdev);10010 rdev2->bdev);
drivers/md/raid1.c+1−0
@@ -680,6 +680,7 @@ static int choose_slow_rdev(struct r1conf *conf, struct r1bio *r1_bio, · 679 unmodified lines
680 len = r1_bio->sectors;680 len = r1_bio->sectors;
681 read_len = raid1_check_read_range(rdev, this_sector, &len);681 read_len = raid1_check_read_range(rdev, this_sector, &len);
682 if (read_len == r1_bio->sectors) {682 if (read_len == r1_bio->sectors) {
683 *max_sectors = read_len;
683 update_read_sectors(conf, disk, this_sector, read_len);684 update_read_sectors(conf, disk, this_sector, read_len);
684 return disk;685 return disk;
685 }686 }
drivers/s390/block/dasd_devmap.c+8−2
@@ -2248,13 +2248,19 @@ static ssize_t dasd_copy_pair_store(struct device *dev, · 2247 unmodified lines
2248 2248
2249 /* allocate primary devmap if needed */2249 /* allocate primary devmap if needed */
2250 prim_devmap = dasd_find_busid(prim_busid);2250 prim_devmap = dasd_find_busid(prim_busid);
2251 if (IS_ERR(prim_devmap))2251 if (IS_ERR(prim_devmap)) {
2252 prim_devmap = dasd_add_busid(prim_busid, DASD_FEATURE_DEFAULT);2252 prim_devmap = dasd_add_busid(prim_busid, DASD_FEATURE_DEFAULT);
2253 if (IS_ERR(prim_devmap))
2254 return PTR_ERR(prim_devmap);
2255 }
2253 2256
2254 /* allocate secondary devmap if needed */2257 /* allocate secondary devmap if needed */
2255 sec_devmap = dasd_find_busid(sec_busid);2258 sec_devmap = dasd_find_busid(sec_busid);
2256 if (IS_ERR(sec_devmap))2259 if (IS_ERR(sec_devmap)) {
2257 sec_devmap = dasd_add_busid(sec_busid, DASD_FEATURE_DEFAULT);2260 sec_devmap = dasd_add_busid(sec_busid, DASD_FEATURE_DEFAULT);
2261 if (IS_ERR(sec_devmap))
2262 return PTR_ERR(sec_devmap);
2263 }
2258 2264
2259 /* setting copy relation is only allowed for offline secondary */2265 /* setting copy relation is only allowed for offline secondary */
2260 if (sec_devmap->device)2266 if (sec_devmap->device)
drivers/s390/block/dasd_diag.c+1−0
@@ -29,6 +29,7 @@ · 28 unmodified lines
29#include "dasd_int.h"29#include "dasd_int.h"
30#include "dasd_diag.h"30#include "dasd_diag.h"
31 31
32MODULE_DESCRIPTION("S/390 Support for DIAG access to DASD Disks");
32MODULE_LICENSE("GPL");33MODULE_LICENSE("GPL");
33 34
34/* The maximum number of blocks per request (max_blocks) is dependent on the35/* The maximum number of blocks per request (max_blocks) is dependent on the
drivers/s390/block/dasd_eckd.c+1−0
@@ -44,6 +44,7 @@ · 43 unmodified lines
44/* 64k are 128 x 512 byte sectors */44/* 64k are 128 x 512 byte sectors */
45#define DASD_RAW_SECTORS_PER_TRACK 12845#define DASD_RAW_SECTORS_PER_TRACK 128
46 46
47MODULE_DESCRIPTION("S/390 DASD ECKD Disks device driver");
47MODULE_LICENSE("GPL");48MODULE_LICENSE("GPL");
48 49
49static struct dasd_discipline dasd_eckd_discipline;50static struct dasd_discipline dasd_eckd_discipline;
drivers/s390/block/dasd_fba.c+1−0
@@ -32,6 +32,7 @@ · 31 unmodified lines
32#define DASD_FBA_CCW_LOCATE 0x4332#define DASD_FBA_CCW_LOCATE 0x43
33#define DASD_FBA_CCW_DEFINE_EXTENT 0x6333#define DASD_FBA_CCW_DEFINE_EXTENT 0x63
34 34
35MODULE_DESCRIPTION("S/390 DASD FBA Disks device driver");
35MODULE_LICENSE("GPL");36MODULE_LICENSE("GPL");
36 37
37static struct dasd_discipline dasd_fba_discipline;38static struct dasd_discipline dasd_fba_discipline;
include/linux/blk-mq.h+78−49
@@ -27,38 +27,61 @@ typedef enum rq_end_io_ret (rq_end_io_fn)(struct request *, blk_status_t); · 26 unmodified lines
27 * request flags */27 * request flags */
28typedef __u32 __bitwise req_flags_t;28typedef __u32 __bitwise req_flags_t;
29 29
30/* drive already may have started this one */30/* Keep rqf_name[] in sync with the definitions below */
31#define RQF_STARTED ((__force req_flags_t)(1 << 1))31enum {
32/* request for flush sequence */32 /* drive already may have started this one */
33#define RQF_FLUSH_SEQ ((__force req_flags_t)(1 << 4))33 __RQF_STARTED,
34/* merge of different types, fail separately */34 /* request for flush sequence */
35#define RQF_MIXED_MERGE ((__force req_flags_t)(1 << 5))35 __RQF_FLUSH_SEQ,
36/* don't call prep for this one */36 /* merge of different types, fail separately */
37#define RQF_DONTPREP ((__force req_flags_t)(1 << 7))37 __RQF_MIXED_MERGE,
38/* use hctx->sched_tags */38 /* don't call prep for this one */
39#define RQF_SCHED_TAGS ((__force req_flags_t)(1 << 8))39 __RQF_DONTPREP,
40/* use an I/O scheduler for this request */40 /* use hctx->sched_tags */
41#define RQF_USE_SCHED ((__force req_flags_t)(1 << 9))41 __RQF_SCHED_TAGS,
42/* vaguely specified driver internal error. Ignored by the block layer */42 /* use an I/O scheduler for this request */
43#define RQF_FAILED ((__force req_flags_t)(1 << 10))43 __RQF_USE_SCHED,
44/* don't warn about errors */44 /* vaguely specified driver internal error. Ignored by block layer */
45#define RQF_QUIET ((__force req_flags_t)(1 << 11))45 __RQF_FAILED,
46/* account into disk and partition IO statistics */46 /* don't warn about errors */
47#define RQF_IO_STAT ((__force req_flags_t)(1 << 13))47 __RQF_QUIET,
48/* runtime pm request */48 /* account into disk and partition IO statistics */
49#define RQF_PM ((__force req_flags_t)(1 << 15))49 __RQF_IO_STAT,
50/* on IO scheduler merge hash */50 /* runtime pm request */
51#define RQF_HASHED ((__force req_flags_t)(1 << 16))51 __RQF_PM,
52/* track IO completion time */52 /* on IO scheduler merge hash */
53#define RQF_STATS ((__force req_flags_t)(1 << 17))53 __RQF_HASHED,
54/* Look at ->special_vec for the actual data payload instead of the54 /* track IO completion time */
55 bio chain. */55 __RQF_STATS,
56#define RQF_SPECIAL_PAYLOAD ((__force req_flags_t)(1 << 18))56 /* Look at ->special_vec for the actual data payload instead of the
57/* The request completion needs to be signaled to zone write pluging. */57 bio chain. */
58#define RQF_ZONE_WRITE_PLUGGING ((__force req_flags_t)(1 << 20))58 __RQF_SPECIAL_PAYLOAD,
59/* ->timeout has been called, don't expire again */59 /* request completion needs to be signaled to zone write plugging. */
60#define RQF_TIMED_OUT ((__force req_flags_t)(1 << 21))60 __RQF_ZONE_WRITE_PLUGGING,
61#define RQF_RESV ((__force req_flags_t)(1 << 23))61 /* ->timeout has been called, don't expire again */
62 __RQF_TIMED_OUT,
63 __RQF_RESV,
64 __RQF_BITS
65};
66
67#define RQF_STARTED ((__force req_flags_t)(1 << __RQF_STARTED))
68#define RQF_FLUSH_SEQ ((__force req_flags_t)(1 << __RQF_FLUSH_SEQ))
69#define RQF_MIXED_MERGE ((__force req_flags_t)(1 << __RQF_MIXED_MERGE))
70#define RQF_DONTPREP ((__force req_flags_t)(1 << __RQF_DONTPREP))
71#define RQF_SCHED_TAGS ((__force req_flags_t)(1 << __RQF_SCHED_TAGS))
72#define RQF_USE_SCHED ((__force req_flags_t)(1 << __RQF_USE_SCHED))
73#define RQF_FAILED ((__force req_flags_t)(1 << __RQF_FAILED))
74#define RQF_QUIET ((__force req_flags_t)(1 << __RQF_QUIET))
75#define RQF_IO_STAT ((__force req_flags_t)(1 << __RQF_IO_STAT))
76#define RQF_PM ((__force req_flags_t)(1 << __RQF_PM))
77#define RQF_HASHED ((__force req_flags_t)(1 << __RQF_HASHED))
78#define RQF_STATS ((__force req_flags_t)(1 << __RQF_STATS))
79#define RQF_SPECIAL_PAYLOAD \
80 ((__force req_flags_t)(1 << __RQF_SPECIAL_PAYLOAD))
81#define RQF_ZONE_WRITE_PLUGGING \
82 ((__force req_flags_t)(1 << __RQF_ZONE_WRITE_PLUGGING))
83#define RQF_TIMED_OUT ((__force req_flags_t)(1 << __RQF_TIMED_OUT))
84#define RQF_RESV ((__force req_flags_t)(1 << __RQF_RESV))
62 85
63/* flags that prevent us from merging requests: */86/* flags that prevent us from merging requests: */
64#define RQF_NOMERGE_FLAGS \87#define RQF_NOMERGE_FLAGS \
@@ -278,8 +301,12 @@ enum blk_eh_timer_return { · 213 unmodified lines
278 BLK_EH_RESET_TIMER,301 BLK_EH_RESET_TIMER,
279};302};
280 303
281#define BLK_TAG_ALLOC_FIFO 0 /* allocate starting from 0 */304/* Keep alloc_policy_name[] in sync with the definitions below */
282#define BLK_TAG_ALLOC_RR 1 /* allocate starting from last allocated tag */305enum {
306 BLK_TAG_ALLOC_FIFO, /* allocate starting from 0 */
307 BLK_TAG_ALLOC_RR, /* allocate starting from last allocated tag */
308 BLK_TAG_ALLOC_MAX
309};
283 310
284/**311/**
285 * struct blk_mq_hw_ctx - State for a hardware queue facing the hardware312 * struct blk_mq_hw_ctx - State for a hardware queue facing the hardware
@@ -644,6 +671,7 @@ struct blk_mq_ops { · 358 unmodified lines
644#endif671#endif
645};672};
646 673
674/* Keep hctx_flag_name[] in sync with the definitions below */
647enum {675enum {
648 BLK_MQ_F_SHOULD_MERGE = 1 << 0,676 BLK_MQ_F_SHOULD_MERGE = 1 << 0,
649 BLK_MQ_F_TAG_QUEUE_SHARED = 1 << 1,677 BLK_MQ_F_TAG_QUEUE_SHARED = 1 << 1,
@@ -653,27 +681,17 @@ enum { · 3 unmodified lines
653 */681 */
654 BLK_MQ_F_STACKING = 1 << 2,682 BLK_MQ_F_STACKING = 1 << 2,
655 BLK_MQ_F_TAG_HCTX_SHARED = 1 << 3,683 BLK_MQ_F_TAG_HCTX_SHARED = 1 << 3,
656 BLK_MQ_F_BLOCKING = 1 << 5,684 BLK_MQ_F_BLOCKING = 1 << 4,
657 /* Do not allow an I/O scheduler to be configured. */685 /* Do not allow an I/O scheduler to be configured. */
658 BLK_MQ_F_NO_SCHED = 1 << 6,686 BLK_MQ_F_NO_SCHED = 1 << 5,
687
659 /*688 /*
660 * Select 'none' during queue registration in case of a single hwq689 * Select 'none' during queue registration in case of a single hwq
661 * or shared hwqs instead of 'mq-deadline'.690 * or shared hwqs instead of 'mq-deadline'.
662 */691 */
663 BLK_MQ_F_NO_SCHED_BY_DEFAULT = 1 << 7,692 BLK_MQ_F_NO_SCHED_BY_DEFAULT = 1 << 6,
664 BLK_MQ_F_ALLOC_POLICY_START_BIT = 8,693 BLK_MQ_F_ALLOC_POLICY_START_BIT = 7,
665 BLK_MQ_F_ALLOC_POLICY_BITS = 1,694 BLK_MQ_F_ALLOC_POLICY_BITS = 1,
666
667 BLK_MQ_S_STOPPED = 0,
668 BLK_MQ_S_TAG_ACTIVE = 1,
669 BLK_MQ_S_SCHED_RESTART = 2,
670
671 /* hw queue is inactive after all its CPUs become offline */
672 BLK_MQ_S_INACTIVE = 3,
673
674 BLK_MQ_MAX_DEPTH = 10240,
675
676 BLK_MQ_CPU_WORK_BATCH = 8,
677};695};
678#define BLK_MQ_FLAG_TO_ALLOC_POLICY(flags) \696#define BLK_MQ_FLAG_TO_ALLOC_POLICY(flags) \
679 ((flags >> BLK_MQ_F_ALLOC_POLICY_START_BIT) & \697 ((flags >> BLK_MQ_F_ALLOC_POLICY_START_BIT) & \
@@ -682,8 +700,19 @@ enum { · 2 unmodified lines
682 ((policy & ((1 << BLK_MQ_F_ALLOC_POLICY_BITS) - 1)) \700 ((policy & ((1 << BLK_MQ_F_ALLOC_POLICY_BITS) - 1)) \
683 << BLK_MQ_F_ALLOC_POLICY_START_BIT)701 << BLK_MQ_F_ALLOC_POLICY_START_BIT)
684 702
703#define BLK_MQ_MAX_DEPTH (10240)
685#define BLK_MQ_NO_HCTX_IDX (-1U)704#define BLK_MQ_NO_HCTX_IDX (-1U)
686 705
706enum {
707 /* Keep hctx_state_name[] in sync with the definitions below */
708 BLK_MQ_S_STOPPED,
709 BLK_MQ_S_TAG_ACTIVE,
710 BLK_MQ_S_SCHED_RESTART,
711 /* hw queue is inactive after all its CPUs become offline */
712 BLK_MQ_S_INACTIVE,
713 BLK_MQ_S_MAX
714};
715
687struct gendisk *__blk_mq_alloc_disk(struct blk_mq_tag_set *set,716struct gendisk *__blk_mq_alloc_disk(struct blk_mq_tag_set *set,
688 struct queue_limits *lim, void *queuedata,717 struct queue_limits *lim, void *queuedata,
689 struct lock_class_key *lkclass);718 struct lock_class_key *lkclass);
include/linux/blk_types.h+1−0
@@ -354,6 +354,7 @@ enum req_op { · 353 unmodified lines
354 REQ_OP_LAST = (__force blk_opf_t)36,354 REQ_OP_LAST = (__force blk_opf_t)36,
355};355};
356 356
357/* Keep cmd_flag_name[] in sync with the definitions below */
357enum req_flag_bits {358enum req_flag_bits {
358 __REQ_FAILFAST_DEV = /* no driver retries of device errors */359 __REQ_FAILFAST_DEV = /* no driver retries of device errors */
359 REQ_OP_BITS,360 REQ_OP_BITS,
include/linux/blkdev.h+16−15
@@ -588,27 +588,28 @@ struct request_queue { · 587 unmodified lines
588};588};
589 589
590/* Keep blk_queue_flag_name[] in sync with the definitions below */590/* Keep blk_queue_flag_name[] in sync with the definitions below */
591#define QUEUE_FLAG_STOPPED 0 /* queue is stopped */591enum {
592#define QUEUE_FLAG_DYING 1 /* queue being torn down */592 QUEUE_FLAG_DYING, /* queue being torn down */
593#define QUEUE_FLAG_NOMERGES 3 /* disable merge attempts */593 QUEUE_FLAG_NOMERGES, /* disable merge attempts */
594#define QUEUE_FLAG_SAME_COMP 4 /* complete on same CPU-group */594 QUEUE_FLAG_SAME_COMP, /* complete on same CPU-group */
595#define QUEUE_FLAG_FAIL_IO 5 /* fake timeout */595 QUEUE_FLAG_FAIL_IO, /* fake timeout */
596#define QUEUE_FLAG_NOXMERGES 9 /* No extended merges */596 QUEUE_FLAG_NOXMERGES, /* No extended merges */
597#define QUEUE_FLAG_SAME_FORCE 12 /* force complete on same CPU */597 QUEUE_FLAG_SAME_FORCE, /* force complete on same CPU */
598#define QUEUE_FLAG_INIT_DONE 14 /* queue is initialized */598 QUEUE_FLAG_INIT_DONE, /* queue is initialized */
599#define QUEUE_FLAG_STATS 20 /* track IO start and completion times */599 QUEUE_FLAG_STATS, /* track IO start and completion times */
600#define QUEUE_FLAG_REGISTERED 22 /* queue has been registered to a disk */600 QUEUE_FLAG_REGISTERED, /* queue has been registered to a disk */
601#define QUEUE_FLAG_QUIESCED 24 /* queue has been quiesced */601 QUEUE_FLAG_QUIESCED, /* queue has been quiesced */
602#define QUEUE_FLAG_RQ_ALLOC_TIME 27 /* record rq->alloc_time_ns */602 QUEUE_FLAG_RQ_ALLOC_TIME, /* record rq->alloc_time_ns */
603#define QUEUE_FLAG_HCTX_ACTIVE 28 /* at least one blk-mq hctx is active */603 QUEUE_FLAG_HCTX_ACTIVE, /* at least one blk-mq hctx is active */
604#define QUEUE_FLAG_SQ_SCHED 30 /* single queue style io dispatch */604 QUEUE_FLAG_SQ_SCHED, /* single queue style io dispatch */
605 QUEUE_FLAG_MAX
606};
605 607
606#define QUEUE_FLAG_MQ_DEFAULT (1UL << QUEUE_FLAG_SAME_COMP)608#define QUEUE_FLAG_MQ_DEFAULT (1UL << QUEUE_FLAG_SAME_COMP)
607 609
608void blk_queue_flag_set(unsigned int flag, struct request_queue *q);610void blk_queue_flag_set(unsigned int flag, struct request_queue *q);
609void blk_queue_flag_clear(unsigned int flag, struct request_queue *q);611void blk_queue_flag_clear(unsigned int flag, struct request_queue *q);
610 612
611#define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags)
612#define blk_queue_dying(q) test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags)613#define blk_queue_dying(q) test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags)
613#define blk_queue_init_done(q) test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags)614#define blk_queue_init_done(q) test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags)
614#define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)615#define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)
include/linux/cgroup-defs.h+0−3
@@ -539,9 +539,6 @@ struct cgroup { · 538 unmodified lines
539 /* used to store eBPF programs */539 /* used to store eBPF programs */
540 struct cgroup_bpf bpf;540 struct cgroup_bpf bpf;
541 541
542 /* If there is block congestion on this cgroup. */
543 atomic_t congestion_count;
544
545 /* Used to store internal freezer state */542 /* Used to store internal freezer state */
546 struct cgroup_freezer_state freezer;543 struct cgroup_freezer_state freezer;
547 544
include/linux/sbitmap.h+5−0
@@ -36,6 +36,11 @@ struct sbitmap_word { · 35 unmodified lines
36 * @cleared: word holding cleared bits36 * @cleared: word holding cleared bits
37 */37 */
38 unsigned long cleared ____cacheline_aligned_in_smp;38 unsigned long cleared ____cacheline_aligned_in_smp;
39
40 /**
41 * @swap_lock: serializes simultaneous updates of ->word and ->cleared
42 */
43 spinlock_t swap_lock;
39} ____cacheline_aligned_in_smp;44} ____cacheline_aligned_in_smp;
40 45
41/**46/**
lib/sbitmap.c+29−7
@@ -60,12 +60,30 @@ static inline void update_alloc_hint_after_get(struct sbitmap *sb, · 59 unmodified lines
60/*60/*
61 * See if we have deferred clears that we can batch move61 * See if we have deferred clears that we can batch move
62 */62 */
63static inline bool sbitmap_deferred_clear(struct sbitmap_word *map)63static inline bool sbitmap_deferred_clear(struct sbitmap_word *map,
64 unsigned int depth, unsigned int alloc_hint, bool wrap)
64{65{
65 unsigned long mask;66 unsigned long mask, word_mask;
66 67
67 if (!READ_ONCE(map->cleared))68 guard(spinlock_irqsave)(&map->swap_lock);
68 return false;69
70 if (!map->cleared) {
71 if (depth == 0)
72 return false;
73
74 word_mask = (~0UL) >> (BITS_PER_LONG - depth);
75 /*
76 * The current behavior is to always retry after moving
77 * ->cleared to word, and we change it to retry in case
78 * of any free bits. To avoid an infinite loop, we need
79 * to take wrap & alloc_hint into account, otherwise a
80 * soft lockup may occur.
81 */
82 if (!wrap && alloc_hint)
83 word_mask &= ~((1UL << alloc_hint) - 1);
84
85 return (READ_ONCE(map->word) & word_mask) != word_mask;
86 }
69 87
70 /*88 /*
71 * First get a stable cleared mask, setting the old mask to 0.89 * First get a stable cleared mask, setting the old mask to 0.
@@ -85,6 +103,7 @@ int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift, · 13 unmodified lines
85 bool alloc_hint)103 bool alloc_hint)
86{104{
87 unsigned int bits_per_word;105 unsigned int bits_per_word;
106 int i;
88 107
89 if (shift < 0)108 if (shift < 0)
90 shift = sbitmap_calculate_shift(depth);109 shift = sbitmap_calculate_shift(depth);
@@ -116,6 +135,9 @@ int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift, · 25 unmodified lines
116 return -ENOMEM;135 return -ENOMEM;
117 }136 }
118 137
138 for (i = 0; i < sb->map_nr; i++)
139 spin_lock_init(&sb->map[i].swap_lock);
140
119 return 0;141 return 0;
120}142}
121EXPORT_SYMBOL_GPL(sbitmap_init_node);143EXPORT_SYMBOL_GPL(sbitmap_init_node);
@@ -126,7 +148,7 @@ void sbitmap_resize(struct sbitmap *sb, unsigned int depth) · 4 unmodified lines
126 unsigned int i;148 unsigned int i;
127 149
128 for (i = 0; i < sb->map_nr; i++)150 for (i = 0; i < sb->map_nr; i++)
129 sbitmap_deferred_clear(&sb->map[i]);151 sbitmap_deferred_clear(&sb->map[i], 0, 0, 0);
130 152
131 sb->depth = depth;153 sb->depth = depth;
132 sb->map_nr = DIV_ROUND_UP(sb->depth, bits_per_word);154 sb->map_nr = DIV_ROUND_UP(sb->depth, bits_per_word);
@@ -179,7 +201,7 @@ static int sbitmap_find_bit_in_word(struct sbitmap_word *map, · 46 unmodified lines
179 alloc_hint, wrap);201 alloc_hint, wrap);
180 if (nr != -1)202 if (nr != -1)
181 break;203 break;
182 if (!sbitmap_deferred_clear(map))204 if (!sbitmap_deferred_clear(map, depth, alloc_hint, wrap))
183 break;205 break;
184 } while (1);206 } while (1);
185 207
@@ -496,7 +518,7 @@ unsigned long __sbitmap_queue_get_batch(struct sbitmap_queue *sbq, int nr_tags, · 310 unmodified lines
496 unsigned int map_depth = __map_depth(sb, index);518 unsigned int map_depth = __map_depth(sb, index);
497 unsigned long val;519 unsigned long val;
498 520
499 sbitmap_deferred_clear(map);521 sbitmap_deferred_clear(map, 0, 0, 0);
500 val = READ_ONCE(map->word);522 val = READ_ONCE(map->word);
501 if (val == (1UL << (map_depth - 1)) - 1)523 if (val == (1UL << (map_depth - 1)) - 1)
502 goto next;524 goto next;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t