Bit spinlocks are problematic if PREEMPT_RT is enabled, because they disable preemption, which is undesired for latency reasons and breaks when regular spinlocks are taken within the bit_spinlock locked region because regular spinlocks are converted to 'sleeping spinlocks' on RT. PREEMPT_RT replaced the bit spinlocks with regular spinlocks to avoid this problem. The replacement was done conditionaly at compile time, but Christoph requested to do an unconditional conversion. Jan suggested to move the spinlock into a existing padding hole which avoids a size increase of struct buffer_head on production kernels. As a benefit the lock gains lockdep coverage. [ bigeasy: Remove the wrapper and use always spinlock_t and move it into the padding hole ]
Unified · Split
| @@ -274,8 +274,7 @@ static void end_buffer_async_read(struct buffer_head *bh, int uptodate) · 273 unmodified lines | |||
| 274 | * decide that the page is now completely done. | 274 | * decide that the page is now completely done. |
| 275 | */ | 275 | */ |
| 276 | first = page_buffers(page); | 276 | first = page_buffers(page); |
| 277 | local_irq_save(flags); | 277 | spin_lock_irqsave(&first->b_uptodate_lock, flags); |
| 278 | bit_spin_lock(BH_Uptodate_Lock, &first->b_state); | ||
| 279 | clear_buffer_async_read(bh); | 278 | clear_buffer_async_read(bh); |
| 280 | unlock_buffer(bh); | 279 | unlock_buffer(bh); |
| 281 | tmp = bh; | 280 | tmp = bh; |
| @@ -288,8 +287,7 @@ static void end_buffer_async_read(struct buffer_head *bh, int uptodate) · 6 unmodified lines | |||
| 288 | } | 287 | } |
| 289 | tmp = tmp->b_this_page; | 288 | tmp = tmp->b_this_page; |
| 290 | } while (tmp != bh); | 289 | } while (tmp != bh); |
| 291 | bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); | 290 | spin_unlock_irqrestore(&first->b_uptodate_lock, flags); |
| 292 | local_irq_restore(flags); | ||
| 293 | 291 | ||
| 294 | /* | 292 | /* |
| 295 | * If none of the buffers had errors and they are all | 293 | * If none of the buffers had errors and they are all |
| @@ -301,8 +299,7 @@ static void end_buffer_async_read(struct buffer_head *bh, int uptodate) · 5 unmodified lines | |||
| 301 | return; | 299 | return; |
| 302 | 300 | ||
| 303 | still_busy: | 301 | still_busy: |
| 304 | bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); | 302 | spin_unlock_irqrestore(&first->b_uptodate_lock, flags); |
| 305 | local_irq_restore(flags); | ||
| 306 | return; | 303 | return; |
| 307 | } | 304 | } |
| 308 | 305 | ||
| @@ -371,8 +368,7 @@ void end_buffer_async_write(struct buffer_head *bh, int uptodate) · 62 unmodified lines | |||
| 371 | } | 368 | } |
| 372 | 369 | ||
| 373 | first = page_buffers(page); | 370 | first = page_buffers(page); |
| 374 | local_irq_save(flags); | 371 | spin_lock_irqsave(&first->b_uptodate_lock, flags); |
| 375 | bit_spin_lock(BH_Uptodate_Lock, &first->b_state); | ||
| 376 | 372 | ||
| 377 | clear_buffer_async_write(bh); | 373 | clear_buffer_async_write(bh); |
| 378 | unlock_buffer(bh); | 374 | unlock_buffer(bh); |
| @@ -384,14 +380,12 @@ void end_buffer_async_write(struct buffer_head *bh, int uptodate) · 5 unmodified lines | |||
| 384 | } | 380 | } |
| 385 | tmp = tmp->b_this_page; | 381 | tmp = tmp->b_this_page; |
| 386 | } | 382 | } |
| 387 | bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); | 383 | spin_unlock_irqrestore(&first->b_uptodate_lock, flags); |
| 388 | local_irq_restore(flags); | ||
| 389 | end_page_writeback(page); | 384 | end_page_writeback(page); |
| 390 | return; | 385 | return; |
| 391 | 386 | ||
| 392 | still_busy: | 387 | still_busy: |
| 393 | bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); | 388 | spin_unlock_irqrestore(&first->b_uptodate_lock, flags); |
| 394 | local_irq_restore(flags); | ||
| 395 | return; | 389 | return; |
| 396 | } | 390 | } |
| 397 | EXPORT_SYMBOL(end_buffer_async_write); | 391 | EXPORT_SYMBOL(end_buffer_async_write); |
| @@ -3385,6 +3379,7 @@ struct buffer_head *alloc_buffer_head(gfp_t gfp_flags) · 2987 unmodified lines | |||
| 3385 | struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags); | 3379 | struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags); |
| 3386 | if (ret) { | 3380 | if (ret) { |
| 3387 | INIT_LIST_HEAD(&ret->b_assoc_buffers); | 3381 | INIT_LIST_HEAD(&ret->b_assoc_buffers); |
| 3382 | spin_lock_init(&ret->b_uptodate_lock); | ||
| 3388 | preempt_disable(); | 3383 | preempt_disable(); |
| 3389 | __this_cpu_inc(bh_accounting.nr); | 3384 | __this_cpu_inc(bh_accounting.nr); |
| 3390 | recalc_bh_state(); | 3385 | recalc_bh_state(); |
| @@ -125,11 +125,10 @@ static void ext4_finish_bio(struct bio *bio) · 124 unmodified lines | |||
| 125 | } | 125 | } |
| 126 | bh = head = page_buffers(page); | 126 | bh = head = page_buffers(page); |
| 127 | /* | 127 | /* |
| 128 | * We check all buffers in the page under BH_Uptodate_Lock | 128 | * We check all buffers in the page under b_uptodate_lock |
| 129 | * to avoid races with other end io clearing async_write flags | 129 | * to avoid races with other end io clearing async_write flags |
| 130 | */ | 130 | */ |
| 131 | local_irq_save(flags); | 131 | spin_lock_irqsave(&head->b_uptodate_lock, flags); |
| 132 | bit_spin_lock(BH_Uptodate_Lock, &head->b_state); | ||
| 133 | do { | 132 | do { |
| 134 | if (bh_offset(bh) < bio_start || | 133 | if (bh_offset(bh) < bio_start || |
| 135 | bh_offset(bh) + bh->b_size > bio_end) { | 134 | bh_offset(bh) + bh->b_size > bio_end) { |
| @@ -141,8 +140,7 @@ static void ext4_finish_bio(struct bio *bio) · 5 unmodified lines | |||
| 141 | if (bio->bi_status) | 140 | if (bio->bi_status) |
| 142 | buffer_io_error(bh); | 141 | buffer_io_error(bh); |
| 143 | } while ((bh = bh->b_this_page) != head); | 142 | } while ((bh = bh->b_this_page) != head); |
| 144 | bit_spin_unlock(BH_Uptodate_Lock, &head->b_state); | 143 | spin_unlock_irqrestore(&head->b_uptodate_lock, flags); |
| 145 | local_irq_restore(flags); | ||
| 146 | if (!under_io) { | 144 | if (!under_io) { |
| 147 | fscrypt_free_bounce_page(bounce_page); | 145 | fscrypt_free_bounce_page(bounce_page); |
| 148 | end_page_writeback(page); | 146 | end_page_writeback(page); |
| @@ -92,8 +92,7 @@ static void ntfs_end_buffer_async_read(struct buffer_head *bh, int uptodate) · 91 unmodified lines | |||
| 92 | "0x%llx.", (unsigned long long)bh->b_blocknr); | 92 | "0x%llx.", (unsigned long long)bh->b_blocknr); |
| 93 | } | 93 | } |
| 94 | first = page_buffers(page); | 94 | first = page_buffers(page); |
| 95 | local_irq_save(flags); | 95 | spin_lock_irqsave(&first->b_uptodate_lock, flags); |
| 96 | bit_spin_lock(BH_Uptodate_Lock, &first->b_state); | ||
| 97 | clear_buffer_async_read(bh); | 96 | clear_buffer_async_read(bh); |
| 98 | unlock_buffer(bh); | 97 | unlock_buffer(bh); |
| 99 | tmp = bh; | 98 | tmp = bh; |
| @@ -108,8 +107,7 @@ static void ntfs_end_buffer_async_read(struct buffer_head *bh, int uptodate) · 8 unmodified lines | |||
| 108 | } | 107 | } |
| 109 | tmp = tmp->b_this_page; | 108 | tmp = tmp->b_this_page; |
| 110 | } while (tmp != bh); | 109 | } while (tmp != bh); |
| 111 | bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); | 110 | spin_unlock_irqrestore(&first->b_uptodate_lock, flags); |
| 112 | local_irq_restore(flags); | ||
| 113 | /* | 111 | /* |
| 114 | * If none of the buffers had errors then we can set the page uptodate, | 112 | * If none of the buffers had errors then we can set the page uptodate, |
| 115 | * but we first have to perform the post read mst fixups, if the | 113 | * but we first have to perform the post read mst fixups, if the |
| @@ -142,8 +140,7 @@ static void ntfs_end_buffer_async_read(struct buffer_head *bh, int uptodate) · 26 unmodified lines | |||
| 142 | unlock_page(page); | 140 | unlock_page(page); |
| 143 | return; | 141 | return; |
| 144 | still_busy: | 142 | still_busy: |
| 145 | bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); | 143 | spin_unlock_irqrestore(&first->b_uptodate_lock, flags); |
| 146 | local_irq_restore(flags); | ||
| 147 | return; | 144 | return; |
| 148 | } | 145 | } |
| 149 | 146 | ||
| @@ -22,9 +22,6 @@ enum bh_state_bits { · 21 unmodified lines | |||
| 22 | BH_Dirty, /* Is dirty */ | 22 | BH_Dirty, /* Is dirty */ |
| 23 | BH_Lock, /* Is locked */ | 23 | BH_Lock, /* Is locked */ |
| 24 | BH_Req, /* Has been submitted for I/O */ | 24 | BH_Req, /* Has been submitted for I/O */ |
| 25 | BH_Uptodate_Lock,/* Used by the first bh in a page, to serialise | ||
| 26 | * IO completion of other buffers in the page | ||
| 27 | */ | ||
| 28 | 25 | ||
| 29 | BH_Mapped, /* Has a disk mapping */ | 26 | BH_Mapped, /* Has a disk mapping */ |
| 30 | BH_New, /* Disk mapping was newly created by get_block */ | 27 | BH_New, /* Disk mapping was newly created by get_block */ |
| @@ -76,6 +73,9 @@ struct buffer_head { · 45 unmodified lines | |||
| 76 | struct address_space *b_assoc_map; /* mapping this buffer is | 73 | struct address_space *b_assoc_map; /* mapping this buffer is |
| 77 | associated with */ | 74 | associated with */ |
| 78 | atomic_t b_count; /* users using this buffer_head */ | 75 | atomic_t b_count; /* users using this buffer_head */ |
| 76 | spinlock_t b_uptodate_lock; /* Used by the first bh in a page, to | ||
| 77 | * serialise IO completion of other | ||
| 78 | * buffers in the page */ | ||
| 79 | }; | 79 | }; |
| 80 | 80 | ||
| 81 | /* | 81 | /* |