// SPDX-License-Identifier: GPL-2.0-only/* * linux/fs/buffer.c * * Copyright (C) 1991, 1992, 2002 Linus Torvalds */ /* * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95 * * Removed a lot of unnecessary code and simplified things now that * the buffer cache isn't our primary cache - Andrew Tridgell 12/96 * * Speed up hash, lru, and free list operations. Use gfp() for allocating * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM * * Added 32k buffer block sizes - these are required older ARM systems. - RMK * * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de> */ #include <linux/kernel.h>#include <linux/sched/signal.h>#include <linux/syscalls.h>#include <linux/fs.h>#include <linux/iomap.h>#include <linux/mm.h>#include <linux/percpu.h>#include <linux/slab.h>#include <linux/capability.h>#include <linux/blkdev.h>#include <linux/blk-crypto.h>#include <linux/file.h>#include <linux/quotaops.h>#include <linux/highmem.h>#include <linux/export.h>#include <linux/backing-dev.h>#include <linux/writeback.h>#include <linux/hash.h>#include <linux/suspend.h>#include <linux/buffer_head.h>#include <linux/task_io_accounting_ops.h>#include <linux/bio.h>#include <linux/cpu.h>#include <linux/bitops.h>#include <linux/mpage.h>#include <linux/bit_spinlock.h>#include <linux/folio_batch.h>#include <linux/sched/mm.h>#include <trace/events/block.h>#include <linux/fscrypt.h>#include <linux/fsverity.h>#include <linux/sched/isolation.h> #include "internal.h" #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers) inline void touch_buffer(struct buffer_head *bh){ trace_block_touch_buffer(bh); folio_mark_accessed(bh->b_folio);}EXPORT_SYMBOL(touch_buffer); void __lock_buffer(struct buffer_head *bh){ wait_on_bit_lock_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);}EXPORT_SYMBOL(__lock_buffer); void unlock_buffer(struct buffer_head *bh){ clear_and_wake_up_bit(BH_Lock, &bh->b_state);}EXPORT_SYMBOL(unlock_buffer); /* * Returns if the folio has dirty or writeback buffers. If all the buffers * are unlocked and clean then the folio_test_dirty information is stale. If * any of the buffers are locked, it is assumed they are locked for IO. */void buffer_check_dirty_writeback(struct folio *folio, bool *dirty, bool *writeback){ struct buffer_head *head, *bh; *dirty = false; *writeback = false; BUG_ON(!folio_test_locked(folio)); head = folio_buffers(folio); if (!head) return; if (folio_test_writeback(folio)) *writeback = true; bh = head; do { if (buffer_locked(bh)) *writeback = true; if (buffer_dirty(bh)) *dirty = true; bh = bh->b_this_page; } while (bh != head);} /* * Block until a buffer comes unlocked. This doesn't stop it * from becoming locked again - you have to lock it yourself * if you want to preserve its state. */void __wait_on_buffer(struct buffer_head * bh){ wait_on_bit_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);}EXPORT_SYMBOL(__wait_on_buffer); static void buffer_io_error(struct buffer_head *bh, char *msg){ if (!test_bit(BH_Quiet, &bh->b_state)) printk_ratelimited(KERN_ERR "Buffer I/O error on dev %pg, logical block %llu%s\n", bh->b_bdev, (unsigned long long)bh->b_blocknr, msg);} /** * bio_endio_bh - Discard the bio used to submit a buffer. * @bio: The bio. * @bhp: Where to return the buffer_head. * * Call this in your bio_end_io handler to retrieve the buffer_head * submitted in bh_submit(). If you did not call bh_submit(), do not * call this function; it will return garbage. * * This function consumes the bio refcount which will probably free the * bio. * * Return: True if the I/O succeeded. */bool bio_endio_bh(struct bio *bio, struct buffer_head **bhp){ bool success = bio->bi_status == BLK_STS_OK; struct buffer_head *bh = bio->bi_private; if (unlikely(bio_flagged(bio, BIO_QUIET))) set_bit(BH_Quiet, &bh->b_state); bio_put(bio); *bhp = bh; return success;}EXPORT_SYMBOL(bio_endio_bh); /** * end_buffer_read_sync - Handle buffer reads finishing * @bh: The buffer. * @uptodate: True if the read was successful. * * If a buffer is read through a mechanism that isn't bh_submit(), you * can call this function to finish the read. */void end_buffer_read_sync(struct buffer_head *bh, int uptodate){ if (uptodate) { set_buffer_uptodate(bh); } else { /* This happens, due to failed read-ahead attempts. */ clear_buffer_uptodate(bh); } unlock_buffer(bh);}EXPORT_SYMBOL(end_buffer_read_sync); /** * bh_end_read - I/O end handler for reads * @bio: The bio being completed. * * Pass this function to bh_submit() if you're reading into the buffer, * unless you need your own special I/O end handler. */void bh_end_read(struct bio *bio){ struct buffer_head *bh; bool uptodate = bio_endio_bh(bio, &bh); end_buffer_read_sync(bh, uptodate);}EXPORT_SYMBOL(bh_end_read); /** * bh_end_write - I/O end handler for writes * @bio: The bio being completed. * * Pass this function to bh_submit() if you're writing from the buffer, * unless you need your own special I/O end handler. */void bh_end_write(struct bio *bio){ struct buffer_head *bh; bool success = bio_endio_bh(bio, &bh); if (success) { set_buffer_uptodate(bh); } else { buffer_io_error(bh, ", lost sync page write"); mark_buffer_write_io_error(bh); clear_buffer_uptodate(bh); } unlock_buffer(bh);}EXPORT_SYMBOL(bh_end_write); static struct buffer_head *__find_get_block_slow(struct block_device *bdev, sector_t block, bool atomic){ struct address_space *bd_mapping = bdev->bd_mapping; const int blkbits = bd_mapping->host->i_blkbits; struct buffer_head *ret = NULL; pgoff_t index; struct buffer_head *bh; struct buffer_head *head; struct folio *folio; int all_mapped = 1; static DEFINE_RATELIMIT_STATE(last_warned, HZ, 1); index = ((loff_t)block << blkbits) / PAGE_SIZE; folio = __filemap_get_folio(bd_mapping, index, FGP_ACCESSED, 0); if (IS_ERR(folio)) goto out; /* * Folio lock protects the buffers. Callers that cannot block * will fallback to serializing vs try_to_free_buffers() via * the i_private_lock. */ if (atomic) spin_lock(&bd_mapping->i_private_lock); else folio_lock(folio); head = folio_buffers(folio); if (!head) goto out_unlock; /* * Upon a noref migration, the folio lock serializes here; * otherwise bail. */ if (test_bit_acquire(BH_Migrate, &head->b_state)) { WARN_ON(!atomic); goto out_unlock; } bh = head; do { if (!buffer_mapped(bh)) all_mapped = 0; else if (bh->b_blocknr == block) { ret = bh; get_bh(bh); goto out_unlock; } bh = bh->b_this_page; } while (bh != head); /* we might be here because some of the buffers on this page are * not mapped. This is due to various races between * file io on the block device and getblk. It gets dealt with * elsewhere, don't buffer_error if we had some unmapped buffers */ ratelimit_set_flags(&last_warned, RATELIMIT_MSG_ON_RELEASE); if (all_mapped && __ratelimit(&last_warned)) { printk("__find_get_block_slow() failed. block=%llu, " "b_blocknr=%llu, b_state=0x%08lx, b_size=%zu, " "device %pg blocksize: %d\n", (unsigned long long)block, (unsigned long long)bh->b_blocknr, bh->b_state, bh->b_size, bdev, 1 << blkbits); }out_unlock: if (atomic) spin_unlock(&bd_mapping->i_private_lock); else folio_unlock(folio); folio_put(folio);out: return ret;} static void end_buffer_async_read(struct buffer_head *bh, int uptodate){ unsigned long flags; struct buffer_head *first; struct buffer_head *tmp; struct folio *folio; int folio_uptodate = 1; BUG_ON(!buffer_async_read(bh)); folio = bh->b_folio; if (uptodate) { set_buffer_uptodate(bh); } else { clear_buffer_uptodate(bh); buffer_io_error(bh, ", async page read"); } /* * Be _very_ careful from here on. Bad things can happen if * two buffer heads end IO at almost the same time and both * decide that the page is now completely done. */ first = folio_buffers(folio); spin_lock_irqsave(&first->b_uptodate_lock, flags); clear_buffer_async_read(bh); unlock_buffer(bh); tmp = bh; do { if (!buffer_uptodate(tmp)) folio_uptodate = 0; if (buffer_async_read(tmp)) { BUG_ON(!buffer_locked(tmp)); goto still_busy; } tmp = tmp->b_this_page; } while (tmp != bh); spin_unlock_irqrestore(&first->b_uptodate_lock, flags); folio_end_read(folio, folio_uptodate); return; still_busy: spin_unlock_irqrestore(&first->b_uptodate_lock, flags);} struct postprocess_bh_ctx { struct work_struct work; struct buffer_head *bh; struct fsverity_info *vi;}; static void verify_bh(struct work_struct *work){ struct postprocess_bh_ctx *ctx = container_of(work, struct postprocess_bh_ctx, work); struct buffer_head *bh = ctx->bh; bool valid; valid = fsverity_verify_blocks(ctx->vi, bh->b_folio, bh->b_size, bh_offset(bh)); end_buffer_async_read(bh, valid); kfree(ctx);} static void decrypt_bh(struct work_struct *work){ struct postprocess_bh_ctx *ctx = container_of(work, struct postprocess_bh_ctx, work); struct buffer_head *bh = ctx->bh; int err; err = fscrypt_decrypt_pagecache_blocks(bh->b_folio, bh->b_size, bh_offset(bh)); if (err == 0 && ctx->vi) { /* * We use different work queues for decryption and for verity * because verity may require reading metadata pages that need * decryption, and we shouldn't recurse to the same workqueue. */ INIT_WORK(&ctx->work, verify_bh); fsverity_enqueue_verify_work(&ctx->work); return; } end_buffer_async_read(bh, err == 0); kfree(ctx);} /* * I/O completion handler for block_read_full_folio() - folios * which come unlocked at the end of I/O. */static void bh_end_async_read(struct bio *bio){ struct buffer_head *bh; bool uptodate = bio_endio_bh(bio, &bh); struct inode *inode = bh->b_folio->mapping->host; bool decrypt = fscrypt_inode_uses_fs_layer_crypto(inode); struct fsverity_info *vi = NULL; /* needed by ext4 */ if (bh->b_folio->index < DIV_ROUND_UP(inode->i_size, PAGE_SIZE)) vi = fsverity_get_info(inode); /* Decrypt (with fscrypt) and/or verify (with fsverity) if needed. */ if (uptodate && (decrypt || vi)) { struct postprocess_bh_ctx *ctx = kmalloc_obj(*ctx, GFP_ATOMIC); if (ctx) { ctx->bh = bh; ctx->vi = vi; if (decrypt) { INIT_WORK(&ctx->work, decrypt_bh); fscrypt_enqueue_decrypt_work(&ctx->work); } else { INIT_WORK(&ctx->work, verify_bh); fsverity_enqueue_verify_work(&ctx->work); } return; } uptodate = false; } end_buffer_async_read(bh, uptodate);} /** * bh_end_async_write - I/O end handler for async folio writes * @bio: The bio being completed. * * Pass this function to bh_submit() if you're doing the equivalent of * block_write_full_folio(). That is, the folio is unlocked, and will * have its writeback flag cleared once all async write buffers have * completed. */void bh_end_async_write(struct bio *bio){ struct buffer_head *bh; bool success = bio_endio_bh(bio, &bh); unsigned long flags; struct buffer_head *first; struct buffer_head *tmp; struct folio *folio; BUG_ON(!buffer_async_write(bh)); folio = bh->b_folio; if (success) { set_buffer_uptodate(bh); } else { buffer_io_error(bh, ", lost async page write"); mark_buffer_write_io_error(bh); clear_buffer_uptodate(bh); } first = folio_buffers(folio); spin_lock_irqsave(&first->b_uptodate_lock, flags); clear_buffer_async_write(bh); unlock_buffer(bh); tmp = bh->b_this_page; while (tmp != bh) { if (buffer_async_write(tmp)) { BUG_ON(!buffer_locked(tmp)); goto still_busy; } tmp = tmp->b_this_page; } spin_unlock_irqrestore(&first->b_uptodate_lock, flags); folio_end_writeback(folio); return; still_busy: spin_unlock_irqrestore(&first->b_uptodate_lock, flags);}EXPORT_SYMBOL(bh_end_async_write); /* * fs/buffer.c contains helper functions for buffer-backed address space's * fsync functions. A common requirement for buffer-based filesystems is * that certain data from the backing blockdev needs to be written out for * a successful fsync(). For example, ext2 indirect blocks need to be * written back and waited upon before fsync() returns. * * The functions mmb_mark_buffer_dirty(), mmb_sync(), mmb_has_buffers() * and mmb_invalidate() are provided for the management of a list of dependent * buffers in mapping_metadata_bhs struct. * * The locking is a little subtle: The list of buffer heads is protected by * the lock in mapping_metadata_bhs so functions coming from bdev mapping * (such as try_to_free_buffers()) need to safely get to mapping_metadata_bhs * using RCU, grab the lock, verify we didn't race with somebody detaching the * bh / moving it to different inode and only then proceeding. */ void mmb_init(struct mapping_metadata_bhs *mmb, struct address_space *mapping){ spin_lock_init(&mmb->lock); INIT_LIST_HEAD(&mmb->list); mmb->mapping = mapping;}EXPORT_SYMBOL(mmb_init); static void __remove_assoc_queue(struct mapping_metadata_bhs *mmb, struct buffer_head *bh){ lockdep_assert_held(&mmb->lock); list_del_init(&bh->b_assoc_buffers);Lines 1–500 of 3095. Load more
// SPDX-License-Identifier: GPL-2.0-only /* * linux/fs/buffer.c * * Copyright (C) 1991, 1992, 2002 Linus Torvalds */ /* * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95 * * Removed a lot of unnecessary code and simplified things now that * the buffer cache isn't our primary cache - Andrew Tridgell 12/96 * * Speed up hash, lru, and free list operations. Use gfp() for allocating * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM * * Added 32k buffer block sizes - these are required older ARM systems. - RMK * * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de> */ #include <linux/kernel.h> #include <linux/sched/signal.h> #include <linux/syscalls.h> #include <linux/fs.h> #include <linux/iomap.h> #include <linux/mm.h> #include <linux/percpu.h> #include <linux/slab.h> #include <linux/capability.h> #include <linux/blkdev.h> #include <linux/blk-crypto.h> #include <linux/file.h> #include <linux/quotaops.h> #include <linux/highmem.h> #include <linux/export.h> #include <linux/backing-dev.h> #include <linux/writeback.h> #include <linux/hash.h> #include <linux/suspend.h> #include <linux/buffer_head.h> #include <linux/task_io_accounting_ops.h> #include <linux/bio.h> #include <linux/cpu.h> #include <linux/bitops.h> #include <linux/mpage.h> #include <linux/bit_spinlock.h> #include <linux/folio_batch.h> #include <linux/sched/mm.h> #include <trace/events/block.h> #include <linux/fscrypt.h> #include <linux/fsverity.h> #include <linux/sched/isolation.h> #include "internal.h" #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers) inline void touch_buffer(struct buffer_head *bh) { trace_block_touch_buffer(bh); folio_mark_accessed(bh->b_folio); } EXPORT_SYMBOL(touch_buffer); void __lock_buffer(struct buffer_head *bh) { wait_on_bit_lock_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE); } EXPORT_SYMBOL(__lock_buffer); void unlock_buffer(struct buffer_head *bh) { clear_and_wake_up_bit(BH_Lock, &bh->b_state); } EXPORT_SYMBOL(unlock_buffer); /* * Returns if the folio has dirty or writeback buffers. If all the buffers * are unlocked and clean then the folio_test_dirty information is stale. If * any of the buffers are locked, it is assumed they are locked for IO. */ void buffer_check_dirty_writeback(struct folio *folio, bool *dirty, bool *writeback) { struct buffer_head *head, *bh; *dirty = false; *writeback = false; BUG_ON(!folio_test_locked(folio)); head = folio_buffers(folio); if (!head) return; if (folio_test_writeback(folio)) *writeback = true; bh = head; do { if (buffer_locked(bh)) *writeback = true; if (buffer_dirty(bh)) *dirty = true; bh = bh->b_this_page; } while (bh != head); } /* * Block until a buffer comes unlocked. This doesn't stop it * from becoming locked again - you have to lock it yourself * if you want to preserve its state. */ void __wait_on_buffer(struct buffer_head * bh) { wait_on_bit_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE); } EXPORT_SYMBOL(__wait_on_buffer); static void buffer_io_error(struct buffer_head *bh, char *msg) { if (!test_bit(BH_Quiet, &bh->b_state)) printk_ratelimited(KERN_ERR "Buffer I/O error on dev %pg, logical block %llu%s\n", bh->b_bdev, (unsigned long long)bh->b_blocknr, msg); } /** * bio_endio_bh - Discard the bio used to submit a buffer. * @bio: The bio. * @bhp: Where to return the buffer_head. * * Call this in your bio_end_io handler to retrieve the buffer_head * submitted in bh_submit(). If you did not call bh_submit(), do not * call this function; it will return garbage. * * This function consumes the bio refcount which will probably free the * bio. * * Return: True if the I/O succeeded. */ bool bio_endio_bh(struct bio *bio, struct buffer_head **bhp) { bool success = bio->bi_status == BLK_STS_OK; struct buffer_head *bh = bio->bi_private; if (unlikely(bio_flagged(bio, BIO_QUIET))) set_bit(BH_Quiet, &bh->b_state); bio_put(bio); *bhp = bh; return success; } EXPORT_SYMBOL(bio_endio_bh); /** * end_buffer_read_sync - Handle buffer reads finishing * @bh: The buffer. * @uptodate: True if the read was successful. * * If a buffer is read through a mechanism that isn't bh_submit(), you * can call this function to finish the read. */ void end_buffer_read_sync(struct buffer_head *bh, int uptodate) { if (uptodate) { set_buffer_uptodate(bh); } else { /* This happens, due to failed read-ahead attempts. */ clear_buffer_uptodate(bh); } unlock_buffer(bh); } EXPORT_SYMBOL(end_buffer_read_sync); /** * bh_end_read - I/O end handler for reads * @bio: The bio being completed. * * Pass this function to bh_submit() if you're reading into the buffer, * unless you need your own special I/O end handler. */ void bh_end_read(struct bio *bio) { struct buffer_head *bh; bool uptodate = bio_endio_bh(bio, &bh); end_buffer_read_sync(bh, uptodate); } EXPORT_SYMBOL(bh_end_read); /** * bh_end_write - I/O end handler for writes * @bio: The bio being completed. * * Pass this function to bh_submit() if you're writing from the buffer, * unless you need your own special I/O end handler. */ void bh_end_write(struct bio *bio) { struct buffer_head *bh; bool success = bio_endio_bh(bio, &bh); if (success) { set_buffer_uptodate(bh); } else { buffer_io_error(bh, ", lost sync page write"); mark_buffer_write_io_error(bh); clear_buffer_uptodate(bh); } unlock_buffer(bh); } EXPORT_SYMBOL(bh_end_write); static struct buffer_head * __find_get_block_slow(struct block_device *bdev, sector_t block, bool atomic) { struct address_space *bd_mapping = bdev->bd_mapping; const int blkbits = bd_mapping->host->i_blkbits; struct buffer_head *ret = NULL; pgoff_t index; struct buffer_head *bh; struct buffer_head *head; struct folio *folio; int all_mapped = 1; static DEFINE_RATELIMIT_STATE(last_warned, HZ, 1); index = ((loff_t)block << blkbits) / PAGE_SIZE; folio = __filemap_get_folio(bd_mapping, index, FGP_ACCESSED, 0); if (IS_ERR(folio)) goto out; /* * Folio lock protects the buffers. Callers that cannot block * will fallback to serializing vs try_to_free_buffers() via * the i_private_lock. */ if (atomic) spin_lock(&bd_mapping->i_private_lock); else folio_lock(folio); head = folio_buffers(folio); if (!head) goto out_unlock; /* * Upon a noref migration, the folio lock serializes here; * otherwise bail. */ if (test_bit_acquire(BH_Migrate, &head->b_state)) { WARN_ON(!atomic); goto out_unlock; } bh = head; do { if (!buffer_mapped(bh)) all_mapped = 0; else if (bh->b_blocknr == block) { ret = bh; get_bh(bh); goto out_unlock; } bh = bh->b_this_page; } while (bh != head); /* we might be here because some of the buffers on this page are * not mapped. This is due to various races between * file io on the block device and getblk. It gets dealt with * elsewhere, don't buffer_error if we had some unmapped buffers */ ratelimit_set_flags(&last_warned, RATELIMIT_MSG_ON_RELEASE); if (all_mapped && __ratelimit(&last_warned)) { printk("__find_get_block_slow() failed. block=%llu, " "b_blocknr=%llu, b_state=0x%08lx, b_size=%zu, " "device %pg blocksize: %d\n", (unsigned long long)block, (unsigned long long)bh->b_blocknr, bh->b_state, bh->b_size, bdev, 1 << blkbits); } out_unlock: if (atomic) spin_unlock(&bd_mapping->i_private_lock); else folio_unlock(folio); folio_put(folio); out: return ret; } static void end_buffer_async_read(struct buffer_head *bh, int uptodate) { unsigned long flags; struct buffer_head *first; struct buffer_head *tmp; struct folio *folio; int folio_uptodate = 1; BUG_ON(!buffer_async_read(bh)); folio = bh->b_folio; if (uptodate) { set_buffer_uptodate(bh); } else { clear_buffer_uptodate(bh); buffer_io_error(bh, ", async page read"); } /* * Be _very_ careful from here on. Bad things can happen if * two buffer heads end IO at almost the same time and both * decide that the page is now completely done. */ first = folio_buffers(folio); spin_lock_irqsave(&first->b_uptodate_lock, flags); clear_buffer_async_read(bh); unlock_buffer(bh); tmp = bh; do { if (!buffer_uptodate(tmp)) folio_uptodate = 0; if (buffer_async_read(tmp)) { BUG_ON(!buffer_locked(tmp)); goto still_busy; } tmp = tmp->b_this_page; } while (tmp != bh); spin_unlock_irqrestore(&first->b_uptodate_lock, flags); folio_end_read(folio, folio_uptodate); return; still_busy: spin_unlock_irqrestore(&first->b_uptodate_lock, flags); } struct postprocess_bh_ctx { struct work_struct work; struct buffer_head *bh; struct fsverity_info *vi; }; static void verify_bh(struct work_struct *work) { struct postprocess_bh_ctx *ctx = container_of(work, struct postprocess_bh_ctx, work); struct buffer_head *bh = ctx->bh; bool valid; valid = fsverity_verify_blocks(ctx->vi, bh->b_folio, bh->b_size, bh_offset(bh)); end_buffer_async_read(bh, valid); kfree(ctx); } static void decrypt_bh(struct work_struct *work) { struct postprocess_bh_ctx *ctx = container_of(work, struct postprocess_bh_ctx, work); struct buffer_head *bh = ctx->bh; int err; err = fscrypt_decrypt_pagecache_blocks(bh->b_folio, bh->b_size, bh_offset(bh)); if (err == 0 && ctx->vi) { /* * We use different work queues for decryption and for verity * because verity may require reading metadata pages that need * decryption, and we shouldn't recurse to the same workqueue. */ INIT_WORK(&ctx->work, verify_bh); fsverity_enqueue_verify_work(&ctx->work); return; } end_buffer_async_read(bh, err == 0); kfree(ctx); } /* * I/O completion handler for block_read_full_folio() - folios * which come unlocked at the end of I/O. */ static void bh_end_async_read(struct bio *bio) { struct buffer_head *bh; bool uptodate = bio_endio_bh(bio, &bh); struct inode *inode = bh->b_folio->mapping->host; bool decrypt = fscrypt_inode_uses_fs_layer_crypto(inode); struct fsverity_info *vi = NULL; /* needed by ext4 */ if (bh->b_folio->index < DIV_ROUND_UP(inode->i_size, PAGE_SIZE)) vi = fsverity_get_info(inode); /* Decrypt (with fscrypt) and/or verify (with fsverity) if needed. */ if (uptodate && (decrypt || vi)) { struct postprocess_bh_ctx *ctx = kmalloc_obj(*ctx, GFP_ATOMIC); if (ctx) { ctx->bh = bh; ctx->vi = vi; if (decrypt) { INIT_WORK(&ctx->work, decrypt_bh); fscrypt_enqueue_decrypt_work(&ctx->work); } else { INIT_WORK(&ctx->work, verify_bh); fsverity_enqueue_verify_work(&ctx->work); } return; } uptodate = false; } end_buffer_async_read(bh, uptodate); } /** * bh_end_async_write - I/O end handler for async folio writes * @bio: The bio being completed. * * Pass this function to bh_submit() if you're doing the equivalent of * block_write_full_folio(). That is, the folio is unlocked, and will * have its writeback flag cleared once all async write buffers have * completed. */ void bh_end_async_write(struct bio *bio) { struct buffer_head *bh; bool success = bio_endio_bh(bio, &bh); unsigned long flags; struct buffer_head *first; struct buffer_head *tmp; struct folio *folio; BUG_ON(!buffer_async_write(bh)); folio = bh->b_folio; if (success) { set_buffer_uptodate(bh); } else { buffer_io_error(bh, ", lost async page write"); mark_buffer_write_io_error(bh); clear_buffer_uptodate(bh); } first = folio_buffers(folio); spin_lock_irqsave(&first->b_uptodate_lock, flags); clear_buffer_async_write(bh); unlock_buffer(bh); tmp = bh->b_this_page; while (tmp != bh) { if (buffer_async_write(tmp)) { BUG_ON(!buffer_locked(tmp)); goto still_busy; } tmp = tmp->b_this_page; } spin_unlock_irqrestore(&first->b_uptodate_lock, flags); folio_end_writeback(folio); return; still_busy: spin_unlock_irqrestore(&first->b_uptodate_lock, flags); } EXPORT_SYMBOL(bh_end_async_write); /* * fs/buffer.c contains helper functions for buffer-backed address space's * fsync functions. A common requirement for buffer-based filesystems is * that certain data from the backing blockdev needs to be written out for * a successful fsync(). For example, ext2 indirect blocks need to be * written back and waited upon before fsync() returns. * * The functions mmb_mark_buffer_dirty(), mmb_sync(), mmb_has_buffers() * and mmb_invalidate() are provided for the management of a list of dependent * buffers in mapping_metadata_bhs struct. * * The locking is a little subtle: The list of buffer heads is protected by * the lock in mapping_metadata_bhs so functions coming from bdev mapping * (such as try_to_free_buffers()) need to safely get to mapping_metadata_bhs * using RCU, grab the lock, verify we didn't race with somebody detaching the * bh / moving it to different inode and only then proceeding. */ void mmb_init(struct mapping_metadata_bhs *mmb, struct address_space *mapping) { spin_lock_init(&mmb->lock); INIT_LIST_HEAD(&mmb->list); mmb->mapping = mapping; } EXPORT_SYMBOL(mmb_init); static void __remove_assoc_queue(struct mapping_metadata_bhs *mmb, struct buffer_head *bh) { lockdep_assert_held(&mmb->lock); list_del_init(&bh->b_assoc_buffers);