// SPDX-License-Identifier: GPL-2.0-or-later/* * raid1.c : Multiple Devices driver for Linux * * Copyright (C) 1999, 2000, 2001 Ingo Molnar, Red Hat * * Copyright (C) 1996, 1997, 1998 Ingo Molnar, Miguel de Icaza, Gadi Oxman * * RAID-1 management functions. * * Better read-balancing code written by Mika Kuoppala <miku@iki.fi>, 2000 * * Fixes to reconstruction by Jakob Østergaard" <jakob@ostenfeld.dk> * Various fixes by Neil Brown <neilb@cse.unsw.edu.au> * * Changes by Peter T. Breuer <ptb@it.uc3m.es> 31/1/2003 to support * bitmapped intelligence in resync: * * - bitmap marked during normal i/o * - bitmap used to skip nondirty blocks during sync * * Additions to bitmap code, (C) 2003-2004 Paul Clements, SteelEye Technology: * - persistent bitmap code */ #include <linux/slab.h>#include <linux/delay.h>#include <linux/blkdev.h>#include <linux/module.h>#include <linux/seq_file.h>#include <linux/ratelimit.h>#include <linux/interval_tree_generic.h> #include <trace/events/block.h> #include "md.h"#include "raid1.h"#include "md-bitmap.h"#include "md-cluster.h" #define UNSUPPORTED_MDDEV_FLAGS \ ((1L << MD_HAS_JOURNAL) | \ (1L << MD_JOURNAL_CLEAN) | \ (1L << MD_HAS_PPL) | \ (1L << MD_HAS_MULTIPLE_PPLS)) static void allow_barrier(struct r1conf *conf, sector_t sector_nr);static void lower_barrier(struct r1conf *conf, sector_t sector_nr);static void raid1_free(struct mddev *mddev, void *priv); #define RAID_1_10_NAME "raid1"#include "raid1-10.c" #define START(node) ((node)->start)#define LAST(node) ((node)->last)INTERVAL_TREE_DEFINE(struct serial_info, node, sector_t, _subtree_last, START, LAST, static inline, raid1_rb); static int check_and_add_serial(struct md_rdev *rdev, struct r1bio *r1_bio, struct serial_info *si){ unsigned long flags; int ret = 0; sector_t lo = r1_bio->sector; sector_t hi = lo + r1_bio->sectors - 1; int idx = sector_to_idx(r1_bio->sector); struct serial_in_rdev *serial = &rdev->serial[idx]; struct serial_info *head_si; spin_lock_irqsave(&serial->serial_lock, flags); /* collision happened */ head_si = raid1_rb_iter_first(&serial->serial_rb, lo, hi); if (head_si && head_si != si) { si->start = lo; si->last = hi; si->wnode_start = head_si->wnode_start; list_add_tail(&si->list_node, &head_si->waiters); ret = -EBUSY; } else if (!head_si) { si->start = lo; si->last = hi; si->wnode_start = si->start; raid1_rb_insert(si, &serial->serial_rb); } spin_unlock_irqrestore(&serial->serial_lock, flags); return ret;} static void wait_for_serialization(struct md_rdev *rdev, struct r1bio *r1_bio){ struct mddev *mddev = rdev->mddev; struct serial_info *si; if (WARN_ON(!mddev->serial_info_pool)) return; si = mempool_alloc(mddev->serial_info_pool, GFP_NOIO); INIT_LIST_HEAD(&si->waiters); INIT_LIST_HEAD(&si->list_node); init_completion(&si->ready); while (check_and_add_serial(rdev, r1_bio, si)) { wait_for_completion(&si->ready); reinit_completion(&si->ready); }} static void remove_serial(struct md_rdev *rdev, sector_t lo, sector_t hi){ struct serial_info *si, *iter_si; unsigned long flags; int found = 0; struct mddev *mddev = rdev->mddev; int idx = sector_to_idx(lo); struct serial_in_rdev *serial = &rdev->serial[idx]; spin_lock_irqsave(&serial->serial_lock, flags); for (si = raid1_rb_iter_first(&serial->serial_rb, lo, hi); si; si = raid1_rb_iter_next(si, lo, hi)) { if (si->start == lo && si->last == hi) { found = 1; break; } } if (found) { raid1_rb_remove(si, &serial->serial_rb); if (!list_empty(&si->waiters)) { list_for_each_entry(iter_si, &si->waiters, list_node) { if (iter_si->wnode_start == si->wnode_start) { list_del_init(&iter_si->list_node); list_splice_init(&si->waiters, &iter_si->waiters); raid1_rb_insert(iter_si, &serial->serial_rb); complete(&iter_si->ready); break; } } } mempool_free(si, mddev->serial_info_pool); } else { WARN(1, "The write IO is not recorded for serialization\n"); } spin_unlock_irqrestore(&serial->serial_lock, flags);} /* * for resync bio, r1bio pointer can be retrieved from the per-bio * 'struct resync_pages'. */static inline struct r1bio *get_resync_r1bio(struct bio *bio){ return get_resync_pages(bio)->raid_bio;} static void *r1bio_pool_alloc(gfp_t gfp_flags, struct r1conf *conf){ int size = offsetof(struct r1bio, bios[conf->raid_disks * 2]); /* allocate a r1bio with room for raid_disks entries in the bios array */ return kzalloc(size, gfp_flags);} #define RESYNC_DEPTH 32#define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9)#define RESYNC_WINDOW (RESYNC_BLOCK_SIZE * RESYNC_DEPTH)#define RESYNC_WINDOW_SECTORS (RESYNC_WINDOW >> 9)#define CLUSTER_RESYNC_WINDOW (16 * RESYNC_WINDOW)#define CLUSTER_RESYNC_WINDOW_SECTORS (CLUSTER_RESYNC_WINDOW >> 9) static void * r1buf_pool_alloc(gfp_t gfp_flags, void *data){ struct r1conf *conf = data; struct r1bio *r1_bio; struct bio *bio; int need_pages; int j; struct resync_pages *rps; r1_bio = r1bio_pool_alloc(gfp_flags, conf); if (!r1_bio) return NULL; rps = kmalloc_objs(struct resync_pages, conf->raid_disks * 2, gfp_flags); if (!rps) goto out_free_r1bio; /* * Allocate bios : 1 for reading, n-1 for writing */ for (j = conf->raid_disks * 2; j-- ; ) { bio = bio_kmalloc(RESYNC_PAGES, gfp_flags); if (!bio) goto out_free_bio; bio_init_inline(bio, NULL, RESYNC_PAGES, 0); r1_bio->bios[j] = bio; } /* * Allocate RESYNC_PAGES data pages and attach them to * the first bio. * If this is a user-requested check/repair, allocate * RESYNC_PAGES for each bio. */ if (test_bit(MD_RECOVERY_REQUESTED, &conf->mddev->recovery)) need_pages = conf->raid_disks * 2; else need_pages = 1; for (j = 0; j < conf->raid_disks * 2; j++) { struct resync_pages *rp = &rps[j]; bio = r1_bio->bios[j]; if (j < need_pages) { if (resync_alloc_pages(rp, gfp_flags)) goto out_free_pages; } else { memcpy(rp, &rps[0], sizeof(*rp)); resync_get_all_pages(rp); } rp->raid_bio = r1_bio; bio->bi_private = rp; } r1_bio->master_bio = NULL; return r1_bio; out_free_pages: while (--j >= 0) resync_free_pages(&rps[j]); out_free_bio: while (++j < conf->raid_disks * 2) { bio_uninit(r1_bio->bios[j]); kfree(r1_bio->bios[j]); } kfree(rps); out_free_r1bio: rbio_pool_free(r1_bio, data); return NULL;} static void r1buf_pool_free(void *__r1_bio, void *data){ struct r1conf *conf = data; int i; struct r1bio *r1bio = __r1_bio; struct resync_pages *rp = NULL; for (i = conf->raid_disks * 2; i--; ) { rp = get_resync_pages(r1bio->bios[i]); resync_free_pages(rp); bio_uninit(r1bio->bios[i]); kfree(r1bio->bios[i]); } /* resync pages array stored in the 1st bio's .bi_private */ kfree(rp); rbio_pool_free(r1bio, data);} static void put_all_bios(struct r1conf *conf, struct r1bio *r1_bio){ int i; for (i = 0; i < conf->raid_disks * 2; i++) { struct bio **bio = r1_bio->bios + i; if (!BIO_SPECIAL(*bio)) bio_put(*bio); *bio = NULL; }} static void free_r1bio(struct r1bio *r1_bio){ struct r1conf *conf = r1_bio->mddev->private; put_all_bios(conf, r1_bio); mempool_free(r1_bio, conf->r1bio_pool);} static void put_buf(struct r1bio *r1_bio){ struct r1conf *conf = r1_bio->mddev->private; sector_t sect = r1_bio->sector; int i; for (i = 0; i < conf->raid_disks * 2; i++) { struct bio *bio = r1_bio->bios[i]; if (bio->bi_end_io) rdev_dec_pending(conf->mirrors[i].rdev, r1_bio->mddev); } mempool_free(r1_bio, &conf->r1buf_pool); lower_barrier(conf, sect);} static void reschedule_retry(struct r1bio *r1_bio){ unsigned long flags; struct mddev *mddev = r1_bio->mddev; struct r1conf *conf = mddev->private; int idx; idx = sector_to_idx(r1_bio->sector); spin_lock_irqsave(&conf->device_lock, flags); list_add(&r1_bio->retry_list, &conf->retry_list); atomic_inc(&conf->nr_queued[idx]); spin_unlock_irqrestore(&conf->device_lock, flags); wake_up(&conf->wait_barrier); md_wakeup_thread(mddev->thread);} /* * raid_end_bio_io() is called when we have finished servicing a mirrored * operation and are ready to return a success/failure code to the buffer * cache layer. */static void call_bio_endio(struct r1bio *r1_bio){ struct bio *bio = r1_bio->master_bio; if (!test_bit(R1BIO_Uptodate, &r1_bio->state)) bio->bi_status = BLK_STS_IOERR; bio_endio(bio);} static void raid_end_bio_io(struct r1bio *r1_bio){ struct bio *bio = r1_bio->master_bio; struct r1conf *conf = r1_bio->mddev->private; sector_t sector = r1_bio->sector; /* if nobody has done the final endio yet, do it now */ if (!test_and_set_bit(R1BIO_Returned, &r1_bio->state)) { pr_debug("raid1: sync end %s on sectors %llu-%llu\n", (bio_data_dir(bio) == WRITE) ? "write" : "read", (unsigned long long) bio->bi_iter.bi_sector, (unsigned long long) bio_end_sector(bio) - 1); call_bio_endio(r1_bio); } free_r1bio(r1_bio); /* * Wake up any possible resync thread that waits for the device * to go idle. All I/Os, even write-behind writes, are done. */ allow_barrier(conf, sector);} /* * Update disk head position estimator based on IRQ completion info. */static inline void update_head_pos(int disk, struct r1bio *r1_bio){ struct r1conf *conf = r1_bio->mddev->private; WRITE_ONCE(conf->mirrors[disk].head_position, r1_bio->sector + r1_bio->sectors);} /* * Find the disk number which triggered given bio */static int find_bio_disk(struct r1bio *r1_bio, struct bio *bio){ int mirror; struct r1conf *conf = r1_bio->mddev->private; int raid_disks = conf->raid_disks; for (mirror = 0; mirror < raid_disks * 2; mirror++) if (r1_bio->bios[mirror] == bio) break; BUG_ON(mirror == raid_disks * 2); update_head_pos(mirror, r1_bio); return mirror;} static void raid1_end_read_request(struct bio *bio){ int uptodate = !bio->bi_status; struct r1bio *r1_bio = bio->bi_private; struct r1conf *conf = r1_bio->mddev->private; struct md_rdev *rdev = conf->mirrors[r1_bio->read_disk].rdev; /* * this branch is our 'one mirror IO has finished' event handler: */ update_head_pos(r1_bio->read_disk, r1_bio); if (uptodate) { set_bit(R1BIO_Uptodate, &r1_bio->state); } else if (test_bit(FailFast, &rdev->flags) && test_bit(R1BIO_FailFast, &r1_bio->state)) { /* This was a fail-fast read so we definitely * want to retry */ ; } else if (!raid1_should_handle_error(bio)) { uptodate = 1; } else { /* If all other devices have failed, we want to return * the error upwards rather than fail the last device. * Here we redefine "uptodate" to mean "Don't want to retry" */ unsigned long flags; spin_lock_irqsave(&conf->device_lock, flags); if (r1_bio->mddev->degraded == conf->raid_disks || (r1_bio->mddev->degraded == conf->raid_disks-1 && test_bit(In_sync, &rdev->flags))) uptodate = 1; spin_unlock_irqrestore(&conf->device_lock, flags); } if (uptodate) { raid_end_bio_io(r1_bio); rdev_dec_pending(rdev, conf->mddev); } else { /* * oops, read error: */ pr_err_ratelimited("md/raid1:%s: %pg: rescheduling sector %llu\n", mdname(conf->mddev), rdev->bdev, (unsigned long long)r1_bio->sector); set_bit(R1BIO_ReadError, &r1_bio->state); reschedule_retry(r1_bio); /* don't drop the reference on read_disk yet */ }} static void close_write(struct r1bio *r1_bio){ struct mddev *mddev = r1_bio->mddev; /* it really is the end of this request */ if (test_bit(R1BIO_BehindIO, &r1_bio->state)) { bio_free_pages(r1_bio->behind_master_bio); bio_put(r1_bio->behind_master_bio); r1_bio->behind_master_bio = NULL; } if (test_bit(R1BIO_BehindIO, &r1_bio->state)) mddev->bitmap_ops->end_behind_write(mddev); md_write_end(mddev);} static void r1_bio_write_done(struct r1bio *r1_bio){ if (!atomic_dec_and_test(&r1_bio->remaining)) return; if (test_bit(R1BIO_WriteError, &r1_bio->state)) reschedule_retry(r1_bio); else { close_write(r1_bio); if (test_bit(R1BIO_MadeGood, &r1_bio->state)) reschedule_retry(r1_bio); else raid_end_bio_io(r1_bio); }} static void raid1_end_write_request(struct bio *bio){ struct r1bio *r1_bio = bio->bi_private; int behind = test_bit(R1BIO_BehindIO, &r1_bio->state); struct r1conf *conf = r1_bio->mddev->private; struct bio *to_put = NULL; int mirror = find_bio_disk(r1_bio, bio); struct md_rdev *rdev = conf->mirrors[mirror].rdev; sector_t lo = r1_bio->sector; sector_t hi = r1_bio->sector + r1_bio->sectors - 1; bool ignore_error = !raid1_should_handle_error(bio) || (bio->bi_status && bio_op(bio) == REQ_OP_DISCARD); /* * 'one mirror IO has finished' event handler: */ if (bio->bi_status && !ignore_error) { set_bit(WriteErrorSeen, &rdev->flags); if (!test_and_set_bit(WantReplacement, &rdev->flags)) set_bit(MD_RECOVERY_NEEDED, & conf->mddev->recovery); if (test_bit(FailFast, &rdev->flags) && (bio->bi_opf & MD_FAILFAST) && /* We never try FailFast to WriteMostly devices */ !test_bit(WriteMostly, &rdev->flags)) { md_error(r1_bio->mddev, rdev); } /* * When the device is faulty, it is not necessary to * handle write error.Lines 1–500 of 3482. Load more
// SPDX-License-Identifier: GPL-2.0-or-later /* * raid1.c : Multiple Devices driver for Linux * * Copyright (C) 1999, 2000, 2001 Ingo Molnar, Red Hat * * Copyright (C) 1996, 1997, 1998 Ingo Molnar, Miguel de Icaza, Gadi Oxman * * RAID-1 management functions. * * Better read-balancing code written by Mika Kuoppala <miku@iki.fi>, 2000 * * Fixes to reconstruction by Jakob Østergaard" <jakob@ostenfeld.dk> * Various fixes by Neil Brown <neilb@cse.unsw.edu.au> * * Changes by Peter T. Breuer <ptb@it.uc3m.es> 31/1/2003 to support * bitmapped intelligence in resync: * * - bitmap marked during normal i/o * - bitmap used to skip nondirty blocks during sync * * Additions to bitmap code, (C) 2003-2004 Paul Clements, SteelEye Technology: * - persistent bitmap code */ #include <linux/slab.h> #include <linux/delay.h> #include <linux/blkdev.h> #include <linux/module.h> #include <linux/seq_file.h> #include <linux/ratelimit.h> #include <linux/interval_tree_generic.h> #include <trace/events/block.h> #include "md.h" #include "raid1.h" #include "md-bitmap.h" #include "md-cluster.h" #define UNSUPPORTED_MDDEV_FLAGS \ ((1L << MD_HAS_JOURNAL) | \ (1L << MD_JOURNAL_CLEAN) | \ (1L << MD_HAS_PPL) | \ (1L << MD_HAS_MULTIPLE_PPLS)) static void allow_barrier(struct r1conf *conf, sector_t sector_nr); static void lower_barrier(struct r1conf *conf, sector_t sector_nr); static void raid1_free(struct mddev *mddev, void *priv); #define RAID_1_10_NAME "raid1" #include "raid1-10.c" #define START(node) ((node)->start) #define LAST(node) ((node)->last) INTERVAL_TREE_DEFINE(struct serial_info, node, sector_t, _subtree_last, START, LAST, static inline, raid1_rb); static int check_and_add_serial(struct md_rdev *rdev, struct r1bio *r1_bio, struct serial_info *si) { unsigned long flags; int ret = 0; sector_t lo = r1_bio->sector; sector_t hi = lo + r1_bio->sectors - 1; int idx = sector_to_idx(r1_bio->sector); struct serial_in_rdev *serial = &rdev->serial[idx]; struct serial_info *head_si; spin_lock_irqsave(&serial->serial_lock, flags); /* collision happened */ head_si = raid1_rb_iter_first(&serial->serial_rb, lo, hi); if (head_si && head_si != si) { si->start = lo; si->last = hi; si->wnode_start = head_si->wnode_start; list_add_tail(&si->list_node, &head_si->waiters); ret = -EBUSY; } else if (!head_si) { si->start = lo; si->last = hi; si->wnode_start = si->start; raid1_rb_insert(si, &serial->serial_rb); } spin_unlock_irqrestore(&serial->serial_lock, flags); return ret; } static void wait_for_serialization(struct md_rdev *rdev, struct r1bio *r1_bio) { struct mddev *mddev = rdev->mddev; struct serial_info *si; if (WARN_ON(!mddev->serial_info_pool)) return; si = mempool_alloc(mddev->serial_info_pool, GFP_NOIO); INIT_LIST_HEAD(&si->waiters); INIT_LIST_HEAD(&si->list_node); init_completion(&si->ready); while (check_and_add_serial(rdev, r1_bio, si)) { wait_for_completion(&si->ready); reinit_completion(&si->ready); } } static void remove_serial(struct md_rdev *rdev, sector_t lo, sector_t hi) { struct serial_info *si, *iter_si; unsigned long flags; int found = 0; struct mddev *mddev = rdev->mddev; int idx = sector_to_idx(lo); struct serial_in_rdev *serial = &rdev->serial[idx]; spin_lock_irqsave(&serial->serial_lock, flags); for (si = raid1_rb_iter_first(&serial->serial_rb, lo, hi); si; si = raid1_rb_iter_next(si, lo, hi)) { if (si->start == lo && si->last == hi) { found = 1; break; } } if (found) { raid1_rb_remove(si, &serial->serial_rb); if (!list_empty(&si->waiters)) { list_for_each_entry(iter_si, &si->waiters, list_node) { if (iter_si->wnode_start == si->wnode_start) { list_del_init(&iter_si->list_node); list_splice_init(&si->waiters, &iter_si->waiters); raid1_rb_insert(iter_si, &serial->serial_rb); complete(&iter_si->ready); break; } } } mempool_free(si, mddev->serial_info_pool); } else { WARN(1, "The write IO is not recorded for serialization\n"); } spin_unlock_irqrestore(&serial->serial_lock, flags); } /* * for resync bio, r1bio pointer can be retrieved from the per-bio * 'struct resync_pages'. */ static inline struct r1bio *get_resync_r1bio(struct bio *bio) { return get_resync_pages(bio)->raid_bio; } static void *r1bio_pool_alloc(gfp_t gfp_flags, struct r1conf *conf) { int size = offsetof(struct r1bio, bios[conf->raid_disks * 2]); /* allocate a r1bio with room for raid_disks entries in the bios array */ return kzalloc(size, gfp_flags); } #define RESYNC_DEPTH 32 #define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9) #define RESYNC_WINDOW (RESYNC_BLOCK_SIZE * RESYNC_DEPTH) #define RESYNC_WINDOW_SECTORS (RESYNC_WINDOW >> 9) #define CLUSTER_RESYNC_WINDOW (16 * RESYNC_WINDOW) #define CLUSTER_RESYNC_WINDOW_SECTORS (CLUSTER_RESYNC_WINDOW >> 9) static void * r1buf_pool_alloc(gfp_t gfp_flags, void *data) { struct r1conf *conf = data; struct r1bio *r1_bio; struct bio *bio; int need_pages; int j; struct resync_pages *rps; r1_bio = r1bio_pool_alloc(gfp_flags, conf); if (!r1_bio) return NULL; rps = kmalloc_objs(struct resync_pages, conf->raid_disks * 2, gfp_flags); if (!rps) goto out_free_r1bio; /* * Allocate bios : 1 for reading, n-1 for writing */ for (j = conf->raid_disks * 2; j-- ; ) { bio = bio_kmalloc(RESYNC_PAGES, gfp_flags); if (!bio) goto out_free_bio; bio_init_inline(bio, NULL, RESYNC_PAGES, 0); r1_bio->bios[j] = bio; } /* * Allocate RESYNC_PAGES data pages and attach them to * the first bio. * If this is a user-requested check/repair, allocate * RESYNC_PAGES for each bio. */ if (test_bit(MD_RECOVERY_REQUESTED, &conf->mddev->recovery)) need_pages = conf->raid_disks * 2; else need_pages = 1; for (j = 0; j < conf->raid_disks * 2; j++) { struct resync_pages *rp = &rps[j]; bio = r1_bio->bios[j]; if (j < need_pages) { if (resync_alloc_pages(rp, gfp_flags)) goto out_free_pages; } else { memcpy(rp, &rps[0], sizeof(*rp)); resync_get_all_pages(rp); } rp->raid_bio = r1_bio; bio->bi_private = rp; } r1_bio->master_bio = NULL; return r1_bio; out_free_pages: while (--j >= 0) resync_free_pages(&rps[j]); out_free_bio: while (++j < conf->raid_disks * 2) { bio_uninit(r1_bio->bios[j]); kfree(r1_bio->bios[j]); } kfree(rps); out_free_r1bio: rbio_pool_free(r1_bio, data); return NULL; } static void r1buf_pool_free(void *__r1_bio, void *data) { struct r1conf *conf = data; int i; struct r1bio *r1bio = __r1_bio; struct resync_pages *rp = NULL; for (i = conf->raid_disks * 2; i--; ) { rp = get_resync_pages(r1bio->bios[i]); resync_free_pages(rp); bio_uninit(r1bio->bios[i]); kfree(r1bio->bios[i]); } /* resync pages array stored in the 1st bio's .bi_private */ kfree(rp); rbio_pool_free(r1bio, data); } static void put_all_bios(struct r1conf *conf, struct r1bio *r1_bio) { int i; for (i = 0; i < conf->raid_disks * 2; i++) { struct bio **bio = r1_bio->bios + i; if (!BIO_SPECIAL(*bio)) bio_put(*bio); *bio = NULL; } } static void free_r1bio(struct r1bio *r1_bio) { struct r1conf *conf = r1_bio->mddev->private; put_all_bios(conf, r1_bio); mempool_free(r1_bio, conf->r1bio_pool); } static void put_buf(struct r1bio *r1_bio) { struct r1conf *conf = r1_bio->mddev->private; sector_t sect = r1_bio->sector; int i; for (i = 0; i < conf->raid_disks * 2; i++) { struct bio *bio = r1_bio->bios[i]; if (bio->bi_end_io) rdev_dec_pending(conf->mirrors[i].rdev, r1_bio->mddev); } mempool_free(r1_bio, &conf->r1buf_pool); lower_barrier(conf, sect); } static void reschedule_retry(struct r1bio *r1_bio) { unsigned long flags; struct mddev *mddev = r1_bio->mddev; struct r1conf *conf = mddev->private; int idx; idx = sector_to_idx(r1_bio->sector); spin_lock_irqsave(&conf->device_lock, flags); list_add(&r1_bio->retry_list, &conf->retry_list); atomic_inc(&conf->nr_queued[idx]); spin_unlock_irqrestore(&conf->device_lock, flags); wake_up(&conf->wait_barrier); md_wakeup_thread(mddev->thread); } /* * raid_end_bio_io() is called when we have finished servicing a mirrored * operation and are ready to return a success/failure code to the buffer * cache layer. */ static void call_bio_endio(struct r1bio *r1_bio) { struct bio *bio = r1_bio->master_bio; if (!test_bit(R1BIO_Uptodate, &r1_bio->state)) bio->bi_status = BLK_STS_IOERR; bio_endio(bio); } static void raid_end_bio_io(struct r1bio *r1_bio) { struct bio *bio = r1_bio->master_bio; struct r1conf *conf = r1_bio->mddev->private; sector_t sector = r1_bio->sector; /* if nobody has done the final endio yet, do it now */ if (!test_and_set_bit(R1BIO_Returned, &r1_bio->state)) { pr_debug("raid1: sync end %s on sectors %llu-%llu\n", (bio_data_dir(bio) == WRITE) ? "write" : "read", (unsigned long long) bio->bi_iter.bi_sector, (unsigned long long) bio_end_sector(bio) - 1); call_bio_endio(r1_bio); } free_r1bio(r1_bio); /* * Wake up any possible resync thread that waits for the device * to go idle. All I/Os, even write-behind writes, are done. */ allow_barrier(conf, sector); } /* * Update disk head position estimator based on IRQ completion info. */ static inline void update_head_pos(int disk, struct r1bio *r1_bio) { struct r1conf *conf = r1_bio->mddev->private; WRITE_ONCE(conf->mirrors[disk].head_position, r1_bio->sector + r1_bio->sectors); } /* * Find the disk number which triggered given bio */ static int find_bio_disk(struct r1bio *r1_bio, struct bio *bio) { int mirror; struct r1conf *conf = r1_bio->mddev->private; int raid_disks = conf->raid_disks; for (mirror = 0; mirror < raid_disks * 2; mirror++) if (r1_bio->bios[mirror] == bio) break; BUG_ON(mirror == raid_disks * 2); update_head_pos(mirror, r1_bio); return mirror; } static void raid1_end_read_request(struct bio *bio) { int uptodate = !bio->bi_status; struct r1bio *r1_bio = bio->bi_private; struct r1conf *conf = r1_bio->mddev->private; struct md_rdev *rdev = conf->mirrors[r1_bio->read_disk].rdev; /* * this branch is our 'one mirror IO has finished' event handler: */ update_head_pos(r1_bio->read_disk, r1_bio); if (uptodate) { set_bit(R1BIO_Uptodate, &r1_bio->state); } else if (test_bit(FailFast, &rdev->flags) && test_bit(R1BIO_FailFast, &r1_bio->state)) { /* This was a fail-fast read so we definitely * want to retry */ ; } else if (!raid1_should_handle_error(bio)) { uptodate = 1; } else { /* If all other devices have failed, we want to return * the error upwards rather than fail the last device. * Here we redefine "uptodate" to mean "Don't want to retry" */ unsigned long flags; spin_lock_irqsave(&conf->device_lock, flags); if (r1_bio->mddev->degraded == conf->raid_disks || (r1_bio->mddev->degraded == conf->raid_disks-1 && test_bit(In_sync, &rdev->flags))) uptodate = 1; spin_unlock_irqrestore(&conf->device_lock, flags); } if (uptodate) { raid_end_bio_io(r1_bio); rdev_dec_pending(rdev, conf->mddev); } else { /* * oops, read error: */ pr_err_ratelimited("md/raid1:%s: %pg: rescheduling sector %llu\n", mdname(conf->mddev), rdev->bdev, (unsigned long long)r1_bio->sector); set_bit(R1BIO_ReadError, &r1_bio->state); reschedule_retry(r1_bio); /* don't drop the reference on read_disk yet */ } } static void close_write(struct r1bio *r1_bio) { struct mddev *mddev = r1_bio->mddev; /* it really is the end of this request */ if (test_bit(R1BIO_BehindIO, &r1_bio->state)) { bio_free_pages(r1_bio->behind_master_bio); bio_put(r1_bio->behind_master_bio); r1_bio->behind_master_bio = NULL; } if (test_bit(R1BIO_BehindIO, &r1_bio->state)) mddev->bitmap_ops->end_behind_write(mddev); md_write_end(mddev); } static void r1_bio_write_done(struct r1bio *r1_bio) { if (!atomic_dec_and_test(&r1_bio->remaining)) return; if (test_bit(R1BIO_WriteError, &r1_bio->state)) reschedule_retry(r1_bio); else { close_write(r1_bio); if (test_bit(R1BIO_MadeGood, &r1_bio->state)) reschedule_retry(r1_bio); else raid_end_bio_io(r1_bio); } } static void raid1_end_write_request(struct bio *bio) { struct r1bio *r1_bio = bio->bi_private; int behind = test_bit(R1BIO_BehindIO, &r1_bio->state); struct r1conf *conf = r1_bio->mddev->private; struct bio *to_put = NULL; int mirror = find_bio_disk(r1_bio, bio); struct md_rdev *rdev = conf->mirrors[mirror].rdev; sector_t lo = r1_bio->sector; sector_t hi = r1_bio->sector + r1_bio->sectors - 1; bool ignore_error = !raid1_should_handle_error(bio) || (bio->bi_status && bio_op(bio) == REQ_OP_DISCARD); /* * 'one mirror IO has finished' event handler: */ if (bio->bi_status && !ignore_error) { set_bit(WriteErrorSeen, &rdev->flags); if (!test_and_set_bit(WantReplacement, &rdev->flags)) set_bit(MD_RECOVERY_NEEDED, & conf->mddev->recovery); if (test_bit(FailFast, &rdev->flags) && (bio->bi_opf & MD_FAILFAST) && /* We never try FailFast to WriteMostly devices */ !test_bit(WriteMostly, &rdev->flags)) { md_error(r1_bio->mddev, rdev); } /* * When the device is faulty, it is not necessary to * handle write error.