// SPDX-License-Identifier: GPL-2.0-or-later/* * raid5.c : Multiple Devices driver for Linux * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman * Copyright (C) 1999, 2000 Ingo Molnar * Copyright (C) 2002, 2003 H. Peter Anvin * * RAID-4/5/6 management functions. * Thanks to Penguin Computing for making the RAID-6 development possible * by donating a test server! */ /* * BITMAP UNPLUGGING: * * The sequencing for updating the bitmap reliably is a little * subtle (and I got it wrong the first time) so it deserves some * explanation. * * We group bitmap updates into batches. Each batch has a number. * We may write out several batches at once, but that isn't very important. * conf->seq_write is the number of the last batch successfully written. * conf->seq_flush is the number of the last batch that was closed to * new additions. * When we discover that we will need to write to any block in a stripe * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq * the number of the batch it will be in. This is seq_flush+1. * When we are ready to do a write, if that batch hasn't been written yet, * we plug the array and queue the stripe for later. * When an unplug happens, we increment bm_flush, thus closing the current * batch. * When we notice that bm_flush > bm_write, we write out all pending updates * to the bitmap, and advance bm_write to where bm_flush was. * This may occasionally write a bit out twice, but is sure never to * miss any bits. */ #include <linux/blkdev.h>#include <linux/kthread.h>#include <linux/raid/pq.h>#include <linux/async_tx.h>#include <linux/module.h>#include <linux/async.h>#include <linux/seq_file.h>#include <linux/cpu.h>#include <linux/slab.h>#include <linux/ratelimit.h>#include <linux/nodemask.h> #include <trace/events/block.h>#include <linux/list_sort.h> #include "md.h"#include "raid5.h"#include "raid0.h"#include "md-bitmap.h"#include "raid5-log.h" #define UNSUPPORTED_MDDEV_FLAGS \ ((1L << MD_FAILFAST_SUPPORTED) | \ (1L << MD_FAILLAST_DEV) | \ (1L << MD_SERIALIZE_POLICY)) #define cpu_to_group(cpu) cpu_to_node(cpu)#define ANY_GROUP NUMA_NO_NODE #define RAID5_MAX_REQ_STRIPES 256 static bool devices_handle_discard_safely = false;module_param(devices_handle_discard_safely, bool, 0644);MODULE_PARM_DESC(devices_handle_discard_safely, "Set to Y if all devices in each array reliably return zeroes on reads from discarded regions");static struct workqueue_struct *raid5_wq; static void raid5_quiesce(struct mddev *mddev, int quiesce); static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect){ int hash = (sect >> RAID5_STRIPE_SHIFT(conf)) & HASH_MASK; return &conf->stripe_hashtbl[hash];} static inline int stripe_hash_locks_hash(struct r5conf *conf, sector_t sect){ return (sect >> RAID5_STRIPE_SHIFT(conf)) & STRIPE_HASH_LOCKS_MASK;} static inline void lock_device_hash_lock(struct r5conf *conf, int hash) __acquires(&conf->device_lock){ spin_lock_irq(conf->hash_locks + hash); spin_lock(&conf->device_lock);} static inline void unlock_device_hash_lock(struct r5conf *conf, int hash) __releases(&conf->device_lock){ spin_unlock(&conf->device_lock); spin_unlock_irq(conf->hash_locks + hash);} static inline void lock_all_device_hash_locks_irq(struct r5conf *conf) __acquires(&conf->device_lock){ int i; spin_lock_irq(conf->hash_locks); for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++) spin_lock_nest_lock(conf->hash_locks + i, conf->hash_locks); spin_lock(&conf->device_lock);} static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf) __releases(&conf->device_lock){ int i; spin_unlock(&conf->device_lock); for (i = NR_STRIPE_HASH_LOCKS - 1; i; i--) spin_unlock(conf->hash_locks + i); spin_unlock_irq(conf->hash_locks);} /* Find first data disk in a raid6 stripe */static inline int raid6_d0(struct stripe_head *sh){ if (sh->ddf_layout) /* ddf always start from first device */ return 0; /* md starts just after Q block */ if (sh->qd_idx == sh->disks - 1) return 0; else return sh->qd_idx + 1;}static inline int raid6_next_disk(int disk, int raid_disks){ disk++; return (disk < raid_disks) ? disk : 0;} /* When walking through the disks in a raid5, starting at raid6_d0, * We need to map each disk to a 'slot', where the data disks are slot * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk * is raid_disks-1. This help does that mapping. */static int raid6_idx_to_slot(int idx, struct stripe_head *sh, int *count, int syndrome_disks){ int slot = *count; if (sh->ddf_layout) (*count)++; if (idx == sh->pd_idx) return syndrome_disks; if (idx == sh->qd_idx) return syndrome_disks + 1; if (!sh->ddf_layout) (*count)++; return slot;} static void print_raid5_conf(struct r5conf *conf); static int stripe_operations_active(struct stripe_head *sh){ return sh->check_state || sh->reconstruct_state || test_bit(STRIPE_BIOFILL_RUN, &sh->state) || test_bit(STRIPE_COMPUTE_RUN, &sh->state);} static bool stripe_is_lowprio(struct stripe_head *sh){ return (test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state) || test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state)) && !test_bit(STRIPE_R5C_CACHING, &sh->state);} static void raid5_wakeup_stripe_thread(struct stripe_head *sh) __must_hold(&sh->raid_conf->device_lock){ struct r5conf *conf = sh->raid_conf; struct r5worker_group *group; int thread_cnt; int i, cpu = sh->cpu; if (!cpu_online(cpu)) { cpu = cpumask_any(cpu_online_mask); sh->cpu = cpu; } if (list_empty(&sh->lru)) { struct r5worker_group *group; group = conf->worker_groups + cpu_to_group(cpu); if (stripe_is_lowprio(sh)) list_add_tail(&sh->lru, &group->loprio_list); else list_add_tail(&sh->lru, &group->handle_list); group->stripes_cnt++; sh->group = group; } if (conf->worker_cnt_per_group == 0) { md_wakeup_thread(conf->mddev->thread); return; } group = conf->worker_groups + cpu_to_group(sh->cpu); group->workers[0].working = true; /* at least one worker should run to avoid race */ queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work); thread_cnt = group->stripes_cnt / MAX_STRIPE_BATCH - 1; /* wakeup more workers */ for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) { if (group->workers[i].working == false) { group->workers[i].working = true; queue_work_on(sh->cpu, raid5_wq, &group->workers[i].work); thread_cnt--; } }} static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh, struct list_head *temp_inactive_list) __must_hold(&conf->device_lock){ int i; int injournal = 0; /* number of date pages with R5_InJournal */ BUG_ON(!list_empty(&sh->lru)); BUG_ON(atomic_read(&conf->active_stripes)==0); if (r5c_is_writeback(conf->log)) for (i = sh->disks; i--; ) if (test_bit(R5_InJournal, &sh->dev[i].flags)) injournal++; /* * In the following cases, the stripe cannot be released to cached * lists. Therefore, we make the stripe write out and set * STRIPE_HANDLE: * 1. when quiesce in r5c write back; * 2. when resync is requested fot the stripe. */ if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) || (conf->quiesce && r5c_is_writeback(conf->log) && !test_bit(STRIPE_HANDLE, &sh->state) && injournal != 0)) { if (test_bit(STRIPE_R5C_CACHING, &sh->state)) r5c_make_stripe_write_out(sh); set_bit(STRIPE_HANDLE, &sh->state); } if (test_bit(STRIPE_HANDLE, &sh->state)) { if (test_bit(STRIPE_DELAYED, &sh->state) && !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) list_add_tail(&sh->lru, &conf->delayed_list); else if (test_bit(STRIPE_BIT_DELAY, &sh->state) && sh->bm_seq - conf->seq_write > 0) list_add_tail(&sh->lru, &conf->bitmap_list); else { clear_bit(STRIPE_DELAYED, &sh->state); clear_bit(STRIPE_BIT_DELAY, &sh->state); if (conf->worker_cnt_per_group == 0) { if (stripe_is_lowprio(sh)) list_add_tail(&sh->lru, &conf->loprio_list); else list_add_tail(&sh->lru, &conf->handle_list); } else { raid5_wakeup_stripe_thread(sh); return; } } md_wakeup_thread(conf->mddev->thread); } else { BUG_ON(stripe_operations_active(sh)); if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) if (atomic_dec_return(&conf->preread_active_stripes) < IO_THRESHOLD) md_wakeup_thread(conf->mddev->thread); atomic_dec(&conf->active_stripes); if (!test_bit(STRIPE_EXPANDING, &sh->state)) { if (!r5c_is_writeback(conf->log)) list_add_tail(&sh->lru, temp_inactive_list); else { WARN_ON(test_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags)); if (injournal == 0) list_add_tail(&sh->lru, temp_inactive_list); else if (injournal == conf->raid_disks - conf->max_degraded) { /* full stripe */ if (!test_and_set_bit(STRIPE_R5C_FULL_STRIPE, &sh->state)) atomic_inc(&conf->r5c_cached_full_stripes); if (test_and_clear_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state)) atomic_dec(&conf->r5c_cached_partial_stripes); list_add_tail(&sh->lru, &conf->r5c_full_stripe_list); r5c_check_cached_full_stripe(conf); } else /* * STRIPE_R5C_PARTIAL_STRIPE is set in * r5c_try_caching_write(). No need to * set it again. */ list_add_tail(&sh->lru, &conf->r5c_partial_stripe_list); } } }} static void __release_stripe(struct r5conf *conf, struct stripe_head *sh, struct list_head *temp_inactive_list) __must_hold(&conf->device_lock){ if (atomic_dec_and_test(&sh->count)) do_release_stripe(conf, sh, temp_inactive_list);} /* * @hash could be NR_STRIPE_HASH_LOCKS, then we have a list of inactive_list * * Be careful: Only one task can add/delete stripes from temp_inactive_list at * given time. Adding stripes only takes device lock, while deleting stripes * only takes hash lock. */static void release_inactive_stripe_list(struct r5conf *conf, struct list_head *temp_inactive_list, int hash){ int size; bool do_wakeup = false; unsigned long flags; if (hash == NR_STRIPE_HASH_LOCKS) { size = NR_STRIPE_HASH_LOCKS; hash = NR_STRIPE_HASH_LOCKS - 1; } else size = 1; while (size) { struct list_head *list = &temp_inactive_list[size - 1]; /* * We don't hold any lock here yet, raid5_get_active_stripe() might * remove stripes from the list */ if (!list_empty_careful(list)) { spin_lock_irqsave(conf->hash_locks + hash, flags); if (list_empty(conf->inactive_list + hash) && !list_empty(list)) atomic_dec(&conf->empty_inactive_list_nr); list_splice_tail_init(list, conf->inactive_list + hash); do_wakeup = true; spin_unlock_irqrestore(conf->hash_locks + hash, flags); } size--; hash--; } if (do_wakeup) { wake_up(&conf->wait_for_stripe); if (atomic_read(&conf->active_stripes) == 0) wake_up(&conf->wait_for_quiescent); if (conf->retry_read_aligned) md_wakeup_thread(conf->mddev->thread); }} static int release_stripe_list(struct r5conf *conf, struct list_head *temp_inactive_list) __must_hold(&conf->device_lock){ struct stripe_head *sh, *t; int count = 0; struct llist_node *head; head = llist_del_all(&conf->released_stripes); head = llist_reverse_order(head); llist_for_each_entry_safe(sh, t, head, release_list) { int hash; /* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */ smp_mb(); clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state); /* * Don't worry the bit is set here, because if the bit is set * again, the count is always > 1. This is true for * STRIPE_ON_UNPLUG_LIST bit too. */ hash = sh->hash_lock_index; __release_stripe(conf, sh, &temp_inactive_list[hash]); count++; } return count;} void raid5_release_stripe(struct stripe_head *sh){ struct r5conf *conf = sh->raid_conf; unsigned long flags; struct list_head list; int hash; bool wakeup; /* Avoid release_list until the last reference. */ if (atomic_add_unless(&sh->count, -1, 1)) return; if (unlikely(!conf->mddev->thread) || test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state)) goto slow_path; wakeup = llist_add(&sh->release_list, &conf->released_stripes); if (wakeup) md_wakeup_thread(conf->mddev->thread); return;slow_path: /* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */ if (atomic_dec_and_lock_irqsave(&sh->count, &conf->device_lock, flags)) { INIT_LIST_HEAD(&list); hash = sh->hash_lock_index; do_release_stripe(conf, sh, &list); spin_unlock_irqrestore(&conf->device_lock, flags); release_inactive_stripe_list(conf, &list, hash); }} static inline void remove_hash(struct stripe_head *sh){ pr_debug("remove_hash(), stripe %llu\n", (unsigned long long)sh->sector); hlist_del_init(&sh->hash);} static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh){ struct hlist_head *hp = stripe_hash(conf, sh->sector); pr_debug("insert_hash(), stripe %llu\n", (unsigned long long)sh->sector); hlist_add_head(&sh->hash, hp);} /* find an idle stripe, make sure it is unhashed, and return it. */static struct stripe_head *get_free_stripe(struct r5conf *conf, int hash){ struct stripe_head *sh = NULL; struct list_head *first; if (list_empty(conf->inactive_list + hash)) goto out; first = (conf->inactive_list + hash)->next; sh = list_entry(first, struct stripe_head, lru); list_del_init(first); remove_hash(sh); atomic_inc(&conf->active_stripes); BUG_ON(hash != sh->hash_lock_index); if (list_empty(conf->inactive_list + hash)) atomic_inc(&conf->empty_inactive_list_nr);out: return sh;} #if PAGE_SIZE != DEFAULT_STRIPE_SIZEstatic void free_stripe_pages(struct stripe_head *sh){ int i; struct page *p; /* Have not allocate page pool */ if (!sh->pages) return; for (i = 0; i < sh->nr_pages; i++) { p = sh->pages[i]; if (p) put_page(p); sh->pages[i] = NULL; }} static int alloc_stripe_pages(struct stripe_head *sh, gfp_t gfp){ int i; struct page *p; for (i = 0; i < sh->nr_pages; i++) { /* The page have allocated. */ if (sh->pages[i]) continue; p = alloc_page(gfp); if (!p) { free_stripe_pages(sh); return -ENOMEM; } sh->pages[i] = p; }Lines 1–500 of 9383. Load more
// SPDX-License-Identifier: GPL-2.0-or-later /* * raid5.c : Multiple Devices driver for Linux * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman * Copyright (C) 1999, 2000 Ingo Molnar * Copyright (C) 2002, 2003 H. Peter Anvin * * RAID-4/5/6 management functions. * Thanks to Penguin Computing for making the RAID-6 development possible * by donating a test server! */ /* * BITMAP UNPLUGGING: * * The sequencing for updating the bitmap reliably is a little * subtle (and I got it wrong the first time) so it deserves some * explanation. * * We group bitmap updates into batches. Each batch has a number. * We may write out several batches at once, but that isn't very important. * conf->seq_write is the number of the last batch successfully written. * conf->seq_flush is the number of the last batch that was closed to * new additions. * When we discover that we will need to write to any block in a stripe * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq * the number of the batch it will be in. This is seq_flush+1. * When we are ready to do a write, if that batch hasn't been written yet, * we plug the array and queue the stripe for later. * When an unplug happens, we increment bm_flush, thus closing the current * batch. * When we notice that bm_flush > bm_write, we write out all pending updates * to the bitmap, and advance bm_write to where bm_flush was. * This may occasionally write a bit out twice, but is sure never to * miss any bits. */ #include <linux/blkdev.h> #include <linux/kthread.h> #include <linux/raid/pq.h> #include <linux/async_tx.h> #include <linux/module.h> #include <linux/async.h> #include <linux/seq_file.h> #include <linux/cpu.h> #include <linux/slab.h> #include <linux/ratelimit.h> #include <linux/nodemask.h> #include <trace/events/block.h> #include <linux/list_sort.h> #include "md.h" #include "raid5.h" #include "raid0.h" #include "md-bitmap.h" #include "raid5-log.h" #define UNSUPPORTED_MDDEV_FLAGS \ ((1L << MD_FAILFAST_SUPPORTED) | \ (1L << MD_FAILLAST_DEV) | \ (1L << MD_SERIALIZE_POLICY)) #define cpu_to_group(cpu) cpu_to_node(cpu) #define ANY_GROUP NUMA_NO_NODE #define RAID5_MAX_REQ_STRIPES 256 static bool devices_handle_discard_safely = false; module_param(devices_handle_discard_safely, bool, 0644); MODULE_PARM_DESC(devices_handle_discard_safely, "Set to Y if all devices in each array reliably return zeroes on reads from discarded regions"); static struct workqueue_struct *raid5_wq; static void raid5_quiesce(struct mddev *mddev, int quiesce); static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect) { int hash = (sect >> RAID5_STRIPE_SHIFT(conf)) & HASH_MASK; return &conf->stripe_hashtbl[hash]; } static inline int stripe_hash_locks_hash(struct r5conf *conf, sector_t sect) { return (sect >> RAID5_STRIPE_SHIFT(conf)) & STRIPE_HASH_LOCKS_MASK; } static inline void lock_device_hash_lock(struct r5conf *conf, int hash) __acquires(&conf->device_lock) { spin_lock_irq(conf->hash_locks + hash); spin_lock(&conf->device_lock); } static inline void unlock_device_hash_lock(struct r5conf *conf, int hash) __releases(&conf->device_lock) { spin_unlock(&conf->device_lock); spin_unlock_irq(conf->hash_locks + hash); } static inline void lock_all_device_hash_locks_irq(struct r5conf *conf) __acquires(&conf->device_lock) { int i; spin_lock_irq(conf->hash_locks); for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++) spin_lock_nest_lock(conf->hash_locks + i, conf->hash_locks); spin_lock(&conf->device_lock); } static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf) __releases(&conf->device_lock) { int i; spin_unlock(&conf->device_lock); for (i = NR_STRIPE_HASH_LOCKS - 1; i; i--) spin_unlock(conf->hash_locks + i); spin_unlock_irq(conf->hash_locks); } /* Find first data disk in a raid6 stripe */ static inline int raid6_d0(struct stripe_head *sh) { if (sh->ddf_layout) /* ddf always start from first device */ return 0; /* md starts just after Q block */ if (sh->qd_idx == sh->disks - 1) return 0; else return sh->qd_idx + 1; } static inline int raid6_next_disk(int disk, int raid_disks) { disk++; return (disk < raid_disks) ? disk : 0; } /* When walking through the disks in a raid5, starting at raid6_d0, * We need to map each disk to a 'slot', where the data disks are slot * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk * is raid_disks-1. This help does that mapping. */ static int raid6_idx_to_slot(int idx, struct stripe_head *sh, int *count, int syndrome_disks) { int slot = *count; if (sh->ddf_layout) (*count)++; if (idx == sh->pd_idx) return syndrome_disks; if (idx == sh->qd_idx) return syndrome_disks + 1; if (!sh->ddf_layout) (*count)++; return slot; } static void print_raid5_conf(struct r5conf *conf); static int stripe_operations_active(struct stripe_head *sh) { return sh->check_state || sh->reconstruct_state || test_bit(STRIPE_BIOFILL_RUN, &sh->state) || test_bit(STRIPE_COMPUTE_RUN, &sh->state); } static bool stripe_is_lowprio(struct stripe_head *sh) { return (test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state) || test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state)) && !test_bit(STRIPE_R5C_CACHING, &sh->state); } static void raid5_wakeup_stripe_thread(struct stripe_head *sh) __must_hold(&sh->raid_conf->device_lock) { struct r5conf *conf = sh->raid_conf; struct r5worker_group *group; int thread_cnt; int i, cpu = sh->cpu; if (!cpu_online(cpu)) { cpu = cpumask_any(cpu_online_mask); sh->cpu = cpu; } if (list_empty(&sh->lru)) { struct r5worker_group *group; group = conf->worker_groups + cpu_to_group(cpu); if (stripe_is_lowprio(sh)) list_add_tail(&sh->lru, &group->loprio_list); else list_add_tail(&sh->lru, &group->handle_list); group->stripes_cnt++; sh->group = group; } if (conf->worker_cnt_per_group == 0) { md_wakeup_thread(conf->mddev->thread); return; } group = conf->worker_groups + cpu_to_group(sh->cpu); group->workers[0].working = true; /* at least one worker should run to avoid race */ queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work); thread_cnt = group->stripes_cnt / MAX_STRIPE_BATCH - 1; /* wakeup more workers */ for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) { if (group->workers[i].working == false) { group->workers[i].working = true; queue_work_on(sh->cpu, raid5_wq, &group->workers[i].work); thread_cnt--; } } } static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh, struct list_head *temp_inactive_list) __must_hold(&conf->device_lock) { int i; int injournal = 0; /* number of date pages with R5_InJournal */ BUG_ON(!list_empty(&sh->lru)); BUG_ON(atomic_read(&conf->active_stripes)==0); if (r5c_is_writeback(conf->log)) for (i = sh->disks; i--; ) if (test_bit(R5_InJournal, &sh->dev[i].flags)) injournal++; /* * In the following cases, the stripe cannot be released to cached * lists. Therefore, we make the stripe write out and set * STRIPE_HANDLE: * 1. when quiesce in r5c write back; * 2. when resync is requested fot the stripe. */ if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) || (conf->quiesce && r5c_is_writeback(conf->log) && !test_bit(STRIPE_HANDLE, &sh->state) && injournal != 0)) { if (test_bit(STRIPE_R5C_CACHING, &sh->state)) r5c_make_stripe_write_out(sh); set_bit(STRIPE_HANDLE, &sh->state); } if (test_bit(STRIPE_HANDLE, &sh->state)) { if (test_bit(STRIPE_DELAYED, &sh->state) && !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) list_add_tail(&sh->lru, &conf->delayed_list); else if (test_bit(STRIPE_BIT_DELAY, &sh->state) && sh->bm_seq - conf->seq_write > 0) list_add_tail(&sh->lru, &conf->bitmap_list); else { clear_bit(STRIPE_DELAYED, &sh->state); clear_bit(STRIPE_BIT_DELAY, &sh->state); if (conf->worker_cnt_per_group == 0) { if (stripe_is_lowprio(sh)) list_add_tail(&sh->lru, &conf->loprio_list); else list_add_tail(&sh->lru, &conf->handle_list); } else { raid5_wakeup_stripe_thread(sh); return; } } md_wakeup_thread(conf->mddev->thread); } else { BUG_ON(stripe_operations_active(sh)); if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) if (atomic_dec_return(&conf->preread_active_stripes) < IO_THRESHOLD) md_wakeup_thread(conf->mddev->thread); atomic_dec(&conf->active_stripes); if (!test_bit(STRIPE_EXPANDING, &sh->state)) { if (!r5c_is_writeback(conf->log)) list_add_tail(&sh->lru, temp_inactive_list); else { WARN_ON(test_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags)); if (injournal == 0) list_add_tail(&sh->lru, temp_inactive_list); else if (injournal == conf->raid_disks - conf->max_degraded) { /* full stripe */ if (!test_and_set_bit(STRIPE_R5C_FULL_STRIPE, &sh->state)) atomic_inc(&conf->r5c_cached_full_stripes); if (test_and_clear_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state)) atomic_dec(&conf->r5c_cached_partial_stripes); list_add_tail(&sh->lru, &conf->r5c_full_stripe_list); r5c_check_cached_full_stripe(conf); } else /* * STRIPE_R5C_PARTIAL_STRIPE is set in * r5c_try_caching_write(). No need to * set it again. */ list_add_tail(&sh->lru, &conf->r5c_partial_stripe_list); } } } } static void __release_stripe(struct r5conf *conf, struct stripe_head *sh, struct list_head *temp_inactive_list) __must_hold(&conf->device_lock) { if (atomic_dec_and_test(&sh->count)) do_release_stripe(conf, sh, temp_inactive_list); } /* * @hash could be NR_STRIPE_HASH_LOCKS, then we have a list of inactive_list * * Be careful: Only one task can add/delete stripes from temp_inactive_list at * given time. Adding stripes only takes device lock, while deleting stripes * only takes hash lock. */ static void release_inactive_stripe_list(struct r5conf *conf, struct list_head *temp_inactive_list, int hash) { int size; bool do_wakeup = false; unsigned long flags; if (hash == NR_STRIPE_HASH_LOCKS) { size = NR_STRIPE_HASH_LOCKS; hash = NR_STRIPE_HASH_LOCKS - 1; } else size = 1; while (size) { struct list_head *list = &temp_inactive_list[size - 1]; /* * We don't hold any lock here yet, raid5_get_active_stripe() might * remove stripes from the list */ if (!list_empty_careful(list)) { spin_lock_irqsave(conf->hash_locks + hash, flags); if (list_empty(conf->inactive_list + hash) && !list_empty(list)) atomic_dec(&conf->empty_inactive_list_nr); list_splice_tail_init(list, conf->inactive_list + hash); do_wakeup = true; spin_unlock_irqrestore(conf->hash_locks + hash, flags); } size--; hash--; } if (do_wakeup) { wake_up(&conf->wait_for_stripe); if (atomic_read(&conf->active_stripes) == 0) wake_up(&conf->wait_for_quiescent); if (conf->retry_read_aligned) md_wakeup_thread(conf->mddev->thread); } } static int release_stripe_list(struct r5conf *conf, struct list_head *temp_inactive_list) __must_hold(&conf->device_lock) { struct stripe_head *sh, *t; int count = 0; struct llist_node *head; head = llist_del_all(&conf->released_stripes); head = llist_reverse_order(head); llist_for_each_entry_safe(sh, t, head, release_list) { int hash; /* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */ smp_mb(); clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state); /* * Don't worry the bit is set here, because if the bit is set * again, the count is always > 1. This is true for * STRIPE_ON_UNPLUG_LIST bit too. */ hash = sh->hash_lock_index; __release_stripe(conf, sh, &temp_inactive_list[hash]); count++; } return count; } void raid5_release_stripe(struct stripe_head *sh) { struct r5conf *conf = sh->raid_conf; unsigned long flags; struct list_head list; int hash; bool wakeup; /* Avoid release_list until the last reference. */ if (atomic_add_unless(&sh->count, -1, 1)) return; if (unlikely(!conf->mddev->thread) || test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state)) goto slow_path; wakeup = llist_add(&sh->release_list, &conf->released_stripes); if (wakeup) md_wakeup_thread(conf->mddev->thread); return; slow_path: /* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */ if (atomic_dec_and_lock_irqsave(&sh->count, &conf->device_lock, flags)) { INIT_LIST_HEAD(&list); hash = sh->hash_lock_index; do_release_stripe(conf, sh, &list); spin_unlock_irqrestore(&conf->device_lock, flags); release_inactive_stripe_list(conf, &list, hash); } } static inline void remove_hash(struct stripe_head *sh) { pr_debug("remove_hash(), stripe %llu\n", (unsigned long long)sh->sector); hlist_del_init(&sh->hash); } static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh) { struct hlist_head *hp = stripe_hash(conf, sh->sector); pr_debug("insert_hash(), stripe %llu\n", (unsigned long long)sh->sector); hlist_add_head(&sh->hash, hp); } /* find an idle stripe, make sure it is unhashed, and return it. */ static struct stripe_head *get_free_stripe(struct r5conf *conf, int hash) { struct stripe_head *sh = NULL; struct list_head *first; if (list_empty(conf->inactive_list + hash)) goto out; first = (conf->inactive_list + hash)->next; sh = list_entry(first, struct stripe_head, lru); list_del_init(first); remove_hash(sh); atomic_inc(&conf->active_stripes); BUG_ON(hash != sh->hash_lock_index); if (list_empty(conf->inactive_list + hash)) atomic_inc(&conf->empty_inactive_list_nr); out: return sh; } #if PAGE_SIZE != DEFAULT_STRIPE_SIZE static void free_stripe_pages(struct stripe_head *sh) { int i; struct page *p; /* Have not allocate page pool */ if (!sh->pages) return; for (i = 0; i < sh->nr_pages; i++) { p = sh->pages[i]; if (p) put_page(p); sh->pages[i] = NULL; } } static int alloc_stripe_pages(struct stripe_head *sh, gfp_t gfp) { int i; struct page *p; for (i = 0; i < sh->nr_pages; i++) { /* The page have allocated. */ if (sh->pages[i]) continue; p = alloc_page(gfp); if (!p) { free_stripe_pages(sh); return -ENOMEM; } sh->pages[i] = p; }