/* * raid5.c : Multiple Devices driver for Linux * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman * Copyright (C) 1999, 2000 Ingo Molnar * Copyright (C) 2002, 2003 H. Peter Anvin * * RAID-4/5/6 management functions. * Thanks to Penguin Computing for making the RAID-6 development possible * by donating a test server! * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2, or (at your option) * any later version. * * You should have received a copy of the GNU General Public License * (for example /usr/src/linux/COPYING); if not, write to the Free * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ /* * BITMAP UNPLUGGING: * * The sequencing for updating the bitmap reliably is a little * subtle (and I got it wrong the first time) so it deserves some * explanation. * * We group bitmap updates into batches. Each batch has a number. * We may write out several batches at once, but that isn't very important. * conf->seq_write is the number of the last batch successfully written. * conf->seq_flush is the number of the last batch that was closed to * new additions. * When we discover that we will need to write to any block in a stripe * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq * the number of the batch it will be in. This is seq_flush+1. * When we are ready to do a write, if that batch hasn't been written yet, * we plug the array and queue the stripe for later. * When an unplug happens, we increment bm_flush, thus closing the current * batch. * When we notice that bm_flush > bm_write, we write out all pending updates * to the bitmap, and advance bm_write to where bm_flush was. * This may occasionally write a bit out twice, but is sure never to * miss any bits. */ #include <linux/blkdev.h>#include <linux/kthread.h>#include <linux/raid/pq.h>#include <linux/async_tx.h>#include <linux/module.h>#include <linux/async.h>#include <linux/seq_file.h>#include <linux/cpu.h>#include <linux/slab.h>#include <linux/ratelimit.h>#include <linux/nodemask.h>#include <trace/events/block.h> #include "md.h"#include "raid5.h"#include "raid0.h"#include "bitmap.h" #define cpu_to_group(cpu) cpu_to_node(cpu)#define ANY_GROUP NUMA_NO_NODE static struct workqueue_struct *raid5_wq;/* * Stripe cache */ #define NR_STRIPES 256#define STRIPE_SIZE PAGE_SIZE#define STRIPE_SHIFT (PAGE_SHIFT - 9)#define STRIPE_SECTORS (STRIPE_SIZE>>9)#define IO_THRESHOLD 1#define BYPASS_THRESHOLD 1#define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head))#define HASH_MASK (NR_HASH - 1)#define MAX_STRIPE_BATCH 8 static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect){ int hash = (sect >> STRIPE_SHIFT) & HASH_MASK; return &conf->stripe_hashtbl[hash];} static inline int stripe_hash_locks_hash(sector_t sect){ return (sect >> STRIPE_SHIFT) & STRIPE_HASH_LOCKS_MASK;} static inline void lock_device_hash_lock(struct r5conf *conf, int hash){ spin_lock_irq(conf->hash_locks + hash); spin_lock(&conf->device_lock);} static inline void unlock_device_hash_lock(struct r5conf *conf, int hash){ spin_unlock(&conf->device_lock); spin_unlock_irq(conf->hash_locks + hash);} static inline void lock_all_device_hash_locks_irq(struct r5conf *conf){ int i; local_irq_disable(); spin_lock(conf->hash_locks); for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++) spin_lock_nest_lock(conf->hash_locks + i, conf->hash_locks); spin_lock(&conf->device_lock);} static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf){ int i; spin_unlock(&conf->device_lock); for (i = NR_STRIPE_HASH_LOCKS; i; i--) spin_unlock(conf->hash_locks + i - 1); local_irq_enable();} /* bio's attached to a stripe+device for I/O are linked together in bi_sector * order without overlap. There may be several bio's per stripe+device, and * a bio could span several devices. * When walking this list for a particular stripe+device, we must never proceed * beyond a bio that extends past this device, as the next bio might no longer * be valid. * This function is used to determine the 'next' bio in the list, given the sector * of the current stripe+device */static inline struct bio *r5_next_bio(struct bio *bio, sector_t sector){ int sectors = bio_sectors(bio); if (bio->bi_sector + sectors < sector + STRIPE_SECTORS) return bio->bi_next; else return NULL;} /* * We maintain a biased count of active stripes in the bottom 16 bits of * bi_phys_segments, and a count of processed stripes in the upper 16 bits */static inline int raid5_bi_processed_stripes(struct bio *bio){ atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; return (atomic_read(segments) >> 16) & 0xffff;} static inline int raid5_dec_bi_active_stripes(struct bio *bio){ atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; return atomic_sub_return(1, segments) & 0xffff;} static inline void raid5_inc_bi_active_stripes(struct bio *bio){ atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; atomic_inc(segments);} static inline void raid5_set_bi_processed_stripes(struct bio *bio, unsigned int cnt){ atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; int old, new; do { old = atomic_read(segments); new = (old & 0xffff) | (cnt << 16); } while (atomic_cmpxchg(segments, old, new) != old);} static inline void raid5_set_bi_stripes(struct bio *bio, unsigned int cnt){ atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; atomic_set(segments, cnt);} /* Find first data disk in a raid6 stripe */static inline int raid6_d0(struct stripe_head *sh){ if (sh->ddf_layout) /* ddf always start from first device */ return 0; /* md starts just after Q block */ if (sh->qd_idx == sh->disks - 1) return 0; else return sh->qd_idx + 1;}static inline int raid6_next_disk(int disk, int raid_disks){ disk++; return (disk < raid_disks) ? disk : 0;} /* When walking through the disks in a raid5, starting at raid6_d0, * We need to map each disk to a 'slot', where the data disks are slot * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk * is raid_disks-1. This help does that mapping. */static int raid6_idx_to_slot(int idx, struct stripe_head *sh, int *count, int syndrome_disks){ int slot = *count; if (sh->ddf_layout) (*count)++; if (idx == sh->pd_idx) return syndrome_disks; if (idx == sh->qd_idx) return syndrome_disks + 1; if (!sh->ddf_layout) (*count)++; return slot;} static void return_io(struct bio *return_bi){ struct bio *bi = return_bi; while (bi) { return_bi = bi->bi_next; bi->bi_next = NULL; bi->bi_size = 0; trace_block_bio_complete(bdev_get_queue(bi->bi_bdev), bi, 0); bio_endio(bi, 0); bi = return_bi; }} static void print_raid5_conf (struct r5conf *conf); static int stripe_operations_active(struct stripe_head *sh){ return sh->check_state || sh->reconstruct_state || test_bit(STRIPE_BIOFILL_RUN, &sh->state) || test_bit(STRIPE_COMPUTE_RUN, &sh->state);} static void raid5_wakeup_stripe_thread(struct stripe_head *sh){ struct r5conf *conf = sh->raid_conf; struct r5worker_group *group; int thread_cnt; int i, cpu = sh->cpu; if (!cpu_online(cpu)) { cpu = cpumask_any(cpu_online_mask); sh->cpu = cpu; } if (list_empty(&sh->lru)) { struct r5worker_group *group; group = conf->worker_groups + cpu_to_group(cpu); list_add_tail(&sh->lru, &group->handle_list); group->stripes_cnt++; sh->group = group; } if (conf->worker_cnt_per_group == 0) { md_wakeup_thread(conf->mddev->thread); return; } group = conf->worker_groups + cpu_to_group(sh->cpu); group->workers[0].working = true; /* at least one worker should run to avoid race */ queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work); thread_cnt = group->stripes_cnt / MAX_STRIPE_BATCH - 1; /* wakeup more workers */ for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) { if (group->workers[i].working == false) { group->workers[i].working = true; queue_work_on(sh->cpu, raid5_wq, &group->workers[i].work); thread_cnt--; } }} static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh, struct list_head *temp_inactive_list){ BUG_ON(!list_empty(&sh->lru)); BUG_ON(atomic_read(&conf->active_stripes)==0); if (test_bit(STRIPE_HANDLE, &sh->state)) { if (test_bit(STRIPE_DELAYED, &sh->state) && !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) list_add_tail(&sh->lru, &conf->delayed_list); else if (test_bit(STRIPE_BIT_DELAY, &sh->state) && sh->bm_seq - conf->seq_write > 0) list_add_tail(&sh->lru, &conf->bitmap_list); else { clear_bit(STRIPE_DELAYED, &sh->state); clear_bit(STRIPE_BIT_DELAY, &sh->state); if (conf->worker_cnt_per_group == 0) { list_add_tail(&sh->lru, &conf->handle_list); } else { raid5_wakeup_stripe_thread(sh); return; } } md_wakeup_thread(conf->mddev->thread); } else { BUG_ON(stripe_operations_active(sh)); if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) if (atomic_dec_return(&conf->preread_active_stripes) < IO_THRESHOLD) md_wakeup_thread(conf->mddev->thread); atomic_dec(&conf->active_stripes); if (!test_bit(STRIPE_EXPANDING, &sh->state)) list_add_tail(&sh->lru, temp_inactive_list); }} static void __release_stripe(struct r5conf *conf, struct stripe_head *sh, struct list_head *temp_inactive_list){ if (atomic_dec_and_test(&sh->count)) do_release_stripe(conf, sh, temp_inactive_list);} /* * @hash could be NR_STRIPE_HASH_LOCKS, then we have a list of inactive_list * * Be careful: Only one task can add/delete stripes from temp_inactive_list at * given time. Adding stripes only takes device lock, while deleting stripes * only takes hash lock. */static void release_inactive_stripe_list(struct r5conf *conf, struct list_head *temp_inactive_list, int hash){ int size; bool do_wakeup = false; unsigned long flags; if (hash == NR_STRIPE_HASH_LOCKS) { size = NR_STRIPE_HASH_LOCKS; hash = NR_STRIPE_HASH_LOCKS - 1; } else size = 1; while (size) { struct list_head *list = &temp_inactive_list[size - 1]; /* * We don't hold any lock here yet, get_active_stripe() might * remove stripes from the list */ if (!list_empty_careful(list)) { spin_lock_irqsave(conf->hash_locks + hash, flags); if (list_empty(conf->inactive_list + hash) && !list_empty(list)) atomic_dec(&conf->empty_inactive_list_nr); list_splice_tail_init(list, conf->inactive_list + hash); do_wakeup = true; spin_unlock_irqrestore(conf->hash_locks + hash, flags); } size--; hash--; } if (do_wakeup) { wake_up(&conf->wait_for_stripe); if (conf->retry_read_aligned) md_wakeup_thread(conf->mddev->thread); }} /* should hold conf->device_lock already */static int release_stripe_list(struct r5conf *conf, struct list_head *temp_inactive_list){ struct stripe_head *sh; int count = 0; struct llist_node *head; head = llist_del_all(&conf->released_stripes); head = llist_reverse_order(head); while (head) { int hash; sh = llist_entry(head, struct stripe_head, release_list); head = llist_next(head); /* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */ smp_mb(); clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state); /* * Don't worry the bit is set here, because if the bit is set * again, the count is always > 1. This is true for * STRIPE_ON_UNPLUG_LIST bit too. */ hash = sh->hash_lock_index; __release_stripe(conf, sh, &temp_inactive_list[hash]); count++; } return count;} static void release_stripe(struct stripe_head *sh){ struct r5conf *conf = sh->raid_conf; unsigned long flags; struct list_head list; int hash; bool wakeup; if (unlikely(!conf->mddev->thread) || test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state)) goto slow_path; wakeup = llist_add(&sh->release_list, &conf->released_stripes); if (wakeup) md_wakeup_thread(conf->mddev->thread); return;slow_path: local_irq_save(flags); /* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */ if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) { INIT_LIST_HEAD(&list); hash = sh->hash_lock_index; do_release_stripe(conf, sh, &list); spin_unlock(&conf->device_lock); release_inactive_stripe_list(conf, &list, hash); } local_irq_restore(flags);} static inline void remove_hash(struct stripe_head *sh){ pr_debug("remove_hash(), stripe %llu\n", (unsigned long long)sh->sector); hlist_del_init(&sh->hash);} static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh){ struct hlist_head *hp = stripe_hash(conf, sh->sector); pr_debug("insert_hash(), stripe %llu\n", (unsigned long long)sh->sector); hlist_add_head(&sh->hash, hp);} /* find an idle stripe, make sure it is unhashed, and return it. */static struct stripe_head *get_free_stripe(struct r5conf *conf, int hash){ struct stripe_head *sh = NULL; struct list_head *first; if (list_empty(conf->inactive_list + hash)) goto out; first = (conf->inactive_list + hash)->next; sh = list_entry(first, struct stripe_head, lru); list_del_init(first); remove_hash(sh); atomic_inc(&conf->active_stripes); BUG_ON(hash != sh->hash_lock_index); if (list_empty(conf->inactive_list + hash)) atomic_inc(&conf->empty_inactive_list_nr);out: return sh;} static void shrink_buffers(struct stripe_head *sh){ struct page *p; int i; int num = sh->raid_conf->pool_size; for (i = 0; i < num ; i++) { p = sh->dev[i].page; if (!p) continue; sh->dev[i].page = NULL; put_page(p); }} static int grow_buffers(struct stripe_head *sh){ int i; int num = sh->raid_conf->pool_size; for (i = 0; i < num; i++) { struct page *page; if (!(page = alloc_page(GFP_KERNEL))) { return 1; }Lines 1–500 of 7047. Load more
/* * raid5.c : Multiple Devices driver for Linux * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman * Copyright (C) 1999, 2000 Ingo Molnar * Copyright (C) 2002, 2003 H. Peter Anvin * * RAID-4/5/6 management functions. * Thanks to Penguin Computing for making the RAID-6 development possible * by donating a test server! * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2, or (at your option) * any later version. * * You should have received a copy of the GNU General Public License * (for example /usr/src/linux/COPYING); if not, write to the Free * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ /* * BITMAP UNPLUGGING: * * The sequencing for updating the bitmap reliably is a little * subtle (and I got it wrong the first time) so it deserves some * explanation. * * We group bitmap updates into batches. Each batch has a number. * We may write out several batches at once, but that isn't very important. * conf->seq_write is the number of the last batch successfully written. * conf->seq_flush is the number of the last batch that was closed to * new additions. * When we discover that we will need to write to any block in a stripe * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq * the number of the batch it will be in. This is seq_flush+1. * When we are ready to do a write, if that batch hasn't been written yet, * we plug the array and queue the stripe for later. * When an unplug happens, we increment bm_flush, thus closing the current * batch. * When we notice that bm_flush > bm_write, we write out all pending updates * to the bitmap, and advance bm_write to where bm_flush was. * This may occasionally write a bit out twice, but is sure never to * miss any bits. */ #include <linux/blkdev.h> #include <linux/kthread.h> #include <linux/raid/pq.h> #include <linux/async_tx.h> #include <linux/module.h> #include <linux/async.h> #include <linux/seq_file.h> #include <linux/cpu.h> #include <linux/slab.h> #include <linux/ratelimit.h> #include <linux/nodemask.h> #include <trace/events/block.h> #include "md.h" #include "raid5.h" #include "raid0.h" #include "bitmap.h" #define cpu_to_group(cpu) cpu_to_node(cpu) #define ANY_GROUP NUMA_NO_NODE static struct workqueue_struct *raid5_wq; /* * Stripe cache */ #define NR_STRIPES 256 #define STRIPE_SIZE PAGE_SIZE #define STRIPE_SHIFT (PAGE_SHIFT - 9) #define STRIPE_SECTORS (STRIPE_SIZE>>9) #define IO_THRESHOLD 1 #define BYPASS_THRESHOLD 1 #define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head)) #define HASH_MASK (NR_HASH - 1) #define MAX_STRIPE_BATCH 8 static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect) { int hash = (sect >> STRIPE_SHIFT) & HASH_MASK; return &conf->stripe_hashtbl[hash]; } static inline int stripe_hash_locks_hash(sector_t sect) { return (sect >> STRIPE_SHIFT) & STRIPE_HASH_LOCKS_MASK; } static inline void lock_device_hash_lock(struct r5conf *conf, int hash) { spin_lock_irq(conf->hash_locks + hash); spin_lock(&conf->device_lock); } static inline void unlock_device_hash_lock(struct r5conf *conf, int hash) { spin_unlock(&conf->device_lock); spin_unlock_irq(conf->hash_locks + hash); } static inline void lock_all_device_hash_locks_irq(struct r5conf *conf) { int i; local_irq_disable(); spin_lock(conf->hash_locks); for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++) spin_lock_nest_lock(conf->hash_locks + i, conf->hash_locks); spin_lock(&conf->device_lock); } static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf) { int i; spin_unlock(&conf->device_lock); for (i = NR_STRIPE_HASH_LOCKS; i; i--) spin_unlock(conf->hash_locks + i - 1); local_irq_enable(); } /* bio's attached to a stripe+device for I/O are linked together in bi_sector * order without overlap. There may be several bio's per stripe+device, and * a bio could span several devices. * When walking this list for a particular stripe+device, we must never proceed * beyond a bio that extends past this device, as the next bio might no longer * be valid. * This function is used to determine the 'next' bio in the list, given the sector * of the current stripe+device */ static inline struct bio *r5_next_bio(struct bio *bio, sector_t sector) { int sectors = bio_sectors(bio); if (bio->bi_sector + sectors < sector + STRIPE_SECTORS) return bio->bi_next; else return NULL; } /* * We maintain a biased count of active stripes in the bottom 16 bits of * bi_phys_segments, and a count of processed stripes in the upper 16 bits */ static inline int raid5_bi_processed_stripes(struct bio *bio) { atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; return (atomic_read(segments) >> 16) & 0xffff; } static inline int raid5_dec_bi_active_stripes(struct bio *bio) { atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; return atomic_sub_return(1, segments) & 0xffff; } static inline void raid5_inc_bi_active_stripes(struct bio *bio) { atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; atomic_inc(segments); } static inline void raid5_set_bi_processed_stripes(struct bio *bio, unsigned int cnt) { atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; int old, new; do { old = atomic_read(segments); new = (old & 0xffff) | (cnt << 16); } while (atomic_cmpxchg(segments, old, new) != old); } static inline void raid5_set_bi_stripes(struct bio *bio, unsigned int cnt) { atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; atomic_set(segments, cnt); } /* Find first data disk in a raid6 stripe */ static inline int raid6_d0(struct stripe_head *sh) { if (sh->ddf_layout) /* ddf always start from first device */ return 0; /* md starts just after Q block */ if (sh->qd_idx == sh->disks - 1) return 0; else return sh->qd_idx + 1; } static inline int raid6_next_disk(int disk, int raid_disks) { disk++; return (disk < raid_disks) ? disk : 0; } /* When walking through the disks in a raid5, starting at raid6_d0, * We need to map each disk to a 'slot', where the data disks are slot * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk * is raid_disks-1. This help does that mapping. */ static int raid6_idx_to_slot(int idx, struct stripe_head *sh, int *count, int syndrome_disks) { int slot = *count; if (sh->ddf_layout) (*count)++; if (idx == sh->pd_idx) return syndrome_disks; if (idx == sh->qd_idx) return syndrome_disks + 1; if (!sh->ddf_layout) (*count)++; return slot; } static void return_io(struct bio *return_bi) { struct bio *bi = return_bi; while (bi) { return_bi = bi->bi_next; bi->bi_next = NULL; bi->bi_size = 0; trace_block_bio_complete(bdev_get_queue(bi->bi_bdev), bi, 0); bio_endio(bi, 0); bi = return_bi; } } static void print_raid5_conf (struct r5conf *conf); static int stripe_operations_active(struct stripe_head *sh) { return sh->check_state || sh->reconstruct_state || test_bit(STRIPE_BIOFILL_RUN, &sh->state) || test_bit(STRIPE_COMPUTE_RUN, &sh->state); } static void raid5_wakeup_stripe_thread(struct stripe_head *sh) { struct r5conf *conf = sh->raid_conf; struct r5worker_group *group; int thread_cnt; int i, cpu = sh->cpu; if (!cpu_online(cpu)) { cpu = cpumask_any(cpu_online_mask); sh->cpu = cpu; } if (list_empty(&sh->lru)) { struct r5worker_group *group; group = conf->worker_groups + cpu_to_group(cpu); list_add_tail(&sh->lru, &group->handle_list); group->stripes_cnt++; sh->group = group; } if (conf->worker_cnt_per_group == 0) { md_wakeup_thread(conf->mddev->thread); return; } group = conf->worker_groups + cpu_to_group(sh->cpu); group->workers[0].working = true; /* at least one worker should run to avoid race */ queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work); thread_cnt = group->stripes_cnt / MAX_STRIPE_BATCH - 1; /* wakeup more workers */ for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) { if (group->workers[i].working == false) { group->workers[i].working = true; queue_work_on(sh->cpu, raid5_wq, &group->workers[i].work); thread_cnt--; } } } static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh, struct list_head *temp_inactive_list) { BUG_ON(!list_empty(&sh->lru)); BUG_ON(atomic_read(&conf->active_stripes)==0); if (test_bit(STRIPE_HANDLE, &sh->state)) { if (test_bit(STRIPE_DELAYED, &sh->state) && !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) list_add_tail(&sh->lru, &conf->delayed_list); else if (test_bit(STRIPE_BIT_DELAY, &sh->state) && sh->bm_seq - conf->seq_write > 0) list_add_tail(&sh->lru, &conf->bitmap_list); else { clear_bit(STRIPE_DELAYED, &sh->state); clear_bit(STRIPE_BIT_DELAY, &sh->state); if (conf->worker_cnt_per_group == 0) { list_add_tail(&sh->lru, &conf->handle_list); } else { raid5_wakeup_stripe_thread(sh); return; } } md_wakeup_thread(conf->mddev->thread); } else { BUG_ON(stripe_operations_active(sh)); if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) if (atomic_dec_return(&conf->preread_active_stripes) < IO_THRESHOLD) md_wakeup_thread(conf->mddev->thread); atomic_dec(&conf->active_stripes); if (!test_bit(STRIPE_EXPANDING, &sh->state)) list_add_tail(&sh->lru, temp_inactive_list); } } static void __release_stripe(struct r5conf *conf, struct stripe_head *sh, struct list_head *temp_inactive_list) { if (atomic_dec_and_test(&sh->count)) do_release_stripe(conf, sh, temp_inactive_list); } /* * @hash could be NR_STRIPE_HASH_LOCKS, then we have a list of inactive_list * * Be careful: Only one task can add/delete stripes from temp_inactive_list at * given time. Adding stripes only takes device lock, while deleting stripes * only takes hash lock. */ static void release_inactive_stripe_list(struct r5conf *conf, struct list_head *temp_inactive_list, int hash) { int size; bool do_wakeup = false; unsigned long flags; if (hash == NR_STRIPE_HASH_LOCKS) { size = NR_STRIPE_HASH_LOCKS; hash = NR_STRIPE_HASH_LOCKS - 1; } else size = 1; while (size) { struct list_head *list = &temp_inactive_list[size - 1]; /* * We don't hold any lock here yet, get_active_stripe() might * remove stripes from the list */ if (!list_empty_careful(list)) { spin_lock_irqsave(conf->hash_locks + hash, flags); if (list_empty(conf->inactive_list + hash) && !list_empty(list)) atomic_dec(&conf->empty_inactive_list_nr); list_splice_tail_init(list, conf->inactive_list + hash); do_wakeup = true; spin_unlock_irqrestore(conf->hash_locks + hash, flags); } size--; hash--; } if (do_wakeup) { wake_up(&conf->wait_for_stripe); if (conf->retry_read_aligned) md_wakeup_thread(conf->mddev->thread); } } /* should hold conf->device_lock already */ static int release_stripe_list(struct r5conf *conf, struct list_head *temp_inactive_list) { struct stripe_head *sh; int count = 0; struct llist_node *head; head = llist_del_all(&conf->released_stripes); head = llist_reverse_order(head); while (head) { int hash; sh = llist_entry(head, struct stripe_head, release_list); head = llist_next(head); /* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */ smp_mb(); clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state); /* * Don't worry the bit is set here, because if the bit is set * again, the count is always > 1. This is true for * STRIPE_ON_UNPLUG_LIST bit too. */ hash = sh->hash_lock_index; __release_stripe(conf, sh, &temp_inactive_list[hash]); count++; } return count; } static void release_stripe(struct stripe_head *sh) { struct r5conf *conf = sh->raid_conf; unsigned long flags; struct list_head list; int hash; bool wakeup; if (unlikely(!conf->mddev->thread) || test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state)) goto slow_path; wakeup = llist_add(&sh->release_list, &conf->released_stripes); if (wakeup) md_wakeup_thread(conf->mddev->thread); return; slow_path: local_irq_save(flags); /* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */ if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) { INIT_LIST_HEAD(&list); hash = sh->hash_lock_index; do_release_stripe(conf, sh, &list); spin_unlock(&conf->device_lock); release_inactive_stripe_list(conf, &list, hash); } local_irq_restore(flags); } static inline void remove_hash(struct stripe_head *sh) { pr_debug("remove_hash(), stripe %llu\n", (unsigned long long)sh->sector); hlist_del_init(&sh->hash); } static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh) { struct hlist_head *hp = stripe_hash(conf, sh->sector); pr_debug("insert_hash(), stripe %llu\n", (unsigned long long)sh->sector); hlist_add_head(&sh->hash, hp); } /* find an idle stripe, make sure it is unhashed, and return it. */ static struct stripe_head *get_free_stripe(struct r5conf *conf, int hash) { struct stripe_head *sh = NULL; struct list_head *first; if (list_empty(conf->inactive_list + hash)) goto out; first = (conf->inactive_list + hash)->next; sh = list_entry(first, struct stripe_head, lru); list_del_init(first); remove_hash(sh); atomic_inc(&conf->active_stripes); BUG_ON(hash != sh->hash_lock_index); if (list_empty(conf->inactive_list + hash)) atomic_inc(&conf->empty_inactive_list_nr); out: return sh; } static void shrink_buffers(struct stripe_head *sh) { struct page *p; int i; int num = sh->raid_conf->pool_size; for (i = 0; i < num ; i++) { p = sh->dev[i].page; if (!p) continue; sh->dev[i].page = NULL; put_page(p); } } static int grow_buffers(struct stripe_head *sh) { int i; int num = sh->raid_conf->pool_size; for (i = 0; i < num; i++) { struct page *page; if (!(page = alloc_page(GFP_KERNEL))) { return 1; }