/* * raid5.c : Multiple Devices driver for Linux * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman * Copyright (C) 1999, 2000 Ingo Molnar * Copyright (C) 2002, 2003 H. Peter Anvin * * RAID-4/5/6 management functions. * Thanks to Penguin Computing for making the RAID-6 development possible * by donating a test server! * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2, or (at your option) * any later version. * * You should have received a copy of the GNU General Public License * (for example /usr/src/linux/COPYING); if not, write to the Free * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ /* * BITMAP UNPLUGGING: * * The sequencing for updating the bitmap reliably is a little * subtle (and I got it wrong the first time) so it deserves some * explanation. * * We group bitmap updates into batches. Each batch has a number. * We may write out several batches at once, but that isn't very important. * conf->bm_write is the number of the last batch successfully written. * conf->bm_flush is the number of the last batch that was closed to * new additions. * When we discover that we will need to write to any block in a stripe * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq * the number of the batch it will be in. This is bm_flush+1. * When we are ready to do a write, if that batch hasn't been written yet, * we plug the array and queue the stripe for later. * When an unplug happens, we increment bm_flush, thus closing the current * batch. * When we notice that bm_flush > bm_write, we write out all pending updates * to the bitmap, and advance bm_write to where bm_flush was. * This may occasionally write a bit out twice, but is sure never to * miss any bits. */ #include <linux/module.h>#include <linux/slab.h>#include <linux/highmem.h>#include <linux/bitops.h>#include <linux/kthread.h>#include <asm/atomic.h>#include "raid6.h" #include <linux/raid/bitmap.h> /* * Stripe cache */ #define NR_STRIPES 256#define STRIPE_SIZE PAGE_SIZE#define STRIPE_SHIFT (PAGE_SHIFT - 9)#define STRIPE_SECTORS (STRIPE_SIZE>>9)#define IO_THRESHOLD 1#define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head))#define HASH_MASK (NR_HASH - 1) #define stripe_hash(conf, sect) (&((conf)->stripe_hashtbl[((sect) >> STRIPE_SHIFT) & HASH_MASK])) /* bio's attached to a stripe+device for I/O are linked together in bi_sector * order without overlap. There may be several bio's per stripe+device, and * a bio could span several devices. * When walking this list for a particular stripe+device, we must never proceed * beyond a bio that extends past this device, as the next bio might no longer * be valid. * This macro is used to determine the 'next' bio in the list, given the sector * of the current stripe+device */#define r5_next_bio(bio, sect) ( ( (bio)->bi_sector + ((bio)->bi_size>>9) < sect + STRIPE_SECTORS) ? (bio)->bi_next : NULL)/* * The following can be used to debug the driver */#define RAID5_DEBUG 0#define RAID5_PARANOIA 1#if RAID5_PARANOIA && defined(CONFIG_SMP)# define CHECK_DEVLOCK() assert_spin_locked(&conf->device_lock)#else# define CHECK_DEVLOCK()#endif #define PRINTK(x...) ((void)(RAID5_DEBUG && printk(x)))#if RAID5_DEBUG#define inline#define __inline__#endif #if !RAID6_USE_EMPTY_ZERO_PAGE/* In .bss so it's zeroed */const char raid6_empty_zero_page[PAGE_SIZE] __attribute__((aligned(256)));#endif static inline int raid6_next_disk(int disk, int raid_disks){ disk++; return (disk < raid_disks) ? disk : 0;}static void print_raid5_conf (raid5_conf_t *conf); static void __release_stripe(raid5_conf_t *conf, struct stripe_head *sh){ if (atomic_dec_and_test(&sh->count)) { BUG_ON(!list_empty(&sh->lru)); BUG_ON(atomic_read(&conf->active_stripes)==0); if (test_bit(STRIPE_HANDLE, &sh->state)) { if (test_bit(STRIPE_DELAYED, &sh->state)) { list_add_tail(&sh->lru, &conf->delayed_list); blk_plug_device(conf->mddev->queue); } else if (test_bit(STRIPE_BIT_DELAY, &sh->state) && sh->bm_seq - conf->seq_write > 0) { list_add_tail(&sh->lru, &conf->bitmap_list); blk_plug_device(conf->mddev->queue); } else { clear_bit(STRIPE_BIT_DELAY, &sh->state); list_add_tail(&sh->lru, &conf->handle_list); } md_wakeup_thread(conf->mddev->thread); } else { if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { atomic_dec(&conf->preread_active_stripes); if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) md_wakeup_thread(conf->mddev->thread); } atomic_dec(&conf->active_stripes); if (!test_bit(STRIPE_EXPANDING, &sh->state)) { list_add_tail(&sh->lru, &conf->inactive_list); wake_up(&conf->wait_for_stripe); if (conf->retry_read_aligned) md_wakeup_thread(conf->mddev->thread); } } }}static void release_stripe(struct stripe_head *sh){ raid5_conf_t *conf = sh->raid_conf; unsigned long flags; spin_lock_irqsave(&conf->device_lock, flags); __release_stripe(conf, sh); spin_unlock_irqrestore(&conf->device_lock, flags);} static inline void remove_hash(struct stripe_head *sh){ PRINTK("remove_hash(), stripe %llu\n", (unsigned long long)sh->sector); hlist_del_init(&sh->hash);} static inline void insert_hash(raid5_conf_t *conf, struct stripe_head *sh){ struct hlist_head *hp = stripe_hash(conf, sh->sector); PRINTK("insert_hash(), stripe %llu\n", (unsigned long long)sh->sector); CHECK_DEVLOCK(); hlist_add_head(&sh->hash, hp);} /* find an idle stripe, make sure it is unhashed, and return it. */static struct stripe_head *get_free_stripe(raid5_conf_t *conf){ struct stripe_head *sh = NULL; struct list_head *first; CHECK_DEVLOCK(); if (list_empty(&conf->inactive_list)) goto out; first = conf->inactive_list.next; sh = list_entry(first, struct stripe_head, lru); list_del_init(first); remove_hash(sh); atomic_inc(&conf->active_stripes);out: return sh;} static void shrink_buffers(struct stripe_head *sh, int num){ struct page *p; int i; for (i=0; i<num ; i++) { p = sh->dev[i].page; if (!p) continue; sh->dev[i].page = NULL; put_page(p); }} static int grow_buffers(struct stripe_head *sh, int num){ int i; for (i=0; i<num; i++) { struct page *page; if (!(page = alloc_page(GFP_KERNEL))) { return 1; } sh->dev[i].page = page; } return 0;} static void raid5_build_block (struct stripe_head *sh, int i); static void init_stripe(struct stripe_head *sh, sector_t sector, int pd_idx, int disks){ raid5_conf_t *conf = sh->raid_conf; int i; BUG_ON(atomic_read(&sh->count) != 0); BUG_ON(test_bit(STRIPE_HANDLE, &sh->state)); CHECK_DEVLOCK(); PRINTK("init_stripe called, stripe %llu\n", (unsigned long long)sh->sector); remove_hash(sh); sh->sector = sector; sh->pd_idx = pd_idx; sh->state = 0; sh->disks = disks; for (i = sh->disks; i--; ) { struct r5dev *dev = &sh->dev[i]; if (dev->toread || dev->towrite || dev->written || test_bit(R5_LOCKED, &dev->flags)) { printk("sector=%llx i=%d %p %p %p %d\n", (unsigned long long)sh->sector, i, dev->toread, dev->towrite, dev->written, test_bit(R5_LOCKED, &dev->flags)); BUG(); } dev->flags = 0; raid5_build_block(sh, i); } insert_hash(conf, sh);} static struct stripe_head *__find_stripe(raid5_conf_t *conf, sector_t sector, int disks){ struct stripe_head *sh; struct hlist_node *hn; CHECK_DEVLOCK(); PRINTK("__find_stripe, sector %llu\n", (unsigned long long)sector); hlist_for_each_entry(sh, hn, stripe_hash(conf, sector), hash) if (sh->sector == sector && sh->disks == disks) return sh; PRINTK("__stripe %llu not in cache\n", (unsigned long long)sector); return NULL;} static void unplug_slaves(mddev_t *mddev);static void raid5_unplug_device(request_queue_t *q); static struct stripe_head *get_active_stripe(raid5_conf_t *conf, sector_t sector, int disks, int pd_idx, int noblock){ struct stripe_head *sh; PRINTK("get_stripe, sector %llu\n", (unsigned long long)sector); spin_lock_irq(&conf->device_lock); do { wait_event_lock_irq(conf->wait_for_stripe, conf->quiesce == 0, conf->device_lock, /* nothing */); sh = __find_stripe(conf, sector, disks); if (!sh) { if (!conf->inactive_blocked) sh = get_free_stripe(conf); if (noblock && sh == NULL) break; if (!sh) { conf->inactive_blocked = 1; wait_event_lock_irq(conf->wait_for_stripe, !list_empty(&conf->inactive_list) && (atomic_read(&conf->active_stripes) < (conf->max_nr_stripes *3/4) || !conf->inactive_blocked), conf->device_lock, raid5_unplug_device(conf->mddev->queue) ); conf->inactive_blocked = 0; } else init_stripe(sh, sector, pd_idx, disks); } else { if (atomic_read(&sh->count)) { BUG_ON(!list_empty(&sh->lru)); } else { if (!test_bit(STRIPE_HANDLE, &sh->state)) atomic_inc(&conf->active_stripes); if (list_empty(&sh->lru) && !test_bit(STRIPE_EXPANDING, &sh->state)) BUG(); list_del_init(&sh->lru); } } } while (sh == NULL); if (sh) atomic_inc(&sh->count); spin_unlock_irq(&conf->device_lock); return sh;} static int grow_one_stripe(raid5_conf_t *conf){ struct stripe_head *sh; sh = kmem_cache_alloc(conf->slab_cache, GFP_KERNEL); if (!sh) return 0; memset(sh, 0, sizeof(*sh) + (conf->raid_disks-1)*sizeof(struct r5dev)); sh->raid_conf = conf; spin_lock_init(&sh->lock); if (grow_buffers(sh, conf->raid_disks)) { shrink_buffers(sh, conf->raid_disks); kmem_cache_free(conf->slab_cache, sh); return 0; } sh->disks = conf->raid_disks; /* we just created an active stripe so... */ atomic_set(&sh->count, 1); atomic_inc(&conf->active_stripes); INIT_LIST_HEAD(&sh->lru); release_stripe(sh); return 1;} static int grow_stripes(raid5_conf_t *conf, int num){ struct kmem_cache *sc; int devs = conf->raid_disks; sprintf(conf->cache_name[0], "raid5/%s", mdname(conf->mddev)); sprintf(conf->cache_name[1], "raid5/%s-alt", mdname(conf->mddev)); conf->active_name = 0; sc = kmem_cache_create(conf->cache_name[conf->active_name], sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev), 0, 0, NULL, NULL); if (!sc) return 1; conf->slab_cache = sc; conf->pool_size = devs; while (num--) if (!grow_one_stripe(conf)) return 1; return 0;} #ifdef CONFIG_MD_RAID5_RESHAPEstatic int resize_stripes(raid5_conf_t *conf, int newsize){ /* Make all the stripes able to hold 'newsize' devices. * New slots in each stripe get 'page' set to a new page. * * This happens in stages: * 1/ create a new kmem_cache and allocate the required number of * stripe_heads. * 2/ gather all the old stripe_heads and tranfer the pages across * to the new stripe_heads. This will have the side effect of * freezing the array as once all stripe_heads have been collected, * no IO will be possible. Old stripe heads are freed once their * pages have been transferred over, and the old kmem_cache is * freed when all stripes are done. * 3/ reallocate conf->disks to be suitable bigger. If this fails, * we simple return a failre status - no need to clean anything up. * 4/ allocate new pages for the new slots in the new stripe_heads. * If this fails, we don't bother trying the shrink the * stripe_heads down again, we just leave them as they are. * As each stripe_head is processed the new one is released into * active service. * * Once step2 is started, we cannot afford to wait for a write, * so we use GFP_NOIO allocations. */ struct stripe_head *osh, *nsh; LIST_HEAD(newstripes); struct disk_info *ndisks; int err = 0; struct kmem_cache *sc; int i; if (newsize <= conf->pool_size) return 0; /* never bother to shrink */ md_allow_write(conf->mddev); /* Step 1 */ sc = kmem_cache_create(conf->cache_name[1-conf->active_name], sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev), 0, 0, NULL, NULL); if (!sc) return -ENOMEM; for (i = conf->max_nr_stripes; i; i--) { nsh = kmem_cache_alloc(sc, GFP_KERNEL); if (!nsh) break; memset(nsh, 0, sizeof(*nsh) + (newsize-1)*sizeof(struct r5dev)); nsh->raid_conf = conf; spin_lock_init(&nsh->lock); list_add(&nsh->lru, &newstripes); } if (i) { /* didn't get enough, give up */ while (!list_empty(&newstripes)) { nsh = list_entry(newstripes.next, struct stripe_head, lru); list_del(&nsh->lru); kmem_cache_free(sc, nsh); } kmem_cache_destroy(sc); return -ENOMEM; } /* Step 2 - Must use GFP_NOIO now. * OK, we have enough stripes, start collecting inactive * stripes and copying them over */ list_for_each_entry(nsh, &newstripes, lru) { spin_lock_irq(&conf->device_lock); wait_event_lock_irq(conf->wait_for_stripe, !list_empty(&conf->inactive_list), conf->device_lock, unplug_slaves(conf->mddev) ); osh = get_free_stripe(conf); spin_unlock_irq(&conf->device_lock); atomic_set(&nsh->count, 1); for(i=0; i<conf->pool_size; i++) nsh->dev[i].page = osh->dev[i].page; for( ; i<newsize; i++) nsh->dev[i].page = NULL; kmem_cache_free(conf->slab_cache, osh); } kmem_cache_destroy(conf->slab_cache); /* Step 3. * At this point, we are holding all the stripes so the array * is completely stalled, so now is a good time to resize * conf->disks. */ ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO); if (ndisks) { for (i=0; i<conf->raid_disks; i++) ndisks[i] = conf->disks[i]; kfree(conf->disks); conf->disks = ndisks; } else err = -ENOMEM; /* Step 4, return new stripes to service */ while(!list_empty(&newstripes)) { nsh = list_entry(newstripes.next, struct stripe_head, lru); list_del_init(&nsh->lru); for (i=conf->raid_disks; i < newsize; i++) if (nsh->dev[i].page == NULL) { struct page *p = alloc_page(GFP_NOIO); nsh->dev[i].page = p; if (!p) err = -ENOMEM; } release_stripe(nsh); } /* critical section pass, GFP_NOIO no longer needed */ conf->slab_cache = sc; conf->active_name = 1-conf->active_name; conf->pool_size = newsize; return err;}#endif static int drop_one_stripe(raid5_conf_t *conf){ struct stripe_head *sh; Lines 1–500 of 4157. Load more
/* * raid5.c : Multiple Devices driver for Linux * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman * Copyright (C) 1999, 2000 Ingo Molnar * Copyright (C) 2002, 2003 H. Peter Anvin * * RAID-4/5/6 management functions. * Thanks to Penguin Computing for making the RAID-6 development possible * by donating a test server! * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2, or (at your option) * any later version. * * You should have received a copy of the GNU General Public License * (for example /usr/src/linux/COPYING); if not, write to the Free * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ /* * BITMAP UNPLUGGING: * * The sequencing for updating the bitmap reliably is a little * subtle (and I got it wrong the first time) so it deserves some * explanation. * * We group bitmap updates into batches. Each batch has a number. * We may write out several batches at once, but that isn't very important. * conf->bm_write is the number of the last batch successfully written. * conf->bm_flush is the number of the last batch that was closed to * new additions. * When we discover that we will need to write to any block in a stripe * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq * the number of the batch it will be in. This is bm_flush+1. * When we are ready to do a write, if that batch hasn't been written yet, * we plug the array and queue the stripe for later. * When an unplug happens, we increment bm_flush, thus closing the current * batch. * When we notice that bm_flush > bm_write, we write out all pending updates * to the bitmap, and advance bm_write to where bm_flush was. * This may occasionally write a bit out twice, but is sure never to * miss any bits. */ #include <linux/module.h> #include <linux/slab.h> #include <linux/highmem.h> #include <linux/bitops.h> #include <linux/kthread.h> #include <asm/atomic.h> #include "raid6.h" #include <linux/raid/bitmap.h> /* * Stripe cache */ #define NR_STRIPES 256 #define STRIPE_SIZE PAGE_SIZE #define STRIPE_SHIFT (PAGE_SHIFT - 9) #define STRIPE_SECTORS (STRIPE_SIZE>>9) #define IO_THRESHOLD 1 #define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head)) #define HASH_MASK (NR_HASH - 1) #define stripe_hash(conf, sect) (&((conf)->stripe_hashtbl[((sect) >> STRIPE_SHIFT) & HASH_MASK])) /* bio's attached to a stripe+device for I/O are linked together in bi_sector * order without overlap. There may be several bio's per stripe+device, and * a bio could span several devices. * When walking this list for a particular stripe+device, we must never proceed * beyond a bio that extends past this device, as the next bio might no longer * be valid. * This macro is used to determine the 'next' bio in the list, given the sector * of the current stripe+device */ #define r5_next_bio(bio, sect) ( ( (bio)->bi_sector + ((bio)->bi_size>>9) < sect + STRIPE_SECTORS) ? (bio)->bi_next : NULL) /* * The following can be used to debug the driver */ #define RAID5_DEBUG 0 #define RAID5_PARANOIA 1 #if RAID5_PARANOIA && defined(CONFIG_SMP) # define CHECK_DEVLOCK() assert_spin_locked(&conf->device_lock) #else # define CHECK_DEVLOCK() #endif #define PRINTK(x...) ((void)(RAID5_DEBUG && printk(x))) #if RAID5_DEBUG #define inline #define __inline__ #endif #if !RAID6_USE_EMPTY_ZERO_PAGE /* In .bss so it's zeroed */ const char raid6_empty_zero_page[PAGE_SIZE] __attribute__((aligned(256))); #endif static inline int raid6_next_disk(int disk, int raid_disks) { disk++; return (disk < raid_disks) ? disk : 0; } static void print_raid5_conf (raid5_conf_t *conf); static void __release_stripe(raid5_conf_t *conf, struct stripe_head *sh) { if (atomic_dec_and_test(&sh->count)) { BUG_ON(!list_empty(&sh->lru)); BUG_ON(atomic_read(&conf->active_stripes)==0); if (test_bit(STRIPE_HANDLE, &sh->state)) { if (test_bit(STRIPE_DELAYED, &sh->state)) { list_add_tail(&sh->lru, &conf->delayed_list); blk_plug_device(conf->mddev->queue); } else if (test_bit(STRIPE_BIT_DELAY, &sh->state) && sh->bm_seq - conf->seq_write > 0) { list_add_tail(&sh->lru, &conf->bitmap_list); blk_plug_device(conf->mddev->queue); } else { clear_bit(STRIPE_BIT_DELAY, &sh->state); list_add_tail(&sh->lru, &conf->handle_list); } md_wakeup_thread(conf->mddev->thread); } else { if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { atomic_dec(&conf->preread_active_stripes); if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) md_wakeup_thread(conf->mddev->thread); } atomic_dec(&conf->active_stripes); if (!test_bit(STRIPE_EXPANDING, &sh->state)) { list_add_tail(&sh->lru, &conf->inactive_list); wake_up(&conf->wait_for_stripe); if (conf->retry_read_aligned) md_wakeup_thread(conf->mddev->thread); } } } } static void release_stripe(struct stripe_head *sh) { raid5_conf_t *conf = sh->raid_conf; unsigned long flags; spin_lock_irqsave(&conf->device_lock, flags); __release_stripe(conf, sh); spin_unlock_irqrestore(&conf->device_lock, flags); } static inline void remove_hash(struct stripe_head *sh) { PRINTK("remove_hash(), stripe %llu\n", (unsigned long long)sh->sector); hlist_del_init(&sh->hash); } static inline void insert_hash(raid5_conf_t *conf, struct stripe_head *sh) { struct hlist_head *hp = stripe_hash(conf, sh->sector); PRINTK("insert_hash(), stripe %llu\n", (unsigned long long)sh->sector); CHECK_DEVLOCK(); hlist_add_head(&sh->hash, hp); } /* find an idle stripe, make sure it is unhashed, and return it. */ static struct stripe_head *get_free_stripe(raid5_conf_t *conf) { struct stripe_head *sh = NULL; struct list_head *first; CHECK_DEVLOCK(); if (list_empty(&conf->inactive_list)) goto out; first = conf->inactive_list.next; sh = list_entry(first, struct stripe_head, lru); list_del_init(first); remove_hash(sh); atomic_inc(&conf->active_stripes); out: return sh; } static void shrink_buffers(struct stripe_head *sh, int num) { struct page *p; int i; for (i=0; i<num ; i++) { p = sh->dev[i].page; if (!p) continue; sh->dev[i].page = NULL; put_page(p); } } static int grow_buffers(struct stripe_head *sh, int num) { int i; for (i=0; i<num; i++) { struct page *page; if (!(page = alloc_page(GFP_KERNEL))) { return 1; } sh->dev[i].page = page; } return 0; } static void raid5_build_block (struct stripe_head *sh, int i); static void init_stripe(struct stripe_head *sh, sector_t sector, int pd_idx, int disks) { raid5_conf_t *conf = sh->raid_conf; int i; BUG_ON(atomic_read(&sh->count) != 0); BUG_ON(test_bit(STRIPE_HANDLE, &sh->state)); CHECK_DEVLOCK(); PRINTK("init_stripe called, stripe %llu\n", (unsigned long long)sh->sector); remove_hash(sh); sh->sector = sector; sh->pd_idx = pd_idx; sh->state = 0; sh->disks = disks; for (i = sh->disks; i--; ) { struct r5dev *dev = &sh->dev[i]; if (dev->toread || dev->towrite || dev->written || test_bit(R5_LOCKED, &dev->flags)) { printk("sector=%llx i=%d %p %p %p %d\n", (unsigned long long)sh->sector, i, dev->toread, dev->towrite, dev->written, test_bit(R5_LOCKED, &dev->flags)); BUG(); } dev->flags = 0; raid5_build_block(sh, i); } insert_hash(conf, sh); } static struct stripe_head *__find_stripe(raid5_conf_t *conf, sector_t sector, int disks) { struct stripe_head *sh; struct hlist_node *hn; CHECK_DEVLOCK(); PRINTK("__find_stripe, sector %llu\n", (unsigned long long)sector); hlist_for_each_entry(sh, hn, stripe_hash(conf, sector), hash) if (sh->sector == sector && sh->disks == disks) return sh; PRINTK("__stripe %llu not in cache\n", (unsigned long long)sector); return NULL; } static void unplug_slaves(mddev_t *mddev); static void raid5_unplug_device(request_queue_t *q); static struct stripe_head *get_active_stripe(raid5_conf_t *conf, sector_t sector, int disks, int pd_idx, int noblock) { struct stripe_head *sh; PRINTK("get_stripe, sector %llu\n", (unsigned long long)sector); spin_lock_irq(&conf->device_lock); do { wait_event_lock_irq(conf->wait_for_stripe, conf->quiesce == 0, conf->device_lock, /* nothing */); sh = __find_stripe(conf, sector, disks); if (!sh) { if (!conf->inactive_blocked) sh = get_free_stripe(conf); if (noblock && sh == NULL) break; if (!sh) { conf->inactive_blocked = 1; wait_event_lock_irq(conf->wait_for_stripe, !list_empty(&conf->inactive_list) && (atomic_read(&conf->active_stripes) < (conf->max_nr_stripes *3/4) || !conf->inactive_blocked), conf->device_lock, raid5_unplug_device(conf->mddev->queue) ); conf->inactive_blocked = 0; } else init_stripe(sh, sector, pd_idx, disks); } else { if (atomic_read(&sh->count)) { BUG_ON(!list_empty(&sh->lru)); } else { if (!test_bit(STRIPE_HANDLE, &sh->state)) atomic_inc(&conf->active_stripes); if (list_empty(&sh->lru) && !test_bit(STRIPE_EXPANDING, &sh->state)) BUG(); list_del_init(&sh->lru); } } } while (sh == NULL); if (sh) atomic_inc(&sh->count); spin_unlock_irq(&conf->device_lock); return sh; } static int grow_one_stripe(raid5_conf_t *conf) { struct stripe_head *sh; sh = kmem_cache_alloc(conf->slab_cache, GFP_KERNEL); if (!sh) return 0; memset(sh, 0, sizeof(*sh) + (conf->raid_disks-1)*sizeof(struct r5dev)); sh->raid_conf = conf; spin_lock_init(&sh->lock); if (grow_buffers(sh, conf->raid_disks)) { shrink_buffers(sh, conf->raid_disks); kmem_cache_free(conf->slab_cache, sh); return 0; } sh->disks = conf->raid_disks; /* we just created an active stripe so... */ atomic_set(&sh->count, 1); atomic_inc(&conf->active_stripes); INIT_LIST_HEAD(&sh->lru); release_stripe(sh); return 1; } static int grow_stripes(raid5_conf_t *conf, int num) { struct kmem_cache *sc; int devs = conf->raid_disks; sprintf(conf->cache_name[0], "raid5/%s", mdname(conf->mddev)); sprintf(conf->cache_name[1], "raid5/%s-alt", mdname(conf->mddev)); conf->active_name = 0; sc = kmem_cache_create(conf->cache_name[conf->active_name], sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev), 0, 0, NULL, NULL); if (!sc) return 1; conf->slab_cache = sc; conf->pool_size = devs; while (num--) if (!grow_one_stripe(conf)) return 1; return 0; } #ifdef CONFIG_MD_RAID5_RESHAPE static int resize_stripes(raid5_conf_t *conf, int newsize) { /* Make all the stripes able to hold 'newsize' devices. * New slots in each stripe get 'page' set to a new page. * * This happens in stages: * 1/ create a new kmem_cache and allocate the required number of * stripe_heads. * 2/ gather all the old stripe_heads and tranfer the pages across * to the new stripe_heads. This will have the side effect of * freezing the array as once all stripe_heads have been collected, * no IO will be possible. Old stripe heads are freed once their * pages have been transferred over, and the old kmem_cache is * freed when all stripes are done. * 3/ reallocate conf->disks to be suitable bigger. If this fails, * we simple return a failre status - no need to clean anything up. * 4/ allocate new pages for the new slots in the new stripe_heads. * If this fails, we don't bother trying the shrink the * stripe_heads down again, we just leave them as they are. * As each stripe_head is processed the new one is released into * active service. * * Once step2 is started, we cannot afford to wait for a write, * so we use GFP_NOIO allocations. */ struct stripe_head *osh, *nsh; LIST_HEAD(newstripes); struct disk_info *ndisks; int err = 0; struct kmem_cache *sc; int i; if (newsize <= conf->pool_size) return 0; /* never bother to shrink */ md_allow_write(conf->mddev); /* Step 1 */ sc = kmem_cache_create(conf->cache_name[1-conf->active_name], sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev), 0, 0, NULL, NULL); if (!sc) return -ENOMEM; for (i = conf->max_nr_stripes; i; i--) { nsh = kmem_cache_alloc(sc, GFP_KERNEL); if (!nsh) break; memset(nsh, 0, sizeof(*nsh) + (newsize-1)*sizeof(struct r5dev)); nsh->raid_conf = conf; spin_lock_init(&nsh->lock); list_add(&nsh->lru, &newstripes); } if (i) { /* didn't get enough, give up */ while (!list_empty(&newstripes)) { nsh = list_entry(newstripes.next, struct stripe_head, lru); list_del(&nsh->lru); kmem_cache_free(sc, nsh); } kmem_cache_destroy(sc); return -ENOMEM; } /* Step 2 - Must use GFP_NOIO now. * OK, we have enough stripes, start collecting inactive * stripes and copying them over */ list_for_each_entry(nsh, &newstripes, lru) { spin_lock_irq(&conf->device_lock); wait_event_lock_irq(conf->wait_for_stripe, !list_empty(&conf->inactive_list), conf->device_lock, unplug_slaves(conf->mddev) ); osh = get_free_stripe(conf); spin_unlock_irq(&conf->device_lock); atomic_set(&nsh->count, 1); for(i=0; i<conf->pool_size; i++) nsh->dev[i].page = osh->dev[i].page; for( ; i<newsize; i++) nsh->dev[i].page = NULL; kmem_cache_free(conf->slab_cache, osh); } kmem_cache_destroy(conf->slab_cache); /* Step 3. * At this point, we are holding all the stripes so the array * is completely stalled, so now is a good time to resize * conf->disks. */ ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO); if (ndisks) { for (i=0; i<conf->raid_disks; i++) ndisks[i] = conf->disks[i]; kfree(conf->disks); conf->disks = ndisks; } else err = -ENOMEM; /* Step 4, return new stripes to service */ while(!list_empty(&newstripes)) { nsh = list_entry(newstripes.next, struct stripe_head, lru); list_del_init(&nsh->lru); for (i=conf->raid_disks; i < newsize; i++) if (nsh->dev[i].page == NULL) { struct page *p = alloc_page(GFP_NOIO); nsh->dev[i].page = p; if (!p) err = -ENOMEM; } release_stripe(nsh); } /* critical section pass, GFP_NOIO no longer needed */ conf->slab_cache = sc; conf->active_name = 1-conf->active_name; conf->pool_size = newsize; return err; } #endif static int drop_one_stripe(raid5_conf_t *conf) { struct stripe_head *sh;