| @@ -634,11 +634,10 @@ static int open_dev(struct block_device *b, fmode_t mode) · 633 unmodified lines |
| 634 | 634 | return 0; |
| 635 | 635 | } |
| 636 | 636 | |
| 637 | | −static int release_dev(struct gendisk *b, fmode_t mode) |
| 637 | +static void release_dev(struct gendisk *b, fmode_t mode) |
| 638 | 638 | { |
| 639 | 639 | struct bcache_device *d = b->private_data; |
| 640 | 640 | closure_put(&d->cl); |
| 641 | | − return 0; |
| 642 | 641 | } |
| 643 | 642 | |
| 644 | 643 | static int ioctl_dev(struct block_device *b, fmode_t mode, |
| @@ -732,8 +731,7 @@ static void bcache_device_free(struct bcache_device *d) · 87 unmodified lines |
| 732 | 731 | |
| 733 | 732 | if (d->c) |
| 734 | 733 | bcache_device_detach(d); |
| 735 | | − |
| 736 | | − if (d->disk) |
| 734 | + if (d->disk && d->disk->flags & GENHD_FL_UP) |
| 737 | 735 | del_gendisk(d->disk); |
| 738 | 736 | if (d->disk && d->disk->queue) |
| 739 | 737 | blk_cleanup_queue(d->disk->queue); |
| @@ -756,12 +754,9 @@ static int bcache_device_init(struct bcache_device *d, unsigned block_size) · 16 unmodified lines |
| 756 | 754 | if (!(d->bio_split = bioset_create(4, offsetof(struct bbio, bio))) || |
| 757 | 755 | !(d->unaligned_bvec = mempool_create_kmalloc_pool(1, |
| 758 | 756 | sizeof(struct bio_vec) * BIO_MAX_PAGES)) || |
| 759 | | − bio_split_pool_init(&d->bio_split_hook)) |
| 760 | | − |
| 761 | | − return -ENOMEM; |
| 762 | | − |
| 763 | | − d->disk = alloc_disk(1); |
| 764 | | − if (!d->disk) |
| 757 | + bio_split_pool_init(&d->bio_split_hook) || |
| 758 | + !(d->disk = alloc_disk(1)) || |
| 759 | + !(q = blk_alloc_queue(GFP_KERNEL))) |
| 765 | 760 | return -ENOMEM; |
| 766 | 761 | |
| 767 | 762 | snprintf(d->disk->disk_name, DISK_NAME_LEN, "bcache%i", bcache_minor); |
| @@ -771,10 +766,6 @@ static int bcache_device_init(struct bcache_device *d, unsigned block_size) · 3 unmodified lines |
| 771 | 766 | d->disk->fops = &bcache_ops; |
| 772 | 767 | d->disk->private_data = d; |
| 773 | 768 | |
| 774 | | − q = blk_alloc_queue(GFP_KERNEL); |
| 775 | | − if (!q) |
| 776 | | − return -ENOMEM; |
| 777 | | − |
| 778 | 769 | blk_queue_make_request(q, NULL); |
| 779 | 770 | d->disk->queue = q; |
| 780 | 771 | q->queuedata = d; |
| @@ -999,14 +990,17 @@ static void cached_dev_free(struct closure *cl) · 218 unmodified lines |
| 999 | 990 | |
| 1000 | 991 | mutex_lock(&bch_register_lock); |
| 1001 | 992 | |
| 1002 | | − bd_unlink_disk_holder(dc->bdev, dc->disk.disk); |
| 993 | + if (atomic_read(&dc->running)) |
| 994 | + bd_unlink_disk_holder(dc->bdev, dc->disk.disk); |
| 1003 | 995 | bcache_device_free(&dc->disk); |
| 1004 | 996 | list_del(&dc->list); |
| 1005 | 997 | |
| 1006 | 998 | mutex_unlock(&bch_register_lock); |
| 1007 | 999 | |
| 1008 | 1000 | if (!IS_ERR_OR_NULL(dc->bdev)) { |
| 1009 | | − blk_sync_queue(bdev_get_queue(dc->bdev)); |
| 1001 | + if (dc->bdev->bd_disk) |
| 1002 | + blk_sync_queue(bdev_get_queue(dc->bdev)); |
| 1003 | + |
| 1010 | 1004 | blkdev_put(dc->bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL); |
| 1011 | 1005 | } |
| 1012 | 1006 | |
| @@ -1028,73 +1022,67 @@ static void cached_dev_flush(struct closure *cl) · 15 unmodified lines |
| 1028 | 1022 | |
| 1029 | 1023 | static int cached_dev_init(struct cached_dev *dc, unsigned block_size) |
| 1030 | 1024 | { |
| 1031 | | − int err; |
| 1025 | + int ret; |
| 1032 | 1026 | struct io *io; |
| 1033 | | − |
| 1034 | | − closure_init(&dc->disk.cl, NULL); |
| 1035 | | − set_closure_fn(&dc->disk.cl, cached_dev_flush, system_wq); |
| 1027 | + struct request_queue *q = bdev_get_queue(dc->bdev); |
| 1036 | 1028 | |
| 1037 | 1029 | __module_get(THIS_MODULE); |
| 1038 | 1030 | INIT_LIST_HEAD(&dc->list); |
| 1031 | + closure_init(&dc->disk.cl, NULL); |
| 1032 | + set_closure_fn(&dc->disk.cl, cached_dev_flush, system_wq); |
| 1039 | 1033 | kobject_init(&dc->disk.kobj, &bch_cached_dev_ktype); |
| 1040 | | − |
| 1041 | | − bch_cache_accounting_init(&dc->accounting, &dc->disk.cl); |
| 1042 | | − |
| 1043 | | − err = bcache_device_init(&dc->disk, block_size); |
| 1044 | | − if (err) |
| 1045 | | − goto err; |
| 1046 | | − |
| 1047 | | − spin_lock_init(&dc->io_lock); |
| 1048 | | − closure_init_unlocked(&dc->sb_write); |
| 1049 | 1034 | INIT_WORK(&dc->detach, cached_dev_detach_finish); |
| 1035 | + closure_init_unlocked(&dc->sb_write); |
| 1036 | + INIT_LIST_HEAD(&dc->io_lru); |
| 1037 | + spin_lock_init(&dc->io_lock); |
| 1038 | + bch_cache_accounting_init(&dc->accounting, &dc->disk.cl); |
| 1050 | 1039 | |
| 1051 | 1040 | dc->sequential_merge = true; |
| 1052 | 1041 | dc->sequential_cutoff = 4 << 20; |
| 1053 | 1042 | |
| 1054 | | − INIT_LIST_HEAD(&dc->io_lru); |
| 1055 | | − dc->sb_bio.bi_max_vecs = 1; |
| 1056 | | − dc->sb_bio.bi_io_vec = dc->sb_bio.bi_inline_vecs; |
| 1057 | | − |
| 1058 | 1043 | for (io = dc->io; io < dc->io + RECENT_IO; io++) { |
| 1059 | 1044 | list_add(&io->lru, &dc->io_lru); |
| 1060 | 1045 | hlist_add_head(&io->hash, dc->io_hash + RECENT_IO); |
| 1061 | 1046 | } |
| 1062 | 1047 | |
| 1063 | | − bch_writeback_init_cached_dev(dc); |
| 1048 | + ret = bcache_device_init(&dc->disk, block_size); |
| 1049 | + if (ret) |
| 1050 | + return ret; |
| 1051 | + |
| 1052 | + set_capacity(dc->disk.disk, |
| 1053 | + dc->bdev->bd_part->nr_sects - dc->sb.data_offset); |
| 1054 | + |
| 1055 | + dc->disk.disk->queue->backing_dev_info.ra_pages = |
| 1056 | + max(dc->disk.disk->queue->backing_dev_info.ra_pages, |
| 1057 | + q->backing_dev_info.ra_pages); |
| 1058 | + |
| 1059 | + bch_cached_dev_request_init(dc); |
| 1060 | + bch_cached_dev_writeback_init(dc); |
| 1064 | 1061 | return 0; |
| 1065 | | −err: |
| 1066 | | − bcache_device_stop(&dc->disk); |
| 1067 | | − return err; |
| 1068 | 1062 | } |
| 1069 | 1063 | |
| 1070 | 1064 | /* Cached device - bcache superblock */ |
| 1071 | 1065 | |
| 1072 | | −static const char *register_bdev(struct cache_sb *sb, struct page *sb_page, |
| 1066 | +static void register_bdev(struct cache_sb *sb, struct page *sb_page, |
| 1073 | 1067 | struct block_device *bdev, |
| 1074 | 1068 | struct cached_dev *dc) |
| 1075 | 1069 | { |
| 1076 | 1070 | char name[BDEVNAME_SIZE]; |
| 1077 | 1071 | const char *err = "cannot allocate memory"; |
| 1078 | | − struct gendisk *g; |
| 1079 | 1072 | struct cache_set *c; |
| 1080 | 1073 | |
| 1081 | | − if (!dc || cached_dev_init(dc, sb->block_size << 9) != 0) |
| 1082 | | − return err; |
| 1083 | | − |
| 1084 | 1074 | memcpy(&dc->sb, sb, sizeof(struct cache_sb)); |
| 1085 | | − dc->sb_bio.bi_io_vec[0].bv_page = sb_page; |
| 1086 | 1075 | dc->bdev = bdev; |
| 1087 | 1076 | dc->bdev->bd_holder = dc; |
| 1088 | 1077 | |
| 1089 | | − g = dc->disk.disk; |
| 1090 | | − |
| 1091 | | − set_capacity(g, dc->bdev->bd_part->nr_sects - dc->sb.data_offset); |
| 1092 | | − |
| 1093 | | − g->queue->backing_dev_info.ra_pages = |
| 1094 | | − max(g->queue->backing_dev_info.ra_pages, |
| 1095 | | − bdev->bd_queue->backing_dev_info.ra_pages); |
| 1078 | + bio_init(&dc->sb_bio); |
| 1079 | + dc->sb_bio.bi_max_vecs = 1; |
| 1080 | + dc->sb_bio.bi_io_vec = dc->sb_bio.bi_inline_vecs; |
| 1081 | + dc->sb_bio.bi_io_vec[0].bv_page = sb_page; |
| 1082 | + get_page(sb_page); |
| 1096 | 1083 | |
| 1097 | | − bch_cached_dev_request_init(dc); |
| 1084 | + if (cached_dev_init(dc, sb->block_size << 9)) |
| 1085 | + goto err; |
| 1098 | 1086 | |
| 1099 | 1087 | err = "error creating kobject"; |
| 1100 | 1088 | if (kobject_add(&dc->disk.kobj, &part_to_dev(bdev->bd_part)->kobj, |
| @@ -1103,6 +1091,8 @@ static const char *register_bdev(struct cache_sb *sb, struct page *sb_page, · 2 unmodified lines |
| 1103 | 1091 | if (bch_cache_accounting_add_kobjs(&dc->accounting, &dc->disk.kobj)) |
| 1104 | 1092 | goto err; |
| 1105 | 1093 | |
| 1094 | + pr_info("registered backing device %s", bdevname(bdev, name)); |
| 1095 | + |
| 1106 | 1096 | list_add(&dc->list, &uncached_devices); |
| 1107 | 1097 | list_for_each_entry(c, &bch_cache_sets, list) |
| 1108 | 1098 | bch_cached_dev_attach(dc, c); |
| @@ -1111,15 +1101,10 @@ static const char *register_bdev(struct cache_sb *sb, struct page *sb_page, · 2 unmodified lines |
| 1111 | 1101 | BDEV_STATE(&dc->sb) == BDEV_STATE_STALE) |
| 1112 | 1102 | bch_cached_dev_run(dc); |
| 1113 | 1103 | |
| 1114 | | − return NULL; |
| 1104 | + return; |
| 1115 | 1105 | err: |
| 1116 | | − kobject_put(&dc->disk.kobj); |
| 1117 | 1106 | pr_notice("error opening %s: %s", bdevname(bdev, name), err); |
| 1118 | | − /* |
| 1119 | | − * Return NULL instead of an error because kobject_put() cleans |
| 1120 | | − * everything up |
| 1121 | | − */ |
| 1122 | | − return NULL; |
| 1107 | + bcache_device_stop(&dc->disk); |
| 1123 | 1108 | } |
| 1124 | 1109 | |
| 1125 | 1110 | /* Flash only volumes */ |
| @@ -1717,20 +1702,11 @@ static int cache_alloc(struct cache_sb *sb, struct cache *ca) · 591 unmodified lines |
| 1717 | 1702 | size_t free; |
| 1718 | 1703 | struct bucket *b; |
| 1719 | 1704 | |
| 1720 | | − if (!ca) |
| 1721 | | − return -ENOMEM; |
| 1722 | | − |
| 1723 | 1705 | __module_get(THIS_MODULE); |
| 1724 | 1706 | kobject_init(&ca->kobj, &bch_cache_ktype); |
| 1725 | 1707 | |
| 1726 | | − memcpy(&ca->sb, sb, sizeof(struct cache_sb)); |
| 1727 | | − |
| 1728 | 1708 | INIT_LIST_HEAD(&ca->discards); |
| 1729 | 1709 | |
| 1730 | | − bio_init(&ca->sb_bio); |
| 1731 | | − ca->sb_bio.bi_max_vecs = 1; |
| 1732 | | − ca->sb_bio.bi_io_vec = ca->sb_bio.bi_inline_vecs; |
| 1733 | | − |
| 1734 | 1710 | bio_init(&ca->journal.bio); |
| 1735 | 1711 | ca->journal.bio.bi_max_vecs = 8; |
| 1736 | 1712 | ca->journal.bio.bi_io_vec = ca->journal.bio.bi_inline_vecs; |
| @@ -1742,18 +1718,17 @@ static int cache_alloc(struct cache_sb *sb, struct cache *ca) · 5 unmodified lines |
| 1742 | 1718 | !init_fifo(&ca->free_inc, free << 2, GFP_KERNEL) || |
| 1743 | 1719 | !init_fifo(&ca->unused, free << 2, GFP_KERNEL) || |
| 1744 | 1720 | !init_heap(&ca->heap, free << 3, GFP_KERNEL) || |
| 1745 | | − !(ca->buckets = vmalloc(sizeof(struct bucket) * |
| 1721 | + !(ca->buckets = vzalloc(sizeof(struct bucket) * |
| 1746 | 1722 | ca->sb.nbuckets)) || |
| 1747 | 1723 | !(ca->prio_buckets = kzalloc(sizeof(uint64_t) * prio_buckets(ca) * |
| 1748 | 1724 | 2, GFP_KERNEL)) || |
| 1749 | 1725 | !(ca->disk_buckets = alloc_bucket_pages(GFP_KERNEL, ca)) || |
| 1750 | 1726 | !(ca->alloc_workqueue = alloc_workqueue("bch_allocator", 0, 1)) || |
| 1751 | 1727 | bio_split_pool_init(&ca->bio_split_hook)) |
| 1752 | | − goto err; |
| 1728 | + return -ENOMEM; |
| 1753 | 1729 | |
| 1754 | 1730 | ca->prio_last_buckets = ca->prio_buckets + prio_buckets(ca); |
| 1755 | 1731 | |
| 1756 | | − memset(ca->buckets, 0, ca->sb.nbuckets * sizeof(struct bucket)); |
| 1757 | 1732 | for_each_bucket(b, ca) |
| 1758 | 1733 | atomic_set(&b->pin, 0); |
| 1759 | 1734 | |
| @@ -1766,22 +1741,28 @@ err: · 6 unmodified lines |
| 1766 | 1741 | return -ENOMEM; |
| 1767 | 1742 | } |
| 1768 | 1743 | |
| 1769 | | −static const char *register_cache(struct cache_sb *sb, struct page *sb_page, |
| 1744 | +static void register_cache(struct cache_sb *sb, struct page *sb_page, |
| 1770 | 1745 | struct block_device *bdev, struct cache *ca) |
| 1771 | 1746 | { |
| 1772 | 1747 | char name[BDEVNAME_SIZE]; |
| 1773 | 1748 | const char *err = "cannot allocate memory"; |
| 1774 | 1749 | |
| 1775 | | − if (cache_alloc(sb, ca) != 0) |
| 1776 | | − return err; |
| 1777 | | − |
| 1778 | | − ca->sb_bio.bi_io_vec[0].bv_page = sb_page; |
| 1750 | + memcpy(&ca->sb, sb, sizeof(struct cache_sb)); |
| 1779 | 1751 | ca->bdev = bdev; |
| 1780 | 1752 | ca->bdev->bd_holder = ca; |
| 1781 | 1753 | |
| 1754 | + bio_init(&ca->sb_bio); |
| 1755 | + ca->sb_bio.bi_max_vecs = 1; |
| 1756 | + ca->sb_bio.bi_io_vec = ca->sb_bio.bi_inline_vecs; |
| 1757 | + ca->sb_bio.bi_io_vec[0].bv_page = sb_page; |
| 1758 | + get_page(sb_page); |
| 1759 | + |
| 1782 | 1760 | if (blk_queue_discard(bdev_get_queue(ca->bdev))) |
| 1783 | 1761 | ca->discard = CACHE_DISCARD(&ca->sb); |
| 1784 | 1762 | |
| 1763 | + if (cache_alloc(sb, ca) != 0) |
| 1764 | + goto err; |
| 1765 | + |
| 1785 | 1766 | err = "error creating kobject"; |
| 1786 | 1767 | if (kobject_add(&ca->kobj, &part_to_dev(bdev->bd_part)->kobj, "bcache")) |
| 1787 | 1768 | goto err; |
| @@ -1791,15 +1772,10 @@ static const char *register_cache(struct cache_sb *sb, struct page *sb_page, · 3 unmodified lines |
| 1791 | 1772 | goto err; |
| 1792 | 1773 | |
| 1793 | 1774 | pr_info("registered cache device %s", bdevname(bdev, name)); |
| 1794 | | − |
| 1795 | | − return NULL; |
| 1775 | + return; |
| 1796 | 1776 | err: |
| 1777 | + pr_notice("error opening %s: %s", bdevname(bdev, name), err); |
| 1797 | 1778 | kobject_put(&ca->kobj); |
| 1798 | | − pr_info("error opening %s: %s", bdevname(bdev, name), err); |
| 1799 | | − /* Return NULL instead of an error because kobject_put() cleans |
| 1800 | | − * everything up |
| 1801 | | − */ |
| 1802 | | − return NULL; |
| 1803 | 1779 | } |
| 1804 | 1780 | |
| 1805 | 1781 | /* Global interfaces/init */ |
| @@ -1833,12 +1809,15 @@ static ssize_t register_bcache(struct kobject *k, struct kobj_attribute *attr, · 27 unmodified lines |
| 1833 | 1809 | bdev = blkdev_get_by_path(strim(path), |
| 1834 | 1810 | FMODE_READ|FMODE_WRITE|FMODE_EXCL, |
| 1835 | 1811 | sb); |
| 1836 | | − if (bdev == ERR_PTR(-EBUSY)) |
| 1837 | | − err = "device busy"; |
| 1838 | | − |
| 1839 | | − if (IS_ERR(bdev) || |
| 1840 | | − set_blocksize(bdev, 4096)) |
| 1812 | + if (IS_ERR(bdev)) { |
| 1813 | + if (bdev == ERR_PTR(-EBUSY)) |
| 1814 | + err = "device busy"; |
| 1841 | 1815 | goto err; |
| 1816 | + } |
| 1817 | + |
| 1818 | + err = "failed to set blocksize"; |
| 1819 | + if (set_blocksize(bdev, 4096)) |
| 1820 | + goto err_close; |
| 1842 | 1821 | |
| 1843 | 1822 | err = read_super(sb, bdev, &sb_page); |
| 1844 | 1823 | if (err) |
| @@ -1846,33 +1825,33 @@ static ssize_t register_bcache(struct kobject *k, struct kobj_attribute *attr, · 1 unmodified line |
| 1846 | 1825 | |
| 1847 | 1826 | if (SB_IS_BDEV(sb)) { |
| 1848 | 1827 | struct cached_dev *dc = kzalloc(sizeof(*dc), GFP_KERNEL); |
| 1828 | + if (!dc) |
| 1829 | + goto err_close; |
| 1849 | 1830 | |
| 1850 | | − err = register_bdev(sb, sb_page, bdev, dc); |
| 1831 | + register_bdev(sb, sb_page, bdev, dc); |
| 1851 | 1832 | } else { |
| 1852 | 1833 | struct cache *ca = kzalloc(sizeof(*ca), GFP_KERNEL); |
| 1834 | + if (!ca) |
| 1835 | + goto err_close; |
| 1853 | 1836 | |
| 1854 | | − err = register_cache(sb, sb_page, bdev, ca); |
| 1837 | + register_cache(sb, sb_page, bdev, ca); |
| 1855 | 1838 | } |
| 1856 | | − |
| 1857 | | − if (err) { |
| 1858 | | − /* register_(bdev|cache) will only return an error if they |
| 1859 | | − * didn't get far enough to create the kobject - if they did, |
| 1860 | | − * the kobject destructor will do this cleanup. |
| 1861 | | − */ |
| 1839 | +out: |
| 1840 | + if (sb_page) |
| 1862 | 1841 | put_page(sb_page); |
| 1863 | | −err_close: |
| 1864 | | − blkdev_put(bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL); |
| 1865 | | −err: |
| 1866 | | − if (attr != &ksysfs_register_quiet) |
| 1867 | | − pr_info("error opening %s: %s", path, err); |
| 1868 | | − ret = -EINVAL; |
| 1869 | | − } |
| 1870 | | − |
| 1871 | 1842 | kfree(sb); |
| 1872 | 1843 | kfree(path); |
| 1873 | 1844 | mutex_unlock(&bch_register_lock); |
| 1874 | 1845 | module_put(THIS_MODULE); |
| 1875 | 1846 | return ret; |
| 1847 | + |
| 1848 | +err_close: |
| 1849 | + blkdev_put(bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL); |
| 1850 | +err: |
| 1851 | + if (attr != &ksysfs_register_quiet) |
| 1852 | + pr_info("error opening %s: %s", path, err); |
| 1853 | + ret = -EINVAL; |
| 1854 | + goto out; |
| 1876 | 1855 | } |
| 1877 | 1856 | |
| 1878 | 1857 | static int bcache_reboot(struct notifier_block *n, unsigned long code, void *x) |