| @@ -213,6 +213,7 @@ struct blkfront_info · 212 unmodified lines |
| 213 | struct blk_mq_tag_set tag_set; | 213 | struct blk_mq_tag_set tag_set; |
| 214 | struct blkfront_ring_info *rinfo; | 214 | struct blkfront_ring_info *rinfo; |
| 215 | unsigned int nr_rings; | 215 | unsigned int nr_rings; |
| | 216 | unsigned int rinfo_size; |
| 216 | /* Save uncomplete reqs and bios for migration. */ | 217 | /* Save uncomplete reqs and bios for migration. */ |
| 217 | struct list_head requests; | 218 | struct list_head requests; |
| 218 | struct bio_list bio_list; | 219 | struct bio_list bio_list; |
| @@ -259,6 +260,18 @@ static int blkfront_setup_indirect(struct blkfront_ring_info *rinfo); · 40 unmodified lines |
| 259 | static void blkfront_gather_backend_features(struct blkfront_info *info); | 260 | static void blkfront_gather_backend_features(struct blkfront_info *info); |
| 260 | static int negotiate_mq(struct blkfront_info *info); | 261 | static int negotiate_mq(struct blkfront_info *info); |
| 261 | | 262 | |
| | 263 | #define for_each_rinfo(info, ptr, idx) \ |
| | 264 | for ((ptr) = (info)->rinfo, (idx) = 0; \ |
| | 265 | (idx) < (info)->nr_rings; \ |
| | 266 | (idx)++, (ptr) = (void *)(ptr) + (info)->rinfo_size) |
| | 267 | |
| | 268 | static inline struct blkfront_ring_info * |
| | 269 | get_rinfo(const struct blkfront_info *info, unsigned int i) |
| | 270 | { |
| | 271 | BUG_ON(i >= info->nr_rings); |
| | 272 | return (void *)info->rinfo + i * info->rinfo_size; |
| | 273 | } |
| | 274 | |
| 262 | static int get_id_from_freelist(struct blkfront_ring_info *rinfo) | 275 | static int get_id_from_freelist(struct blkfront_ring_info *rinfo) |
| 263 | { | 276 | { |
| 264 | unsigned long free = rinfo->shadow_free; | 277 | unsigned long free = rinfo->shadow_free; |
| @@ -883,8 +896,7 @@ static blk_status_t blkif_queue_rq(struct blk_mq_hw_ctx *hctx, · 618 unmodified lines |
| 883 | struct blkfront_info *info = hctx->queue->queuedata; | 896 | struct blkfront_info *info = hctx->queue->queuedata; |
| 884 | struct blkfront_ring_info *rinfo = NULL; | 897 | struct blkfront_ring_info *rinfo = NULL; |
| 885 | | 898 | |
| 886 | BUG_ON(info->nr_rings <= qid); | 899 | rinfo = get_rinfo(info, qid); |
| 887 | rinfo = &info->rinfo[qid]; | | |
| 888 | blk_mq_start_request(qd->rq); | 900 | blk_mq_start_request(qd->rq); |
| 889 | spin_lock_irqsave(&rinfo->ring_lock, flags); | 901 | spin_lock_irqsave(&rinfo->ring_lock, flags); |
| 890 | if (RING_FULL(&rinfo->ring)) | 902 | if (RING_FULL(&rinfo->ring)) |
| @@ -1181,6 +1193,7 @@ static int xlvbd_alloc_gendisk(blkif_sector_t capacity, · 290 unmodified lines |
| 1181 | static void xlvbd_release_gendisk(struct blkfront_info *info) | 1193 | static void xlvbd_release_gendisk(struct blkfront_info *info) |
| 1182 | { | 1194 | { |
| 1183 | unsigned int minor, nr_minors, i; | 1195 | unsigned int minor, nr_minors, i; |
| | 1196 | struct blkfront_ring_info *rinfo; |
| 1184 | | 1197 | |
| 1185 | if (info->rq == NULL) | 1198 | if (info->rq == NULL) |
| 1186 | return; | 1199 | return; |
| @@ -1188,9 +1201,7 @@ static void xlvbd_release_gendisk(struct blkfront_info *info) · 1 unmodified line |
| 1188 | /* No more blkif_request(). */ | 1201 | /* No more blkif_request(). */ |
| 1189 | blk_mq_stop_hw_queues(info->rq); | 1202 | blk_mq_stop_hw_queues(info->rq); |
| 1190 | | 1203 | |
| 1191 | for (i = 0; i < info->nr_rings; i++) { | 1204 | for_each_rinfo(info, rinfo, i) { |
| 1192 | struct blkfront_ring_info *rinfo = &info->rinfo[i]; | | |
| 1193 | | | |
| 1194 | /* No more gnttab callback work. */ | 1205 | /* No more gnttab callback work. */ |
| 1195 | gnttab_cancel_free_callback(&rinfo->callback); | 1206 | gnttab_cancel_free_callback(&rinfo->callback); |
| 1196 | | 1207 | |
| @@ -1339,6 +1350,7 @@ free_shadow: · 142 unmodified lines |
| 1339 | static void blkif_free(struct blkfront_info *info, int suspend) | 1350 | static void blkif_free(struct blkfront_info *info, int suspend) |
| 1340 | { | 1351 | { |
| 1341 | unsigned int i; | 1352 | unsigned int i; |
| | 1353 | struct blkfront_ring_info *rinfo; |
| 1342 | | 1354 | |
| 1343 | /* Prevent new requests being issued until we fix things up. */ | 1355 | /* Prevent new requests being issued until we fix things up. */ |
| 1344 | info->connected = suspend ? | 1356 | info->connected = suspend ? |
| @@ -1347,8 +1359,8 @@ static void blkif_free(struct blkfront_info *info, int suspend) · 2 unmodified lines |
| 1347 | if (info->rq) | 1359 | if (info->rq) |
| 1348 | blk_mq_stop_hw_queues(info->rq); | 1360 | blk_mq_stop_hw_queues(info->rq); |
| 1349 | | 1361 | |
| 1350 | for (i = 0; i < info->nr_rings; i++) | 1362 | for_each_rinfo(info, rinfo, i) |
| 1351 | blkif_free_ring(&info->rinfo[i]); | 1363 | blkif_free_ring(rinfo); |
| 1352 | | 1364 | |
| 1353 | kvfree(info->rinfo); | 1365 | kvfree(info->rinfo); |
| 1354 | info->rinfo = NULL; | 1366 | info->rinfo = NULL; |
| @@ -1775,6 +1787,7 @@ static int talk_to_blkback(struct xenbus_device *dev, · 420 unmodified lines |
| 1775 | int err; | 1787 | int err; |
| 1776 | unsigned int i, max_page_order; | 1788 | unsigned int i, max_page_order; |
| 1777 | unsigned int ring_page_order; | 1789 | unsigned int ring_page_order; |
| | 1790 | struct blkfront_ring_info *rinfo; |
| 1778 | | 1791 | |
| 1779 | if (!info) | 1792 | if (!info) |
| 1780 | return -ENODEV; | 1793 | return -ENODEV; |
| @@ -1788,9 +1801,7 @@ static int talk_to_blkback(struct xenbus_device *dev, · 7 unmodified lines |
| 1788 | if (err) | 1801 | if (err) |
| 1789 | goto destroy_blkring; | 1802 | goto destroy_blkring; |
| 1790 | | 1803 | |
| 1791 | for (i = 0; i < info->nr_rings; i++) { | 1804 | for_each_rinfo(info, rinfo, i) { |
| 1792 | struct blkfront_ring_info *rinfo = &info->rinfo[i]; | | |
| 1793 | | | |
| 1794 | /* Create shared ring, alloc event channel. */ | 1805 | /* Create shared ring, alloc event channel. */ |
| 1795 | err = setup_blkring(dev, rinfo); | 1806 | err = setup_blkring(dev, rinfo); |
| 1796 | if (err) | 1807 | if (err) |
| @@ -1815,7 +1826,7 @@ again: · 18 unmodified lines |
| 1815 | | 1826 | |
| 1816 | /* We already got the number of queues/rings in _probe */ | 1827 | /* We already got the number of queues/rings in _probe */ |
| 1817 | if (info->nr_rings == 1) { | 1828 | if (info->nr_rings == 1) { |
| 1818 | err = write_per_ring_nodes(xbt, &info->rinfo[0], dev->nodename); | 1829 | err = write_per_ring_nodes(xbt, info->rinfo, dev->nodename); |
| 1819 | if (err) | 1830 | if (err) |
| 1820 | goto destroy_blkring; | 1831 | goto destroy_blkring; |
| 1821 | } else { | 1832 | } else { |
| @@ -1837,10 +1848,10 @@ again: · 15 unmodified lines |
| 1837 | goto abort_transaction; | 1848 | goto abort_transaction; |
| 1838 | } | 1849 | } |
| 1839 | | 1850 | |
| 1840 | for (i = 0; i < info->nr_rings; i++) { | 1851 | for_each_rinfo(info, rinfo, i) { |
| 1841 | memset(path, 0, pathsize); | 1852 | memset(path, 0, pathsize); |
| 1842 | snprintf(path, pathsize, "%s/queue-%u", dev->nodename, i); | 1853 | snprintf(path, pathsize, "%s/queue-%u", dev->nodename, i); |
| 1843 | err = write_per_ring_nodes(xbt, &info->rinfo[i], path); | 1854 | err = write_per_ring_nodes(xbt, rinfo, path); |
| 1844 | if (err) { | 1855 | if (err) { |
| 1845 | kfree(path); | 1856 | kfree(path); |
| 1846 | goto destroy_blkring; | 1857 | goto destroy_blkring; |
| @@ -1868,9 +1879,8 @@ again: · 21 unmodified lines |
| 1868 | goto destroy_blkring; | 1879 | goto destroy_blkring; |
| 1869 | } | 1880 | } |
| 1870 | | 1881 | |
| 1871 | for (i = 0; i < info->nr_rings; i++) { | 1882 | for_each_rinfo(info, rinfo, i) { |
| 1872 | unsigned int j; | 1883 | unsigned int j; |
| 1873 | struct blkfront_ring_info *rinfo = &info->rinfo[i]; | | |
| 1874 | | 1884 | |
| 1875 | for (j = 0; j < BLK_RING_SIZE(info); j++) | 1885 | for (j = 0; j < BLK_RING_SIZE(info); j++) |
| 1876 | rinfo->shadow[j].req.u.rw.id = j + 1; | 1886 | rinfo->shadow[j].req.u.rw.id = j + 1; |
| @@ -1900,6 +1910,7 @@ static int negotiate_mq(struct blkfront_info *info) · 23 unmodified lines |
| 1900 | { | 1910 | { |
| 1901 | unsigned int backend_max_queues; | 1911 | unsigned int backend_max_queues; |
| 1902 | unsigned int i; | 1912 | unsigned int i; |
| | 1913 | struct blkfront_ring_info *rinfo; |
| 1903 | | 1914 | |
| 1904 | BUG_ON(info->nr_rings); | 1915 | BUG_ON(info->nr_rings); |
| 1905 | | 1916 | |
| @@ -1911,20 +1922,16 @@ static int negotiate_mq(struct blkfront_info *info) · 5 unmodified lines |
| 1911 | if (!info->nr_rings) | 1922 | if (!info->nr_rings) |
| 1912 | info->nr_rings = 1; | 1923 | info->nr_rings = 1; |
| 1913 | | 1924 | |
| 1914 | info->rinfo = kvcalloc(info->nr_rings, | 1925 | info->rinfo_size = struct_size(info->rinfo, shadow, |
| 1915 | struct_size(info->rinfo, shadow, | 1926 | BLK_RING_SIZE(info)); |
| 1916 | BLK_RING_SIZE(info)), | 1927 | info->rinfo = kvcalloc(info->nr_rings, info->rinfo_size, GFP_KERNEL); |
| 1917 | GFP_KERNEL); | | |
| 1918 | if (!info->rinfo) { | 1928 | if (!info->rinfo) { |
| 1919 | xenbus_dev_fatal(info->xbdev, -ENOMEM, "allocating ring_info structure"); | 1929 | xenbus_dev_fatal(info->xbdev, -ENOMEM, "allocating ring_info structure"); |
| 1920 | info->nr_rings = 0; | 1930 | info->nr_rings = 0; |
| 1921 | return -ENOMEM; | 1931 | return -ENOMEM; |
| 1922 | } | 1932 | } |
| 1923 | | 1933 | |
| 1924 | for (i = 0; i < info->nr_rings; i++) { | 1934 | for_each_rinfo(info, rinfo, i) { |
| 1925 | struct blkfront_ring_info *rinfo; | | |
| 1926 | | | |
| 1927 | rinfo = &info->rinfo[i]; | | |
| 1928 | INIT_LIST_HEAD(&rinfo->indirect_pages); | 1935 | INIT_LIST_HEAD(&rinfo->indirect_pages); |
| 1929 | INIT_LIST_HEAD(&rinfo->grants); | 1936 | INIT_LIST_HEAD(&rinfo->grants); |
| 1930 | rinfo->dev_info = info; | 1937 | rinfo->dev_info = info; |
| @@ -2017,6 +2024,7 @@ static int blkif_recover(struct blkfront_info *info) · 86 unmodified lines |
| 2017 | int rc; | 2024 | int rc; |
| 2018 | struct bio *bio; | 2025 | struct bio *bio; |
| 2019 | unsigned int segs; | 2026 | unsigned int segs; |
| | 2027 | struct blkfront_ring_info *rinfo; |
| 2020 | | 2028 | |
| 2021 | blkfront_gather_backend_features(info); | 2029 | blkfront_gather_backend_features(info); |
| 2022 | /* Reset limits changed by blk_mq_update_nr_hw_queues(). */ | 2030 | /* Reset limits changed by blk_mq_update_nr_hw_queues(). */ |
| @@ -2024,9 +2032,7 @@ static int blkif_recover(struct blkfront_info *info) · 1 unmodified line |
| 2024 | segs = info->max_indirect_segments ? : BLKIF_MAX_SEGMENTS_PER_REQUEST; | 2032 | segs = info->max_indirect_segments ? : BLKIF_MAX_SEGMENTS_PER_REQUEST; |
| 2025 | blk_queue_max_segments(info->rq, segs / GRANTS_PER_PSEG); | 2033 | blk_queue_max_segments(info->rq, segs / GRANTS_PER_PSEG); |
| 2026 | | 2034 | |
| 2027 | for (r_index = 0; r_index < info->nr_rings; r_index++) { | 2035 | for_each_rinfo(info, rinfo, r_index) { |
| 2028 | struct blkfront_ring_info *rinfo = &info->rinfo[r_index]; | | |
| 2029 | | | |
| 2030 | rc = blkfront_setup_indirect(rinfo); | 2036 | rc = blkfront_setup_indirect(rinfo); |
| 2031 | if (rc) | 2037 | if (rc) |
| 2032 | return rc; | 2038 | return rc; |
| @@ -2036,10 +2042,7 @@ static int blkif_recover(struct blkfront_info *info) · 3 unmodified lines |
| 2036 | /* Now safe for us to use the shared ring */ | 2042 | /* Now safe for us to use the shared ring */ |
| 2037 | info->connected = BLKIF_STATE_CONNECTED; | 2043 | info->connected = BLKIF_STATE_CONNECTED; |
| 2038 | | 2044 | |
| 2039 | for (r_index = 0; r_index < info->nr_rings; r_index++) { | 2045 | for_each_rinfo(info, rinfo, r_index) { |
| 2040 | struct blkfront_ring_info *rinfo; | | |
| 2041 | | | |
| 2042 | rinfo = &info->rinfo[r_index]; | | |
| 2043 | /* Kick any other new requests queued since we resumed */ | 2046 | /* Kick any other new requests queued since we resumed */ |
| 2044 | kick_pending_request_queues(rinfo); | 2047 | kick_pending_request_queues(rinfo); |
| 2045 | } | 2048 | } |
| @@ -2072,13 +2075,13 @@ static int blkfront_resume(struct xenbus_device *dev) · 26 unmodified lines |
| 2072 | struct blkfront_info *info = dev_get_drvdata(&dev->dev); | 2075 | struct blkfront_info *info = dev_get_drvdata(&dev->dev); |
| 2073 | int err = 0; | 2076 | int err = 0; |
| 2074 | unsigned int i, j; | 2077 | unsigned int i, j; |
| | 2078 | struct blkfront_ring_info *rinfo; |
| 2075 | | 2079 | |
| 2076 | dev_dbg(&dev->dev, "blkfront_resume: %s\n", dev->nodename); | 2080 | dev_dbg(&dev->dev, "blkfront_resume: %s\n", dev->nodename); |
| 2077 | | 2081 | |
| 2078 | bio_list_init(&info->bio_list); | 2082 | bio_list_init(&info->bio_list); |
| 2079 | INIT_LIST_HEAD(&info->requests); | 2083 | INIT_LIST_HEAD(&info->requests); |
| 2080 | for (i = 0; i < info->nr_rings; i++) { | 2084 | for_each_rinfo(info, rinfo, i) { |
| 2081 | struct blkfront_ring_info *rinfo = &info->rinfo[i]; | | |
| 2082 | struct bio_list merge_bio; | 2085 | struct bio_list merge_bio; |
| 2083 | struct blk_shadow *shadow = rinfo->shadow; | 2086 | struct blk_shadow *shadow = rinfo->shadow; |
| 2084 | | 2087 | |
| @@ -2337,6 +2340,7 @@ static void blkfront_connect(struct blkfront_info *info) · 252 unmodified lines |
| 2337 | unsigned int binfo; | 2340 | unsigned int binfo; |
| 2338 | char *envp[] = { "RESIZE=1", NULL }; | 2341 | char *envp[] = { "RESIZE=1", NULL }; |
| 2339 | int err, i; | 2342 | int err, i; |
| | 2343 | struct blkfront_ring_info *rinfo; |
| 2340 | | 2344 | |
| 2341 | switch (info->connected) { | 2345 | switch (info->connected) { |
| 2342 | case BLKIF_STATE_CONNECTED: | 2346 | case BLKIF_STATE_CONNECTED: |
| @@ -2394,8 +2398,8 @@ static void blkfront_connect(struct blkfront_info *info) · 51 unmodified lines |
| 2394 | "physical-sector-size", | 2398 | "physical-sector-size", |
| 2395 | sector_size); | 2399 | sector_size); |
| 2396 | blkfront_gather_backend_features(info); | 2400 | blkfront_gather_backend_features(info); |
| 2397 | for (i = 0; i < info->nr_rings; i++) { | 2401 | for_each_rinfo(info, rinfo, i) { |
| 2398 | err = blkfront_setup_indirect(&info->rinfo[i]); | 2402 | err = blkfront_setup_indirect(rinfo); |
| 2399 | if (err) { | 2403 | if (err) { |
| 2400 | xenbus_dev_fatal(info->xbdev, err, "setup_indirect at %s", | 2404 | xenbus_dev_fatal(info->xbdev, err, "setup_indirect at %s", |
| 2401 | info->xbdev->otherend); | 2405 | info->xbdev->otherend); |
| @@ -2416,8 +2420,8 @@ static void blkfront_connect(struct blkfront_info *info) · 14 unmodified lines |
| 2416 | | 2420 | |
| 2417 | /* Kick pending requests. */ | 2421 | /* Kick pending requests. */ |
| 2418 | info->connected = BLKIF_STATE_CONNECTED; | 2422 | info->connected = BLKIF_STATE_CONNECTED; |
| 2419 | for (i = 0; i < info->nr_rings; i++) | 2423 | for_each_rinfo(info, rinfo, i) |
| 2420 | kick_pending_request_queues(&info->rinfo[i]); | 2424 | kick_pending_request_queues(rinfo); |
| 2421 | | 2425 | |
| 2422 | device_add_disk(&info->xbdev->dev, info->gd, NULL); | 2426 | device_add_disk(&info->xbdev->dev, info->gd, NULL); |
| 2423 | | 2427 | |
| @@ -2652,9 +2656,9 @@ static void purge_persistent_grants(struct blkfront_info *info) · 228 unmodified lines |
| 2652 | { | 2656 | { |
| 2653 | unsigned int i; | 2657 | unsigned int i; |
| 2654 | unsigned long flags; | 2658 | unsigned long flags; |
| | 2659 | struct blkfront_ring_info *rinfo; |
| 2655 | | 2660 | |
| 2656 | for (i = 0; i < info->nr_rings; i++) { | 2661 | for_each_rinfo(info, rinfo, i) { |
| 2657 | struct blkfront_ring_info *rinfo = &info->rinfo[i]; | | |
| 2658 | struct grant *gnt_list_entry, *tmp; | 2662 | struct grant *gnt_list_entry, *tmp; |
| 2659 | | 2663 | |
| 2660 | spin_lock_irqsave(&rinfo->ring_lock, flags); | 2664 | spin_lock_irqsave(&rinfo->ring_lock, flags); |