* for-5.9/drivers: (38 commits) block: add max_active_zones to blk-sysfs block: add max_open_zones to blk-sysfs s390/dasd: Use struct_size() helper s390/dasd: fix inability to use DASD with DIAG driver md-cluster: fix wild pointer of unlock_all_bitmaps() md/raid5-cache: clear MD_SB_CHANGE_PENDING before flushing stripes md: fix deadlock causing by sysfs_notify md: improve io stats accounting md: raid0/linear: fix dereference before null check on pointer mddev rsxx: switch from 'pci_free_consistent()' to 'dma_free_coherent()' nvme: remove ns->disk checks nvme-pci: use standard block status symbolic names nvme-pci: use the consistent return type of nvme_pci_iod_alloc_size() nvme-pci: add a blank line after declarations nvme-pci: fix some comments issues nvme-pci: remove redundant segment validation nvme: document quirked Intel models nvme: expose reconnect_delay and ctrl_loss_tmo via sysfs nvme: support for zoned namespaces nvme: support for multiple Command Sets Supported and Effects log pages ...
Unified · Split
| @@ -273,6 +273,24 @@ Description: · 272 unmodified lines | |||
| 273 | device ("host-aware" or "host-managed" zone model). For regular | 273 | device ("host-aware" or "host-managed" zone model). For regular |
| 274 | block devices, the value is always 0. | 274 | block devices, the value is always 0. |
| 275 | 275 | ||
| 276 | What: /sys/block/<disk>/queue/max_active_zones | ||
| 277 | Date: July 2020 | ||
| 278 | Contact: Niklas Cassel <niklas.cassel@wdc.com> | ||
| 279 | Description: | ||
| 280 | For zoned block devices (zoned attribute indicating | ||
| 281 | "host-managed" or "host-aware"), the sum of zones belonging to | ||
| 282 | any of the zone states: EXPLICIT OPEN, IMPLICIT OPEN or CLOSED, | ||
| 283 | is limited by this value. If this value is 0, there is no limit. | ||
| 284 | |||
| 285 | What: /sys/block/<disk>/queue/max_open_zones | ||
| 286 | Date: July 2020 | ||
| 287 | Contact: Niklas Cassel <niklas.cassel@wdc.com> | ||
| 288 | Description: | ||
| 289 | For zoned block devices (zoned attribute indicating | ||
| 290 | "host-managed" or "host-aware"), the sum of zones belonging to | ||
| 291 | any of the zone states: EXPLICIT OPEN or IMPLICIT OPEN, | ||
| 292 | is limited by this value. If this value is 0, there is no limit. | ||
| 293 | |||
| 276 | What: /sys/block/<disk>/queue/chunk_sectors | 294 | What: /sys/block/<disk>/queue/chunk_sectors |
| 277 | Date: September 2016 | 295 | Date: September 2016 |
| 278 | Contact: Hannes Reinecke <hare@suse.com> | 296 | Contact: Hannes Reinecke <hare@suse.com> |
| @@ -117,6 +117,20 @@ Maximum number of elements in a DMA scatter/gather list with integrity · 116 unmodified lines | |||
| 117 | data that will be submitted by the block layer core to the associated | 117 | data that will be submitted by the block layer core to the associated |
| 118 | block driver. | 118 | block driver. |
| 119 | 119 | ||
| 120 | max_active_zones (RO) | ||
| 121 | --------------------- | ||
| 122 | For zoned block devices (zoned attribute indicating "host-managed" or | ||
| 123 | "host-aware"), the sum of zones belonging to any of the zone states: | ||
| 124 | EXPLICIT OPEN, IMPLICIT OPEN or CLOSED, is limited by this value. | ||
| 125 | If this value is 0, there is no limit. | ||
| 126 | |||
| 127 | max_open_zones (RO) | ||
| 128 | ------------------- | ||
| 129 | For zoned block devices (zoned attribute indicating "host-managed" or | ||
| 130 | "host-aware"), the sum of zones belonging to any of the zone states: | ||
| 131 | EXPLICIT OPEN or IMPLICIT OPEN, is limited by this value. | ||
| 132 | If this value is 0, there is no limit. | ||
| 133 | |||
| 120 | max_sectors_kb (RW) | 134 | max_sectors_kb (RW) |
| 121 | ------------------- | 135 | ------------------- |
| 122 | This is the maximum number of kilobytes that the block layer will allow | 136 | This is the maximum number of kilobytes that the block layer will allow |
| @@ -86,9 +86,10 @@ config BLK_DEV_ZONED · 85 unmodified lines | |||
| 86 | select MQ_IOSCHED_DEADLINE | 86 | select MQ_IOSCHED_DEADLINE |
| 87 | help | 87 | help |
| 88 | Block layer zoned block device support. This option enables | 88 | Block layer zoned block device support. This option enables |
| 89 | support for ZAC/ZBC host-managed and host-aware zoned block devices. | 89 | support for ZAC/ZBC/ZNS host-managed and host-aware zoned block |
| 90 | devices. | ||
| 90 | 91 | ||
| 91 | Say yes here if you have a ZAC or ZBC storage device. | 92 | Say yes here if you have a ZAC, ZBC, or ZNS storage device. |
| 92 | 93 | ||
| 93 | config BLK_DEV_THROTTLING | 94 | config BLK_DEV_THROTTLING |
| 94 | bool "Block layer bio throttling support" | 95 | bool "Block layer bio throttling support" |
| @@ -306,6 +306,16 @@ static ssize_t queue_nr_zones_show(struct request_queue *q, char *page) · 305 unmodified lines | |||
| 306 | return queue_var_show(blk_queue_nr_zones(q), page); | 306 | return queue_var_show(blk_queue_nr_zones(q), page); |
| 307 | } | 307 | } |
| 308 | 308 | ||
| 309 | static ssize_t queue_max_open_zones_show(struct request_queue *q, char *page) | ||
| 310 | { | ||
| 311 | return queue_var_show(queue_max_open_zones(q), page); | ||
| 312 | } | ||
| 313 | |||
| 314 | static ssize_t queue_max_active_zones_show(struct request_queue *q, char *page) | ||
| 315 | { | ||
| 316 | return queue_var_show(queue_max_active_zones(q), page); | ||
| 317 | } | ||
| 318 | |||
| 309 | static ssize_t queue_nomerges_show(struct request_queue *q, char *page) | 319 | static ssize_t queue_nomerges_show(struct request_queue *q, char *page) |
| 310 | { | 320 | { |
| 311 | return queue_var_show((blk_queue_nomerges(q) << 1) | | 321 | return queue_var_show((blk_queue_nomerges(q) << 1) | |
| @@ -668,6 +678,16 @@ static struct queue_sysfs_entry queue_nr_zones_entry = { · 356 unmodified lines | |||
| 668 | .show = queue_nr_zones_show, | 678 | .show = queue_nr_zones_show, |
| 669 | }; | 679 | }; |
| 670 | 680 | ||
| 681 | static struct queue_sysfs_entry queue_max_open_zones_entry = { | ||
| 682 | .attr = {.name = "max_open_zones", .mode = 0444 }, | ||
| 683 | .show = queue_max_open_zones_show, | ||
| 684 | }; | ||
| 685 | |||
| 686 | static struct queue_sysfs_entry queue_max_active_zones_entry = { | ||
| 687 | .attr = {.name = "max_active_zones", .mode = 0444 }, | ||
| 688 | .show = queue_max_active_zones_show, | ||
| 689 | }; | ||
| 690 | |||
| 671 | static struct queue_sysfs_entry queue_nomerges_entry = { | 691 | static struct queue_sysfs_entry queue_nomerges_entry = { |
| 672 | .attr = {.name = "nomerges", .mode = 0644 }, | 692 | .attr = {.name = "nomerges", .mode = 0644 }, |
| 673 | .show = queue_nomerges_show, | 693 | .show = queue_nomerges_show, |
| @@ -766,6 +786,8 @@ static struct attribute *queue_attrs[] = { · 92 unmodified lines | |||
| 766 | &queue_nonrot_entry.attr, | 786 | &queue_nonrot_entry.attr, |
| 767 | &queue_zoned_entry.attr, | 787 | &queue_zoned_entry.attr, |
| 768 | &queue_nr_zones_entry.attr, | 788 | &queue_nr_zones_entry.attr, |
| 789 | &queue_max_open_zones_entry.attr, | ||
| 790 | &queue_max_active_zones_entry.attr, | ||
| 769 | &queue_nomerges_entry.attr, | 791 | &queue_nomerges_entry.attr, |
| 770 | &queue_rq_affinity_entry.attr, | 792 | &queue_rq_affinity_entry.attr, |
| 771 | &queue_iostats_entry.attr, | 793 | &queue_iostats_entry.attr, |
| @@ -793,6 +815,11 @@ static umode_t queue_attr_visible(struct kobject *kobj, struct attribute *attr, · 21 unmodified lines | |||
| 793 | (!q->mq_ops || !q->mq_ops->timeout)) | 815 | (!q->mq_ops || !q->mq_ops->timeout)) |
| 794 | return 0; | 816 | return 0; |
| 795 | 817 | ||
| 818 | if ((attr == &queue_max_open_zones_entry.attr || | ||
| 819 | attr == &queue_max_active_zones_entry.attr) && | ||
| 820 | !blk_queue_is_zoned(q)) | ||
| 821 | return 0; | ||
| 822 | |||
| 796 | return attr->mode; | 823 | return attr->mode; |
| 797 | } | 824 | } |
| 798 | 825 | ||
| @@ -312,6 +312,7 @@ int blkdev_report_zones_ioctl(struct block_device *bdev, fmode_t mode, · 311 unmodified lines | |||
| 312 | return ret; | 312 | return ret; |
| 313 | 313 | ||
| 314 | rep.nr_zones = ret; | 314 | rep.nr_zones = ret; |
| 315 | rep.flags = BLK_ZONE_REP_CAPACITY; | ||
| 315 | if (copy_to_user(argp, &rep, sizeof(struct blk_zone_report))) | 316 | if (copy_to_user(argp, &rep, sizeof(struct blk_zone_report))) |
| 316 | return -EFAULT; | 317 | return -EFAULT; |
| 317 | return 0; | 318 | return 0; |
| @@ -49,6 +49,7 @@ struct nullb_device { · 48 unmodified lines | |||
| 49 | unsigned long completion_nsec; /* time in ns to complete a request */ | 49 | unsigned long completion_nsec; /* time in ns to complete a request */ |
| 50 | unsigned long cache_size; /* disk cache size in MB */ | 50 | unsigned long cache_size; /* disk cache size in MB */ |
| 51 | unsigned long zone_size; /* zone size in MB if device is zoned */ | 51 | unsigned long zone_size; /* zone size in MB if device is zoned */ |
| 52 | unsigned long zone_capacity; /* zone capacity in MB if device is zoned */ | ||
| 52 | unsigned int zone_nr_conv; /* number of conventional zones */ | 53 | unsigned int zone_nr_conv; /* number of conventional zones */ |
| 53 | unsigned int submit_queues; /* number of submission queues */ | 54 | unsigned int submit_queues; /* number of submission queues */ |
| 54 | unsigned int home_node; /* home node for the device */ | 55 | unsigned int home_node; /* home node for the device */ |
| @@ -200,6 +200,10 @@ static unsigned long g_zone_size = 256; · 199 unmodified lines | |||
| 200 | module_param_named(zone_size, g_zone_size, ulong, S_IRUGO); | 200 | module_param_named(zone_size, g_zone_size, ulong, S_IRUGO); |
| 201 | MODULE_PARM_DESC(zone_size, "Zone size in MB when block device is zoned. Must be power-of-two: Default: 256"); | 201 | MODULE_PARM_DESC(zone_size, "Zone size in MB when block device is zoned. Must be power-of-two: Default: 256"); |
| 202 | 202 | ||
| 203 | static unsigned long g_zone_capacity; | ||
| 204 | module_param_named(zone_capacity, g_zone_capacity, ulong, 0444); | ||
| 205 | MODULE_PARM_DESC(zone_capacity, "Zone capacity in MB when block device is zoned. Can be less than or equal to zone size. Default: Zone size"); | ||
| 206 | |||
| 203 | static unsigned int g_zone_nr_conv; | 207 | static unsigned int g_zone_nr_conv; |
| 204 | module_param_named(zone_nr_conv, g_zone_nr_conv, uint, 0444); | 208 | module_param_named(zone_nr_conv, g_zone_nr_conv, uint, 0444); |
| 205 | MODULE_PARM_DESC(zone_nr_conv, "Number of conventional zones when block device is zoned. Default: 0"); | 209 | MODULE_PARM_DESC(zone_nr_conv, "Number of conventional zones when block device is zoned. Default: 0"); |
| @@ -341,6 +345,7 @@ NULLB_DEVICE_ATTR(mbps, uint, NULL); · 135 unmodified lines | |||
| 341 | NULLB_DEVICE_ATTR(cache_size, ulong, NULL); | 345 | NULLB_DEVICE_ATTR(cache_size, ulong, NULL); |
| 342 | NULLB_DEVICE_ATTR(zoned, bool, NULL); | 346 | NULLB_DEVICE_ATTR(zoned, bool, NULL); |
| 343 | NULLB_DEVICE_ATTR(zone_size, ulong, NULL); | 347 | NULLB_DEVICE_ATTR(zone_size, ulong, NULL); |
| 348 | NULLB_DEVICE_ATTR(zone_capacity, ulong, NULL); | ||
| 344 | NULLB_DEVICE_ATTR(zone_nr_conv, uint, NULL); | 349 | NULLB_DEVICE_ATTR(zone_nr_conv, uint, NULL); |
| 345 | 350 | ||
| 346 | static ssize_t nullb_device_power_show(struct config_item *item, char *page) | 351 | static ssize_t nullb_device_power_show(struct config_item *item, char *page) |
| @@ -457,6 +462,7 @@ static struct configfs_attribute *nullb_device_attrs[] = { · 110 unmodified lines | |||
| 457 | &nullb_device_attr_badblocks, | 462 | &nullb_device_attr_badblocks, |
| 458 | &nullb_device_attr_zoned, | 463 | &nullb_device_attr_zoned, |
| 459 | &nullb_device_attr_zone_size, | 464 | &nullb_device_attr_zone_size, |
| 465 | &nullb_device_attr_zone_capacity, | ||
| 460 | &nullb_device_attr_zone_nr_conv, | 466 | &nullb_device_attr_zone_nr_conv, |
| 461 | NULL, | 467 | NULL, |
| 462 | }; | 468 | }; |
| @@ -510,7 +516,8 @@ nullb_group_drop_item(struct config_group *group, struct config_item *item) · 47 unmodified lines | |||
| 510 | 516 | ||
| 511 | static ssize_t memb_group_features_show(struct config_item *item, char *page) | 517 | static ssize_t memb_group_features_show(struct config_item *item, char *page) |
| 512 | { | 518 | { |
| 513 | return snprintf(page, PAGE_SIZE, "memory_backed,discard,bandwidth,cache,badblocks,zoned,zone_size,zone_nr_conv\n"); | 519 | return snprintf(page, PAGE_SIZE, |
| 520 | "memory_backed,discard,bandwidth,cache,badblocks,zoned,zone_size,zone_capacity,zone_nr_conv\n"); | ||
| 514 | } | 521 | } |
| 515 | 522 | ||
| 516 | CONFIGFS_ATTR_RO(memb_group_, features); | 523 | CONFIGFS_ATTR_RO(memb_group_, features); |
| @@ -571,6 +578,7 @@ static struct nullb_device *null_alloc_dev(void) · 54 unmodified lines | |||
| 571 | dev->use_per_node_hctx = g_use_per_node_hctx; | 578 | dev->use_per_node_hctx = g_use_per_node_hctx; |
| 572 | dev->zoned = g_zoned; | 579 | dev->zoned = g_zoned; |
| 573 | dev->zone_size = g_zone_size; | 580 | dev->zone_size = g_zone_size; |
| 581 | dev->zone_capacity = g_zone_capacity; | ||
| 574 | dev->zone_nr_conv = g_zone_nr_conv; | 582 | dev->zone_nr_conv = g_zone_nr_conv; |
| 575 | return dev; | 583 | return dev; |
| 576 | } | 584 | } |
| @@ -28,6 +28,15 @@ int null_init_zoned_dev(struct nullb_device *dev, struct request_queue *q) · 27 unmodified lines | |||
| 28 | return -EINVAL; | 28 | return -EINVAL; |
| 29 | } | 29 | } |
| 30 | 30 | ||
| 31 | if (!dev->zone_capacity) | ||
| 32 | dev->zone_capacity = dev->zone_size; | ||
| 33 | |||
| 34 | if (dev->zone_capacity > dev->zone_size) { | ||
| 35 | pr_err("null_blk: zone capacity (%lu MB) larger than zone size (%lu MB)\n", | ||
| 36 | dev->zone_capacity, dev->zone_size); | ||
| 37 | return -EINVAL; | ||
| 38 | } | ||
| 39 | |||
| 31 | dev->zone_size_sects = dev->zone_size << ZONE_SIZE_SHIFT; | 40 | dev->zone_size_sects = dev->zone_size << ZONE_SIZE_SHIFT; |
| 32 | dev->nr_zones = dev_size >> | 41 | dev->nr_zones = dev_size >> |
| 33 | (SECTOR_SHIFT + ilog2(dev->zone_size_sects)); | 42 | (SECTOR_SHIFT + ilog2(dev->zone_size_sects)); |
| @@ -47,6 +56,7 @@ int null_init_zoned_dev(struct nullb_device *dev, struct request_queue *q) · 13 unmodified lines | |||
| 47 | 56 | ||
| 48 | zone->start = sector; | 57 | zone->start = sector; |
| 49 | zone->len = dev->zone_size_sects; | 58 | zone->len = dev->zone_size_sects; |
| 59 | zone->capacity = zone->len; | ||
| 50 | zone->wp = zone->start + zone->len; | 60 | zone->wp = zone->start + zone->len; |
| 51 | zone->type = BLK_ZONE_TYPE_CONVENTIONAL; | 61 | zone->type = BLK_ZONE_TYPE_CONVENTIONAL; |
| 52 | zone->cond = BLK_ZONE_COND_NOT_WP; | 62 | zone->cond = BLK_ZONE_COND_NOT_WP; |
| @@ -59,6 +69,7 @@ int null_init_zoned_dev(struct nullb_device *dev, struct request_queue *q) · 6 unmodified lines | |||
| 59 | 69 | ||
| 60 | zone->start = zone->wp = sector; | 70 | zone->start = zone->wp = sector; |
| 61 | zone->len = dev->zone_size_sects; | 71 | zone->len = dev->zone_size_sects; |
| 72 | zone->capacity = dev->zone_capacity << ZONE_SIZE_SHIFT; | ||
| 62 | zone->type = BLK_ZONE_TYPE_SEQWRITE_REQ; | 73 | zone->type = BLK_ZONE_TYPE_SEQWRITE_REQ; |
| 63 | zone->cond = BLK_ZONE_COND_EMPTY; | 74 | zone->cond = BLK_ZONE_COND_EMPTY; |
| 64 | 75 | ||
| @@ -185,6 +196,9 @@ static blk_status_t null_zone_write(struct nullb_cmd *cmd, sector_t sector, · 120 unmodified lines | |||
| 185 | return BLK_STS_IOERR; | 196 | return BLK_STS_IOERR; |
| 186 | } | 197 | } |
| 187 | 198 | ||
| 199 | if (zone->wp + nr_sectors > zone->start + zone->capacity) | ||
| 200 | return BLK_STS_IOERR; | ||
| 201 | |||
| 188 | if (zone->cond != BLK_ZONE_COND_EXP_OPEN) | 202 | if (zone->cond != BLK_ZONE_COND_EXP_OPEN) |
| 189 | zone->cond = BLK_ZONE_COND_IMP_OPEN; | 203 | zone->cond = BLK_ZONE_COND_IMP_OPEN; |
| 190 | 204 | ||
| @@ -193,7 +207,7 @@ static blk_status_t null_zone_write(struct nullb_cmd *cmd, sector_t sector, · 2 unmodified lines | |||
| 193 | return ret; | 207 | return ret; |
| 194 | 208 | ||
| 195 | zone->wp += nr_sectors; | 209 | zone->wp += nr_sectors; |
| 196 | if (zone->wp == zone->start + zone->len) | 210 | if (zone->wp == zone->start + zone->capacity) |
| 197 | zone->cond = BLK_ZONE_COND_FULL; | 211 | zone->cond = BLK_ZONE_COND_FULL; |
| 198 | return BLK_STS_OK; | 212 | return BLK_STS_OK; |
| 199 | default: | 213 | default: |
| @@ -562,13 +562,15 @@ static int rsxx_eeh_frozen(struct pci_dev *dev) · 561 unmodified lines | |||
| 562 | 562 | ||
| 563 | for (i = 0; i < card->n_targets; i++) { | 563 | for (i = 0; i < card->n_targets; i++) { |
| 564 | if (card->ctrl[i].status.buf) | 564 | if (card->ctrl[i].status.buf) |
| 565 | pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8, | 565 | dma_free_coherent(&card->dev->dev, |
| 566 | card->ctrl[i].status.buf, | 566 | STATUS_BUFFER_SIZE8, |
| 567 | card->ctrl[i].status.dma_addr); | 567 | card->ctrl[i].status.buf, |
| 568 | card->ctrl[i].status.dma_addr); | ||
| 568 | if (card->ctrl[i].cmd.buf) | 569 | if (card->ctrl[i].cmd.buf) |
| 569 | pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8, | 570 | dma_free_coherent(&card->dev->dev, |
| 570 | card->ctrl[i].cmd.buf, | 571 | COMMAND_BUFFER_SIZE8, |
| 571 | card->ctrl[i].cmd.dma_addr); | 572 | card->ctrl[i].cmd.buf, |
| 573 | card->ctrl[i].cmd.dma_addr); | ||
| 572 | } | 574 | } |
| 573 | 575 | ||
| 574 | return 0; | 576 | return 0; |
| @@ -711,15 +713,15 @@ static pci_ers_result_t rsxx_slot_reset(struct pci_dev *dev) · 136 unmodified lines | |||
| 711 | failed_hw_buffers_init: | 713 | failed_hw_buffers_init: |
| 712 | for (i = 0; i < card->n_targets; i++) { | 714 | for (i = 0; i < card->n_targets; i++) { |
| 713 | if (card->ctrl[i].status.buf) | 715 | if (card->ctrl[i].status.buf) |
| 714 | pci_free_consistent(card->dev, | 716 | dma_free_coherent(&card->dev->dev, |
| 715 | STATUS_BUFFER_SIZE8, | 717 | STATUS_BUFFER_SIZE8, |
| 716 | card->ctrl[i].status.buf, | 718 | card->ctrl[i].status.buf, |
| 717 | card->ctrl[i].status.dma_addr); | 719 | card->ctrl[i].status.dma_addr); |
| 718 | if (card->ctrl[i].cmd.buf) | 720 | if (card->ctrl[i].cmd.buf) |
| 719 | pci_free_consistent(card->dev, | 721 | dma_free_coherent(&card->dev->dev, |
| 720 | COMMAND_BUFFER_SIZE8, | 722 | COMMAND_BUFFER_SIZE8, |
| 721 | card->ctrl[i].cmd.buf, | 723 | card->ctrl[i].cmd.buf, |
| 722 | card->ctrl[i].cmd.dma_addr); | 724 | card->ctrl[i].cmd.dma_addr); |
| 723 | } | 725 | } |
| 724 | failed_hw_setup: | 726 | failed_hw_setup: |
| 725 | rsxx_eeh_failure(dev); | 727 | rsxx_eeh_failure(dev); |
| @@ -1631,7 +1631,7 @@ void md_bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector, bool force) · 1630 unmodified lines | |||
| 1631 | s += blocks; | 1631 | s += blocks; |
| 1632 | } | 1632 | } |
| 1633 | bitmap->last_end_sync = jiffies; | 1633 | bitmap->last_end_sync = jiffies; |
| 1634 | sysfs_notify(&bitmap->mddev->kobj, NULL, "sync_completed"); | 1634 | sysfs_notify_dirent_safe(bitmap->mddev->sysfs_completed); |
| 1635 | } | 1635 | } |
| 1636 | EXPORT_SYMBOL(md_bitmap_cond_end_sync); | 1636 | EXPORT_SYMBOL(md_bitmap_cond_end_sync); |
| 1637 | 1637 | ||
| @@ -1518,6 +1518,7 @@ static void unlock_all_bitmaps(struct mddev *mddev) · 1517 unmodified lines | |||
| 1518 | } | 1518 | } |
| 1519 | } | 1519 | } |
| 1520 | kfree(cinfo->other_bitmap_lockres); | 1520 | kfree(cinfo->other_bitmap_lockres); |
| 1521 | cinfo->other_bitmap_lockres = NULL; | ||
| 1521 | } | 1522 | } |
| 1522 | } | 1523 | } |
| 1523 | 1524 | ||
| @@ -463,24 +463,46 @@ check_suspended: · 462 unmodified lines | |||
| 463 | } | 463 | } |
| 464 | EXPORT_SYMBOL(md_handle_request); | 464 | EXPORT_SYMBOL(md_handle_request); |
| 465 | 465 | ||
| 466 | struct md_io { | ||
| 467 | struct mddev *mddev; | ||
| 468 | bio_end_io_t *orig_bi_end_io; | ||
| 469 | void *orig_bi_private; | ||
| 470 | unsigned long start_time; | ||
| 471 | }; | ||
| 472 | |||
| 473 | static void md_end_io(struct bio *bio) | ||
| 474 | { | ||
| 475 | struct md_io *md_io = bio->bi_private; | ||
| 476 | struct mddev *mddev = md_io->mddev; | ||
| 477 | |||
| 478 | disk_end_io_acct(mddev->gendisk, bio_op(bio), md_io->start_time); | ||
| 479 | |||
| 480 | bio->bi_end_io = md_io->orig_bi_end_io; | ||
| 481 | bio->bi_private = md_io->orig_bi_private; | ||
| 482 | |||
| 483 | mempool_free(md_io, &mddev->md_io_pool); | ||
| 484 | |||
| 485 | if (bio->bi_end_io) | ||
| 486 | bio->bi_end_io(bio); | ||
| 487 | } | ||
| 488 | |||
| 466 | static blk_qc_t md_submit_bio(struct bio *bio) | 489 | static blk_qc_t md_submit_bio(struct bio *bio) |
| 467 | { | 490 | { |
| 468 | const int rw = bio_data_dir(bio); | 491 | const int rw = bio_data_dir(bio); |
| 469 | const int sgrp = op_stat_group(bio_op(bio)); | ||
| 470 | struct mddev *mddev = bio->bi_disk->private_data; | 492 | struct mddev *mddev = bio->bi_disk->private_data; |
| 471 | unsigned int sectors; | ||
| 472 | 493 | ||
| 473 | if (unlikely(test_bit(MD_BROKEN, &mddev->flags)) && (rw == WRITE)) { | 494 | if (mddev == NULL || mddev->pers == NULL) { |
| 474 | bio_io_error(bio); | 495 | bio_io_error(bio); |
| 475 | return BLK_QC_T_NONE; | 496 | return BLK_QC_T_NONE; |
| 476 | } | 497 | } |
| 477 | 498 | ||
| 478 | blk_queue_split(&bio); | 499 | if (unlikely(test_bit(MD_BROKEN, &mddev->flags)) && (rw == WRITE)) { |
| 479 | |||
| 480 | if (mddev == NULL || mddev->pers == NULL) { | ||
| 481 | bio_io_error(bio); | 500 | bio_io_error(bio); |
| 482 | return BLK_QC_T_NONE; | 501 | return BLK_QC_T_NONE; |
| 483 | } | 502 | } |
| 503 | |||
| 504 | blk_queue_split(&bio); | ||
| 505 | |||
| 484 | if (mddev->ro == 1 && unlikely(rw == WRITE)) { | 506 | if (mddev->ro == 1 && unlikely(rw == WRITE)) { |
| 485 | if (bio_sectors(bio) != 0) | 507 | if (bio_sectors(bio) != 0) |
| 486 | bio->bi_status = BLK_STS_IOERR; | 508 | bio->bi_status = BLK_STS_IOERR; |
| @@ -488,21 +510,27 @@ static blk_qc_t md_submit_bio(struct bio *bio) · 1 unmodified line | |||
| 488 | return BLK_QC_T_NONE; | 510 | return BLK_QC_T_NONE; |
| 489 | } | 511 | } |
| 490 | 512 | ||
| 491 | /* | 513 | if (bio->bi_end_io != md_end_io) { |
| 492 | * save the sectors now since our bio can | 514 | struct md_io *md_io; |
| 493 | * go away inside make_request | 515 | |
| 494 | */ | 516 | md_io = mempool_alloc(&mddev->md_io_pool, GFP_NOIO); |
| 495 | sectors = bio_sectors(bio); | 517 | md_io->mddev = mddev; |
| 518 | md_io->orig_bi_end_io = bio->bi_end_io; | ||
| 519 | md_io->orig_bi_private = bio->bi_private; | ||
| 520 | |||
| 521 | bio->bi_end_io = md_end_io; | ||
| 522 | bio->bi_private = md_io; | ||
| 523 | |||
| 524 | md_io->start_time = disk_start_io_acct(mddev->gendisk, | ||
| 525 | bio_sectors(bio), | ||
| 526 | bio_op(bio)); | ||
| 527 | } | ||
| 528 | |||
| 496 | /* bio could be mergeable after passing to underlayer */ | 529 | /* bio could be mergeable after passing to underlayer */ |
| 497 | bio->bi_opf &= ~REQ_NOMERGE; | 530 | bio->bi_opf &= ~REQ_NOMERGE; |
| 498 | 531 | ||
| 499 | md_handle_request(mddev, bio); | 532 | md_handle_request(mddev, bio); |
| 500 | 533 | ||
| 501 | part_stat_lock(); | ||
| 502 | part_stat_inc(&mddev->gendisk->part0, ios[sgrp]); | ||
| 503 | part_stat_add(&mddev->gendisk->part0, sectors[sgrp], sectors); | ||
| 504 | part_stat_unlock(); | ||
| 505 | |||
| 506 | return BLK_QC_T_NONE; | 534 | return BLK_QC_T_NONE; |
| 507 | } | 535 | } |
| 508 | 536 | ||
| @@ -2424,6 +2452,10 @@ static int bind_rdev_to_array(struct md_rdev *rdev, struct mddev *mddev) · 1915 unmodified lines | |||
| 2424 | if (sysfs_create_link(&rdev->kobj, ko, "block")) | 2452 | if (sysfs_create_link(&rdev->kobj, ko, "block")) |
| 2425 | /* failure here is OK */; | 2453 | /* failure here is OK */; |
| 2426 | rdev->sysfs_state = sysfs_get_dirent_safe(rdev->kobj.sd, "state"); | 2454 | rdev->sysfs_state = sysfs_get_dirent_safe(rdev->kobj.sd, "state"); |
| 2455 | rdev->sysfs_unack_badblocks = | ||
| 2456 | sysfs_get_dirent_safe(rdev->kobj.sd, "unacknowledged_bad_blocks"); | ||
| 2457 | rdev->sysfs_badblocks = | ||
| 2458 | sysfs_get_dirent_safe(rdev->kobj.sd, "bad_blocks"); | ||
| 2427 | 2459 | ||
| 2428 | list_add_rcu(&rdev->same_set, &mddev->disks); | 2460 | list_add_rcu(&rdev->same_set, &mddev->disks); |
| 2429 | bd_link_disk_holder(rdev->bdev, mddev->gendisk); | 2461 | bd_link_disk_holder(rdev->bdev, mddev->gendisk); |
| @@ -2457,7 +2489,11 @@ static void unbind_rdev_from_array(struct md_rdev *rdev) · 27 unmodified lines | |||
| 2457 | rdev->mddev = NULL; | 2489 | rdev->mddev = NULL; |
| 2458 | sysfs_remove_link(&rdev->kobj, "block"); | 2490 | sysfs_remove_link(&rdev->kobj, "block"); |
| 2459 | sysfs_put(rdev->sysfs_state); | 2491 | sysfs_put(rdev->sysfs_state); |
| 2492 | sysfs_put(rdev->sysfs_unack_badblocks); | ||
| 2493 | sysfs_put(rdev->sysfs_badblocks); | ||
| 2460 | rdev->sysfs_state = NULL; | 2494 | rdev->sysfs_state = NULL; |
| 2495 | rdev->sysfs_unack_badblocks = NULL; | ||
| 2496 | rdev->sysfs_badblocks = NULL; | ||
| 2461 | rdev->badblocks.count = 0; | 2497 | rdev->badblocks.count = 0; |
| 2462 | /* We need to delay this, otherwise we can deadlock when | 2498 | /* We need to delay this, otherwise we can deadlock when |
| 2463 | * writing to 'remove' to "dev/state". We also need | 2499 | * writing to 'remove' to "dev/state". We also need |
| @@ -2802,7 +2838,7 @@ rewrite: · 338 unmodified lines | |||
| 2802 | goto repeat; | 2838 | goto repeat; |
| 2803 | wake_up(&mddev->sb_wait); | 2839 | wake_up(&mddev->sb_wait); |
| 2804 | if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) | 2840 | if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) |
| 2805 | sysfs_notify(&mddev->kobj, NULL, "sync_completed"); | 2841 | sysfs_notify_dirent_safe(mddev->sysfs_completed); |
| 2806 | 2842 | ||
| 2807 | rdev_for_each(rdev, mddev) { | 2843 | rdev_for_each(rdev, mddev) { |
| 2808 | if (test_and_clear_bit(FaultRecorded, &rdev->flags)) | 2844 | if (test_and_clear_bit(FaultRecorded, &rdev->flags)) |
| @@ -4055,7 +4091,7 @@ level_store(struct mddev *mddev, const char *buf, size_t len) · 1246 unmodified lines | |||
| 4055 | mddev_resume(mddev); | 4091 | mddev_resume(mddev); |
| 4056 | if (!mddev->thread) | 4092 | if (!mddev->thread) |
| 4057 | md_update_sb(mddev, 1); | 4093 | md_update_sb(mddev, 1); |
| 4058 | sysfs_notify(&mddev->kobj, NULL, "level"); | 4094 | sysfs_notify_dirent_safe(mddev->sysfs_level); |
| 4059 | md_new_event(mddev); | 4095 | md_new_event(mddev); |
| 4060 | rv = len; | 4096 | rv = len; |
| 4061 | out_unlock: | 4097 | out_unlock: |
| @@ -4808,7 +4844,7 @@ action_store(struct mddev *mddev, const char *page, size_t len) · 746 unmodified lines | |||
| 4808 | } | 4844 | } |
| 4809 | if (err) | 4845 | if (err) |
| 4810 | return err; | 4846 | return err; |
| 4811 | sysfs_notify(&mddev->kobj, NULL, "degraded"); | 4847 | sysfs_notify_dirent_safe(mddev->sysfs_degraded); |
| 4812 | } else { | 4848 | } else { |
| 4813 | if (cmd_match(page, "check")) | 4849 | if (cmd_match(page, "check")) |
| 4814 | set_bit(MD_RECOVERY_CHECK, &mddev->recovery); | 4850 | set_bit(MD_RECOVERY_CHECK, &mddev->recovery); |
| @@ -5514,6 +5550,13 @@ static void md_free(struct kobject *ko) · 699 unmodified lines | |||
| 5514 | 5550 | ||
| 5515 | if (mddev->sysfs_state) | 5551 | if (mddev->sysfs_state) |
| 5516 | sysfs_put(mddev->sysfs_state); | 5552 | sysfs_put(mddev->sysfs_state); |
| 5553 | if (mddev->sysfs_completed) | ||
| 5554 | sysfs_put(mddev->sysfs_completed); | ||
| 5555 | if (mddev->sysfs_degraded) | ||
| 5556 | sysfs_put(mddev->sysfs_degraded); | ||
| 5557 | if (mddev->sysfs_level) | ||
| 5558 | sysfs_put(mddev->sysfs_level); | ||
| 5559 | |||
| 5517 | 5560 | ||
| 5518 | if (mddev->gendisk) | 5561 | if (mddev->gendisk) |
| 5519 | del_gendisk(mddev->gendisk); | 5562 | del_gendisk(mddev->gendisk); |
| @@ -5525,6 +5568,7 @@ static void md_free(struct kobject *ko) · 5 unmodified lines | |||
| 5525 | 5568 | ||
| 5526 | bioset_exit(&mddev->bio_set); | 5569 | bioset_exit(&mddev->bio_set); |
| 5527 | bioset_exit(&mddev->sync_set); | 5570 | bioset_exit(&mddev->sync_set); |
| 5571 | mempool_exit(&mddev->md_io_pool); | ||
| 5528 | kfree(mddev); | 5572 | kfree(mddev); |
| 5529 | } | 5573 | } |
| 5530 | 5574 | ||
| @@ -5620,6 +5664,11 @@ static int md_alloc(dev_t dev, char *name) · 89 unmodified lines | |||
| 5620 | */ | 5664 | */ |
| 5621 | mddev->hold_active = UNTIL_STOP; | 5665 | mddev->hold_active = UNTIL_STOP; |
| 5622 | 5666 | ||
| 5667 | error = mempool_init_kmalloc_pool(&mddev->md_io_pool, BIO_POOL_SIZE, | ||
| 5668 | sizeof(struct md_io)); | ||
| 5669 | if (error) | ||
| 5670 | goto abort; | ||
| 5671 | |||
| 5623 | error = -ENOMEM; | 5672 | error = -ENOMEM; |
| 5624 | mddev->queue = blk_alloc_queue(NUMA_NO_NODE); | 5673 | mddev->queue = blk_alloc_queue(NUMA_NO_NODE); |
| 5625 | if (!mddev->queue) | 5674 | if (!mddev->queue) |
| @@ -5676,6 +5725,9 @@ static int md_alloc(dev_t dev, char *name) · 50 unmodified lines | |||
| 5676 | if (!error && mddev->kobj.sd) { | 5725 | if (!error && mddev->kobj.sd) { |
| 5677 | kobject_uevent(&mddev->kobj, KOBJ_ADD); | 5726 | kobject_uevent(&mddev->kobj, KOBJ_ADD); |
| 5678 | mddev->sysfs_state = sysfs_get_dirent_safe(mddev->kobj.sd, "array_state"); | 5727 | mddev->sysfs_state = sysfs_get_dirent_safe(mddev->kobj.sd, "array_state"); |
| 5728 | mddev->sysfs_completed = sysfs_get_dirent_safe(mddev->kobj.sd, "sync_completed"); | ||
| 5729 | mddev->sysfs_degraded = sysfs_get_dirent_safe(mddev->kobj.sd, "degraded"); | ||
| 5730 | mddev->sysfs_level = sysfs_get_dirent_safe(mddev->kobj.sd, "level"); | ||
| 5679 | } | 5731 | } |
| 5680 | mddev_put(mddev); | 5732 | mddev_put(mddev); |
| 5681 | return error; | 5733 | return error; |
| @@ -6028,7 +6080,7 @@ static int do_md_run(struct mddev *mddev) · 346 unmodified lines | |||
| 6028 | kobject_uevent(&disk_to_dev(mddev->gendisk)->kobj, KOBJ_CHANGE); | 6080 | kobject_uevent(&disk_to_dev(mddev->gendisk)->kobj, KOBJ_CHANGE); |
| 6029 | sysfs_notify_dirent_safe(mddev->sysfs_state); | 6081 | sysfs_notify_dirent_safe(mddev->sysfs_state); |
| 6030 | sysfs_notify_dirent_safe(mddev->sysfs_action); | 6082 | sysfs_notify_dirent_safe(mddev->sysfs_action); |
| 6031 | sysfs_notify(&mddev->kobj, NULL, "degraded"); | 6083 | sysfs_notify_dirent_safe(mddev->sysfs_degraded); |
| 6032 | out: | 6084 | out: |
| 6033 | clear_bit(MD_NOT_READY, &mddev->flags); | 6085 | clear_bit(MD_NOT_READY, &mddev->flags); |
| 6034 | return err; | 6086 | return err; |
| @@ -8742,7 +8794,7 @@ void md_do_sync(struct md_thread *thread) · 2707 unmodified lines | |||
| 8742 | } else | 8794 | } else |
| 8743 | mddev->curr_resync = 3; /* no longer delayed */ | 8795 | mddev->curr_resync = 3; /* no longer delayed */ |
| 8744 | mddev->curr_resync_completed = j; | 8796 | mddev->curr_resync_completed = j; |
| 8745 | sysfs_notify(&mddev->kobj, NULL, "sync_completed"); | 8797 | sysfs_notify_dirent_safe(mddev->sysfs_completed); |
| 8746 | md_new_event(mddev); | 8798 | md_new_event(mddev); |
| 8747 | update_time = jiffies; | 8799 | update_time = jiffies; |
| 8748 | 8800 | ||
| @@ -8770,7 +8822,7 @@ void md_do_sync(struct md_thread *thread) · 21 unmodified lines | |||
| 8770 | mddev->recovery_cp = j; | 8822 | mddev->recovery_cp = j; |
| 8771 | update_time = jiffies; | 8823 | update_time = jiffies; |
| 8772 | set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags); | 8824 | set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags); |
| 8773 | sysfs_notify(&mddev->kobj, NULL, "sync_completed"); | 8825 | sysfs_notify_dirent_safe(mddev->sysfs_completed); |
| 8774 | } | 8826 | } |
| 8775 | 8827 | ||
| 8776 | while (j >= mddev->resync_max && | 8828 | while (j >= mddev->resync_max && |
| @@ -8877,7 +8929,7 @@ void md_do_sync(struct md_thread *thread) · 100 unmodified lines | |||
| 8877 | !test_bit(MD_RECOVERY_INTR, &mddev->recovery) && | 8929 | !test_bit(MD_RECOVERY_INTR, &mddev->recovery) && |
| 8878 | mddev->curr_resync > 3) { | 8930 | mddev->curr_resync > 3) { |
| 8879 | mddev->curr_resync_completed = mddev->curr_resync; | 8931 | mddev->curr_resync_completed = mddev->curr_resync; |
| 8880 | sysfs_notify(&mddev->kobj, NULL, "sync_completed"); | 8932 | sysfs_notify_dirent_safe(mddev->sysfs_completed); |
| 8881 | } | 8933 | } |
| 8882 | mddev->pers->sync_request(mddev, max_sectors, &skipped); | 8934 | mddev->pers->sync_request(mddev, max_sectors, &skipped); |
| 8883 | 8935 | ||
| @@ -9007,7 +9059,7 @@ static int remove_and_add_spares(struct mddev *mddev, · 123 unmodified lines | |||
| 9007 | } | 9059 | } |
| 9008 | 9060 | ||
| 9009 | if (removed && mddev->kobj.sd) | 9061 | if (removed && mddev->kobj.sd) |
| 9010 | sysfs_notify(&mddev->kobj, NULL, "degraded"); | 9062 | sysfs_notify_dirent_safe(mddev->sysfs_degraded); |
| 9011 | 9063 | ||
| 9012 | if (this && removed) | 9064 | if (this && removed) |
| 9013 | goto no_add; | 9065 | goto no_add; |
| @@ -9290,8 +9342,7 @@ void md_reap_sync_thread(struct mddev *mddev) · 276 unmodified lines | |||
| 9290 | /* success...*/ | 9342 | /* success...*/ |
| 9291 | /* activate any spares */ | 9343 | /* activate any spares */ |
| 9292 | if (mddev->pers->spare_active(mddev)) { | 9344 | if (mddev->pers->spare_active(mddev)) { |
| 9293 | sysfs_notify(&mddev->kobj, NULL, | 9345 | sysfs_notify_dirent_safe(mddev->sysfs_degraded); |
| 9294 | "degraded"); | ||
| 9295 | set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); | 9346 | set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); |
| 9296 | } | 9347 | } |
| 9297 | } | 9348 | } |
| @@ -9381,8 +9432,7 @@ int rdev_set_badblocks(struct md_rdev *rdev, sector_t s, int sectors, · 83 unmodified lines | |||
| 9381 | if (rv == 0) { | 9432 | if (rv == 0) { |
| 9382 | /* Make sure they get written out promptly */ | 9433 | /* Make sure they get written out promptly */ |
| 9383 | if (test_bit(ExternalBbl, &rdev->flags)) | 9434 | if (test_bit(ExternalBbl, &rdev->flags)) |
| 9384 | sysfs_notify(&rdev->kobj, NULL, | 9435 | sysfs_notify_dirent_safe(rdev->sysfs_unack_badblocks); |
| 9385 | "unacknowledged_bad_blocks"); | ||
| 9386 | sysfs_notify_dirent_safe(rdev->sysfs_state); | 9436 | sysfs_notify_dirent_safe(rdev->sysfs_state); |
| 9387 | set_mask_bits(&mddev->sb_flags, 0, | 9437 | set_mask_bits(&mddev->sb_flags, 0, |
| 9388 | BIT(MD_SB_CHANGE_CLEAN) | BIT(MD_SB_CHANGE_PENDING)); | 9438 | BIT(MD_SB_CHANGE_CLEAN) | BIT(MD_SB_CHANGE_PENDING)); |
| @@ -9403,7 +9453,7 @@ int rdev_clear_badblocks(struct md_rdev *rdev, sector_t s, int sectors, · 14 unmodified lines | |||
| 9403 | s += rdev->data_offset; | 9453 | s += rdev->data_offset; |
| 9404 | rv = badblocks_clear(&rdev->badblocks, s, sectors); | 9454 | rv = badblocks_clear(&rdev->badblocks, s, sectors); |
| 9405 | if ((rv == 0) && test_bit(ExternalBbl, &rdev->flags)) | 9455 | if ((rv == 0) && test_bit(ExternalBbl, &rdev->flags)) |
| 9406 | sysfs_notify(&rdev->kobj, NULL, "bad_blocks"); | 9456 | sysfs_notify_dirent_safe(rdev->sysfs_badblocks); |
| 9407 | return rv; | 9457 | return rv; |
| 9408 | } | 9458 | } |
| 9409 | EXPORT_SYMBOL_GPL(rdev_clear_badblocks); | 9459 | EXPORT_SYMBOL_GPL(rdev_clear_badblocks); |
| @@ -9633,7 +9683,7 @@ static int read_rdev(struct mddev *mddev, struct md_rdev *rdev) · 223 unmodified lines | |||
| 9633 | if (rdev->recovery_offset == MaxSector && | 9683 | if (rdev->recovery_offset == MaxSector && |
| 9634 | !test_bit(In_sync, &rdev->flags) && | 9684 | !test_bit(In_sync, &rdev->flags) && |
| 9635 | mddev->pers->spare_active(mddev)) | 9685 | mddev->pers->spare_active(mddev)) |
| 9636 | sysfs_notify(&mddev->kobj, NULL, "degraded"); | 9686 | sysfs_notify_dirent_safe(mddev->sysfs_degraded); |
| 9637 | 9687 | ||
| 9638 | put_page(swapout); | 9688 | put_page(swapout); |
| 9639 | return 0; | 9689 | return 0; |
| @@ -126,7 +126,10 @@ struct md_rdev { · 125 unmodified lines | |||
| 126 | 126 | ||
| 127 | struct kernfs_node *sysfs_state; /* handle for 'state' | 127 | struct kernfs_node *sysfs_state; /* handle for 'state' |
| 128 | * sysfs entry */ | 128 | * sysfs entry */ |
| 129 | 129 | /* handle for 'unacknowledged_bad_blocks' sysfs dentry */ | |
| 130 | struct kernfs_node *sysfs_unack_badblocks; | ||
| 131 | /* handle for 'bad_blocks' sysfs dentry */ | ||
| 132 | struct kernfs_node *sysfs_badblocks; | ||
| 130 | struct badblocks badblocks; | 133 | struct badblocks badblocks; |
| 131 | 134 | ||
| 132 | struct { | 135 | struct { |
| @@ -420,6 +423,9 @@ struct mddev { · 287 unmodified lines | |||
| 420 | * file in sysfs. | 423 | * file in sysfs. |
| 421 | */ | 424 | */ |
| 422 | struct kernfs_node *sysfs_action; /* handle for 'sync_action' */ | 425 | struct kernfs_node *sysfs_action; /* handle for 'sync_action' */ |
| 426 | struct kernfs_node *sysfs_completed; /*handle for 'sync_completed' */ | ||
| 427 | struct kernfs_node *sysfs_degraded; /*handle for 'degraded' */ | ||
| 428 | struct kernfs_node *sysfs_level; /*handle for 'level' */ | ||
| 423 | 429 | ||
| 424 | struct work_struct del_work; /* used for delayed sysfs removal */ | 430 | struct work_struct del_work; /* used for delayed sysfs removal */ |
| 425 | 431 | ||
| @@ -481,6 +487,7 @@ struct mddev { · 55 unmodified lines | |||
| 481 | struct bio_set sync_set; /* for sync operations like | 487 | struct bio_set sync_set; /* for sync operations like |
| 482 | * metadata and bitmap writes | 488 | * metadata and bitmap writes |
| 483 | */ | 489 | */ |
| 490 | mempool_t md_io_pool; | ||
| 484 | 491 | ||
| 485 | /* Generic flush handling. | 492 | /* Generic flush handling. |
| 486 | * The last to finish preflush schedules a worker to submit | 493 | * The last to finish preflush schedules a worker to submit |
| @@ -4429,7 +4429,7 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, · 4428 unmodified lines | |||
| 4429 | sector_nr = conf->reshape_progress; | 4429 | sector_nr = conf->reshape_progress; |
| 4430 | if (sector_nr) { | 4430 | if (sector_nr) { |
| 4431 | mddev->curr_resync_completed = sector_nr; | 4431 | mddev->curr_resync_completed = sector_nr; |
| 4432 | sysfs_notify(&mddev->kobj, NULL, "sync_completed"); | 4432 | sysfs_notify_dirent_safe(mddev->sysfs_completed); |
| 4433 | *skipped = 1; | 4433 | *skipped = 1; |
| 4434 | return sector_nr; | 4434 | return sector_nr; |
| 4435 | } | 4435 | } |
| @@ -2430,10 +2430,15 @@ static void r5c_recovery_flush_data_only_stripes(struct r5l_log *log, · 2429 unmodified lines | |||
| 2430 | struct mddev *mddev = log->rdev->mddev; | 2430 | struct mddev *mddev = log->rdev->mddev; |
| 2431 | struct r5conf *conf = mddev->private; | 2431 | struct r5conf *conf = mddev->private; |
| 2432 | struct stripe_head *sh, *next; | 2432 | struct stripe_head *sh, *next; |
| 2433 | bool cleared_pending = false; | ||
| 2433 | 2434 | ||
| 2434 | if (ctx->data_only_stripes == 0) | 2435 | if (ctx->data_only_stripes == 0) |
| 2435 | return; | 2436 | return; |
| 2436 | 2437 | ||
| 2438 | if (test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) { | ||
| 2439 | cleared_pending = true; | ||
| 2440 | clear_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags); | ||
| 2441 | } | ||
| 2437 | log->r5c_journal_mode = R5C_JOURNAL_MODE_WRITE_BACK; | 2442 | log->r5c_journal_mode = R5C_JOURNAL_MODE_WRITE_BACK; |
| 2438 | 2443 | ||
| 2439 | list_for_each_entry_safe(sh, next, &ctx->cached_list, lru) { | 2444 | list_for_each_entry_safe(sh, next, &ctx->cached_list, lru) { |
| @@ -2448,6 +2453,8 @@ static void r5c_recovery_flush_data_only_stripes(struct r5l_log *log, · 8 unmodified lines | |||
| 2448 | atomic_read(&conf->active_stripes) == 0); | 2453 | atomic_read(&conf->active_stripes) == 0); |
| 2449 | 2454 | ||
| 2450 | log->r5c_journal_mode = R5C_JOURNAL_MODE_WRITE_THROUGH; | 2455 | log->r5c_journal_mode = R5C_JOURNAL_MODE_WRITE_THROUGH; |
| 2456 | if (cleared_pending) | ||
| 2457 | set_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags); | ||
| 2451 | } | 2458 | } |
| 2452 | 2459 | ||
| 2453 | static int r5l_recovery_log(struct r5l_log *log) | 2460 | static int r5l_recovery_log(struct r5l_log *log) |
| @@ -5777,7 +5777,7 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *sk · 5776 unmodified lines | |||
| 5777 | sector_div(sector_nr, new_data_disks); | 5777 | sector_div(sector_nr, new_data_disks); |
| 5778 | if (sector_nr) { | 5778 | if (sector_nr) { |
| 5779 | mddev->curr_resync_completed = sector_nr; | 5779 | mddev->curr_resync_completed = sector_nr; |
| 5780 | sysfs_notify(&mddev->kobj, NULL, "sync_completed"); | 5780 | sysfs_notify_dirent_safe(mddev->sysfs_completed); |
| 5781 | *skipped = 1; | 5781 | *skipped = 1; |
| 5782 | retn = sector_nr; | 5782 | retn = sector_nr; |
| 5783 | goto finish; | 5783 | goto finish; |
| @@ -5891,7 +5891,7 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *sk · 107 unmodified lines | |||
| 5891 | conf->reshape_safe = mddev->reshape_position; | 5891 | conf->reshape_safe = mddev->reshape_position; |
| 5892 | spin_unlock_irq(&conf->device_lock); | 5892 | spin_unlock_irq(&conf->device_lock); |
| 5893 | wake_up(&conf->wait_for_overlap); | 5893 | wake_up(&conf->wait_for_overlap); |
| 5894 | sysfs_notify(&mddev->kobj, NULL, "sync_completed"); | 5894 | sysfs_notify_dirent_safe(mddev->sysfs_completed); |
| 5895 | } | 5895 | } |
| 5896 | 5896 | ||
| 5897 | INIT_LIST_HEAD(&stripes); | 5897 | INIT_LIST_HEAD(&stripes); |
| @@ -5998,7 +5998,7 @@ finish: · 100 unmodified lines | |||
| 5998 | conf->reshape_safe = mddev->reshape_position; | 5998 | conf->reshape_safe = mddev->reshape_position; |
| 5999 | spin_unlock_irq(&conf->device_lock); | 5999 | spin_unlock_irq(&conf->device_lock); |
| 6000 | wake_up(&conf->wait_for_overlap); | 6000 | wake_up(&conf->wait_for_overlap); |
| 6001 | sysfs_notify(&mddev->kobj, NULL, "sync_completed"); | 6001 | sysfs_notify_dirent_safe(mddev->sysfs_completed); |
| 6002 | } | 6002 | } |
| 6003 | ret: | 6003 | ret: |
| 6004 | return retn; | 6004 | return retn; |
| @@ -13,6 +13,7 @@ nvme-core-y := core.o · 12 unmodified lines | |||
| 13 | nvme-core-$(CONFIG_TRACING) += trace.o | 13 | nvme-core-$(CONFIG_TRACING) += trace.o |
| 14 | nvme-core-$(CONFIG_NVME_MULTIPATH) += multipath.o | 14 | nvme-core-$(CONFIG_NVME_MULTIPATH) += multipath.o |
| 15 | nvme-core-$(CONFIG_NVM) += lightnvm.o | 15 | nvme-core-$(CONFIG_NVM) += lightnvm.o |
| 16 | nvme-core-$(CONFIG_BLK_DEV_ZONED) += zns.o | ||
| 16 | nvme-core-$(CONFIG_FAULT_INJECTION_DEBUG_FS) += fault_inject.o | 17 | nvme-core-$(CONFIG_FAULT_INJECTION_DEBUG_FS) += fault_inject.o |
| 17 | nvme-core-$(CONFIG_NVME_HWMON) += hwmon.o | 18 | nvme-core-$(CONFIG_NVME_HWMON) += hwmon.o |
| 18 | 19 | ||
| @@ -89,7 +89,7 @@ static dev_t nvme_chr_devt; · 88 unmodified lines | |||
| 89 | static struct class *nvme_class; | 89 | static struct class *nvme_class; |
| 90 | static struct class *nvme_subsys_class; | 90 | static struct class *nvme_subsys_class; |
| 91 | 91 | ||
| 92 | static int nvme_revalidate_disk(struct gendisk *disk); | 92 | static int _nvme_revalidate_disk(struct gendisk *disk); |
| 93 | static void nvme_put_subsystem(struct nvme_subsystem *subsys); | 93 | static void nvme_put_subsystem(struct nvme_subsystem *subsys); |
| 94 | static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl, | 94 | static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl, |
| 95 | unsigned nsid); | 95 | unsigned nsid); |
| @@ -100,7 +100,7 @@ static void nvme_set_queue_dying(struct nvme_ns *ns) · 4 unmodified lines | |||
| 100 | * Revalidating a dead namespace sets capacity to 0. This will end | 100 | * Revalidating a dead namespace sets capacity to 0. This will end |
| 101 | * buffered writers dirtying pages that can't be synced. | 101 | * buffered writers dirtying pages that can't be synced. |
| 102 | */ | 102 | */ |
| 103 | if (!ns->disk || test_and_set_bit(NVME_NS_DEAD, &ns->flags)) | 103 | if (test_and_set_bit(NVME_NS_DEAD, &ns->flags)) |
| 104 | return; | 104 | return; |
| 105 | blk_set_queue_dying(ns->queue); | 105 | blk_set_queue_dying(ns->queue); |
| 106 | /* Forcibly unquiesce queues to avoid blocking dispatch */ | 106 | /* Forcibly unquiesce queues to avoid blocking dispatch */ |
| @@ -287,6 +287,10 @@ void nvme_complete_rq(struct request *req) · 180 unmodified lines | |||
| 287 | nvme_retry_req(req); | 287 | nvme_retry_req(req); |
| 288 | return; | 288 | return; |
| 289 | } | 289 | } |
| 290 | } else if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) && | ||
| 291 | req_op(req) == REQ_OP_ZONE_APPEND) { | ||
| 292 | req->__sector = nvme_lba_to_sect(req->q->queuedata, | ||
| 293 | le64_to_cpu(nvme_req(req)->result.u64)); | ||
| 290 | } | 294 | } |
| 291 | 295 | ||
| 292 | nvme_trace_bio_complete(req, status); | 296 | nvme_trace_bio_complete(req, status); |
| @@ -555,7 +559,7 @@ static int nvme_configure_directives(struct nvme_ctrl *ctrl) · 262 unmodified lines | |||
| 555 | goto out_disable_stream; | 559 | goto out_disable_stream; |
| 556 | } | 560 | } |
| 557 | 561 | ||
| 558 | ctrl->nr_streams = min_t(unsigned, ctrl->nssa, BLK_MAX_WRITE_HINTS - 1); | 562 | ctrl->nr_streams = min_t(u16, ctrl->nssa, BLK_MAX_WRITE_HINTS - 1); |
| 559 | dev_info(ctrl->device, "Using %u streams\n", ctrl->nr_streams); | 563 | dev_info(ctrl->device, "Using %u streams\n", ctrl->nr_streams); |
| 560 | return 0; | 564 | return 0; |
| 561 | 565 | ||
| @@ -673,7 +677,8 @@ static inline blk_status_t nvme_setup_write_zeroes(struct nvme_ns *ns, · 111 unmodified lines | |||
| 673 | } | 677 | } |
| 674 | 678 | ||
| 675 | static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns, | 679 | static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns, |
| 676 | struct request *req, struct nvme_command *cmnd) | 680 | struct request *req, struct nvme_command *cmnd, |
| 681 | enum nvme_opcode op) | ||
| 677 | { | 682 | { |
| 678 | struct nvme_ctrl *ctrl = ns->ctrl; | 683 | struct nvme_ctrl *ctrl = ns->ctrl; |
| 679 | u16 control = 0; | 684 | u16 control = 0; |
| @@ -687,7 +692,7 @@ static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns, · 7 unmodified lines | |||
| 687 | if (req->cmd_flags & REQ_RAHEAD) | 692 | if (req->cmd_flags & REQ_RAHEAD) |
| 688 | dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH; | 693 | dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH; |
| 689 | 694 | ||
| 690 | cmnd->rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read); | 695 | cmnd->rw.opcode = op; |
| 691 | cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id); | 696 | cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id); |
| 692 | cmnd->rw.slba = cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req))); | 697 | cmnd->rw.slba = cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req))); |
| 693 | cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1); | 698 | cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1); |
| @@ -716,6 +721,8 @@ static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns, · 22 unmodified lines | |||
| 716 | case NVME_NS_DPS_PI_TYPE2: | 721 | case NVME_NS_DPS_PI_TYPE2: |
| 717 | control |= NVME_RW_PRINFO_PRCHK_GUARD | | 722 | control |= NVME_RW_PRINFO_PRCHK_GUARD | |
| 718 | NVME_RW_PRINFO_PRCHK_REF; | 723 | NVME_RW_PRINFO_PRCHK_REF; |
| 724 | if (op == nvme_cmd_zone_append) | ||
| 725 | control |= NVME_RW_APPEND_PIREMAP; | ||
| 719 | cmnd->rw.reftag = cpu_to_le32(t10_pi_ref_tag(req)); | 726 | cmnd->rw.reftag = cpu_to_le32(t10_pi_ref_tag(req)); |
| 720 | break; | 727 | break; |
| 721 | } | 728 | } |
| @@ -756,6 +763,19 @@ blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req, · 34 unmodified lines | |||
| 756 | case REQ_OP_FLUSH: | 763 | case REQ_OP_FLUSH: |
| 757 | nvme_setup_flush(ns, cmd); | 764 | nvme_setup_flush(ns, cmd); |
| 758 | break; | 765 | break; |
| 766 | case REQ_OP_ZONE_RESET_ALL: | ||
| 767 | case REQ_OP_ZONE_RESET: | ||
| 768 | ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_RESET); | ||
| 769 | break; | ||
| 770 | case REQ_OP_ZONE_OPEN: | ||
| 771 | ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_OPEN); | ||
| 772 | break; | ||
| 773 | case REQ_OP_ZONE_CLOSE: | ||
| 774 | ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_CLOSE); | ||
| 775 | break; | ||
| 776 | case REQ_OP_ZONE_FINISH: | ||
| 777 | ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_FINISH); | ||
| 778 | break; | ||
| 759 | case REQ_OP_WRITE_ZEROES: | 779 | case REQ_OP_WRITE_ZEROES: |
| 760 | ret = nvme_setup_write_zeroes(ns, req, cmd); | 780 | ret = nvme_setup_write_zeroes(ns, req, cmd); |
| 761 | break; | 781 | break; |
| @@ -763,8 +783,13 @@ blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req, · 1 unmodified line | |||
| 763 | ret = nvme_setup_discard(ns, req, cmd); | 783 | ret = nvme_setup_discard(ns, req, cmd); |
| 764 | break; | 784 | break; |
| 765 | case REQ_OP_READ: | 785 | case REQ_OP_READ: |
| 786 | ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_read); | ||
| 787 | break; | ||
| 766 | case REQ_OP_WRITE: | 788 | case REQ_OP_WRITE: |
| 767 | ret = nvme_setup_rw(ns, req, cmd); | 789 | ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_write); |
| 790 | break; | ||
| 791 | case REQ_OP_ZONE_APPEND: | ||
| 792 | ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_zone_append); | ||
| 768 | break; | 793 | break; |
| 769 | default: | 794 | default: |
| 770 | WARN_ON_ONCE(1); | 795 | WARN_ON_ONCE(1); |
| @@ -1056,8 +1081,13 @@ static int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id) · 285 unmodified lines | |||
| 1056 | return error; | 1081 | return error; |
| 1057 | } | 1082 | } |
| 1058 | 1083 | ||
| 1084 | static bool nvme_multi_css(struct nvme_ctrl *ctrl) | ||
| 1085 | { | ||
| 1086 | return (ctrl->ctrl_config & NVME_CC_CSS_MASK) == NVME_CC_CSS_CSI; | ||
| 1087 | } | ||
| 1088 | |||
| 1059 | static int nvme_process_ns_desc(struct nvme_ctrl *ctrl, struct nvme_ns_ids *ids, | 1089 | static int nvme_process_ns_desc(struct nvme_ctrl *ctrl, struct nvme_ns_ids *ids, |
| 1060 | struct nvme_ns_id_desc *cur) | 1090 | struct nvme_ns_id_desc *cur, bool *csi_seen) |
| 1061 | { | 1091 | { |
| 1062 | const char *warn_str = "ctrl returned bogus length:"; | 1092 | const char *warn_str = "ctrl returned bogus length:"; |
| 1063 | void *data = cur; | 1093 | void *data = cur; |
| @@ -1087,6 +1117,15 @@ static int nvme_process_ns_desc(struct nvme_ctrl *ctrl, struct nvme_ns_ids *ids, · 23 unmodified lines | |||
| 1087 | } | 1117 | } |
| 1088 | uuid_copy(&ids->uuid, data + sizeof(*cur)); | 1118 | uuid_copy(&ids->uuid, data + sizeof(*cur)); |
| 1089 | return NVME_NIDT_UUID_LEN; | 1119 | return NVME_NIDT_UUID_LEN; |
| 1120 | case NVME_NIDT_CSI: | ||
| 1121 | if (cur->nidl != NVME_NIDT_CSI_LEN) { | ||
| 1122 | dev_warn(ctrl->device, "%s %d for NVME_NIDT_CSI\n", | ||
| 1123 | warn_str, cur->nidl); | ||
| 1124 | return -1; | ||
| 1125 | } | ||
| 1126 | memcpy(&ids->csi, data + sizeof(*cur), NVME_NIDT_CSI_LEN); | ||
| 1127 | *csi_seen = true; | ||
| 1128 | return NVME_NIDT_CSI_LEN; | ||
| 1090 | default: | 1129 | default: |
| 1091 | /* Skip unknown types */ | 1130 | /* Skip unknown types */ |
| 1092 | return cur->nidl; | 1131 | return cur->nidl; |
| @@ -1097,10 +1136,9 @@ static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl, unsigned nsid, · 4 unmodified lines | |||
| 1097 | struct nvme_ns_ids *ids) | 1136 | struct nvme_ns_ids *ids) |
| 1098 | { | 1137 | { |
| 1099 | struct nvme_command c = { }; | 1138 | struct nvme_command c = { }; |
| 1100 | int status; | 1139 | bool csi_seen = false; |
| 1140 | int status, pos, len; | ||
| 1101 | void *data; | 1141 | void *data; |
| 1102 | int pos; | ||
| 1103 | int len; | ||
| 1104 | 1142 | ||
| 1105 | c.identify.opcode = nvme_admin_identify; | 1143 | c.identify.opcode = nvme_admin_identify; |
| 1106 | c.identify.nsid = cpu_to_le32(nsid); | 1144 | c.identify.nsid = cpu_to_le32(nsid); |
| @@ -1125,7 +1163,7 @@ static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl, unsigned nsid, · 18 unmodified lines | |||
| 1125 | * device just because of a temporal retry-able error (such | 1163 | * device just because of a temporal retry-able error (such |
| 1126 | * as path of transport errors). | 1164 | * as path of transport errors). |
| 1127 | */ | 1165 | */ |
| 1128 | if (status > 0 && (status & NVME_SC_DNR)) | 1166 | if (status > 0 && (status & NVME_SC_DNR) && !nvme_multi_css(ctrl)) |
| 1129 | status = 0; | 1167 | status = 0; |
| 1130 | goto free_data; | 1168 | goto free_data; |
| 1131 | } | 1169 | } |
| @@ -1136,12 +1174,19 @@ static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl, unsigned nsid, · 4 unmodified lines | |||
| 1136 | if (cur->nidl == 0) | 1174 | if (cur->nidl == 0) |
| 1137 | break; | 1175 | break; |
| 1138 | 1176 | ||
| 1139 | len = nvme_process_ns_desc(ctrl, ids, cur); | 1177 | len = nvme_process_ns_desc(ctrl, ids, cur, &csi_seen); |
| 1140 | if (len < 0) | 1178 | if (len < 0) |
| 1141 | goto free_data; | 1179 | break; |
| 1142 | 1180 | ||
| 1143 | len += sizeof(*cur); | 1181 | len += sizeof(*cur); |
| 1144 | } | 1182 | } |
| 1183 | |||
| 1184 | if (nvme_multi_css(ctrl) && !csi_seen) { | ||
| 1185 | dev_warn(ctrl->device, "Command set not reported for nsid:%d\n", | ||
| 1186 | nsid); | ||
| 1187 | status = -EINVAL; | ||
| 1188 | } | ||
| 1189 | |||
| 1145 | free_data: | 1190 | free_data: |
| 1146 | kfree(data); | 1191 | kfree(data); |
| 1147 | return status; | 1192 | return status; |
| @@ -1350,8 +1395,8 @@ static u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, · 202 unmodified lines | |||
| 1350 | u32 effects = 0; | 1395 | u32 effects = 0; |
| 1351 | 1396 | ||
| 1352 | if (ns) { | 1397 | if (ns) { |
| 1353 | if (ctrl->effects) | 1398 | if (ns->head->effects) |
| 1354 | effects = le32_to_cpu(ctrl->effects->iocs[opcode]); | 1399 | effects = le32_to_cpu(ns->head->effects->iocs[opcode]); |
| 1355 | if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC)) | 1400 | if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC)) |
| 1356 | dev_warn(ctrl->device, | 1401 | dev_warn(ctrl->device, |
| 1357 | "IO command:%02x has unhandled effects:%08x\n", | 1402 | "IO command:%02x has unhandled effects:%08x\n", |
| @@ -1378,14 +1423,23 @@ static u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, · 20 unmodified lines | |||
| 1378 | return effects; | 1423 | return effects; |
| 1379 | } | 1424 | } |
| 1380 | 1425 | ||
| 1381 | static void nvme_update_formats(struct nvme_ctrl *ctrl) | 1426 | static void nvme_update_formats(struct nvme_ctrl *ctrl, u32 *effects) |
| 1382 | { | 1427 | { |
| 1383 | struct nvme_ns *ns; | 1428 | struct nvme_ns *ns; |
| 1384 | 1429 | ||
| 1385 | down_read(&ctrl->namespaces_rwsem); | 1430 | down_read(&ctrl->namespaces_rwsem); |
| 1386 | list_for_each_entry(ns, &ctrl->namespaces, list) | 1431 | list_for_each_entry(ns, &ctrl->namespaces, list) |
| 1387 | if (ns->disk && nvme_revalidate_disk(ns->disk)) | 1432 | if (_nvme_revalidate_disk(ns->disk)) |
| 1388 | nvme_set_queue_dying(ns); | 1433 | nvme_set_queue_dying(ns); |
| 1434 | else if (blk_queue_is_zoned(ns->disk->queue)) { | ||
| 1435 | /* | ||
| 1436 | * IO commands are required to fully revalidate a zoned | ||
| 1437 | * device. Force the command effects to trigger rescan | ||
| 1438 | * work so report zones can run in a context with | ||
| 1439 | * unfrozen IO queues. | ||
| 1440 | */ | ||
| 1441 | *effects |= NVME_CMD_EFFECTS_NCC; | ||
| 1442 | } | ||
| 1389 | up_read(&ctrl->namespaces_rwsem); | 1443 | up_read(&ctrl->namespaces_rwsem); |
| 1390 | } | 1444 | } |
| 1391 | 1445 | ||
| @@ -1397,7 +1451,7 @@ static void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects) · 5 unmodified lines | |||
| 1397 | * this command. | 1451 | * this command. |
| 1398 | */ | 1452 | */ |
| 1399 | if (effects & NVME_CMD_EFFECTS_LBCC) | 1453 | if (effects & NVME_CMD_EFFECTS_LBCC) |
| 1400 | nvme_update_formats(ctrl); | 1454 | nvme_update_formats(ctrl, &effects); |
| 1401 | if (effects & (NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK)) { | 1455 | if (effects & (NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK)) { |
| 1402 | nvme_unfreeze(ctrl); | 1456 | nvme_unfreeze(ctrl); |
| 1403 | nvme_mpath_unfreeze(ctrl->subsys); | 1457 | nvme_mpath_unfreeze(ctrl->subsys); |
| @@ -1512,7 +1566,7 @@ static int nvme_user_cmd64(struct nvme_ctrl *ctrl, struct nvme_ns *ns, · 108 unmodified lines | |||
| 1512 | * Issue ioctl requests on the first available path. Note that unlike normal | 1566 | * Issue ioctl requests on the first available path. Note that unlike normal |
| 1513 | * block layer requests we will not retry failed request on another controller. | 1567 | * block layer requests we will not retry failed request on another controller. |
| 1514 | */ | 1568 | */ |
| 1515 | static struct nvme_ns *nvme_get_ns_from_disk(struct gendisk *disk, | 1569 | struct nvme_ns *nvme_get_ns_from_disk(struct gendisk *disk, |
| 1516 | struct nvme_ns_head **head, int *srcu_idx) | 1570 | struct nvme_ns_head **head, int *srcu_idx) |
| 1517 | { | 1571 | { |
| 1518 | #ifdef CONFIG_NVME_MULTIPATH | 1572 | #ifdef CONFIG_NVME_MULTIPATH |
| @@ -1532,7 +1586,7 @@ static struct nvme_ns *nvme_get_ns_from_disk(struct gendisk *disk, · 13 unmodified lines | |||
| 1532 | return disk->private_data; | 1586 | return disk->private_data; |
| 1533 | } | 1587 | } |
| 1534 | 1588 | ||
| 1535 | static void nvme_put_ns_from_disk(struct nvme_ns_head *head, int idx) | 1589 | void nvme_put_ns_from_disk(struct nvme_ns_head *head, int idx) |
| 1536 | { | 1590 | { |
| 1537 | if (head) | 1591 | if (head) |
| 1538 | srcu_read_unlock(&head->srcu, idx); | 1592 | srcu_read_unlock(&head->srcu, idx); |
| @@ -1798,7 +1852,7 @@ static int nvme_report_ns_ids(struct nvme_ctrl *ctrl, unsigned int nsid, · 259 unmodified lines | |||
| 1798 | memcpy(ids->eui64, id->eui64, sizeof(id->eui64)); | 1852 | memcpy(ids->eui64, id->eui64, sizeof(id->eui64)); |
| 1799 | if (ctrl->vs >= NVME_VS(1, 2, 0)) | 1853 | if (ctrl->vs >= NVME_VS(1, 2, 0)) |
| 1800 | memcpy(ids->nguid, id->nguid, sizeof(id->nguid)); | 1854 | memcpy(ids->nguid, id->nguid, sizeof(id->nguid)); |
| 1801 | if (ctrl->vs >= NVME_VS(1, 3, 0)) | 1855 | if (ctrl->vs >= NVME_VS(1, 3, 0) || nvme_multi_css(ctrl)) |
| 1802 | return nvme_identify_ns_descs(ctrl, nsid, ids); | 1856 | return nvme_identify_ns_descs(ctrl, nsid, ids); |
| 1803 | return 0; | 1857 | return 0; |
| 1804 | } | 1858 | } |
| @@ -1814,7 +1868,8 @@ static bool nvme_ns_ids_equal(struct nvme_ns_ids *a, struct nvme_ns_ids *b) · 9 unmodified lines | |||
| 1814 | { | 1868 | { |
| 1815 | return uuid_equal(&a->uuid, &b->uuid) && | 1869 | return uuid_equal(&a->uuid, &b->uuid) && |
| 1816 | memcmp(&a->nguid, &b->nguid, sizeof(a->nguid)) == 0 && | 1870 | memcmp(&a->nguid, &b->nguid, sizeof(a->nguid)) == 0 && |
| 1817 | memcmp(&a->eui64, &b->eui64, sizeof(a->eui64)) == 0; | 1871 | memcmp(&a->eui64, &b->eui64, sizeof(a->eui64)) == 0 && |
| 1872 | a->csi == b->csi; | ||
| 1818 | } | 1873 | } |
| 1819 | 1874 | ||
| 1820 | static int nvme_setup_streams_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns, | 1875 | static int nvme_setup_streams_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns, |
| @@ -1924,18 +1979,38 @@ static void nvme_update_disk_info(struct gendisk *disk, · 103 unmodified lines | |||
| 1924 | 1979 | ||
| 1925 | static int __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id) | 1980 | static int __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id) |
| 1926 | { | 1981 | { |
| 1982 | unsigned lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK; | ||
| 1927 | struct nvme_ns *ns = disk->private_data; | 1983 | struct nvme_ns *ns = disk->private_data; |
| 1928 | struct nvme_ctrl *ctrl = ns->ctrl; | 1984 | struct nvme_ctrl *ctrl = ns->ctrl; |
| 1985 | int ret; | ||
| 1929 | u32 iob; | 1986 | u32 iob; |
| 1930 | 1987 | ||
| 1931 | /* | 1988 | /* |
| 1932 | * If identify namespace failed, use default 512 byte block size so | 1989 | * If identify namespace failed, use default 512 byte block size so |
| 1933 | * block layer can use before failing read/write for 0 capacity. | 1990 | * block layer can use before failing read/write for 0 capacity. |
| 1934 | */ | 1991 | */ |
| 1935 | ns->lba_shift = id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ds; | 1992 | ns->lba_shift = id->lbaf[lbaf].ds; |
| 1936 | if (ns->lba_shift == 0) | 1993 | if (ns->lba_shift == 0) |
| 1937 | ns->lba_shift = 9; | 1994 | ns->lba_shift = 9; |
| 1938 | 1995 | ||
| 1996 | switch (ns->head->ids.csi) { | ||
| 1997 | case NVME_CSI_NVM: | ||
| 1998 | break; | ||
| 1999 | case NVME_CSI_ZNS: | ||
| 2000 | ret = nvme_update_zone_info(disk, ns, lbaf); | ||
| 2001 | if (ret) { | ||
| 2002 | dev_warn(ctrl->device, | ||
| 2003 | "failed to add zoned namespace:%u ret:%d\n", | ||
| 2004 | ns->head->ns_id, ret); | ||
| 2005 | return ret; | ||
| 2006 | } | ||
| 2007 | break; | ||
| 2008 | default: | ||
| 2009 | dev_warn(ctrl->device, "unknown csi:%u ns:%u\n", | ||
| 2010 | ns->head->ids.csi, ns->head->ns_id); | ||
| 2011 | return -ENODEV; | ||
| 2012 | } | ||
| 2013 | |||
| 1939 | if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) && | 2014 | if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) && |
| 1940 | is_power_of_2(ctrl->max_hw_sectors)) | 2015 | is_power_of_2(ctrl->max_hw_sectors)) |
| 1941 | iob = ctrl->max_hw_sectors; | 2016 | iob = ctrl->max_hw_sectors; |
| @@ -1943,7 +2018,7 @@ static int __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id) · 1 unmodified line | |||
| 1943 | iob = nvme_lba_to_sect(ns, le16_to_cpu(id->noiob)); | 2018 | iob = nvme_lba_to_sect(ns, le16_to_cpu(id->noiob)); |
| 1944 | 2019 | ||
| 1945 | ns->features = 0; | 2020 | ns->features = 0; |
| 1946 | ns->ms = le16_to_cpu(id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ms); | 2021 | ns->ms = le16_to_cpu(id->lbaf[lbaf].ms); |
| 1947 | /* the PI implementation requires metadata equal t10 pi tuple size */ | 2022 | /* the PI implementation requires metadata equal t10 pi tuple size */ |
| 1948 | if (ns->ms == sizeof(struct t10_pi_tuple)) | 2023 | if (ns->ms == sizeof(struct t10_pi_tuple)) |
| 1949 | ns->pi_type = id->dps & NVME_NS_DPS_PI_MASK; | 2024 | ns->pi_type = id->dps & NVME_NS_DPS_PI_MASK; |
| @@ -1986,7 +2061,7 @@ static int __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id) · 36 unmodified lines | |||
| 1986 | return 0; | 2061 | return 0; |
| 1987 | } | 2062 | } |
| 1988 | 2063 | ||
| 1989 | static int nvme_revalidate_disk(struct gendisk *disk) | 2064 | static int _nvme_revalidate_disk(struct gendisk *disk) |
| 1990 | { | 2065 | { |
| 1991 | struct nvme_ns *ns = disk->private_data; | 2066 | struct nvme_ns *ns = disk->private_data; |
| 1992 | struct nvme_ctrl *ctrl = ns->ctrl; | 2067 | struct nvme_ctrl *ctrl = ns->ctrl; |
| @@ -2034,6 +2109,28 @@ out: · 41 unmodified lines | |||
| 2034 | return ret; | 2109 | return ret; |
| 2035 | } | 2110 | } |
| 2036 | 2111 | ||
| 2112 | static int nvme_revalidate_disk(struct gendisk *disk) | ||
| 2113 | { | ||
| 2114 | int ret; | ||
| 2115 | |||
| 2116 | ret = _nvme_revalidate_disk(disk); | ||
| 2117 | if (ret) | ||
| 2118 | return ret; | ||
| 2119 | |||
| 2120 | #ifdef CONFIG_BLK_DEV_ZONED | ||
| 2121 | if (blk_queue_is_zoned(disk->queue)) { | ||
| 2122 | struct nvme_ns *ns = disk->private_data; | ||
| 2123 | struct nvme_ctrl *ctrl = ns->ctrl; | ||
| 2124 | |||
| 2125 | ret = blk_revalidate_disk_zones(disk, NULL); | ||
| 2126 | if (!ret) | ||
| 2127 | blk_queue_max_zone_append_sectors(disk->queue, | ||
| 2128 | ctrl->max_zone_append); | ||
| 2129 | } | ||
| 2130 | #endif | ||
| 2131 | return ret; | ||
| 2132 | } | ||
| 2133 | |||
| 2037 | static char nvme_pr_type(enum pr_type type) | 2134 | static char nvme_pr_type(enum pr_type type) |
| 2038 | { | 2135 | { |
| 2039 | switch (type) { | 2136 | switch (type) { |
| @@ -2164,6 +2261,7 @@ static const struct block_device_operations nvme_fops = { · 124 unmodified lines | |||
| 2164 | .release = nvme_release, | 2261 | .release = nvme_release, |
| 2165 | .getgeo = nvme_getgeo, | 2262 | .getgeo = nvme_getgeo, |
| 2166 | .revalidate_disk= nvme_revalidate_disk, | 2263 | .revalidate_disk= nvme_revalidate_disk, |
| 2264 | .report_zones = nvme_report_zones, | ||
| 2167 | .pr_ops = &nvme_pr_ops, | 2265 | .pr_ops = &nvme_pr_ops, |
| 2168 | }; | 2266 | }; |
| 2169 | 2267 | ||
| @@ -2190,6 +2288,7 @@ const struct block_device_operations nvme_ns_head_ops = { · 20 unmodified lines | |||
| 2190 | .ioctl = nvme_ioctl, | 2288 | .ioctl = nvme_ioctl, |
| 2191 | .compat_ioctl = nvme_compat_ioctl, | 2289 | .compat_ioctl = nvme_compat_ioctl, |
| 2192 | .getgeo = nvme_getgeo, | 2290 | .getgeo = nvme_getgeo, |
| 2291 | .report_zones = nvme_report_zones, | ||
| 2193 | .pr_ops = &nvme_pr_ops, | 2292 | .pr_ops = &nvme_pr_ops, |
| 2194 | }; | 2293 | }; |
| 2195 | #endif /* CONFIG_NVME_MULTIPATH */ | 2294 | #endif /* CONFIG_NVME_MULTIPATH */ |
| @@ -2271,7 +2370,10 @@ int nvme_enable_ctrl(struct nvme_ctrl *ctrl) · 75 unmodified lines | |||
| 2271 | 2370 | ||
| 2272 | ctrl->page_size = 1 << page_shift; | 2371 | ctrl->page_size = 1 << page_shift; |
| 2273 | 2372 | ||
| 2274 | ctrl->ctrl_config = NVME_CC_CSS_NVM; | 2373 | if (NVME_CAP_CSS(ctrl->cap) & NVME_CAP_CSS_CSI) |
| 2374 | ctrl->ctrl_config = NVME_CC_CSS_CSI; | ||
| 2375 | else | ||
| 2376 | ctrl->ctrl_config = NVME_CC_CSS_NVM; | ||
| 2275 | ctrl->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT; | 2377 | ctrl->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT; |
| 2276 | ctrl->ctrl_config |= NVME_CC_AMS_RR | NVME_CC_SHN_NONE; | 2378 | ctrl->ctrl_config |= NVME_CC_AMS_RR | NVME_CC_SHN_NONE; |
| 2277 | ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES; | 2379 | ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES; |
| @@ -2819,7 +2921,7 @@ out_unlock: · 541 unmodified lines | |||
| 2819 | return ret; | 2921 | return ret; |
| 2820 | } | 2922 | } |
| 2821 | 2923 | ||
| 2822 | int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, | 2924 | int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi, |
| 2823 | void *log, size_t size, u64 offset) | 2925 | void *log, size_t size, u64 offset) |
| 2824 | { | 2926 | { |
| 2825 | struct nvme_command c = { }; | 2927 | struct nvme_command c = { }; |
| @@ -2833,27 +2935,55 @@ int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, · 7 unmodified lines | |||
| 2833 | c.get_log_page.numdu = cpu_to_le16(dwlen >> 16); | 2935 | c.get_log_page.numdu = cpu_to_le16(dwlen >> 16); |
| 2834 | c.get_log_page.lpol = cpu_to_le32(lower_32_bits(offset)); | 2936 | c.get_log_page.lpol = cpu_to_le32(lower_32_bits(offset)); |
| 2835 | c.get_log_page.lpou = cpu_to_le32(upper_32_bits(offset)); | 2937 | c.get_log_page.lpou = cpu_to_le32(upper_32_bits(offset)); |
| 2938 | c.get_log_page.csi = csi; | ||
| 2836 | 2939 | ||
| 2837 | return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size); | 2940 | return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size); |
| 2838 | } | 2941 | } |
| 2839 | 2942 | ||
| 2840 | static int nvme_get_effects_log(struct nvme_ctrl *ctrl) | 2943 | static struct nvme_cel *nvme_find_cel(struct nvme_ctrl *ctrl, u8 csi) |
| 2841 | { | 2944 | { |
| 2945 | struct nvme_cel *cel, *ret = NULL; | ||
| 2946 | |||
| 2947 | spin_lock(&ctrl->lock); | ||
| 2948 | list_for_each_entry(cel, &ctrl->cels, entry) { | ||
| 2949 | if (cel->csi == csi) { | ||
| 2950 | ret = cel; | ||
| 2951 | break; | ||
| 2952 | } | ||
| 2953 | } | ||
| 2954 | spin_unlock(&ctrl->lock); | ||
| 2955 | |||
| 2956 | return ret; | ||
| 2957 | } | ||
| 2958 | |||
| 2959 | static int nvme_get_effects_log(struct nvme_ctrl *ctrl, u8 csi, | ||
| 2960 | struct nvme_effects_log **log) | ||
| 2961 | { | ||
| 2962 | struct nvme_cel *cel = nvme_find_cel(ctrl, csi); | ||
| 2842 | int ret; | 2963 | int ret; |
| 2843 | 2964 | ||
| 2844 | if (!ctrl->effects) | 2965 | if (cel) |
| 2845 | ctrl->effects = kzalloc(sizeof(*ctrl->effects), GFP_KERNEL); | 2966 | goto out; |
| 2846 | 2967 | ||
| 2847 | if (!ctrl->effects) | 2968 | cel = kzalloc(sizeof(*cel), GFP_KERNEL); |
| 2848 | return 0; | 2969 | if (!cel) |
| 2970 | return -ENOMEM; | ||
| 2849 | 2971 | ||
| 2850 | ret = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CMD_EFFECTS, 0, | 2972 | ret = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CMD_EFFECTS, 0, csi, |
| 2851 | ctrl->effects, sizeof(*ctrl->effects), 0); | 2973 | &cel->log, sizeof(cel->log), 0); |
| 2852 | if (ret) { | 2974 | if (ret) { |
| 2853 | kfree(ctrl->effects); | 2975 | kfree(cel); |
| 2854 | ctrl->effects = NULL; | 2976 | return ret; |
| 2855 | } | 2977 | } |
| 2856 | return ret; | 2978 | |
| 2979 | cel->csi = csi; | ||
| 2980 | |||
| 2981 | spin_lock(&ctrl->lock); | ||
| 2982 | list_add_tail(&cel->entry, &ctrl->cels); | ||
| 2983 | spin_unlock(&ctrl->lock); | ||
| 2984 | out: | ||
| 2985 | *log = &cel->log; | ||
| 2986 | return 0; | ||
| 2857 | } | 2987 | } |
| 2858 | 2988 | ||
| 2859 | /* | 2989 | /* |
| @@ -2874,7 +3004,7 @@ int nvme_init_identify(struct nvme_ctrl *ctrl) · 14 unmodified lines | |||
| 2874 | return ret; | 3004 | return ret; |
| 2875 | } | 3005 | } |
| 2876 | page_shift = NVME_CAP_MPSMIN(ctrl->cap) + 12; | 3006 | page_shift = NVME_CAP_MPSMIN(ctrl->cap) + 12; |
| 2877 | ctrl->sqsize = min_t(int, NVME_CAP_MQES(ctrl->cap), ctrl->sqsize); | 3007 | ctrl->sqsize = min_t(u16, NVME_CAP_MQES(ctrl->cap), ctrl->sqsize); |
| 2878 | 3008 | ||
| 2879 | if (ctrl->vs >= NVME_VS(1, 1, 0)) | 3009 | if (ctrl->vs >= NVME_VS(1, 1, 0)) |
| 2880 | ctrl->subsystem = NVME_CAP_NSSRC(ctrl->cap); | 3010 | ctrl->subsystem = NVME_CAP_NSSRC(ctrl->cap); |
| @@ -2886,7 +3016,7 @@ int nvme_init_identify(struct nvme_ctrl *ctrl) · 5 unmodified lines | |||
| 2886 | } | 3016 | } |
| 2887 | 3017 | ||
| 2888 | if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) { | 3018 | if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) { |
| 2889 | ret = nvme_get_effects_log(ctrl); | 3019 | ret = nvme_get_effects_log(ctrl, NVME_CSI_NVM, &ctrl->effects); |
| 2890 | if (ret < 0) | 3020 | if (ret < 0) |
| 2891 | goto out_free; | 3021 | goto out_free; |
| 2892 | } | 3022 | } |
| @@ -2948,7 +3078,7 @@ int nvme_init_identify(struct nvme_ctrl *ctrl) · 55 unmodified lines | |||
| 2948 | 3078 | ||
| 2949 | if (id->rtd3e) { | 3079 | if (id->rtd3e) { |
| 2950 | /* us -> s */ | 3080 | /* us -> s */ |
| 2951 | u32 transition_time = le32_to_cpu(id->rtd3e) / 1000000; | 3081 | u32 transition_time = le32_to_cpu(id->rtd3e) / USEC_PER_SEC; |
| 2952 | 3082 | ||
| 2953 | ctrl->shutdown_timeout = clamp_t(unsigned int, transition_time, | 3083 | ctrl->shutdown_timeout = clamp_t(unsigned int, transition_time, |
| 2954 | shutdown_timeout, 60); | 3084 | shutdown_timeout, 60); |
| @@ -3406,6 +3536,66 @@ static ssize_t nvme_sysfs_show_address(struct device *dev, · 451 unmodified lines | |||
| 3406 | } | 3536 | } |
| 3407 | static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL); | 3537 | static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL); |
| 3408 | 3538 | ||
| 3539 | static ssize_t nvme_ctrl_loss_tmo_show(struct device *dev, | ||
| 3540 | struct device_attribute *attr, char *buf) | ||
| 3541 | { | ||
| 3542 | struct nvme_ctrl *ctrl = dev_get_drvdata(dev); | ||
| 3543 | struct nvmf_ctrl_options *opts = ctrl->opts; | ||
| 3544 | |||
| 3545 | if (ctrl->opts->max_reconnects == -1) | ||
| 3546 | return sprintf(buf, "off\n"); | ||
| 3547 | return sprintf(buf, "%d\n", | ||
| 3548 | opts->max_reconnects * opts->reconnect_delay); | ||
| 3549 | } | ||
| 3550 | |||
| 3551 | static ssize_t nvme_ctrl_loss_tmo_store(struct device *dev, | ||
| 3552 | struct device_attribute *attr, const char *buf, size_t count) | ||
| 3553 | { | ||
| 3554 | struct nvme_ctrl *ctrl = dev_get_drvdata(dev); | ||
| 3555 | struct nvmf_ctrl_options *opts = ctrl->opts; | ||
| 3556 | int ctrl_loss_tmo, err; | ||
| 3557 | |||
| 3558 | err = kstrtoint(buf, 10, &ctrl_loss_tmo); | ||
| 3559 | if (err) | ||
| 3560 | return -EINVAL; | ||
| 3561 | |||
| 3562 | else if (ctrl_loss_tmo < 0) | ||
| 3563 | opts->max_reconnects = -1; | ||
| 3564 | else | ||
| 3565 | opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo, | ||
| 3566 | opts->reconnect_delay); | ||
| 3567 | return count; | ||
| 3568 | } | ||
| 3569 | static DEVICE_ATTR(ctrl_loss_tmo, S_IRUGO | S_IWUSR, | ||
| 3570 | nvme_ctrl_loss_tmo_show, nvme_ctrl_loss_tmo_store); | ||
| 3571 | |||
| 3572 | static ssize_t nvme_ctrl_reconnect_delay_show(struct device *dev, | ||
| 3573 | struct device_attribute *attr, char *buf) | ||
| 3574 | { | ||
| 3575 | struct nvme_ctrl *ctrl = dev_get_drvdata(dev); | ||
| 3576 | |||
| 3577 | if (ctrl->opts->reconnect_delay == -1) | ||
| 3578 | return sprintf(buf, "off\n"); | ||
| 3579 | return sprintf(buf, "%d\n", ctrl->opts->reconnect_delay); | ||
| 3580 | } | ||
| 3581 | |||
| 3582 | static ssize_t nvme_ctrl_reconnect_delay_store(struct device *dev, | ||
| 3583 | struct device_attribute *attr, const char *buf, size_t count) | ||
| 3584 | { | ||
| 3585 | struct nvme_ctrl *ctrl = dev_get_drvdata(dev); | ||
| 3586 | unsigned int v; | ||
| 3587 | int err; | ||
| 3588 | |||
| 3589 | err = kstrtou32(buf, 10, &v); | ||
| 3590 | if (err || v > UINT_MAX) | ||
| 3591 | return -EINVAL; | ||
| 3592 | |||
| 3593 | ctrl->opts->reconnect_delay = v; | ||
| 3594 | return count; | ||
| 3595 | } | ||
| 3596 | static DEVICE_ATTR(reconnect_delay, S_IRUGO | S_IWUSR, | ||
| 3597 | nvme_ctrl_reconnect_delay_show, nvme_ctrl_reconnect_delay_store); | ||
| 3598 | |||
| 3409 | static struct attribute *nvme_dev_attrs[] = { | 3599 | static struct attribute *nvme_dev_attrs[] = { |
| 3410 | &dev_attr_reset_controller.attr, | 3600 | &dev_attr_reset_controller.attr, |
| 3411 | &dev_attr_rescan_controller.attr, | 3601 | &dev_attr_rescan_controller.attr, |
| @@ -3423,6 +3613,8 @@ static struct attribute *nvme_dev_attrs[] = { · 11 unmodified lines | |||
| 3423 | &dev_attr_sqsize.attr, | 3613 | &dev_attr_sqsize.attr, |
| 3424 | &dev_attr_hostnqn.attr, | 3614 | &dev_attr_hostnqn.attr, |
| 3425 | &dev_attr_hostid.attr, | 3615 | &dev_attr_hostid.attr, |
| 3616 | &dev_attr_ctrl_loss_tmo.attr, | ||
| 3617 | &dev_attr_reconnect_delay.attr, | ||
| 3426 | NULL | 3618 | NULL |
| 3427 | }; | 3619 | }; |
| 3428 | 3620 | ||
| @@ -3519,6 +3711,13 @@ static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl, · 90 unmodified lines | |||
| 3519 | goto out_cleanup_srcu; | 3711 | goto out_cleanup_srcu; |
| 3520 | } | 3712 | } |
| 3521 | 3713 | ||
| 3714 | if (head->ids.csi) { | ||
| 3715 | ret = nvme_get_effects_log(ctrl, head->ids.csi, &head->effects); | ||
| 3716 | if (ret) | ||
| 3717 | goto out_cleanup_srcu; | ||
| 3718 | } else | ||
| 3719 | head->effects = ctrl->effects; | ||
| 3720 | |||
| 3522 | ret = nvme_mpath_alloc_disk(ctrl, head); | 3721 | ret = nvme_mpath_alloc_disk(ctrl, head); |
| 3523 | if (ret) | 3722 | if (ret) |
| 3524 | goto out_cleanup_srcu; | 3723 | goto out_cleanup_srcu; |
| @@ -3735,7 +3934,7 @@ static void nvme_ns_remove(struct nvme_ns *ns) · 210 unmodified lines | |||
| 3735 | nvme_mpath_clear_current_path(ns); | 3934 | nvme_mpath_clear_current_path(ns); |
| 3736 | synchronize_srcu(&ns->head->srcu); /* wait for concurrent submissions */ | 3935 | synchronize_srcu(&ns->head->srcu); /* wait for concurrent submissions */ |
| 3737 | 3936 | ||
| 3738 | if (ns->disk && ns->disk->flags & GENHD_FL_UP) { | 3937 | if (ns->disk->flags & GENHD_FL_UP) { |
| 3739 | del_gendisk(ns->disk); | 3938 | del_gendisk(ns->disk); |
| 3740 | blk_cleanup_queue(ns->queue); | 3939 | blk_cleanup_queue(ns->queue); |
| 3741 | if (blk_get_integrity(ns->disk)) | 3940 | if (blk_get_integrity(ns->disk)) |
| @@ -3766,7 +3965,7 @@ static void nvme_validate_ns(struct nvme_ctrl *ctrl, unsigned nsid) · 24 unmodified lines | |||
| 3766 | 3965 | ||
| 3767 | ns = nvme_find_get_ns(ctrl, nsid); | 3966 | ns = nvme_find_get_ns(ctrl, nsid); |
| 3768 | if (ns) { | 3967 | if (ns) { |
| 3769 | if (ns->disk && revalidate_disk(ns->disk)) | 3968 | if (revalidate_disk(ns->disk)) |
| 3770 | nvme_ns_remove(ns); | 3969 | nvme_ns_remove(ns); |
| 3771 | nvme_put_ns(ns); | 3970 | nvme_put_ns(ns); |
| 3772 | } else | 3971 | } else |
| @@ -3859,8 +4058,8 @@ static void nvme_clear_changed_ns_log(struct nvme_ctrl *ctrl) · 86 unmodified lines | |||
| 3859 | * raced with us in reading the log page, which could cause us to miss | 4058 | * raced with us in reading the log page, which could cause us to miss |
| 3860 | * updates. | 4059 | * updates. |
| 3861 | */ | 4060 | */ |
| 3862 | error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CHANGED_NS, 0, log, | 4061 | error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CHANGED_NS, 0, |
| 3863 | log_size, 0); | 4062 | NVME_CSI_NVM, log, log_size, 0); |
| 3864 | if (error) | 4063 | if (error) |
| 3865 | dev_warn(ctrl->device, | 4064 | dev_warn(ctrl->device, |
| 3866 | "reading changed ns log failed: %d\n", error); | 4065 | "reading changed ns log failed: %d\n", error); |
| @@ -4004,8 +4203,8 @@ static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl) · 137 unmodified lines | |||
| 4004 | if (!log) | 4203 | if (!log) |
| 4005 | return; | 4204 | return; |
| 4006 | 4205 | ||
| 4007 | if (nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_FW_SLOT, 0, log, | 4206 | if (nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_FW_SLOT, 0, NVME_CSI_NVM, |
| 4008 | sizeof(*log), 0)) | 4207 | log, sizeof(*log), 0)) |
| 4009 | dev_warn(ctrl->device, "Get FW SLOT INFO log error\n"); | 4208 | dev_warn(ctrl->device, "Get FW SLOT INFO log error\n"); |
| 4010 | kfree(log); | 4209 | kfree(log); |
| 4011 | } | 4210 | } |
| @@ -4142,11 +4341,16 @@ static void nvme_free_ctrl(struct device *dev) · 130 unmodified lines | |||
| 4142 | struct nvme_ctrl *ctrl = | 4341 | struct nvme_ctrl *ctrl = |
| 4143 | container_of(dev, struct nvme_ctrl, ctrl_device); | 4342 | container_of(dev, struct nvme_ctrl, ctrl_device); |
| 4144 | struct nvme_subsystem *subsys = ctrl->subsys; | 4343 | struct nvme_subsystem *subsys = ctrl->subsys; |
| 4344 | struct nvme_cel *cel, *next; | ||
| 4145 | 4345 | ||
| 4146 | if (subsys && ctrl->instance != subsys->instance) | 4346 | if (subsys && ctrl->instance != subsys->instance) |
| 4147 | ida_simple_remove(&nvme_instance_ida, ctrl->instance); | 4347 | ida_simple_remove(&nvme_instance_ida, ctrl->instance); |
| 4148 | 4348 | ||
| 4149 | kfree(ctrl->effects); | 4349 | list_for_each_entry_safe(cel, next, &ctrl->cels, entry) { |
| 4350 | list_del(&cel->entry); | ||
| 4351 | kfree(cel); | ||
| 4352 | } | ||
| 4353 | |||
| 4150 | nvme_mpath_uninit(ctrl); | 4354 | nvme_mpath_uninit(ctrl); |
| 4151 | __free_page(ctrl->discard_page); | 4355 | __free_page(ctrl->discard_page); |
| 4152 | 4356 | ||
| @@ -4177,6 +4381,7 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev, · 24 unmodified lines | |||
| 4177 | spin_lock_init(&ctrl->lock); | 4381 | spin_lock_init(&ctrl->lock); |
| 4178 | mutex_init(&ctrl->scan_lock); | 4382 | mutex_init(&ctrl->scan_lock); |
| 4179 | INIT_LIST_HEAD(&ctrl->namespaces); | 4383 | INIT_LIST_HEAD(&ctrl->namespaces); |
| 4384 | INIT_LIST_HEAD(&ctrl->cels); | ||
| 4180 | init_rwsem(&ctrl->namespaces_rwsem); | 4385 | init_rwsem(&ctrl->namespaces_rwsem); |
| 4181 | ctrl->dev = dev; | 4386 | ctrl->dev = dev; |
| 4182 | ctrl->ops = ops; | 4387 | ctrl->ops = ops; |
| @@ -4373,6 +4578,8 @@ static inline void _nvme_check_size(void) · 190 unmodified lines | |||
| 4373 | BUILD_BUG_ON(sizeof(struct nvme_command) != 64); | 4578 | BUILD_BUG_ON(sizeof(struct nvme_command) != 64); |
| 4374 | BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE); | 4579 | BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE); |
| 4375 | BUILD_BUG_ON(sizeof(struct nvme_id_ns) != NVME_IDENTIFY_DATA_SIZE); | 4580 | BUILD_BUG_ON(sizeof(struct nvme_id_ns) != NVME_IDENTIFY_DATA_SIZE); |
| 4581 | BUILD_BUG_ON(sizeof(struct nvme_id_ns_zns) != NVME_IDENTIFY_DATA_SIZE); | ||
| 4582 | BUILD_BUG_ON(sizeof(struct nvme_id_ctrl_zns) != NVME_IDENTIFY_DATA_SIZE); | ||
| 4376 | BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64); | 4583 | BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64); |
| 4377 | BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512); | 4584 | BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512); |
| 4378 | BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64); | 4585 | BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64); |
| @@ -62,7 +62,7 @@ static int nvme_hwmon_get_smart_log(struct nvme_hwmon_data *data) · 61 unmodified lines | |||
| 62 | int ret; | 62 | int ret; |
| 63 | 63 | ||
| 64 | ret = nvme_get_log(data->ctrl, NVME_NSID_ALL, NVME_LOG_SMART, 0, | 64 | ret = nvme_get_log(data->ctrl, NVME_NSID_ALL, NVME_LOG_SMART, 0, |
| 65 | &data->log, sizeof(data->log), 0); | 65 | NVME_CSI_NVM, &data->log, sizeof(data->log), 0); |
| 66 | 66 | ||
| 67 | return ret <= 0 ? ret : -EIO; | 67 | return ret <= 0 ? ret : -EIO; |
| 68 | } | 68 | } |
| @@ -593,8 +593,8 @@ static int nvme_nvm_get_chk_meta(struct nvm_dev *ndev, · 592 unmodified lines | |||
| 593 | dev_meta_off = dev_meta; | 593 | dev_meta_off = dev_meta; |
| 594 | 594 | ||
| 595 | ret = nvme_get_log(ctrl, ns->head->ns_id, | 595 | ret = nvme_get_log(ctrl, ns->head->ns_id, |
| 596 | NVME_NVM_LOG_REPORT_CHUNK, 0, dev_meta, len, | 596 | NVME_NVM_LOG_REPORT_CHUNK, 0, NVME_CSI_NVM, |
| 597 | offset); | 597 | dev_meta, len, offset); |
| 598 | if (ret) { | 598 | if (ret) { |
| 599 | dev_err(ctrl->device, "Get REPORT CHUNK log error\n"); | 599 | dev_err(ctrl->device, "Get REPORT CHUNK log error\n"); |
| 600 | break; | 600 | break; |
| @@ -527,7 +527,7 @@ static int nvme_read_ana_log(struct nvme_ctrl *ctrl) · 526 unmodified lines | |||
| 527 | int error; | 527 | int error; |
| 528 | 528 | ||
| 529 | mutex_lock(&ctrl->ana_lock); | 529 | mutex_lock(&ctrl->ana_lock); |
| 530 | error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_ANA, 0, | 530 | error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_ANA, 0, NVME_CSI_NVM, |
| 531 | ctrl->ana_log_buf, ctrl->ana_log_size, 0); | 531 | ctrl->ana_log_buf, ctrl->ana_log_size, 0); |
| 532 | if (error) { | 532 | if (error) { |
| 533 | dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error); | 533 | dev_warn(ctrl->device, "Failed to get ANA log: %d\n", error); |
| @@ -191,6 +191,12 @@ struct nvme_fault_inject { · 190 unmodified lines | |||
| 191 | #endif | 191 | #endif |
| 192 | }; | 192 | }; |
| 193 | 193 | ||
| 194 | struct nvme_cel { | ||
| 195 | struct list_head entry; | ||
| 196 | struct nvme_effects_log log; | ||
| 197 | u8 csi; | ||
| 198 | }; | ||
| 199 | |||
| 194 | struct nvme_ctrl { | 200 | struct nvme_ctrl { |
| 195 | bool comp_seen; | 201 | bool comp_seen; |
| 196 | enum nvme_ctrl_state state; | 202 | enum nvme_ctrl_state state; |
| @@ -232,6 +238,9 @@ struct nvme_ctrl { · 35 unmodified lines | |||
| 232 | u32 max_hw_sectors; | 238 | u32 max_hw_sectors; |
| 233 | u32 max_segments; | 239 | u32 max_segments; |
| 234 | u32 max_integrity_segments; | 240 | u32 max_integrity_segments; |
| 241 | #ifdef CONFIG_BLK_DEV_ZONED | ||
| 242 | u32 max_zone_append; | ||
| 243 | #endif | ||
| 235 | u16 crdt[3]; | 244 | u16 crdt[3]; |
| 236 | u16 oncs; | 245 | u16 oncs; |
| 237 | u16 oacs; | 246 | u16 oacs; |
| @@ -257,6 +266,7 @@ struct nvme_ctrl { · 19 unmodified lines | |||
| 257 | unsigned long quirks; | 266 | unsigned long quirks; |
| 258 | struct nvme_id_power_state psd[32]; | 267 | struct nvme_id_power_state psd[32]; |
| 259 | struct nvme_effects_log *effects; | 268 | struct nvme_effects_log *effects; |
| 269 | struct list_head cels; | ||
| 260 | struct work_struct scan_work; | 270 | struct work_struct scan_work; |
| 261 | struct work_struct async_event_work; | 271 | struct work_struct async_event_work; |
| 262 | struct delayed_work ka_work; | 272 | struct delayed_work ka_work; |
| @@ -339,6 +349,7 @@ struct nvme_ns_ids { · 76 unmodified lines | |||
| 339 | u8 eui64[8]; | 349 | u8 eui64[8]; |
| 340 | u8 nguid[16]; | 350 | u8 nguid[16]; |
| 341 | uuid_t uuid; | 351 | uuid_t uuid; |
| 352 | u8 csi; | ||
| 342 | }; | 353 | }; |
| 343 | 354 | ||
| 344 | /* | 355 | /* |
| @@ -358,6 +369,7 @@ struct nvme_ns_head { · 13 unmodified lines | |||
| 358 | struct kref ref; | 369 | struct kref ref; |
| 359 | bool shared; | 370 | bool shared; |
| 360 | int instance; | 371 | int instance; |
| 372 | struct nvme_effects_log *effects; | ||
| 361 | #ifdef CONFIG_NVME_MULTIPATH | 373 | #ifdef CONFIG_NVME_MULTIPATH |
| 362 | struct gendisk *disk; | 374 | struct gendisk *disk; |
| 363 | struct bio_list requeue_list; | 375 | struct bio_list requeue_list; |
| @@ -395,6 +407,9 @@ struct nvme_ns { · 31 unmodified lines | |||
| 395 | u16 sgs; | 407 | u16 sgs; |
| 396 | u32 sws; | 408 | u32 sws; |
| 397 | u8 pi_type; | 409 | u8 pi_type; |
| 410 | #ifdef CONFIG_BLK_DEV_ZONED | ||
| 411 | u64 zsze; | ||
| 412 | #endif | ||
| 398 | unsigned long features; | 413 | unsigned long features; |
| 399 | unsigned long flags; | 414 | unsigned long flags; |
| 400 | #define NVME_NS_REMOVING 0 | 415 | #define NVME_NS_REMOVING 0 |
| @@ -560,8 +575,11 @@ int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl); · 159 unmodified lines | |||
| 560 | int nvme_try_sched_reset(struct nvme_ctrl *ctrl); | 575 | int nvme_try_sched_reset(struct nvme_ctrl *ctrl); |
| 561 | int nvme_delete_ctrl(struct nvme_ctrl *ctrl); | 576 | int nvme_delete_ctrl(struct nvme_ctrl *ctrl); |
| 562 | 577 | ||
| 563 | int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, | 578 | int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi, |
| 564 | void *log, size_t size, u64 offset); | 579 | void *log, size_t size, u64 offset); |
| 580 | struct nvme_ns *nvme_get_ns_from_disk(struct gendisk *disk, | ||
| 581 | struct nvme_ns_head **head, int *srcu_idx); | ||
| 582 | void nvme_put_ns_from_disk(struct nvme_ns_head *head, int idx); | ||
| 565 | 583 | ||
| 566 | extern const struct attribute_group *nvme_ns_id_attr_groups[]; | 584 | extern const struct attribute_group *nvme_ns_id_attr_groups[]; |
| 567 | extern const struct block_device_operations nvme_ns_head_ops; | 585 | extern const struct block_device_operations nvme_ns_head_ops; |
| @@ -697,6 +715,36 @@ static inline void nvme_mpath_update_disk_size(struct gendisk *disk) · 129 unmodified lines | |||
| 697 | } | 715 | } |
| 698 | #endif /* CONFIG_NVME_MULTIPATH */ | 716 | #endif /* CONFIG_NVME_MULTIPATH */ |
| 699 | 717 | ||
| 718 | #ifdef CONFIG_BLK_DEV_ZONED | ||
| 719 | int nvme_update_zone_info(struct gendisk *disk, struct nvme_ns *ns, | ||
| 720 | unsigned lbaf); | ||
| 721 | |||
| 722 | int nvme_report_zones(struct gendisk *disk, sector_t sector, | ||
| 723 | unsigned int nr_zones, report_zones_cb cb, void *data); | ||
| 724 | |||
| 725 | blk_status_t nvme_setup_zone_mgmt_send(struct nvme_ns *ns, struct request *req, | ||
| 726 | struct nvme_command *cmnd, | ||
| 727 | enum nvme_zone_mgmt_action action); | ||
| 728 | #else | ||
| 729 | #define nvme_report_zones NULL | ||
| 730 | |||
| 731 | static inline blk_status_t nvme_setup_zone_mgmt_send(struct nvme_ns *ns, | ||
| 732 | struct request *req, struct nvme_command *cmnd, | ||
| 733 | enum nvme_zone_mgmt_action action) | ||
| 734 | { | ||
| 735 | return BLK_STS_NOTSUPP; | ||
| 736 | } | ||
| 737 | |||
| 738 | static inline int nvme_update_zone_info(struct gendisk *disk, | ||
| 739 | struct nvme_ns *ns, | ||
| 740 | unsigned lbaf) | ||
| 741 | { | ||
| 742 | dev_warn(ns->ctrl->device, | ||
| 743 | "Please enable CONFIG_BLK_DEV_ZONED to support ZNS devices\n"); | ||
| 744 | return -EPROTONOSUPPORT; | ||
| 745 | } | ||
| 746 | #endif | ||
| 747 | |||
| 700 | #ifdef CONFIG_NVM | 748 | #ifdef CONFIG_NVM |
| 701 | int nvme_nvm_register(struct nvme_ns *ns, char *disk_name, int node); | 749 | int nvme_nvm_register(struct nvme_ns *ns, char *disk_name, int node); |
| 702 | void nvme_nvm_unregister(struct nvme_ns *ns); | 750 | void nvme_nvm_unregister(struct nvme_ns *ns); |
| @@ -61,10 +61,10 @@ MODULE_PARM_DESC(sgl_threshold, · 60 unmodified lines | |||
| 61 | static int io_queue_depth_set(const char *val, const struct kernel_param *kp); | 61 | static int io_queue_depth_set(const char *val, const struct kernel_param *kp); |
| 62 | static const struct kernel_param_ops io_queue_depth_ops = { | 62 | static const struct kernel_param_ops io_queue_depth_ops = { |
| 63 | .set = io_queue_depth_set, | 63 | .set = io_queue_depth_set, |
| 64 | .get = param_get_int, | 64 | .get = param_get_uint, |
| 65 | }; | 65 | }; |
| 66 | 66 | ||
| 67 | static int io_queue_depth = 1024; | 67 | static unsigned int io_queue_depth = 1024; |
| 68 | module_param_cb(io_queue_depth, &io_queue_depth_ops, &io_queue_depth, 0644); | 68 | module_param_cb(io_queue_depth, &io_queue_depth_ops, &io_queue_depth, 0644); |
| 69 | MODULE_PARM_DESC(io_queue_depth, "set io queue depth, should >= 2"); | 69 | MODULE_PARM_DESC(io_queue_depth, "set io queue depth, should >= 2"); |
| 70 | 70 | ||
| @@ -115,7 +115,7 @@ struct nvme_dev { · 44 unmodified lines | |||
| 115 | unsigned max_qid; | 115 | unsigned max_qid; |
| 116 | unsigned io_queues[HCTX_MAX_TYPES]; | 116 | unsigned io_queues[HCTX_MAX_TYPES]; |
| 117 | unsigned int num_vecs; | 117 | unsigned int num_vecs; |
| 118 | int q_depth; | 118 | u16 q_depth; |
| 119 | int io_sqes; | 119 | int io_sqes; |
| 120 | u32 db_stride; | 120 | u32 db_stride; |
| 121 | void __iomem *bar; | 121 | void __iomem *bar; |
| @@ -151,13 +151,14 @@ struct nvme_dev { · 29 unmodified lines | |||
| 151 | 151 | ||
| 152 | static int io_queue_depth_set(const char *val, const struct kernel_param *kp) | 152 | static int io_queue_depth_set(const char *val, const struct kernel_param *kp) |
| 153 | { | 153 | { |
| 154 | int n = 0, ret; | 154 | int ret; |
| 155 | u16 n; | ||
| 155 | 156 | ||
| 156 | ret = kstrtoint(val, 10, &n); | 157 | ret = kstrtou16(val, 10, &n); |
| 157 | if (ret != 0 || n < 2) | 158 | if (ret != 0 || n < 2) |
| 158 | return -EINVAL; | 159 | return -EINVAL; |
| 159 | 160 | ||
| 160 | return param_set_int(val, kp); | 161 | return param_set_ushort(val, kp); |
| 161 | } | 162 | } |
| 162 | 163 | ||
| 163 | static inline unsigned int sq_idx(unsigned int qid, u32 stride) | 164 | static inline unsigned int sq_idx(unsigned int qid, u32 stride) |
| @@ -361,7 +362,7 @@ static int nvme_pci_npages_sgl(unsigned int num_seg) · 197 unmodified lines | |||
| 361 | return DIV_ROUND_UP(num_seg * sizeof(struct nvme_sgl_desc), PAGE_SIZE); | 362 | return DIV_ROUND_UP(num_seg * sizeof(struct nvme_sgl_desc), PAGE_SIZE); |
| 362 | } | 363 | } |
| 363 | 364 | ||
| 364 | static unsigned int nvme_pci_iod_alloc_size(struct nvme_dev *dev, | 365 | static size_t nvme_pci_iod_alloc_size(struct nvme_dev *dev, |
| 365 | unsigned int size, unsigned int nseg, bool use_sgl) | 366 | unsigned int size, unsigned int nseg, bool use_sgl) |
| 366 | { | 367 | { |
| 367 | size_t alloc_size; | 368 | size_t alloc_size; |
| @@ -500,9 +501,6 @@ static inline bool nvme_pci_use_sgls(struct nvme_dev *dev, struct request *req) · 132 unmodified lines | |||
| 500 | int nseg = blk_rq_nr_phys_segments(req); | 501 | int nseg = blk_rq_nr_phys_segments(req); |
| 501 | unsigned int avg_seg_size; | 502 | unsigned int avg_seg_size; |
| 502 | 503 | ||
| 503 | if (nseg == 0) | ||
| 504 | return false; | ||
| 505 | |||
| 506 | avg_seg_size = DIV_ROUND_UP(blk_rq_payload_bytes(req), nseg); | 504 | avg_seg_size = DIV_ROUND_UP(blk_rq_payload_bytes(req), nseg); |
| 507 | 505 | ||
| 508 | if (!(dev->ctrl.sgls & ((1 << 0) | (1 << 1)))) | 506 | if (!(dev->ctrl.sgls & ((1 << 0) | (1 << 1)))) |
| @@ -764,7 +762,7 @@ static blk_status_t nvme_setup_prp_simple(struct nvme_dev *dev, · 255 unmodified lines | |||
| 764 | cmnd->dptr.prp1 = cpu_to_le64(iod->first_dma); | 762 | cmnd->dptr.prp1 = cpu_to_le64(iod->first_dma); |
| 765 | if (bv->bv_len > first_prp_len) | 763 | if (bv->bv_len > first_prp_len) |
| 766 | cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma + first_prp_len); | 764 | cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma + first_prp_len); |
| 767 | return 0; | 765 | return BLK_STS_OK; |
| 768 | } | 766 | } |
| 769 | 767 | ||
| 770 | static blk_status_t nvme_setup_sgl_simple(struct nvme_dev *dev, | 768 | static blk_status_t nvme_setup_sgl_simple(struct nvme_dev *dev, |
| @@ -782,7 +780,7 @@ static blk_status_t nvme_setup_sgl_simple(struct nvme_dev *dev, · 11 unmodified lines | |||
| 782 | cmnd->dptr.sgl.addr = cpu_to_le64(iod->first_dma); | 780 | cmnd->dptr.sgl.addr = cpu_to_le64(iod->first_dma); |
| 783 | cmnd->dptr.sgl.length = cpu_to_le32(iod->dma_len); | 781 | cmnd->dptr.sgl.length = cpu_to_le32(iod->dma_len); |
| 784 | cmnd->dptr.sgl.type = NVME_SGL_FMT_DATA_DESC << 4; | 782 | cmnd->dptr.sgl.type = NVME_SGL_FMT_DATA_DESC << 4; |
| 785 | return 0; | 783 | return BLK_STS_OK; |
| 786 | } | 784 | } |
| 787 | 785 | ||
| 788 | static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req, | 786 | static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req, |
| @@ -846,7 +844,7 @@ static blk_status_t nvme_map_metadata(struct nvme_dev *dev, struct request *req, · 57 unmodified lines | |||
| 846 | if (dma_mapping_error(dev->dev, iod->meta_dma)) | 844 | if (dma_mapping_error(dev->dev, iod->meta_dma)) |
| 847 | return BLK_STS_IOERR; | 845 | return BLK_STS_IOERR; |
| 848 | cmnd->rw.metadata = cpu_to_le64(iod->meta_dma); | 846 | cmnd->rw.metadata = cpu_to_le64(iod->meta_dma); |
| 849 | return 0; | 847 | return BLK_STS_OK; |
| 850 | } | 848 | } |
| 851 | 849 | ||
| 852 | /* | 850 | /* |
| @@ -1019,6 +1017,7 @@ static irqreturn_t nvme_irq(int irq, void *data) · 166 unmodified lines | |||
| 1019 | static irqreturn_t nvme_irq_check(int irq, void *data) | 1017 | static irqreturn_t nvme_irq_check(int irq, void *data) |
| 1020 | { | 1018 | { |
| 1021 | struct nvme_queue *nvmeq = data; | 1019 | struct nvme_queue *nvmeq = data; |
| 1020 | |||
| 1022 | if (nvme_cqe_pending(nvmeq)) | 1021 | if (nvme_cqe_pending(nvmeq)) |
| 1023 | return IRQ_WAKE_THREAD; | 1022 | return IRQ_WAKE_THREAD; |
| 1024 | return IRQ_NONE; | 1023 | return IRQ_NONE; |
| @@ -1154,7 +1153,6 @@ static void abort_endio(struct request *req, blk_status_t error) · 129 unmodified lines | |||
| 1154 | 1153 | ||
| 1155 | static bool nvme_should_reset(struct nvme_dev *dev, u32 csts) | 1154 | static bool nvme_should_reset(struct nvme_dev *dev, u32 csts) |
| 1156 | { | 1155 | { |
| 1157 | |||
| 1158 | /* If true, indicates loss of adapter communication, possibly by a | 1156 | /* If true, indicates loss of adapter communication, possibly by a |
| 1159 | * NVMe Subsystem reset. | 1157 | * NVMe Subsystem reset. |
| 1160 | */ | 1158 | */ |
| @@ -1261,9 +1259,9 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved) · 100 unmodified lines | |||
| 1261 | } | 1259 | } |
| 1262 | 1260 | ||
| 1263 | /* | 1261 | /* |
| 1264 | * Shutdown the controller immediately and schedule a reset if the | 1262 | * Shutdown the controller immediately and schedule a reset if the |
| 1265 | * command was already aborted once before and still hasn't been | 1263 | * command was already aborted once before and still hasn't been |
| 1266 | * returned to the driver, or if this is the admin queue. | 1264 | * returned to the driver, or if this is the admin queue. |
| 1267 | */ | 1265 | */ |
| 1268 | if (!nvmeq->qid || iod->aborted) { | 1266 | if (!nvmeq->qid || iod->aborted) { |
| 1269 | dev_warn(dev->ctrl.device, | 1267 | dev_warn(dev->ctrl.device, |
| @@ -1402,6 +1400,7 @@ static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues, · 132 unmodified lines | |||
| 1402 | 1400 | ||
| 1403 | if (q_size_aligned * nr_io_queues > dev->cmb_size) { | 1401 | if (q_size_aligned * nr_io_queues > dev->cmb_size) { |
| 1404 | u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues); | 1402 | u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues); |
| 1403 | |||
| 1405 | mem_per_q = round_down(mem_per_q, dev->ctrl.page_size); | 1404 | mem_per_q = round_down(mem_per_q, dev->ctrl.page_size); |
| 1406 | q_depth = div_u64(mem_per_q, entry_size); | 1405 | q_depth = div_u64(mem_per_q, entry_size); |
| 1407 | 1406 | ||
| @@ -1932,12 +1931,12 @@ out: · 524 unmodified lines | |||
| 1932 | 1931 | ||
| 1933 | static int nvme_alloc_host_mem(struct nvme_dev *dev, u64 min, u64 preferred) | 1932 | static int nvme_alloc_host_mem(struct nvme_dev *dev, u64 min, u64 preferred) |
| 1934 | { | 1933 | { |
| 1935 | u32 chunk_size; | 1934 | u64 min_chunk = min_t(u64, preferred, PAGE_SIZE * MAX_ORDER_NR_PAGES); |
| 1935 | u64 hmminds = max_t(u32, dev->ctrl.hmminds * 4096, PAGE_SIZE * 2); | ||
| 1936 | u64 chunk_size; | ||
| 1936 | 1937 | ||
| 1937 | /* start big and work our way down */ | 1938 | /* start big and work our way down */ |
| 1938 | for (chunk_size = min_t(u64, preferred, PAGE_SIZE * MAX_ORDER_NR_PAGES); | 1939 | for (chunk_size = min_chunk; chunk_size >= hmminds; chunk_size /= 2) { |
| 1939 | chunk_size >= max_t(u32, dev->ctrl.hmminds * 4096, PAGE_SIZE * 2); | ||
| 1940 | chunk_size /= 2) { | ||
| 1941 | if (!__nvme_alloc_host_mem(dev, preferred, chunk_size)) { | 1940 | if (!__nvme_alloc_host_mem(dev, preferred, chunk_size)) { |
| 1942 | if (!min || dev->host_mem_size >= min) | 1941 | if (!min || dev->host_mem_size >= min) |
| 1943 | return 0; | 1942 | return 0; |
| @@ -2003,7 +2002,7 @@ static void nvme_calc_irq_sets(struct irq_affinity *affd, unsigned int nrirqs) · 59 unmodified lines | |||
| 2003 | unsigned int nr_read_queues, nr_write_queues = dev->nr_write_queues; | 2002 | unsigned int nr_read_queues, nr_write_queues = dev->nr_write_queues; |
| 2004 | 2003 | ||
| 2005 | /* | 2004 | /* |
| 2006 | * If there is no interupt available for queues, ensure that | 2005 | * If there is no interrupt available for queues, ensure that |
| 2007 | * the default queue is set to 1. The affinity set size is | 2006 | * the default queue is set to 1. The affinity set size is |
| 2008 | * also set to one, but the irq core ignores it for this case. | 2007 | * also set to one, but the irq core ignores it for this case. |
| 2009 | * | 2008 | * |
| @@ -2261,8 +2260,8 @@ static void nvme_dev_add(struct nvme_dev *dev) · 251 unmodified lines | |||
| 2261 | dev->tagset.nr_maps++; | 2260 | dev->tagset.nr_maps++; |
| 2262 | dev->tagset.timeout = NVME_IO_TIMEOUT; | 2261 | dev->tagset.timeout = NVME_IO_TIMEOUT; |
| 2263 | dev->tagset.numa_node = dev->ctrl.numa_node; | 2262 | dev->tagset.numa_node = dev->ctrl.numa_node; |
| 2264 | dev->tagset.queue_depth = | 2263 | dev->tagset.queue_depth = min_t(unsigned int, dev->q_depth, |
| 2265 | min_t(int, dev->q_depth, BLK_MQ_MAX_DEPTH) - 1; | 2264 | BLK_MQ_MAX_DEPTH) - 1; |
| 2266 | dev->tagset.cmd_size = sizeof(struct nvme_iod); | 2265 | dev->tagset.cmd_size = sizeof(struct nvme_iod); |
| 2267 | dev->tagset.flags = BLK_MQ_F_SHOULD_MERGE; | 2266 | dev->tagset.flags = BLK_MQ_F_SHOULD_MERGE; |
| 2268 | dev->tagset.driver_data = dev; | 2267 | dev->tagset.driver_data = dev; |
| @@ -2321,7 +2320,7 @@ static int nvme_pci_enable(struct nvme_dev *dev) · 52 unmodified lines | |||
| 2321 | 2320 | ||
| 2322 | dev->ctrl.cap = lo_hi_readq(dev->bar + NVME_REG_CAP); | 2321 | dev->ctrl.cap = lo_hi_readq(dev->bar + NVME_REG_CAP); |
| 2323 | 2322 | ||
| 2324 | dev->q_depth = min_t(int, NVME_CAP_MQES(dev->ctrl.cap) + 1, | 2323 | dev->q_depth = min_t(u16, NVME_CAP_MQES(dev->ctrl.cap) + 1, |
| 2325 | io_queue_depth); | 2324 | io_queue_depth); |
| 2326 | dev->ctrl.sqsize = dev->q_depth - 1; /* 0's based queue depth */ | 2325 | dev->ctrl.sqsize = dev->q_depth - 1; /* 0's based queue depth */ |
| 2327 | dev->db_stride = 1 << NVME_CAP_STRIDE(dev->ctrl.cap); | 2326 | dev->db_stride = 1 << NVME_CAP_STRIDE(dev->ctrl.cap); |
| @@ -2876,6 +2875,7 @@ static void nvme_reset_done(struct pci_dev *pdev) · 548 unmodified lines | |||
| 2876 | static void nvme_shutdown(struct pci_dev *pdev) | 2875 | static void nvme_shutdown(struct pci_dev *pdev) |
| 2877 | { | 2876 | { |
| 2878 | struct nvme_dev *dev = pci_get_drvdata(pdev); | 2877 | struct nvme_dev *dev = pci_get_drvdata(pdev); |
| 2878 | |||
| 2879 | nvme_disable_prepare_reset(dev, true); | 2879 | nvme_disable_prepare_reset(dev, true); |
| 2880 | } | 2880 | } |
| 2881 | 2881 | ||
| @@ -3006,6 +3006,7 @@ unfreeze: · 124 unmodified lines | |||
| 3006 | static int nvme_simple_suspend(struct device *dev) | 3006 | static int nvme_simple_suspend(struct device *dev) |
| 3007 | { | 3007 | { |
| 3008 | struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev)); | 3008 | struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev)); |
| 3009 | |||
| 3009 | return nvme_disable_prepare_reset(ndev, true); | 3010 | return nvme_disable_prepare_reset(ndev, true); |
| 3010 | } | 3011 | } |
| 3011 | 3012 | ||
| @@ -3079,16 +3080,16 @@ static const struct pci_error_handlers nvme_err_handler = { · 67 unmodified lines | |||
| 3079 | }; | 3080 | }; |
| 3080 | 3081 | ||
| 3081 | static const struct pci_device_id nvme_id_table[] = { | 3082 | static const struct pci_device_id nvme_id_table[] = { |
| 3082 | { PCI_VDEVICE(INTEL, 0x0953), | 3083 | { PCI_VDEVICE(INTEL, 0x0953), /* Intel 750/P3500/P3600/P3700 */ |
| 3083 | .driver_data = NVME_QUIRK_STRIPE_SIZE | | 3084 | .driver_data = NVME_QUIRK_STRIPE_SIZE | |
| 3084 | NVME_QUIRK_DEALLOCATE_ZEROES, }, | 3085 | NVME_QUIRK_DEALLOCATE_ZEROES, }, |
| 3085 | { PCI_VDEVICE(INTEL, 0x0a53), | 3086 | { PCI_VDEVICE(INTEL, 0x0a53), /* Intel P3520 */ |
| 3086 | .driver_data = NVME_QUIRK_STRIPE_SIZE | | 3087 | .driver_data = NVME_QUIRK_STRIPE_SIZE | |
| 3087 | NVME_QUIRK_DEALLOCATE_ZEROES, }, | 3088 | NVME_QUIRK_DEALLOCATE_ZEROES, }, |
| 3088 | { PCI_VDEVICE(INTEL, 0x0a54), | 3089 | { PCI_VDEVICE(INTEL, 0x0a54), /* Intel P4500/P4600 */ |
| 3089 | .driver_data = NVME_QUIRK_STRIPE_SIZE | | 3090 | .driver_data = NVME_QUIRK_STRIPE_SIZE | |
| 3090 | NVME_QUIRK_DEALLOCATE_ZEROES, }, | 3091 | NVME_QUIRK_DEALLOCATE_ZEROES, }, |
| 3091 | { PCI_VDEVICE(INTEL, 0x0a55), | 3092 | { PCI_VDEVICE(INTEL, 0x0a55), /* Dell Express Flash P4600 */ |
| 3092 | .driver_data = NVME_QUIRK_STRIPE_SIZE | | 3093 | .driver_data = NVME_QUIRK_STRIPE_SIZE | |
| 3093 | NVME_QUIRK_DEALLOCATE_ZEROES, }, | 3094 | NVME_QUIRK_DEALLOCATE_ZEROES, }, |
| 3094 | { PCI_VDEVICE(INTEL, 0xf1a5), /* Intel 600P/P3100 */ | 3095 | { PCI_VDEVICE(INTEL, 0xf1a5), /* Intel 600P/P3100 */ |
| @@ -46,6 +46,7 @@ struct nvme_tcp_request { · 45 unmodified lines | |||
| 46 | u32 pdu_sent; | 46 | u32 pdu_sent; |
| 47 | u16 ttag; | 47 | u16 ttag; |
| 48 | struct list_head entry; | 48 | struct list_head entry; |
| 49 | struct llist_node lentry; | ||
| 49 | __le32 ddgst; | 50 | __le32 ddgst; |
| 50 | 51 | ||
| 51 | struct bio *curr_bio; | 52 | struct bio *curr_bio; |
| @@ -75,9 +76,10 @@ struct nvme_tcp_queue { · 23 unmodified lines | |||
| 75 | struct work_struct io_work; | 76 | struct work_struct io_work; |
| 76 | int io_cpu; | 77 | int io_cpu; |
| 77 | 78 | ||
| 78 | spinlock_t lock; | ||
| 79 | struct mutex send_mutex; | 79 | struct mutex send_mutex; |
| 80 | struct llist_head req_list; | ||
| 80 | struct list_head send_list; | 81 | struct list_head send_list; |
| 82 | bool more_requests; | ||
| 81 | 83 | ||
| 82 | /* recv state */ | 84 | /* recv state */ |
| 83 | void *pdu; | 85 | void *pdu; |
| @@ -261,15 +263,13 @@ static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req, · 177 unmodified lines | |||
| 261 | } | 263 | } |
| 262 | 264 | ||
| 263 | static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req, | 265 | static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req, |
| 264 | bool sync) | 266 | bool sync, bool last) |
| 265 | { | 267 | { |
| 266 | struct nvme_tcp_queue *queue = req->queue; | 268 | struct nvme_tcp_queue *queue = req->queue; |
| 267 | bool empty; | 269 | bool empty; |
| 268 | 270 | ||
| 269 | spin_lock(&queue->lock); | 271 | empty = llist_add(&req->lentry, &queue->req_list) && |
| 270 | empty = list_empty(&queue->send_list) && !queue->request; | 272 | list_empty(&queue->send_list) && !queue->request; |
| 271 | list_add_tail(&req->entry, &queue->send_list); | ||
| 272 | spin_unlock(&queue->lock); | ||
| 273 | 273 | ||
| 274 | /* | 274 | /* |
| 275 | * if we're the first on the send_list and we can try to send | 275 | * if we're the first on the send_list and we can try to send |
| @@ -278,25 +278,42 @@ static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req, · 2 unmodified lines | |||
| 278 | */ | 278 | */ |
| 279 | if (queue->io_cpu == smp_processor_id() && | 279 | if (queue->io_cpu == smp_processor_id() && |
| 280 | sync && empty && mutex_trylock(&queue->send_mutex)) { | 280 | sync && empty && mutex_trylock(&queue->send_mutex)) { |
| 281 | queue->more_requests = !last; | ||
| 281 | nvme_tcp_try_send(queue); | 282 | nvme_tcp_try_send(queue); |
| 283 | queue->more_requests = false; | ||
| 282 | mutex_unlock(&queue->send_mutex); | 284 | mutex_unlock(&queue->send_mutex); |
| 283 | } else { | 285 | } else if (last) { |
| 284 | queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); | 286 | queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); |
| 285 | } | 287 | } |
| 286 | } | 288 | } |
| 287 | 289 | ||
| 290 | static void nvme_tcp_process_req_list(struct nvme_tcp_queue *queue) | ||
| 291 | { | ||
| 292 | struct nvme_tcp_request *req; | ||
| 293 | struct llist_node *node; | ||
| 294 | |||
| 295 | for (node = llist_del_all(&queue->req_list); node; node = node->next) { | ||
| 296 | req = llist_entry(node, struct nvme_tcp_request, lentry); | ||
| 297 | list_add(&req->entry, &queue->send_list); | ||
| 298 | } | ||
| 299 | } | ||
| 300 | |||
| 288 | static inline struct nvme_tcp_request * | 301 | static inline struct nvme_tcp_request * |
| 289 | nvme_tcp_fetch_request(struct nvme_tcp_queue *queue) | 302 | nvme_tcp_fetch_request(struct nvme_tcp_queue *queue) |
| 290 | { | 303 | { |
| 291 | struct nvme_tcp_request *req; | 304 | struct nvme_tcp_request *req; |
| 292 | 305 | ||
| 293 | spin_lock(&queue->lock); | ||
| 294 | req = list_first_entry_or_null(&queue->send_list, | 306 | req = list_first_entry_or_null(&queue->send_list, |
| 295 | struct nvme_tcp_request, entry); | 307 | struct nvme_tcp_request, entry); |
| 296 | if (req) | 308 | if (!req) { |
| 297 | list_del(&req->entry); | 309 | nvme_tcp_process_req_list(queue); |
| 298 | spin_unlock(&queue->lock); | 310 | req = list_first_entry_or_null(&queue->send_list, |
| 311 | struct nvme_tcp_request, entry); | ||
| 312 | if (unlikely(!req)) | ||
| 313 | return NULL; | ||
| 314 | } | ||
| 299 | 315 | ||
| 316 | list_del(&req->entry); | ||
| 300 | return req; | 317 | return req; |
| 301 | } | 318 | } |
| 302 | 319 | ||
| @@ -596,7 +613,7 @@ static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue, · 293 unmodified lines | |||
| 596 | req->state = NVME_TCP_SEND_H2C_PDU; | 613 | req->state = NVME_TCP_SEND_H2C_PDU; |
| 597 | req->offset = 0; | 614 | req->offset = 0; |
| 598 | 615 | ||
| 599 | nvme_tcp_queue_request(req, false); | 616 | nvme_tcp_queue_request(req, false, true); |
| 600 | 617 | ||
| 601 | return 0; | 618 | return 0; |
| 602 | } | 619 | } |
| @@ -863,6 +880,12 @@ done: · 260 unmodified lines | |||
| 863 | read_unlock(&sk->sk_callback_lock); | 880 | read_unlock(&sk->sk_callback_lock); |
| 864 | } | 881 | } |
| 865 | 882 | ||
| 883 | static inline bool nvme_tcp_queue_more(struct nvme_tcp_queue *queue) | ||
| 884 | { | ||
| 885 | return !list_empty(&queue->send_list) || | ||
| 886 | !llist_empty(&queue->req_list) || queue->more_requests; | ||
| 887 | } | ||
| 888 | |||
| 866 | static inline void nvme_tcp_done_send_req(struct nvme_tcp_queue *queue) | 889 | static inline void nvme_tcp_done_send_req(struct nvme_tcp_queue *queue) |
| 867 | { | 890 | { |
| 868 | queue->request = NULL; | 891 | queue->request = NULL; |
| @@ -884,7 +907,7 @@ static int nvme_tcp_try_send_data(struct nvme_tcp_request *req) · 15 unmodified lines | |||
| 884 | bool last = nvme_tcp_pdu_last_send(req, len); | 907 | bool last = nvme_tcp_pdu_last_send(req, len); |
| 885 | int ret, flags = MSG_DONTWAIT; | 908 | int ret, flags = MSG_DONTWAIT; |
| 886 | 909 | ||
| 887 | if (last && !queue->data_digest) | 910 | if (last && !queue->data_digest && !nvme_tcp_queue_more(queue)) |
| 888 | flags |= MSG_EOR; | 911 | flags |= MSG_EOR; |
| 889 | else | 912 | else |
| 890 | flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST; | 913 | flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST; |
| @@ -931,7 +954,7 @@ static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req) · 40 unmodified lines | |||
| 931 | int flags = MSG_DONTWAIT; | 954 | int flags = MSG_DONTWAIT; |
| 932 | int ret; | 955 | int ret; |
| 933 | 956 | ||
| 934 | if (inline_data) | 957 | if (inline_data || nvme_tcp_queue_more(queue)) |
| 935 | flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST; | 958 | flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST; |
| 936 | else | 959 | else |
| 937 | flags |= MSG_EOR; | 960 | flags |= MSG_EOR; |
| @@ -996,12 +1019,17 @@ static int nvme_tcp_try_send_ddgst(struct nvme_tcp_request *req) · 58 unmodified lines | |||
| 996 | { | 1019 | { |
| 997 | struct nvme_tcp_queue *queue = req->queue; | 1020 | struct nvme_tcp_queue *queue = req->queue; |
| 998 | int ret; | 1021 | int ret; |
| 999 | struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_EOR }; | 1022 | struct msghdr msg = { .msg_flags = MSG_DONTWAIT }; |
| 1000 | struct kvec iov = { | 1023 | struct kvec iov = { |
| 1001 | .iov_base = &req->ddgst + req->offset, | 1024 | .iov_base = &req->ddgst + req->offset, |
| 1002 | .iov_len = NVME_TCP_DIGEST_LENGTH - req->offset | 1025 | .iov_len = NVME_TCP_DIGEST_LENGTH - req->offset |
| 1003 | }; | 1026 | }; |
| 1004 | 1027 | ||
| 1028 | if (nvme_tcp_queue_more(queue)) | ||
| 1029 | msg.msg_flags |= MSG_MORE; | ||
| 1030 | else | ||
| 1031 | msg.msg_flags |= MSG_EOR; | ||
| 1032 | |||
| 1005 | ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); | 1033 | ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); |
| 1006 | if (unlikely(ret <= 0)) | 1034 | if (unlikely(ret <= 0)) |
| 1007 | return ret; | 1035 | return ret; |
| @@ -1344,8 +1372,8 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, · 336 unmodified lines | |||
| 1344 | int ret, rcv_pdu_size; | 1372 | int ret, rcv_pdu_size; |
| 1345 | 1373 | ||
| 1346 | queue->ctrl = ctrl; | 1374 | queue->ctrl = ctrl; |
| 1375 | init_llist_head(&queue->req_list); | ||
| 1347 | INIT_LIST_HEAD(&queue->send_list); | 1376 | INIT_LIST_HEAD(&queue->send_list); |
| 1348 | spin_lock_init(&queue->lock); | ||
| 1349 | mutex_init(&queue->send_mutex); | 1377 | mutex_init(&queue->send_mutex); |
| 1350 | INIT_WORK(&queue->io_work, nvme_tcp_io_work); | 1378 | INIT_WORK(&queue->io_work, nvme_tcp_io_work); |
| 1351 | queue->queue_size = queue_size; | 1379 | queue->queue_size = queue_size; |
| @@ -2106,7 +2134,7 @@ static void nvme_tcp_submit_async_event(struct nvme_ctrl *arg) · 754 unmodified lines | |||
| 2106 | ctrl->async_req.curr_bio = NULL; | 2134 | ctrl->async_req.curr_bio = NULL; |
| 2107 | ctrl->async_req.data_len = 0; | 2135 | ctrl->async_req.data_len = 0; |
| 2108 | 2136 | ||
| 2109 | nvme_tcp_queue_request(&ctrl->async_req, true); | 2137 | nvme_tcp_queue_request(&ctrl->async_req, true, true); |
| 2110 | } | 2138 | } |
| 2111 | 2139 | ||
| 2112 | static enum blk_eh_timer_return | 2140 | static enum blk_eh_timer_return |
| @@ -2218,6 +2246,14 @@ static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns, · 105 unmodified lines | |||
| 2218 | return 0; | 2246 | return 0; |
| 2219 | } | 2247 | } |
| 2220 | 2248 | ||
| 2249 | static void nvme_tcp_commit_rqs(struct blk_mq_hw_ctx *hctx) | ||
| 2250 | { | ||
| 2251 | struct nvme_tcp_queue *queue = hctx->driver_data; | ||
| 2252 | |||
| 2253 | if (!llist_empty(&queue->req_list)) | ||
| 2254 | queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); | ||
| 2255 | } | ||
| 2256 | |||
| 2221 | static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx, | 2257 | static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx, |
| 2222 | const struct blk_mq_queue_data *bd) | 2258 | const struct blk_mq_queue_data *bd) |
| 2223 | { | 2259 | { |
| @@ -2237,7 +2273,7 @@ static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx, · 13 unmodified lines | |||
| 2237 | 2273 | ||
| 2238 | blk_mq_start_request(rq); | 2274 | blk_mq_start_request(rq); |
| 2239 | 2275 | ||
| 2240 | nvme_tcp_queue_request(req, true); | 2276 | nvme_tcp_queue_request(req, true, bd->last); |
| 2241 | 2277 | ||
| 2242 | return BLK_STS_OK; | 2278 | return BLK_STS_OK; |
| 2243 | } | 2279 | } |
| @@ -2305,6 +2341,7 @@ static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx) · 61 unmodified lines | |||
| 2305 | 2341 | ||
| 2306 | static const struct blk_mq_ops nvme_tcp_mq_ops = { | 2342 | static const struct blk_mq_ops nvme_tcp_mq_ops = { |
| 2307 | .queue_rq = nvme_tcp_queue_rq, | 2343 | .queue_rq = nvme_tcp_queue_rq, |
| 2344 | .commit_rqs = nvme_tcp_commit_rqs, | ||
| 2308 | .complete = nvme_complete_rq, | 2345 | .complete = nvme_complete_rq, |
| 2309 | .init_request = nvme_tcp_init_request, | 2346 | .init_request = nvme_tcp_init_request, |
| 2310 | .exit_request = nvme_tcp_exit_request, | 2347 | .exit_request = nvme_tcp_exit_request, |
| @@ -0,0 +1,256 @@ | |||
| 1 | // SPDX-License-Identifier: GPL-2.0 | ||
| 2 | /* | ||
| 3 | * Copyright (C) 2020 Western Digital Corporation or its affiliates. | ||
| 4 | */ | ||
| 5 | |||
| 6 | #include <linux/blkdev.h> | ||
| 7 | #include <linux/vmalloc.h> | ||
| 8 | #include "nvme.h" | ||
| 9 | |||
| 10 | static int nvme_set_max_append(struct nvme_ctrl *ctrl) | ||
| 11 | { | ||
| 12 | struct nvme_command c = { }; | ||
| 13 | struct nvme_id_ctrl_zns *id; | ||
| 14 | int status; | ||
| 15 | |||
| 16 | id = kzalloc(sizeof(*id), GFP_KERNEL); | ||
| 17 | if (!id) | ||
| 18 | return -ENOMEM; | ||
| 19 | |||
| 20 | c.identify.opcode = nvme_admin_identify; | ||
| 21 | c.identify.cns = NVME_ID_CNS_CS_CTRL; | ||
| 22 | c.identify.csi = NVME_CSI_ZNS; | ||
| 23 | |||
| 24 | status = nvme_submit_sync_cmd(ctrl->admin_q, &c, id, sizeof(*id)); | ||
| 25 | if (status) { | ||
| 26 | kfree(id); | ||
| 27 | return status; | ||
| 28 | } | ||
| 29 | |||
| 30 | if (id->zasl) | ||
| 31 | ctrl->max_zone_append = 1 << (id->zasl + 3); | ||
| 32 | else | ||
| 33 | ctrl->max_zone_append = ctrl->max_hw_sectors; | ||
| 34 | kfree(id); | ||
| 35 | return 0; | ||
| 36 | } | ||
| 37 | |||
| 38 | int nvme_update_zone_info(struct gendisk *disk, struct nvme_ns *ns, | ||
| 39 | unsigned lbaf) | ||
| 40 | { | ||
| 41 | struct nvme_effects_log *log = ns->head->effects; | ||
| 42 | struct request_queue *q = disk->queue; | ||
| 43 | struct nvme_command c = { }; | ||
| 44 | struct nvme_id_ns_zns *id; | ||
| 45 | int status; | ||
| 46 | |||
| 47 | /* Driver requires zone append support */ | ||
| 48 | if (!(le32_to_cpu(log->iocs[nvme_cmd_zone_append]) & | ||
| 49 | NVME_CMD_EFFECTS_CSUPP)) { | ||
| 50 | dev_warn(ns->ctrl->device, | ||
| 51 | "append not supported for zoned namespace:%d\n", | ||
| 52 | ns->head->ns_id); | ||
| 53 | return -EINVAL; | ||
| 54 | } | ||
| 55 | |||
| 56 | /* Lazily query controller append limit for the first zoned namespace */ | ||
| 57 | if (!ns->ctrl->max_zone_append) { | ||
| 58 | status = nvme_set_max_append(ns->ctrl); | ||
| 59 | if (status) | ||
| 60 | return status; | ||
| 61 | } | ||
| 62 | |||
| 63 | id = kzalloc(sizeof(*id), GFP_KERNEL); | ||
| 64 | if (!id) | ||
| 65 | return -ENOMEM; | ||
| 66 | |||
| 67 | c.identify.opcode = nvme_admin_identify; | ||
| 68 | c.identify.nsid = cpu_to_le32(ns->head->ns_id); | ||
| 69 | c.identify.cns = NVME_ID_CNS_CS_NS; | ||
| 70 | c.identify.csi = NVME_CSI_ZNS; | ||
| 71 | |||
| 72 | status = nvme_submit_sync_cmd(ns->ctrl->admin_q, &c, id, sizeof(*id)); | ||
| 73 | if (status) | ||
| 74 | goto free_data; | ||
| 75 | |||
| 76 | /* | ||
| 77 | * We currently do not handle devices requiring any of the zoned | ||
| 78 | * operation characteristics. | ||
| 79 | */ | ||
| 80 | if (id->zoc) { | ||
| 81 | dev_warn(ns->ctrl->device, | ||
| 82 | "zone operations:%x not supported for namespace:%u\n", | ||
| 83 | le16_to_cpu(id->zoc), ns->head->ns_id); | ||
| 84 | status = -EINVAL; | ||
| 85 | goto free_data; | ||
| 86 | } | ||
| 87 | |||
| 88 | ns->zsze = nvme_lba_to_sect(ns, le64_to_cpu(id->lbafe[lbaf].zsze)); | ||
| 89 | if (!is_power_of_2(ns->zsze)) { | ||
| 90 | dev_warn(ns->ctrl->device, | ||
| 91 | "invalid zone size:%llu for namespace:%u\n", | ||
| 92 | ns->zsze, ns->head->ns_id); | ||
| 93 | status = -EINVAL; | ||
| 94 | goto free_data; | ||
| 95 | } | ||
| 96 | |||
| 97 | q->limits.zoned = BLK_ZONED_HM; | ||
| 98 | blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, q); | ||
| 99 | blk_queue_max_open_zones(q, le32_to_cpu(id->mor) + 1); | ||
| 100 | blk_queue_max_active_zones(q, le32_to_cpu(id->mar) + 1); | ||
| 101 | free_data: | ||
| 102 | kfree(id); | ||
| 103 | return status; | ||
| 104 | } | ||
| 105 | |||
| 106 | static void *nvme_zns_alloc_report_buffer(struct nvme_ns *ns, | ||
| 107 | unsigned int nr_zones, size_t *buflen) | ||
| 108 | { | ||
| 109 | struct request_queue *q = ns->disk->queue; | ||
| 110 | size_t bufsize; | ||
| 111 | void *buf; | ||
| 112 | |||
| 113 | const size_t min_bufsize = sizeof(struct nvme_zone_report) + | ||
| 114 | sizeof(struct nvme_zone_descriptor); | ||
| 115 | |||
| 116 | nr_zones = min_t(unsigned int, nr_zones, | ||
| 117 | get_capacity(ns->disk) >> ilog2(ns->zsze)); | ||
| 118 | |||
| 119 | bufsize = sizeof(struct nvme_zone_report) + | ||
| 120 | nr_zones * sizeof(struct nvme_zone_descriptor); | ||
| 121 | bufsize = min_t(size_t, bufsize, | ||
| 122 | queue_max_hw_sectors(q) << SECTOR_SHIFT); | ||
| 123 | bufsize = min_t(size_t, bufsize, queue_max_segments(q) << PAGE_SHIFT); | ||
| 124 | |||
| 125 | while (bufsize >= min_bufsize) { | ||
| 126 | buf = __vmalloc(bufsize, GFP_KERNEL | __GFP_NORETRY); | ||
| 127 | if (buf) { | ||
| 128 | *buflen = bufsize; | ||
| 129 | return buf; | ||
| 130 | } | ||
| 131 | bufsize >>= 1; | ||
| 132 | } | ||
| 133 | return NULL; | ||
| 134 | } | ||
| 135 | |||
| 136 | static int __nvme_ns_report_zones(struct nvme_ns *ns, sector_t sector, | ||
| 137 | struct nvme_zone_report *report, | ||
| 138 | size_t buflen) | ||
| 139 | { | ||
| 140 | struct nvme_command c = { }; | ||
| 141 | int ret; | ||
| 142 | |||
| 143 | c.zmr.opcode = nvme_cmd_zone_mgmt_recv; | ||
| 144 | c.zmr.nsid = cpu_to_le32(ns->head->ns_id); | ||
| 145 | c.zmr.slba = cpu_to_le64(nvme_sect_to_lba(ns, sector)); | ||
| 146 | c.zmr.numd = cpu_to_le32(nvme_bytes_to_numd(buflen)); | ||
| 147 | c.zmr.zra = NVME_ZRA_ZONE_REPORT; | ||
| 148 | c.zmr.zrasf = NVME_ZRASF_ZONE_REPORT_ALL; | ||
| 149 | c.zmr.pr = NVME_REPORT_ZONE_PARTIAL; | ||
| 150 | |||
| 151 | ret = nvme_submit_sync_cmd(ns->queue, &c, report, buflen); | ||
| 152 | if (ret) | ||
| 153 | return ret; | ||
| 154 | |||
| 155 | return le64_to_cpu(report->nr_zones); | ||
| 156 | } | ||
| 157 | |||
| 158 | static int nvme_zone_parse_entry(struct nvme_ns *ns, | ||
| 159 | struct nvme_zone_descriptor *entry, | ||
| 160 | unsigned int idx, report_zones_cb cb, | ||
| 161 | void *data) | ||
| 162 | { | ||
| 163 | struct blk_zone zone = { }; | ||
| 164 | |||
| 165 | if ((entry->zt & 0xf) != NVME_ZONE_TYPE_SEQWRITE_REQ) { | ||
| 166 | dev_err(ns->ctrl->device, "invalid zone type %#x\n", | ||
| 167 | entry->zt); | ||
| 168 | return -EINVAL; | ||
| 169 | } | ||
| 170 | |||
| 171 | zone.type = BLK_ZONE_TYPE_SEQWRITE_REQ; | ||
| 172 | zone.cond = entry->zs >> 4; | ||
| 173 | zone.len = ns->zsze; | ||
| 174 | zone.capacity = nvme_lba_to_sect(ns, le64_to_cpu(entry->zcap)); | ||
| 175 | zone.start = nvme_lba_to_sect(ns, le64_to_cpu(entry->zslba)); | ||
| 176 | zone.wp = nvme_lba_to_sect(ns, le64_to_cpu(entry->wp)); | ||
| 177 | |||
| 178 | return cb(&zone, idx, data); | ||
| 179 | } | ||
| 180 | |||
| 181 | static int nvme_ns_report_zones(struct nvme_ns *ns, sector_t sector, | ||
| 182 | unsigned int nr_zones, report_zones_cb cb, void *data) | ||
| 183 | { | ||
| 184 | struct nvme_zone_report *report; | ||
| 185 | int ret, zone_idx = 0; | ||
| 186 | unsigned int nz, i; | ||
| 187 | size_t buflen; | ||
| 188 | |||
| 189 | report = nvme_zns_alloc_report_buffer(ns, nr_zones, &buflen); | ||
| 190 | if (!report) | ||
| 191 | return -ENOMEM; | ||
| 192 | |||
| 193 | sector &= ~(ns->zsze - 1); | ||
| 194 | while (zone_idx < nr_zones && sector < get_capacity(ns->disk)) { | ||
| 195 | memset(report, 0, buflen); | ||
| 196 | ret = __nvme_ns_report_zones(ns, sector, report, buflen); | ||
| 197 | if (ret < 0) | ||
| 198 | goto out_free; | ||
| 199 | |||
| 200 | nz = min_t(unsigned int, ret, nr_zones); | ||
| 201 | if (!nz) | ||
| 202 | break; | ||
| 203 | |||
| 204 | for (i = 0; i < nz && zone_idx < nr_zones; i++) { | ||
| 205 | ret = nvme_zone_parse_entry(ns, &report->entries[i], | ||
| 206 | zone_idx, cb, data); | ||
| 207 | if (ret) | ||
| 208 | goto out_free; | ||
| 209 | zone_idx++; | ||
| 210 | } | ||
| 211 | |||
| 212 | sector += ns->zsze * nz; | ||
| 213 | } | ||
| 214 | |||
| 215 | if (zone_idx > 0) | ||
| 216 | ret = zone_idx; | ||
| 217 | else | ||
| 218 | ret = -EINVAL; | ||
| 219 | out_free: | ||
| 220 | kvfree(report); | ||
| 221 | return ret; | ||
| 222 | } | ||
| 223 | |||
| 224 | int nvme_report_zones(struct gendisk *disk, sector_t sector, | ||
| 225 | unsigned int nr_zones, report_zones_cb cb, void *data) | ||
| 226 | { | ||
| 227 | struct nvme_ns_head *head = NULL; | ||
| 228 | struct nvme_ns *ns; | ||
| 229 | int srcu_idx, ret; | ||
| 230 | |||
| 231 | ns = nvme_get_ns_from_disk(disk, &head, &srcu_idx); | ||
| 232 | if (unlikely(!ns)) | ||
| 233 | return -EWOULDBLOCK; | ||
| 234 | |||
| 235 | if (ns->head->ids.csi == NVME_CSI_ZNS) | ||
| 236 | ret = nvme_ns_report_zones(ns, sector, nr_zones, cb, data); | ||
| 237 | else | ||
| 238 | ret = -EINVAL; | ||
| 239 | nvme_put_ns_from_disk(head, srcu_idx); | ||
| 240 | |||
| 241 | return ret; | ||
| 242 | } | ||
| 243 | |||
| 244 | blk_status_t nvme_setup_zone_mgmt_send(struct nvme_ns *ns, struct request *req, | ||
| 245 | struct nvme_command *c, enum nvme_zone_mgmt_action action) | ||
| 246 | { | ||
| 247 | c->zms.opcode = nvme_cmd_zone_mgmt_send; | ||
| 248 | c->zms.nsid = cpu_to_le32(ns->head->ns_id); | ||
| 249 | c->zms.slba = cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req))); | ||
| 250 | c->zms.zsa = action; | ||
| 251 | |||
| 252 | if (req_op(req) == REQ_OP_ZONE_RESET_ALL) | ||
| 253 | c->zms.select_all = 1; | ||
| 254 | |||
| 255 | return BLK_STS_OK; | ||
| 256 | } | ||
| @@ -427,7 +427,7 @@ static void nvmet_execute_identify_ctrl(struct nvmet_req *req) · 426 unmodified lines | |||
| 427 | id->awupf = 0; | 427 | id->awupf = 0; |
| 428 | 428 | ||
| 429 | id->sgls = cpu_to_le32(1 << 0); /* we always support SGLs */ | 429 | id->sgls = cpu_to_le32(1 << 0); /* we always support SGLs */ |
| 430 | if (ctrl->ops->has_keyed_sgls) | 430 | if (ctrl->ops->flags & NVMF_KEYED_SGLS) |
| 431 | id->sgls |= cpu_to_le32(1 << 2); | 431 | id->sgls |= cpu_to_le32(1 << 2); |
| 432 | if (req->port->inline_data_size) | 432 | if (req->port->inline_data_size) |
| 433 | id->sgls |= cpu_to_le32(1 << 20); | 433 | id->sgls |= cpu_to_le32(1 << 20); |
| @@ -862,14 +862,14 @@ static ssize_t nvmet_subsys_attr_version_show(struct config_item *item, · 861 unmodified lines | |||
| 862 | struct nvmet_subsys *subsys = to_subsys(item); | 862 | struct nvmet_subsys *subsys = to_subsys(item); |
| 863 | 863 | ||
| 864 | if (NVME_TERTIARY(subsys->ver)) | 864 | if (NVME_TERTIARY(subsys->ver)) |
| 865 | return snprintf(page, PAGE_SIZE, "%d.%d.%d\n", | 865 | return snprintf(page, PAGE_SIZE, "%llu.%llu.%llu\n", |
| 866 | (int)NVME_MAJOR(subsys->ver), | 866 | NVME_MAJOR(subsys->ver), |
| 867 | (int)NVME_MINOR(subsys->ver), | 867 | NVME_MINOR(subsys->ver), |
| 868 | (int)NVME_TERTIARY(subsys->ver)); | 868 | NVME_TERTIARY(subsys->ver)); |
| 869 | 869 | ||
| 870 | return snprintf(page, PAGE_SIZE, "%d.%d\n", | 870 | return snprintf(page, PAGE_SIZE, "%llu.%llu\n", |
| 871 | (int)NVME_MAJOR(subsys->ver), | 871 | NVME_MAJOR(subsys->ver), |
| 872 | (int)NVME_MINOR(subsys->ver)); | 872 | NVME_MINOR(subsys->ver)); |
| 873 | } | 873 | } |
| 874 | 874 | ||
| 875 | static ssize_t nvmet_subsys_attr_version_store(struct config_item *item, | 875 | static ssize_t nvmet_subsys_attr_version_store(struct config_item *item, |
| @@ -336,7 +336,7 @@ int nvmet_enable_port(struct nvmet_port *port) · 335 unmodified lines | |||
| 336 | * If the user requested PI support and the transport isn't pi capable, | 336 | * If the user requested PI support and the transport isn't pi capable, |
| 337 | * don't enable the port. | 337 | * don't enable the port. |
| 338 | */ | 338 | */ |
| 339 | if (port->pi_enable && !ops->metadata_support) { | 339 | if (port->pi_enable && !(ops->flags & NVMF_METADATA_SUPPORTED)) { |
| 340 | pr_err("T10-PI is not supported by transport type %d\n", | 340 | pr_err("T10-PI is not supported by transport type %d\n", |
| 341 | port->disc_addr.trtype); | 341 | port->disc_addr.trtype); |
| 342 | ret = -EINVAL; | 342 | ret = -EINVAL; |
| @@ -277,7 +277,7 @@ static void nvmet_execute_disc_identify(struct nvmet_req *req) · 276 unmodified lines | |||
| 277 | id->maxcmd = cpu_to_le16(NVMET_MAX_CMD); | 277 | id->maxcmd = cpu_to_le16(NVMET_MAX_CMD); |
| 278 | 278 | ||
| 279 | id->sgls = cpu_to_le32(1 << 0); /* we always support SGLs */ | 279 | id->sgls = cpu_to_le32(1 << 0); /* we always support SGLs */ |
| 280 | if (ctrl->ops->has_keyed_sgls) | 280 | if (ctrl->ops->flags & NVMF_KEYED_SGLS) |
| 281 | id->sgls |= cpu_to_le32(1 << 2); | 281 | id->sgls |= cpu_to_le32(1 << 2); |
| 282 | if (req->port->inline_data_size) | 282 | if (req->port->inline_data_size) |
| 283 | id->sgls |= cpu_to_le32(1 << 20); | 283 | id->sgls |= cpu_to_le32(1 << 20); |
| @@ -43,6 +43,17 @@ static const match_table_t opt_tokens = { · 42 unmodified lines | |||
| 43 | { NVMF_OPT_ERR, NULL } | 43 | { NVMF_OPT_ERR, NULL } |
| 44 | }; | 44 | }; |
| 45 | 45 | ||
| 46 | static int fcloop_verify_addr(substring_t *s) | ||
| 47 | { | ||
| 48 | size_t blen = s->to - s->from + 1; | ||
| 49 | |||
| 50 | if (strnlen(s->from, blen) != NVME_FC_TRADDR_HEXNAMELEN + 2 || | ||
| 51 | strncmp(s->from, "0x", 2)) | ||
| 52 | return -EINVAL; | ||
| 53 | |||
| 54 | return 0; | ||
| 55 | } | ||
| 56 | |||
| 46 | static int | 57 | static int |
| 47 | fcloop_parse_options(struct fcloop_ctrl_options *opts, | 58 | fcloop_parse_options(struct fcloop_ctrl_options *opts, |
| 48 | const char *buf) | 59 | const char *buf) |
| @@ -64,14 +75,16 @@ fcloop_parse_options(struct fcloop_ctrl_options *opts, · 15 unmodified lines | |||
| 64 | opts->mask |= token; | 75 | opts->mask |= token; |
| 65 | switch (token) { | 76 | switch (token) { |
| 66 | case NVMF_OPT_WWNN: | 77 | case NVMF_OPT_WWNN: |
| 67 | if (match_u64(args, &token64)) { | 78 | if (fcloop_verify_addr(args) || |
| 79 | match_u64(args, &token64)) { | ||
| 68 | ret = -EINVAL; | 80 | ret = -EINVAL; |
| 69 | goto out_free_options; | 81 | goto out_free_options; |
| 70 | } | 82 | } |
| 71 | opts->wwnn = token64; | 83 | opts->wwnn = token64; |
| 72 | break; | 84 | break; |
| 73 | case NVMF_OPT_WWPN: | 85 | case NVMF_OPT_WWPN: |
| 74 | if (match_u64(args, &token64)) { | 86 | if (fcloop_verify_addr(args) || |
| 87 | match_u64(args, &token64)) { | ||
| 75 | ret = -EINVAL; | 88 | ret = -EINVAL; |
| 76 | goto out_free_options; | 89 | goto out_free_options; |
| 77 | } | 90 | } |
| @@ -92,14 +105,16 @@ fcloop_parse_options(struct fcloop_ctrl_options *opts, · 14 unmodified lines | |||
| 92 | opts->fcaddr = token; | 105 | opts->fcaddr = token; |
| 93 | break; | 106 | break; |
| 94 | case NVMF_OPT_LPWWNN: | 107 | case NVMF_OPT_LPWWNN: |
| 95 | if (match_u64(args, &token64)) { | 108 | if (fcloop_verify_addr(args) || |
| 109 | match_u64(args, &token64)) { | ||
| 96 | ret = -EINVAL; | 110 | ret = -EINVAL; |
| 97 | goto out_free_options; | 111 | goto out_free_options; |
| 98 | } | 112 | } |
| 99 | opts->lpwwnn = token64; | 113 | opts->lpwwnn = token64; |
| 100 | break; | 114 | break; |
| 101 | case NVMF_OPT_LPWWPN: | 115 | case NVMF_OPT_LPWWPN: |
| 102 | if (match_u64(args, &token64)) { | 116 | if (fcloop_verify_addr(args) || |
| 117 | match_u64(args, &token64)) { | ||
| 103 | ret = -EINVAL; | 118 | ret = -EINVAL; |
| 104 | goto out_free_options; | 119 | goto out_free_options; |
| 105 | } | 120 | } |
| @@ -141,14 +156,16 @@ fcloop_parse_nm_options(struct device *dev, u64 *nname, u64 *pname, · 35 unmodified lines | |||
| 141 | token = match_token(p, opt_tokens, args); | 156 | token = match_token(p, opt_tokens, args); |
| 142 | switch (token) { | 157 | switch (token) { |
| 143 | case NVMF_OPT_WWNN: | 158 | case NVMF_OPT_WWNN: |
| 144 | if (match_u64(args, &token64)) { | 159 | if (fcloop_verify_addr(args) || |
| 160 | match_u64(args, &token64)) { | ||
| 145 | ret = -EINVAL; | 161 | ret = -EINVAL; |
| 146 | goto out_free_options; | 162 | goto out_free_options; |
| 147 | } | 163 | } |
| 148 | *nname = token64; | 164 | *nname = token64; |
| 149 | break; | 165 | break; |
| 150 | case NVMF_OPT_WWPN: | 166 | case NVMF_OPT_WWPN: |
| 151 | if (match_u64(args, &token64)) { | 167 | if (fcloop_verify_addr(args) || |
| 168 | match_u64(args, &token64)) { | ||
| 152 | ret = -EINVAL; | 169 | ret = -EINVAL; |
| 153 | goto out_free_options; | 170 | goto out_free_options; |
| 154 | } | 171 | } |
| @@ -36,7 +36,6 @@ struct nvme_loop_ctrl { · 35 unmodified lines | |||
| 36 | struct nvme_loop_iod async_event_iod; | 36 | struct nvme_loop_iod async_event_iod; |
| 37 | struct nvme_ctrl ctrl; | 37 | struct nvme_ctrl ctrl; |
| 38 | 38 | ||
| 39 | struct nvmet_ctrl *target_ctrl; | ||
| 40 | struct nvmet_port *port; | 39 | struct nvmet_port *port; |
| 41 | }; | 40 | }; |
| 42 | 41 | ||
| @@ -286,8 +286,9 @@ struct nvmet_fabrics_ops { · 285 unmodified lines | |||
| 286 | struct module *owner; | 286 | struct module *owner; |
| 287 | unsigned int type; | 287 | unsigned int type; |
| 288 | unsigned int msdbd; | 288 | unsigned int msdbd; |
| 289 | bool has_keyed_sgls : 1; | 289 | unsigned int flags; |
| 290 | bool metadata_support : 1; | 290 | #define NVMF_KEYED_SGLS (1 << 0) |
| 291 | #define NVMF_METADATA_SUPPORTED (1 << 1) | ||
| 291 | void (*queue_response)(struct nvmet_req *req); | 292 | void (*queue_response)(struct nvmet_req *req); |
| 292 | int (*add_port)(struct nvmet_port *port); | 293 | int (*add_port)(struct nvmet_port *port); |
| 293 | void (*remove_port)(struct nvmet_port *port); | 294 | void (*remove_port)(struct nvmet_port *port); |
| @@ -1970,8 +1970,7 @@ static const struct nvmet_fabrics_ops nvmet_rdma_ops = { · 1969 unmodified lines | |||
| 1970 | .owner = THIS_MODULE, | 1970 | .owner = THIS_MODULE, |
| 1971 | .type = NVMF_TRTYPE_RDMA, | 1971 | .type = NVMF_TRTYPE_RDMA, |
| 1972 | .msdbd = 1, | 1972 | .msdbd = 1, |
| 1973 | .has_keyed_sgls = 1, | 1973 | .flags = NVMF_KEYED_SGLS | NVMF_METADATA_SUPPORTED, |
| 1974 | .metadata_support = 1, | ||
| 1975 | .add_port = nvmet_rdma_add_port, | 1974 | .add_port = nvmet_rdma_add_port, |
| 1976 | .remove_port = nvmet_rdma_remove_port, | 1975 | .remove_port = nvmet_rdma_remove_port, |
| 1977 | .queue_response = nvmet_rdma_queue_response, | 1976 | .queue_response = nvmet_rdma_queue_response, |
| @@ -459,17 +459,11 @@ static void nvmet_setup_response_pdu(struct nvmet_tcp_cmd *cmd) · 458 unmodified lines | |||
| 459 | static void nvmet_tcp_process_resp_list(struct nvmet_tcp_queue *queue) | 459 | static void nvmet_tcp_process_resp_list(struct nvmet_tcp_queue *queue) |
| 460 | { | 460 | { |
| 461 | struct llist_node *node; | 461 | struct llist_node *node; |
| 462 | struct nvmet_tcp_cmd *cmd; | ||
| 462 | 463 | ||
| 463 | node = llist_del_all(&queue->resp_list); | 464 | for (node = llist_del_all(&queue->resp_list); node; node = node->next) { |
| 464 | if (!node) | 465 | cmd = llist_entry(node, struct nvmet_tcp_cmd, lentry); |
| 465 | return; | ||
| 466 | |||
| 467 | while (node) { | ||
| 468 | struct nvmet_tcp_cmd *cmd = llist_entry(node, | ||
| 469 | struct nvmet_tcp_cmd, lentry); | ||
| 470 | |||
| 471 | list_add(&cmd->entry, &queue->resp_send_list); | 466 | list_add(&cmd->entry, &queue->resp_send_list); |
| 472 | node = node->next; | ||
| 473 | queue->send_list_len++; | 467 | queue->send_list_len++; |
| 474 | } | 468 | } |
| 475 | } | 469 | } |
| @@ -1717,7 +1711,6 @@ static const struct nvmet_fabrics_ops nvmet_tcp_ops = { · 1241 unmodified lines | |||
| 1717 | .owner = THIS_MODULE, | 1711 | .owner = THIS_MODULE, |
| 1718 | .type = NVMF_TRTYPE_TCP, | 1712 | .type = NVMF_TRTYPE_TCP, |
| 1719 | .msdbd = 1, | 1713 | .msdbd = 1, |
| 1720 | .has_keyed_sgls = 0, | ||
| 1721 | .add_port = nvmet_tcp_add_port, | 1714 | .add_port = nvmet_tcp_add_port, |
| 1722 | .remove_port = nvmet_tcp_remove_port, | 1715 | .remove_port = nvmet_tcp_remove_port, |
| 1723 | .queue_response = nvmet_tcp_queue_response, | 1716 | .queue_response = nvmet_tcp_queue_response, |
| @@ -319,7 +319,7 @@ dasd_diag_check_device(struct dasd_device *device) · 318 unmodified lines | |||
| 319 | struct dasd_diag_characteristics *rdc_data; | 319 | struct dasd_diag_characteristics *rdc_data; |
| 320 | struct vtoc_cms_label *label; | 320 | struct vtoc_cms_label *label; |
| 321 | struct dasd_block *block; | 321 | struct dasd_block *block; |
| 322 | struct dasd_diag_bio bio; | 322 | struct dasd_diag_bio *bio; |
| 323 | unsigned int sb, bsize; | 323 | unsigned int sb, bsize; |
| 324 | blocknum_t end_block; | 324 | blocknum_t end_block; |
| 325 | int rc; | 325 | int rc; |
| @@ -395,29 +395,36 @@ dasd_diag_check_device(struct dasd_device *device) · 69 unmodified lines | |||
| 395 | rc = -ENOMEM; | 395 | rc = -ENOMEM; |
| 396 | goto out; | 396 | goto out; |
| 397 | } | 397 | } |
| 398 | bio = kzalloc(sizeof(*bio), GFP_KERNEL); | ||
| 399 | if (bio == NULL) { | ||
| 400 | DBF_DEV_EVENT(DBF_WARNING, device, "%s", | ||
| 401 | "No memory to allocate initialization bio"); | ||
| 402 | rc = -ENOMEM; | ||
| 403 | goto out_label; | ||
| 404 | } | ||
| 398 | rc = 0; | 405 | rc = 0; |
| 399 | end_block = 0; | 406 | end_block = 0; |
| 400 | /* try all sizes - needed for ECKD devices */ | 407 | /* try all sizes - needed for ECKD devices */ |
| 401 | for (bsize = 512; bsize <= PAGE_SIZE; bsize <<= 1) { | 408 | for (bsize = 512; bsize <= PAGE_SIZE; bsize <<= 1) { |
| 402 | mdsk_init_io(device, bsize, 0, &end_block); | 409 | mdsk_init_io(device, bsize, 0, &end_block); |
| 403 | memset(&bio, 0, sizeof (struct dasd_diag_bio)); | 410 | memset(bio, 0, sizeof(*bio)); |
| 404 | bio.type = MDSK_READ_REQ; | 411 | bio->type = MDSK_READ_REQ; |
| 405 | bio.block_number = private->pt_block + 1; | 412 | bio->block_number = private->pt_block + 1; |
| 406 | bio.buffer = label; | 413 | bio->buffer = label; |
| 407 | memset(&private->iob, 0, sizeof (struct dasd_diag_rw_io)); | 414 | memset(&private->iob, 0, sizeof (struct dasd_diag_rw_io)); |
| 408 | private->iob.dev_nr = rdc_data->dev_nr; | 415 | private->iob.dev_nr = rdc_data->dev_nr; |
| 409 | private->iob.key = 0; | 416 | private->iob.key = 0; |
| 410 | private->iob.flags = 0; /* do synchronous io */ | 417 | private->iob.flags = 0; /* do synchronous io */ |
| 411 | private->iob.block_count = 1; | 418 | private->iob.block_count = 1; |
| 412 | private->iob.interrupt_params = 0; | 419 | private->iob.interrupt_params = 0; |
| 413 | private->iob.bio_list = &bio; | 420 | private->iob.bio_list = bio; |
| 414 | private->iob.flaga = DASD_DIAG_FLAGA_DEFAULT; | 421 | private->iob.flaga = DASD_DIAG_FLAGA_DEFAULT; |
| 415 | rc = dia250(&private->iob, RW_BIO); | 422 | rc = dia250(&private->iob, RW_BIO); |
| 416 | if (rc == 3) { | 423 | if (rc == 3) { |
| 417 | pr_warn("%s: A 64-bit DIAG call failed\n", | 424 | pr_warn("%s: A 64-bit DIAG call failed\n", |
| 418 | dev_name(&device->cdev->dev)); | 425 | dev_name(&device->cdev->dev)); |
| 419 | rc = -EOPNOTSUPP; | 426 | rc = -EOPNOTSUPP; |
| 420 | goto out_label; | 427 | goto out_bio; |
| 421 | } | 428 | } |
| 422 | mdsk_term_io(device); | 429 | mdsk_term_io(device); |
| 423 | if (rc == 0) | 430 | if (rc == 0) |
| @@ -427,7 +434,7 @@ dasd_diag_check_device(struct dasd_device *device) · 3 unmodified lines | |||
| 427 | pr_warn("%s: Accessing the DASD failed because of an incorrect format (rc=%d)\n", | 434 | pr_warn("%s: Accessing the DASD failed because of an incorrect format (rc=%d)\n", |
| 428 | dev_name(&device->cdev->dev), rc); | 435 | dev_name(&device->cdev->dev), rc); |
| 429 | rc = -EIO; | 436 | rc = -EIO; |
| 430 | goto out_label; | 437 | goto out_bio; |
| 431 | } | 438 | } |
| 432 | /* check for label block */ | 439 | /* check for label block */ |
| 433 | if (memcmp(label->label_id, DASD_DIAG_CMS1, | 440 | if (memcmp(label->label_id, DASD_DIAG_CMS1, |
| @@ -457,6 +464,8 @@ dasd_diag_check_device(struct dasd_device *device) · 23 unmodified lines | |||
| 457 | (rc == 4) ? ", read-only device" : ""); | 464 | (rc == 4) ? ", read-only device" : ""); |
| 458 | rc = 0; | 465 | rc = 0; |
| 459 | } | 466 | } |
| 467 | out_bio: | ||
| 468 | kfree(bio); | ||
| 460 | out_label: | 469 | out_label: |
| 461 | free_page((long) label); | 470 | free_page((long) label); |
| 462 | out: | 471 | out: |
| @@ -506,7 +515,7 @@ static struct dasd_ccw_req *dasd_diag_build_cp(struct dasd_device *memdev, · 43 unmodified lines | |||
| 506 | struct req_iterator iter; | 515 | struct req_iterator iter; |
| 507 | struct bio_vec bv; | 516 | struct bio_vec bv; |
| 508 | char *dst; | 517 | char *dst; |
| 509 | unsigned int count, datasize; | 518 | unsigned int count; |
| 510 | sector_t recid, first_rec, last_rec; | 519 | sector_t recid, first_rec, last_rec; |
| 511 | unsigned int blksize, off; | 520 | unsigned int blksize, off; |
| 512 | unsigned char rw_cmd; | 521 | unsigned char rw_cmd; |
| @@ -534,10 +543,8 @@ static struct dasd_ccw_req *dasd_diag_build_cp(struct dasd_device *memdev, · 21 unmodified lines | |||
| 534 | if (count != last_rec - first_rec + 1) | 543 | if (count != last_rec - first_rec + 1) |
| 535 | return ERR_PTR(-EINVAL); | 544 | return ERR_PTR(-EINVAL); |
| 536 | /* Build the request */ | 545 | /* Build the request */ |
| 537 | datasize = sizeof(struct dasd_diag_req) + | 546 | cqr = dasd_smalloc_request(DASD_DIAG_MAGIC, 0, struct_size(dreq, bio, count), |
| 538 | count*sizeof(struct dasd_diag_bio); | 547 | memdev, blk_mq_rq_to_pdu(req)); |
| 539 | cqr = dasd_smalloc_request(DASD_DIAG_MAGIC, 0, datasize, memdev, | ||
| 540 | blk_mq_rq_to_pdu(req)); | ||
| 541 | if (IS_ERR(cqr)) | 548 | if (IS_ERR(cqr)) |
| 542 | return cqr; | 549 | return cqr; |
| 543 | 550 | ||
| @@ -59,6 +59,7 @@ static int sd_zbc_parse_report(struct scsi_disk *sdkp, u8 *buf, · 58 unmodified lines | |||
| 59 | zone.non_seq = 1; | 59 | zone.non_seq = 1; |
| 60 | 60 | ||
| 61 | zone.len = logical_to_sectors(sdp, get_unaligned_be64(&buf[8])); | 61 | zone.len = logical_to_sectors(sdp, get_unaligned_be64(&buf[8])); |
| 62 | zone.capacity = zone.len; | ||
| 62 | zone.start = logical_to_sectors(sdp, get_unaligned_be64(&buf[16])); | 63 | zone.start = logical_to_sectors(sdp, get_unaligned_be64(&buf[16])); |
| 63 | zone.wp = logical_to_sectors(sdp, get_unaligned_be64(&buf[24])); | 64 | zone.wp = logical_to_sectors(sdp, get_unaligned_be64(&buf[24])); |
| 64 | if (zone.type != ZBC_ZONE_TYPE_CONV && | 65 | if (zone.type != ZBC_ZONE_TYPE_CONV && |
| @@ -716,6 +717,11 @@ int sd_zbc_read_zones(struct scsi_disk *sdkp, unsigned char *buf) · 651 unmodified lines | |||
| 716 | /* The drive satisfies the kernel restrictions: set it up */ | 717 | /* The drive satisfies the kernel restrictions: set it up */ |
| 717 | blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, q); | 718 | blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, q); |
| 718 | blk_queue_required_elevator_features(q, ELEVATOR_F_ZBD_SEQ_WRITE); | 719 | blk_queue_required_elevator_features(q, ELEVATOR_F_ZBD_SEQ_WRITE); |
| 720 | if (sdkp->zones_max_open == U32_MAX) | ||
| 721 | blk_queue_max_open_zones(q, 0); | ||
| 722 | else | ||
| 723 | blk_queue_max_open_zones(q, sdkp->zones_max_open); | ||
| 724 | blk_queue_max_active_zones(q, 0); | ||
| 719 | nr_zones = round_up(sdkp->capacity, zone_blocks) >> ilog2(zone_blocks); | 725 | nr_zones = round_up(sdkp->capacity, zone_blocks) >> ilog2(zone_blocks); |
| 720 | 726 | ||
| 721 | /* READ16/WRITE16 is mandatory for ZBC disks */ | 727 | /* READ16/WRITE16 is mandatory for ZBC disks */ |
| @@ -513,6 +513,8 @@ struct request_queue { · 512 unmodified lines | |||
| 513 | unsigned int nr_zones; | 513 | unsigned int nr_zones; |
| 514 | unsigned long *conv_zones_bitmap; | 514 | unsigned long *conv_zones_bitmap; |
| 515 | unsigned long *seq_zones_wlock; | 515 | unsigned long *seq_zones_wlock; |
| 516 | unsigned int max_open_zones; | ||
| 517 | unsigned int max_active_zones; | ||
| 516 | #endif /* CONFIG_BLK_DEV_ZONED */ | 518 | #endif /* CONFIG_BLK_DEV_ZONED */ |
| 517 | 519 | ||
| 518 | /* | 520 | /* |
| @@ -722,6 +724,28 @@ static inline bool blk_queue_zone_is_seq(struct request_queue *q, · 203 unmodified lines | |||
| 722 | return true; | 724 | return true; |
| 723 | return !test_bit(blk_queue_zone_no(q, sector), q->conv_zones_bitmap); | 725 | return !test_bit(blk_queue_zone_no(q, sector), q->conv_zones_bitmap); |
| 724 | } | 726 | } |
| 727 | |||
| 728 | static inline void blk_queue_max_open_zones(struct request_queue *q, | ||
| 729 | unsigned int max_open_zones) | ||
| 730 | { | ||
| 731 | q->max_open_zones = max_open_zones; | ||
| 732 | } | ||
| 733 | |||
| 734 | static inline unsigned int queue_max_open_zones(const struct request_queue *q) | ||
| 735 | { | ||
| 736 | return q->max_open_zones; | ||
| 737 | } | ||
| 738 | |||
| 739 | static inline void blk_queue_max_active_zones(struct request_queue *q, | ||
| 740 | unsigned int max_active_zones) | ||
| 741 | { | ||
| 742 | q->max_active_zones = max_active_zones; | ||
| 743 | } | ||
| 744 | |||
| 745 | static inline unsigned int queue_max_active_zones(const struct request_queue *q) | ||
| 746 | { | ||
| 747 | return q->max_active_zones; | ||
| 748 | } | ||
| 725 | #else /* CONFIG_BLK_DEV_ZONED */ | 749 | #else /* CONFIG_BLK_DEV_ZONED */ |
| 726 | static inline unsigned int blk_queue_nr_zones(struct request_queue *q) | 750 | static inline unsigned int blk_queue_nr_zones(struct request_queue *q) |
| 727 | { | 751 | { |
| @@ -737,6 +761,14 @@ static inline unsigned int blk_queue_zone_no(struct request_queue *q, · 9 unmodified lines | |||
| 737 | { | 761 | { |
| 738 | return 0; | 762 | return 0; |
| 739 | } | 763 | } |
| 764 | static inline unsigned int queue_max_open_zones(const struct request_queue *q) | ||
| 765 | { | ||
| 766 | return 0; | ||
| 767 | } | ||
| 768 | static inline unsigned int queue_max_active_zones(const struct request_queue *q) | ||
| 769 | { | ||
| 770 | return 0; | ||
| 771 | } | ||
| 740 | #endif /* CONFIG_BLK_DEV_ZONED */ | 772 | #endif /* CONFIG_BLK_DEV_ZONED */ |
| 741 | 773 | ||
| 742 | static inline bool rq_is_sync(struct request *rq) | 774 | static inline bool rq_is_sync(struct request *rq) |
| @@ -1519,6 +1551,24 @@ static inline sector_t bdev_zone_sectors(struct block_device *bdev) · 776 unmodified lines | |||
| 1519 | return 0; | 1551 | return 0; |
| 1520 | } | 1552 | } |
| 1521 | 1553 | ||
| 1554 | static inline unsigned int bdev_max_open_zones(struct block_device *bdev) | ||
| 1555 | { | ||
| 1556 | struct request_queue *q = bdev_get_queue(bdev); | ||
| 1557 | |||
| 1558 | if (q) | ||
| 1559 | return queue_max_open_zones(q); | ||
| 1560 | return 0; | ||
| 1561 | } | ||
| 1562 | |||
| 1563 | static inline unsigned int bdev_max_active_zones(struct block_device *bdev) | ||
| 1564 | { | ||
| 1565 | struct request_queue *q = bdev_get_queue(bdev); | ||
| 1566 | |||
| 1567 | if (q) | ||
| 1568 | return queue_max_active_zones(q); | ||
| 1569 | return 0; | ||
| 1570 | } | ||
| 1571 | |||
| 1522 | static inline int queue_dma_alignment(const struct request_queue *q) | 1572 | static inline int queue_dma_alignment(const struct request_queue *q) |
| 1523 | { | 1573 | { |
| 1524 | return q ? q->dma_alignment : 511; | 1574 | return q ? q->dma_alignment : 511; |
| @@ -132,6 +132,7 @@ enum { · 131 unmodified lines | |||
| 132 | #define NVME_CAP_TIMEOUT(cap) (((cap) >> 24) & 0xff) | 132 | #define NVME_CAP_TIMEOUT(cap) (((cap) >> 24) & 0xff) |
| 133 | #define NVME_CAP_STRIDE(cap) (((cap) >> 32) & 0xf) | 133 | #define NVME_CAP_STRIDE(cap) (((cap) >> 32) & 0xf) |
| 134 | #define NVME_CAP_NSSRC(cap) (((cap) >> 36) & 0x1) | 134 | #define NVME_CAP_NSSRC(cap) (((cap) >> 36) & 0x1) |
| 135 | #define NVME_CAP_CSS(cap) (((cap) >> 37) & 0xff) | ||
| 135 | #define NVME_CAP_MPSMIN(cap) (((cap) >> 48) & 0xf) | 136 | #define NVME_CAP_MPSMIN(cap) (((cap) >> 48) & 0xf) |
| 136 | #define NVME_CAP_MPSMAX(cap) (((cap) >> 52) & 0xf) | 137 | #define NVME_CAP_MPSMAX(cap) (((cap) >> 52) & 0xf) |
| 137 | 138 | ||
| @@ -162,7 +163,6 @@ enum { · 24 unmodified lines | |||
| 162 | 163 | ||
| 163 | enum { | 164 | enum { |
| 164 | NVME_CC_ENABLE = 1 << 0, | 165 | NVME_CC_ENABLE = 1 << 0, |
| 165 | NVME_CC_CSS_NVM = 0 << 4, | ||
| 166 | NVME_CC_EN_SHIFT = 0, | 166 | NVME_CC_EN_SHIFT = 0, |
| 167 | NVME_CC_CSS_SHIFT = 4, | 167 | NVME_CC_CSS_SHIFT = 4, |
| 168 | NVME_CC_MPS_SHIFT = 7, | 168 | NVME_CC_MPS_SHIFT = 7, |
| @@ -170,6 +170,9 @@ enum { · 1 unmodified line | |||
| 170 | NVME_CC_SHN_SHIFT = 14, | 170 | NVME_CC_SHN_SHIFT = 14, |
| 171 | NVME_CC_IOSQES_SHIFT = 16, | 171 | NVME_CC_IOSQES_SHIFT = 16, |
| 172 | NVME_CC_IOCQES_SHIFT = 20, | 172 | NVME_CC_IOCQES_SHIFT = 20, |
| 173 | NVME_CC_CSS_NVM = 0 << NVME_CC_CSS_SHIFT, | ||
| 174 | NVME_CC_CSS_CSI = 6 << NVME_CC_CSS_SHIFT, | ||
| 175 | NVME_CC_CSS_MASK = 7 << NVME_CC_CSS_SHIFT, | ||
| 173 | NVME_CC_AMS_RR = 0 << NVME_CC_AMS_SHIFT, | 176 | NVME_CC_AMS_RR = 0 << NVME_CC_AMS_SHIFT, |
| 174 | NVME_CC_AMS_WRRU = 1 << NVME_CC_AMS_SHIFT, | 177 | NVME_CC_AMS_WRRU = 1 << NVME_CC_AMS_SHIFT, |
| 175 | NVME_CC_AMS_VS = 7 << NVME_CC_AMS_SHIFT, | 178 | NVME_CC_AMS_VS = 7 << NVME_CC_AMS_SHIFT, |
| @@ -179,6 +182,8 @@ enum { · 3 unmodified lines | |||
| 179 | NVME_CC_SHN_MASK = 3 << NVME_CC_SHN_SHIFT, | 182 | NVME_CC_SHN_MASK = 3 << NVME_CC_SHN_SHIFT, |
| 180 | NVME_CC_IOSQES = NVME_NVM_IOSQES << NVME_CC_IOSQES_SHIFT, | 183 | NVME_CC_IOSQES = NVME_NVM_IOSQES << NVME_CC_IOSQES_SHIFT, |
| 181 | NVME_CC_IOCQES = NVME_NVM_IOCQES << NVME_CC_IOCQES_SHIFT, | 184 | NVME_CC_IOCQES = NVME_NVM_IOCQES << NVME_CC_IOCQES_SHIFT, |
| 185 | NVME_CAP_CSS_NVM = 1 << 0, | ||
| 186 | NVME_CAP_CSS_CSI = 1 << 6, | ||
| 182 | NVME_CSTS_RDY = 1 << 0, | 187 | NVME_CSTS_RDY = 1 << 0, |
| 183 | NVME_CSTS_CFS = 1 << 1, | 188 | NVME_CSTS_CFS = 1 << 1, |
| 184 | NVME_CSTS_NSSRO = 1 << 4, | 189 | NVME_CSTS_NSSRO = 1 << 4, |
| @@ -369,11 +374,37 @@ struct nvme_id_ns { · 184 unmodified lines | |||
| 369 | __u8 vs[3712]; | 374 | __u8 vs[3712]; |
| 370 | }; | 375 | }; |
| 371 | 376 | ||
| 377 | struct nvme_zns_lbafe { | ||
| 378 | __le64 zsze; | ||
| 379 | __u8 zdes; | ||
| 380 | __u8 rsvd9[7]; | ||
| 381 | }; | ||
| 382 | |||
| 383 | struct nvme_id_ns_zns { | ||
| 384 | __le16 zoc; | ||
| 385 | __le16 ozcs; | ||
| 386 | __le32 mar; | ||
| 387 | __le32 mor; | ||
| 388 | __le32 rrl; | ||
| 389 | __le32 frl; | ||
| 390 | __u8 rsvd20[2796]; | ||
| 391 | struct nvme_zns_lbafe lbafe[16]; | ||
| 392 | __u8 rsvd3072[768]; | ||
| 393 | __u8 vs[256]; | ||
| 394 | }; | ||
| 395 | |||
| 396 | struct nvme_id_ctrl_zns { | ||
| 397 | __u8 zasl; | ||
| 398 | __u8 rsvd1[4095]; | ||
| 399 | }; | ||
| 400 | |||
| 372 | enum { | 401 | enum { |
| 373 | NVME_ID_CNS_NS = 0x00, | 402 | NVME_ID_CNS_NS = 0x00, |
| 374 | NVME_ID_CNS_CTRL = 0x01, | 403 | NVME_ID_CNS_CTRL = 0x01, |
| 375 | NVME_ID_CNS_NS_ACTIVE_LIST = 0x02, | 404 | NVME_ID_CNS_NS_ACTIVE_LIST = 0x02, |
| 376 | NVME_ID_CNS_NS_DESC_LIST = 0x03, | 405 | NVME_ID_CNS_NS_DESC_LIST = 0x03, |
| 406 | NVME_ID_CNS_CS_NS = 0x05, | ||
| 407 | NVME_ID_CNS_CS_CTRL = 0x06, | ||
| 377 | NVME_ID_CNS_NS_PRESENT_LIST = 0x10, | 408 | NVME_ID_CNS_NS_PRESENT_LIST = 0x10, |
| 378 | NVME_ID_CNS_NS_PRESENT = 0x11, | 409 | NVME_ID_CNS_NS_PRESENT = 0x11, |
| 379 | NVME_ID_CNS_CTRL_NS_LIST = 0x12, | 410 | NVME_ID_CNS_CTRL_NS_LIST = 0x12, |
| @@ -383,6 +414,11 @@ enum { · 3 unmodified lines | |||
| 383 | NVME_ID_CNS_UUID_LIST = 0x17, | 414 | NVME_ID_CNS_UUID_LIST = 0x17, |
| 384 | }; | 415 | }; |
| 385 | 416 | ||
| 417 | enum { | ||
| 418 | NVME_CSI_NVM = 0, | ||
| 419 | NVME_CSI_ZNS = 2, | ||
| 420 | }; | ||
| 421 | |||
| 386 | enum { | 422 | enum { |
| 387 | NVME_DIR_IDENTIFY = 0x00, | 423 | NVME_DIR_IDENTIFY = 0x00, |
| 388 | NVME_DIR_STREAMS = 0x01, | 424 | NVME_DIR_STREAMS = 0x01, |
| @@ -435,11 +471,13 @@ struct nvme_ns_id_desc { · 46 unmodified lines | |||
| 435 | #define NVME_NIDT_EUI64_LEN 8 | 471 | #define NVME_NIDT_EUI64_LEN 8 |
| 436 | #define NVME_NIDT_NGUID_LEN 16 | 472 | #define NVME_NIDT_NGUID_LEN 16 |
| 437 | #define NVME_NIDT_UUID_LEN 16 | 473 | #define NVME_NIDT_UUID_LEN 16 |
| 474 | #define NVME_NIDT_CSI_LEN 1 | ||
| 438 | 475 | ||
| 439 | enum { | 476 | enum { |
| 440 | NVME_NIDT_EUI64 = 0x01, | 477 | NVME_NIDT_EUI64 = 0x01, |
| 441 | NVME_NIDT_NGUID = 0x02, | 478 | NVME_NIDT_NGUID = 0x02, |
| 442 | NVME_NIDT_UUID = 0x03, | 479 | NVME_NIDT_UUID = 0x03, |
| 480 | NVME_NIDT_CSI = 0x04, | ||
| 443 | }; | 481 | }; |
| 444 | 482 | ||
| 445 | struct nvme_smart_log { | 483 | struct nvme_smart_log { |
| @@ -519,6 +557,27 @@ struct nvme_ana_rsp_hdr { · 73 unmodified lines | |||
| 519 | __le16 rsvd10[3]; | 557 | __le16 rsvd10[3]; |
| 520 | }; | 558 | }; |
| 521 | 559 | ||
| 560 | struct nvme_zone_descriptor { | ||
| 561 | __u8 zt; | ||
| 562 | __u8 zs; | ||
| 563 | __u8 za; | ||
| 564 | __u8 rsvd3[5]; | ||
| 565 | __le64 zcap; | ||
| 566 | __le64 zslba; | ||
| 567 | __le64 wp; | ||
| 568 | __u8 rsvd32[32]; | ||
| 569 | }; | ||
| 570 | |||
| 571 | enum { | ||
| 572 | NVME_ZONE_TYPE_SEQWRITE_REQ = 0x2, | ||
| 573 | }; | ||
| 574 | |||
| 575 | struct nvme_zone_report { | ||
| 576 | __le64 nr_zones; | ||
| 577 | __u8 resv8[56]; | ||
| 578 | struct nvme_zone_descriptor entries[]; | ||
| 579 | }; | ||
| 580 | |||
| 522 | enum { | 581 | enum { |
| 523 | NVME_SMART_CRIT_SPARE = 1 << 0, | 582 | NVME_SMART_CRIT_SPARE = 1 << 0, |
| 524 | NVME_SMART_CRIT_TEMPERATURE = 1 << 1, | 583 | NVME_SMART_CRIT_TEMPERATURE = 1 << 1, |
| @@ -613,6 +672,9 @@ enum nvme_opcode { · 88 unmodified lines | |||
| 613 | nvme_cmd_resv_report = 0x0e, | 672 | nvme_cmd_resv_report = 0x0e, |
| 614 | nvme_cmd_resv_acquire = 0x11, | 673 | nvme_cmd_resv_acquire = 0x11, |
| 615 | nvme_cmd_resv_release = 0x15, | 674 | nvme_cmd_resv_release = 0x15, |
| 675 | nvme_cmd_zone_mgmt_send = 0x79, | ||
| 676 | nvme_cmd_zone_mgmt_recv = 0x7a, | ||
| 677 | nvme_cmd_zone_append = 0x7d, | ||
| 616 | }; | 678 | }; |
| 617 | 679 | ||
| 618 | #define nvme_opcode_name(opcode) { opcode, #opcode } | 680 | #define nvme_opcode_name(opcode) { opcode, #opcode } |
| @@ -751,6 +813,7 @@ struct nvme_rw_command { · 132 unmodified lines | |||
| 751 | enum { | 813 | enum { |
| 752 | NVME_RW_LR = 1 << 15, | 814 | NVME_RW_LR = 1 << 15, |
| 753 | NVME_RW_FUA = 1 << 14, | 815 | NVME_RW_FUA = 1 << 14, |
| 816 | NVME_RW_APPEND_PIREMAP = 1 << 9, | ||
| 754 | NVME_RW_DSM_FREQ_UNSPEC = 0, | 817 | NVME_RW_DSM_FREQ_UNSPEC = 0, |
| 755 | NVME_RW_DSM_FREQ_TYPICAL = 1, | 818 | NVME_RW_DSM_FREQ_TYPICAL = 1, |
| 756 | NVME_RW_DSM_FREQ_RARE = 2, | 819 | NVME_RW_DSM_FREQ_RARE = 2, |
| @@ -816,6 +879,53 @@ struct nvme_write_zeroes_cmd { · 59 unmodified lines | |||
| 816 | __le16 appmask; | 879 | __le16 appmask; |
| 817 | }; | 880 | }; |
| 818 | 881 | ||
| 882 | enum nvme_zone_mgmt_action { | ||
| 883 | NVME_ZONE_CLOSE = 0x1, | ||
| 884 | NVME_ZONE_FINISH = 0x2, | ||
| 885 | NVME_ZONE_OPEN = 0x3, | ||
| 886 | NVME_ZONE_RESET = 0x4, | ||
| 887 | NVME_ZONE_OFFLINE = 0x5, | ||
| 888 | NVME_ZONE_SET_DESC_EXT = 0x10, | ||
| 889 | }; | ||
| 890 | |||
| 891 | struct nvme_zone_mgmt_send_cmd { | ||
| 892 | __u8 opcode; | ||
| 893 | __u8 flags; | ||
| 894 | __u16 command_id; | ||
| 895 | __le32 nsid; | ||
| 896 | __le32 cdw2[2]; | ||
| 897 | __le64 metadata; | ||
| 898 | union nvme_data_ptr dptr; | ||
| 899 | __le64 slba; | ||
| 900 | __le32 cdw12; | ||
| 901 | __u8 zsa; | ||
| 902 | __u8 select_all; | ||
| 903 | __u8 rsvd13[2]; | ||
| 904 | __le32 cdw14[2]; | ||
| 905 | }; | ||
| 906 | |||
| 907 | struct nvme_zone_mgmt_recv_cmd { | ||
| 908 | __u8 opcode; | ||
| 909 | __u8 flags; | ||
| 910 | __u16 command_id; | ||
| 911 | __le32 nsid; | ||
| 912 | __le64 rsvd2[2]; | ||
| 913 | union nvme_data_ptr dptr; | ||
| 914 | __le64 slba; | ||
| 915 | __le32 numd; | ||
| 916 | __u8 zra; | ||
| 917 | __u8 zrasf; | ||
| 918 | __u8 pr; | ||
| 919 | __u8 rsvd13; | ||
| 920 | __le32 cdw14[2]; | ||
| 921 | }; | ||
| 922 | |||
| 923 | enum { | ||
| 924 | NVME_ZRA_ZONE_REPORT = 0, | ||
| 925 | NVME_ZRASF_ZONE_REPORT_ALL = 0, | ||
| 926 | NVME_REPORT_ZONE_PARTIAL = 1, | ||
| 927 | }; | ||
| 928 | |||
| 819 | /* Features */ | 929 | /* Features */ |
| 820 | 930 | ||
| 821 | enum { | 931 | enum { |
| @@ -972,7 +1082,9 @@ struct nvme_identify { · 150 unmodified lines | |||
| 972 | __u8 cns; | 1082 | __u8 cns; |
| 973 | __u8 rsvd3; | 1083 | __u8 rsvd3; |
| 974 | __le16 ctrlid; | 1084 | __le16 ctrlid; |
| 975 | __u32 rsvd11[5]; | 1085 | __u8 rsvd11[3]; |
| 1086 | __u8 csi; | ||
| 1087 | __u32 rsvd12[4]; | ||
| 976 | }; | 1088 | }; |
| 977 | 1089 | ||
| 978 | #define NVME_IDENTIFY_DATA_SIZE 4096 | 1090 | #define NVME_IDENTIFY_DATA_SIZE 4096 |
| @@ -1086,7 +1198,9 @@ struct nvme_get_log_page_command { · 107 unmodified lines | |||
| 1086 | }; | 1198 | }; |
| 1087 | __le64 lpo; | 1199 | __le64 lpo; |
| 1088 | }; | 1200 | }; |
| 1089 | __u32 rsvd14[2]; | 1201 | __u8 rsvd14[3]; |
| 1202 | __u8 csi; | ||
| 1203 | __u32 rsvd15; | ||
| 1090 | }; | 1204 | }; |
| 1091 | 1205 | ||
| 1092 | struct nvme_directive_cmd { | 1206 | struct nvme_directive_cmd { |
| @@ -1283,6 +1397,8 @@ struct nvme_command { · 190 unmodified lines | |||
| 1283 | struct nvme_format_cmd format; | 1397 | struct nvme_format_cmd format; |
| 1284 | struct nvme_dsm_cmd dsm; | 1398 | struct nvme_dsm_cmd dsm; |
| 1285 | struct nvme_write_zeroes_cmd write_zeroes; | 1399 | struct nvme_write_zeroes_cmd write_zeroes; |
| 1400 | struct nvme_zone_mgmt_send_cmd zms; | ||
| 1401 | struct nvme_zone_mgmt_recv_cmd zmr; | ||
| 1286 | struct nvme_abort_cmd abort; | 1402 | struct nvme_abort_cmd abort; |
| 1287 | struct nvme_get_log_page_command get_log_page; | 1403 | struct nvme_get_log_page_command get_log_page; |
| 1288 | struct nvmf_common_command fabrics; | 1404 | struct nvmf_common_command fabrics; |
| @@ -1416,6 +1532,18 @@ enum { · 127 unmodified lines | |||
| 1416 | NVME_SC_DISCOVERY_RESTART = 0x190, | 1532 | NVME_SC_DISCOVERY_RESTART = 0x190, |
| 1417 | NVME_SC_AUTH_REQUIRED = 0x191, | 1533 | NVME_SC_AUTH_REQUIRED = 0x191, |
| 1418 | 1534 | ||
| 1535 | /* | ||
| 1536 | * I/O Command Set Specific - Zoned commands: | ||
| 1537 | */ | ||
| 1538 | NVME_SC_ZONE_BOUNDARY_ERROR = 0x1b8, | ||
| 1539 | NVME_SC_ZONE_FULL = 0x1b9, | ||
| 1540 | NVME_SC_ZONE_READ_ONLY = 0x1ba, | ||
| 1541 | NVME_SC_ZONE_OFFLINE = 0x1bb, | ||
| 1542 | NVME_SC_ZONE_INVALID_WRITE = 0x1bc, | ||
| 1543 | NVME_SC_ZONE_TOO_MANY_ACTIVE = 0x1bd, | ||
| 1544 | NVME_SC_ZONE_TOO_MANY_OPEN = 0x1be, | ||
| 1545 | NVME_SC_ZONE_INVALID_TRANSITION = 0x1bf, | ||
| 1546 | |||
| 1419 | /* | 1547 | /* |
| 1420 | * Media and Data Integrity Errors: | 1548 | * Media and Data Integrity Errors: |
| 1421 | */ | 1549 | */ |
| @@ -73,6 +73,15 @@ enum blk_zone_cond { · 72 unmodified lines | |||
| 73 | BLK_ZONE_COND_OFFLINE = 0xF, | 73 | BLK_ZONE_COND_OFFLINE = 0xF, |
| 74 | }; | 74 | }; |
| 75 | 75 | ||
| 76 | /** | ||
| 77 | * enum blk_zone_report_flags - Feature flags of reported zone descriptors. | ||
| 78 | * | ||
| 79 | * @BLK_ZONE_REP_CAPACITY: Zone descriptor has capacity field. | ||
| 80 | */ | ||
| 81 | enum blk_zone_report_flags { | ||
| 82 | BLK_ZONE_REP_CAPACITY = (1 << 0), | ||
| 83 | }; | ||
| 84 | |||
| 76 | /** | 85 | /** |
| 77 | * struct blk_zone - Zone descriptor for BLKREPORTZONE ioctl. | 86 | * struct blk_zone - Zone descriptor for BLKREPORTZONE ioctl. |
| 78 | * | 87 | * |
| @@ -99,7 +108,9 @@ struct blk_zone { · 20 unmodified lines | |||
| 99 | __u8 cond; /* Zone condition */ | 108 | __u8 cond; /* Zone condition */ |
| 100 | __u8 non_seq; /* Non-sequential write resources active */ | 109 | __u8 non_seq; /* Non-sequential write resources active */ |
| 101 | __u8 reset; /* Reset write pointer recommended */ | 110 | __u8 reset; /* Reset write pointer recommended */ |
| 102 | __u8 reserved[36]; | 111 | __u8 resv[4]; |
| 112 | __u64 capacity; /* Zone capacity in number of sectors */ | ||
| 113 | __u8 reserved[24]; | ||
| 103 | }; | 114 | }; |
| 104 | 115 | ||
| 105 | /** | 116 | /** |
| @@ -115,7 +126,7 @@ struct blk_zone { · 9 unmodified lines | |||
| 115 | struct blk_zone_report { | 126 | struct blk_zone_report { |
| 116 | __u64 sector; | 127 | __u64 sector; |
| 117 | __u32 nr_zones; | 128 | __u32 nr_zones; |
| 118 | __u8 reserved[4]; | 129 | __u32 flags; |
| 119 | struct blk_zone zones[0]; | 130 | struct blk_zone zones[0]; |
| 120 | }; | 131 | }; |
| 121 | 132 | ||