History

Merge tag 'for-5.17/drivers-2022-01-11' of git://git.kernel.dk/linux-block

Pull block driver updates from Jens Axboe: - mtip32xx pci cleanups (Bjorn) - mtip32xx conversion to generic power management (Vaibhav) - rsxx pci powermanagement cleanups (Bjorn) - Remove the rsxx driver. This hardware never saw much adoption, and it's been end of lifed for a while. (Christoph) - MD pull request from Song: - REQ_NOWAIT support (Vishal Verma) - raid6 benchmark optimization (Dirk Müller) - Fix for acct bioset (Xiao Ni) - Clean up max_queued_requests (Mariusz Tkaczyk) - PREEMPT_RT optimization (Davidlohr Bueso) - Use default_groups in kobj_type (Greg Kroah-Hartman) - Use attribute groups in pktcdvd and rnbd (Greg) - NVMe pull request from Christoph: - increment request genctr on completion (Keith Busch, Geliang Tang) - add a 'iopolicy' module parameter (Hannes Reinecke) - print out valid arguments when reading from /dev/nvme-fabrics (Hannes Reinecke) - Use struct_group() in drbd (Kees) - null_blk fixes (Ming) - Get rid of congestion logic in pktcdvd (Neil) - Floppy ejection hang fix (Tasos) - Floppy max user request size fix (Xiongwei) - Loop locking fix (Tetsuo) * tag 'for-5.17/drivers-2022-01-11' of git://git.kernel.dk/linux-block: (32 commits) md: use default_groups in kobj_type md: Move alloc/free acct bioset in to personality lib/raid6: Use strict priority ranking for pq gen() benchmarking lib/raid6: skip benchmark of non-chosen xor_syndrome functions md: fix spelling of "its" md: raid456 add nowait support md: raid10 add nowait support md: raid1 add nowait support md: add support for REQ_NOWAIT md: drop queue limitation for RAID1 and RAID10 md/raid5: play nice with PREEMPT_RT block/rnbd-clt-sysfs: use default_groups in kobj_type pktcdvd: convert to use attribute groups block: null_blk: only set set->nr_maps as 3 if active poll_queues is > 0 nvme: add 'iopolicy' module parameter nvme: drop unused variable ctrl in nvme_setup_cmd nvme: increment request genctr on completion nvme-fabrics: print out valid arguments when reading from /dev/nvme-fabrics block: remove the rsxx driver rsxx: Drop PCI legacy power management ...

Linus Torvalds committed 4 years ago

Commit c9193f42 parents d3c8108 d85bd8240 files · +612 −4501Browse files

Unified · Split

40 files
  1. MAINTAINERS +0 −6
  2. drivers/block/Kconfig +0 −11
  3. drivers/block/Makefile +0 −1
  4. drivers/block/drbd/drbd_main.c +2 −1
  5. drivers/block/drbd/drbd_protocol.h +4 −2
  6. drivers/block/drbd/drbd_receiver.c +2 −1
  7. drivers/block/floppy.c +4 −2
  8. drivers/block/loop.c +52 −56
  9. drivers/block/loop.h +1 −0
  10. drivers/block/mtip32xx/mtip32xx.c +22 −64
  11. drivers/block/null_blk/main.c +6 −6
  12. drivers/block/pktcdvd.c +154 −152
  13. drivers/block/rnbd/rnbd-clt-sysfs.c +2 −1
  14. drivers/block/rsxx/Makefile +0 −3
  15. drivers/block/rsxx/config.c +0 −197
  16. drivers/block/rsxx/core.c +0 −1126
  17. drivers/block/rsxx/cregs.c +0 −789
  18. drivers/block/rsxx/dev.c +0 −306
  19. drivers/block/rsxx/dma.c +0 −1085
  20. drivers/block/rsxx/rsxx.h +0 −33
  21. drivers/block/rsxx/rsxx_cfg.h +0 −58
  22. drivers/block/rsxx/rsxx_priv.h +0 −418
  23. drivers/md/md-cluster.c +1 −1
  24. drivers/md/md.c +41 −12
  25. drivers/md/md.h +2 −0
  26. drivers/md/raid0.c +28 −10
  27. drivers/md/raid1-10.c +0 −6
  28. drivers/md/raid1.c +56 −27
  29. drivers/md/raid10.c +67 −40
  30. drivers/md/raid5.c +51 −16
  31. drivers/md/raid5.h +3 −1
  32. drivers/nvme/host/core.c +1 −6
  33. drivers/nvme/host/fabrics.c +21 −1
  34. drivers/nvme/host/multipath.c +36 −5
  35. drivers/nvme/host/nvme.h +8 −0
  36. include/linux/pktcdvd.h +2 −10
  37. include/linux/raid/pq.h +1 −1
  38. lib/raid6/algos.c +38 −40
  39. lib/raid6/avx2.c +4 −4
  40. lib/raid6/avx512.c +3 −3
MAINTAINERS+0−6
@@ -7489,12 +7489,6 @@ F: Documentation/firmware_class/ · 7488 unmodified lines
74897489 F: drivers/base/firmware_loader/
74907490 F: include/linux/firmware.h
74917491
7492−FLASH ADAPTER DRIVER (IBM Flash Adapter 900GB Full Height PCI Flash Card)
7493−M: Joshua Morris <josh.h.morris@us.ibm.com>
7494−M: Philip Kelleher <pjk1939@linux.ibm.com>
7495−S: Maintained
7496−F: drivers/block/rsxx/
7497−
74987492 FLEXTIMER FTM-QUADDEC DRIVER
74997493 M: Patrick Havelange <patrick.havelange@essensium.com>
75007494 L: linux-iio@vger.kernel.org
drivers/block/Kconfig+0−11
@@ -392,17 +392,6 @@ config BLK_DEV_RBD · 391 unmodified lines
392392
393393 If unsure, say N.
394394
395−config BLK_DEV_RSXX
396− tristate "IBM Flash Adapter 900GB Full Height PCIe Device Driver"
397− depends on PCI
398− select CRC32
399− help
400− Device driver for IBM's high speed PCIe SSD
401− storage device: Flash Adapter 900GB Full Height.
402−
403− To compile this driver as a module, choose M here: the
404− module will be called rsxx.
405−
406395 source "drivers/block/rnbd/Kconfig"
407396
408397 endif # BLK_DEV
drivers/block/Makefile+0−1
@@ -34,7 +34,6 @@ obj-$(CONFIG_BLK_DEV_DRBD) += drbd/ · 33 unmodified lines
3434 obj-$(CONFIG_BLK_DEV_RBD) += rbd.o
3535 obj-$(CONFIG_BLK_DEV_PCIESSD_MTIP32XX) += mtip32xx/
3636
37−obj-$(CONFIG_BLK_DEV_RSXX) += rsxx/
3837 obj-$(CONFIG_ZRAM) += zram/
3938 obj-$(CONFIG_BLK_DEV_RNBD) += rnbd/
4039
drivers/block/drbd/drbd_main.c+2−1
@@ -729,7 +729,8 @@ int drbd_send_sync_param(struct drbd_peer_device *peer_device) · 728 unmodified lines
729729 cmd = apv >= 89 ? P_SYNC_PARAM89 : P_SYNC_PARAM;
730730
731731 /* initialize verify_alg and csums_alg */
732− memset(p->verify_alg, 0, 2 * SHARED_SECRET_MAX);
732+ BUILD_BUG_ON(sizeof(p->algs) != 2 * SHARED_SECRET_MAX);
733+ memset(&p->algs, 0, sizeof(p->algs));
733734
734735 if (get_ldev(peer_device->device)) {
735736 dc = rcu_dereference(peer_device->device->ldev->disk_conf);
drivers/block/drbd/drbd_protocol.h+4−2
@@ -283,8 +283,10 @@ struct p_rs_param_89 { · 282 unmodified lines
283283
284284 struct p_rs_param_95 {
285285 u32 resync_rate;
286− char verify_alg[SHARED_SECRET_MAX];
287− char csums_alg[SHARED_SECRET_MAX];
286+ struct_group(algs,
287+ char verify_alg[SHARED_SECRET_MAX];
288+ char csums_alg[SHARED_SECRET_MAX];
289+ );
288290 u32 c_plan_ahead;
289291 u32 c_delay_target;
290292 u32 c_fill_target;
drivers/block/drbd/drbd_receiver.c+2−1
@@ -3921,7 +3921,8 @@ static int receive_SyncParam(struct drbd_connection *connection, struct packet_i · 3920 unmodified lines
39213921
39223922 /* initialize verify_alg and csums_alg */
39233923 p = pi->data;
3924− memset(p->verify_alg, 0, 2 * SHARED_SECRET_MAX);
3924+ BUILD_BUG_ON(sizeof(p->algs) != 2 * SHARED_SECRET_MAX);
3925+ memset(&p->algs, 0, sizeof(p->algs));
39253926
39263927 err = drbd_recv_all(peer_device->connection, p, header_size);
39273928 if (err)
drivers/block/floppy.c+4−2
@@ -1015,7 +1015,7 @@ static DECLARE_DELAYED_WORK(fd_timer, fd_timer_workfn); · 1014 unmodified lines
10151015 static void cancel_activity(void)
10161016 {
10171017 do_floppy = NULL;
1018− cancel_delayed_work_sync(&fd_timer);
1018+ cancel_delayed_work(&fd_timer);
10191019 cancel_work_sync(&floppy_work);
10201020 }
10211021
@@ -3081,6 +3081,8 @@ static void raw_cmd_free(struct floppy_raw_cmd **ptr) · 2059 unmodified lines
30813081 }
30823082 }
30833083
3084+#define MAX_LEN (1UL << MAX_ORDER << PAGE_SHIFT)
3085+
30843086 static int raw_cmd_copyin(int cmd, void __user *param,
30853087 struct floppy_raw_cmd **rcmd)
30863088 {
@@ -3108,7 +3110,7 @@ loop: · 21 unmodified lines
31083110 ptr->resultcode = 0;
31093111
31103112 if (ptr->flags & (FD_RAW_READ | FD_RAW_WRITE)) {
3111− if (ptr->length <= 0)
3113+ if (ptr->length <= 0 || ptr->length >= MAX_LEN)
31123114 return -EINVAL;
31133115 ptr->kernel_data = (char *)fd_dma_mem_alloc(ptr->length);
31143116 fallback_on_nodma_alloc(&ptr->kernel_data, ptr->length);
drivers/block/loop.c+52−56
@@ -1082,13 +1082,10 @@ out_putf: · 1081 unmodified lines
10821082 return error;
10831083 }
10841084
1085−static int __loop_clr_fd(struct loop_device *lo, bool release)
1085+static void __loop_clr_fd(struct loop_device *lo)
10861086 {
1087− struct file *filp = NULL;
1087+ struct file *filp;
10881088 gfp_t gfp = lo->old_gfp_mask;
1089− int err = 0;
1090− bool partscan = false;
1091− int lo_number;
10921089 struct loop_worker *pos, *worker;
10931090
10941091 /*
@@ -1103,17 +1100,14 @@ static int __loop_clr_fd(struct loop_device *lo, bool release) · 8 unmodified lines
11031100 * became visible.
11041101 */
11051102
1103+ /*
1104+ * Since this function is called upon "ioctl(LOOP_CLR_FD)" xor "close()
1105+ * after ioctl(LOOP_CLR_FD)", it is a sign of something going wrong if
1106+ * lo->lo_state has changed while waiting for lo->lo_mutex.
1107+ */
11061108 mutex_lock(&lo->lo_mutex);
1107− if (WARN_ON_ONCE(lo->lo_state != Lo_rundown)) {
1108− err = -ENXIO;
1109− goto out_unlock;
1110− }
1111−
1112− filp = lo->lo_backing_file;
1113− if (filp == NULL) {
1114− err = -EINVAL;
1115− goto out_unlock;
1116− }
1109+ BUG_ON(lo->lo_state != Lo_rundown);
1110+ mutex_unlock(&lo->lo_mutex);
11171111
11181112 if (test_bit(QUEUE_FLAG_WC, &lo->lo_queue->queue_flags))
11191113 blk_queue_write_cache(lo->lo_queue, false, false);
@@ -1134,6 +1128,7 @@ static int __loop_clr_fd(struct loop_device *lo, bool release) · 14 unmodified lines
11341128 del_timer_sync(&lo->timer);
11351129
11361130 spin_lock_irq(&lo->lo_lock);
1131+ filp = lo->lo_backing_file;
11371132 lo->lo_backing_file = NULL;
11381133 spin_unlock_irq(&lo->lo_lock);
11391134
@@ -1149,60 +1144,59 @@ static int __loop_clr_fd(struct loop_device *lo, bool release) · 9 unmodified lines
11491144 /* let user-space know about this change */
11501145 kobject_uevent(&disk_to_dev(lo->lo_disk)->kobj, KOBJ_CHANGE);
11511146 mapping_set_gfp_mask(filp->f_mapping, gfp);
1152− /* This is safe: open() is still holding a reference. */
1153− module_put(THIS_MODULE);
11541147 blk_mq_unfreeze_queue(lo->lo_queue);
11551148
1156− partscan = lo->lo_flags & LO_FLAGS_PARTSCAN;
1157− lo_number = lo->lo_number;
11581149 disk_force_media_change(lo->lo_disk, DISK_EVENT_MEDIA_CHANGE);
1159−out_unlock:
1160− mutex_unlock(&lo->lo_mutex);
1161− if (partscan) {
1162− /*
1163− * open_mutex has been held already in release path, so don't
1164− * acquire it if this function is called in such case.
1165− *
1166− * If the reread partition isn't from release path, lo_refcnt
1167− * must be at least one and it can only become zero when the
1168− * current holder is released.
1169− */
1170− if (!release)
1171− mutex_lock(&lo->lo_disk->open_mutex);
1150+
1151+ if (lo->lo_flags & LO_FLAGS_PARTSCAN) {
1152+ int err;
1153+
1154+ mutex_lock(&lo->lo_disk->open_mutex);
11721155 err = bdev_disk_changed(lo->lo_disk, false);
1173− if (!release)
1174− mutex_unlock(&lo->lo_disk->open_mutex);
1156+ mutex_unlock(&lo->lo_disk->open_mutex);
11751157 if (err)
11761158 pr_warn("%s: partition scan of loop%d failed (rc=%d)\n",
1177− __func__, lo_number, err);
1159+ __func__, lo->lo_number, err);
11781160 /* Device is gone, no point in returning error */
1179− err = 0;
11801161 }
11811162
1182− /*
1183− * lo->lo_state is set to Lo_unbound here after above partscan has
1184− * finished.
1185− *
1186− * There cannot be anybody else entering __loop_clr_fd() as
1187− * lo->lo_backing_file is already cleared and Lo_rundown state
1188− * protects us from all the other places trying to change the 'lo'
1189− * device.
1190− */
1191− mutex_lock(&lo->lo_mutex);
11921163 lo->lo_flags = 0;
11931164 if (!part_shift)
11941165 lo->lo_disk->flags |= GENHD_FL_NO_PART;
1166+
1167+ fput(filp);
1168+}
1169+
1170+static void loop_rundown_completed(struct loop_device *lo)
1171+{
1172+ mutex_lock(&lo->lo_mutex);
11951173 lo->lo_state = Lo_unbound;
11961174 mutex_unlock(&lo->lo_mutex);
1175+ module_put(THIS_MODULE);
1176+}
11971177
1198− /*
1199− * Need not hold lo_mutex to fput backing file. Calling fput holding
1200− * lo_mutex triggers a circular lock dependency possibility warning as
1201− * fput can take open_mutex which is usually taken before lo_mutex.
1202− */
1203− if (filp)
1204− fput(filp);
1205− return err;
1178+static void loop_rundown_workfn(struct work_struct *work)
1179+{
1180+ struct loop_device *lo = container_of(work, struct loop_device,
1181+ rundown_work);
1182+ struct block_device *bdev = lo->lo_device;
1183+ struct gendisk *disk = lo->lo_disk;
1184+
1185+ __loop_clr_fd(lo);
1186+ kobject_put(&bdev->bd_device.kobj);
1187+ module_put(disk->fops->owner);
1188+ loop_rundown_completed(lo);
1189+}
1190+
1191+static void loop_schedule_rundown(struct loop_device *lo)
1192+{
1193+ struct block_device *bdev = lo->lo_device;
1194+ struct gendisk *disk = lo->lo_disk;
1195+
1196+ __module_get(disk->fops->owner);
1197+ kobject_get(&bdev->bd_device.kobj);
1198+ INIT_WORK(&lo->rundown_work, loop_rundown_workfn);
1199+ queue_work(system_long_wq, &lo->rundown_work);
12061200 }
12071201
12081202 static int loop_clr_fd(struct loop_device *lo)
@@ -1234,7 +1228,9 @@ static int loop_clr_fd(struct loop_device *lo) · 25 unmodified lines
12341228 lo->lo_state = Lo_rundown;
12351229 mutex_unlock(&lo->lo_mutex);
12361230
1237− return __loop_clr_fd(lo, false);
1231+ __loop_clr_fd(lo);
1232+ loop_rundown_completed(lo);
1233+ return 0;
12381234 }
12391235
12401236 static int
@@ -1758,7 +1754,7 @@ static void lo_release(struct gendisk *disk, fmode_t mode) · 517 unmodified lines
17581754 * In autoclear mode, stop the loop thread
17591755 * and remove configuration after last close.
17601756 */
1761− __loop_clr_fd(lo, true);
1757+ loop_schedule_rundown(lo);
17621758 return;
17631759 } else if (lo->lo_state == Lo_bound) {
17641760 /*
drivers/block/loop.h+1−0
@@ -56,6 +56,7 @@ struct loop_device { · 55 unmodified lines
5656 struct gendisk *lo_disk;
5757 struct mutex lo_mutex;
5858 bool idr_visible;
59+ struct work_struct rundown_work;
5960 };
6061
6162 struct loop_cmd {
drivers/block/mtip32xx/mtip32xx.c+22−64
@@ -136,16 +136,15 @@ struct mtip_compat_ide_task_request_s { · 135 unmodified lines
136136 * return value
137137 * true if device removed, else false
138138 */
139−static bool mtip_check_surprise_removal(struct pci_dev *pdev)
139+static bool mtip_check_surprise_removal(struct driver_data *dd)
140140 {
141141 u16 vendor_id = 0;
142− struct driver_data *dd = pci_get_drvdata(pdev);
143142
144143 if (dd->sr)
145144 return true;
146145
147146 /* Read the vendorID from the configuration space */
148− pci_read_config_word(pdev, 0x00, &vendor_id);
147+ pci_read_config_word(dd->pdev, 0x00, &vendor_id);
149148 if (vendor_id == 0xFFFF) {
150149 dd->sr = true;
151150 if (dd->queue)
@@ -447,7 +446,7 @@ static int mtip_device_reset(struct driver_data *dd) · 295 unmodified lines
447446 {
448447 int rv = 0;
449448
450− if (mtip_check_surprise_removal(dd->pdev))
449+ if (mtip_check_surprise_removal(dd))
451450 return 0;
452451
453452 if (mtip_hba_reset(dd) < 0)
@@ -727,7 +726,7 @@ static inline void mtip_process_errors(struct driver_data *dd, u32 port_stat) · 273 unmodified lines
727726 dev_warn(&dd->pdev->dev,
728727 "Port stat errors %x unhandled\n",
729728 (port_stat & ~PORT_IRQ_HANDLED));
730− if (mtip_check_surprise_removal(dd->pdev))
729+ if (mtip_check_surprise_removal(dd))
731730 return;
732731 }
733732 if (likely(port_stat & (PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR))) {
@@ -752,7 +751,7 @@ static inline irqreturn_t mtip_handle_irq(struct driver_data *data) · 18 unmodified lines
752751 /* Acknowledge the interrupt status on the port.*/
753752 port_stat = readl(port->mmio + PORT_IRQ_STAT);
754753 if (unlikely(port_stat == 0xFFFFFFFF)) {
755− mtip_check_surprise_removal(dd->pdev);
754+ mtip_check_surprise_removal(dd);
756755 return IRQ_HANDLED;
757756 }
758757 writel(port_stat, port->mmio + PORT_IRQ_STAT);
@@ -796,7 +795,7 @@ static inline irqreturn_t mtip_handle_irq(struct driver_data *data) · 37 unmodified lines
796795 }
797796
798797 if (unlikely(port_stat & PORT_IRQ_ERR)) {
799− if (unlikely(mtip_check_surprise_removal(dd->pdev))) {
798+ if (unlikely(mtip_check_surprise_removal(dd))) {
800799 /* don't proceed further */
801800 return IRQ_HANDLED;
802801 }
@@ -915,7 +914,7 @@ static int mtip_quiesce_io(struct mtip_port *port, unsigned long timeout) · 112 unmodified lines
915914
916915 msleep(100);
917916
918− if (mtip_check_surprise_removal(port->dd->pdev))
917+ if (mtip_check_surprise_removal(port->dd))
919918 goto err_fault;
920919
921920 active = mtip_commands_active(port);
@@ -980,7 +979,7 @@ static int mtip_exec_internal_command(struct mtip_port *port, · 58 unmodified lines
980979 return -EFAULT;
981980 }
982981
983− if (mtip_check_surprise_removal(dd->pdev))
982+ if (mtip_check_surprise_removal(dd))
984983 return -EFAULT;
985984
986985 rq = blk_mq_alloc_request(dd->queue, REQ_OP_DRV_IN, BLK_MQ_REQ_RESERVED);
@@ -1022,7 +1021,7 @@ static int mtip_exec_internal_command(struct mtip_port *port, · 35 unmodified lines
10221021 fis->command, int_cmd->status);
10231022 rv = -EIO;
10241023
1025− if (mtip_check_surprise_removal(dd->pdev) ||
1024+ if (mtip_check_surprise_removal(dd) ||
10261025 test_bit(MTIP_DDF_REMOVE_PENDING_BIT,
10271026 &dd->dd_flag)) {
10281027 dev_err(&dd->pdev->dev,
@@ -2513,7 +2512,7 @@ static int mtip_ftl_rebuild_poll(struct driver_data *dd) · 1484 unmodified lines
25132512 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT,
25142513 &dd->dd_flag)))
25152514 return -EFAULT;
2516− if (mtip_check_surprise_removal(dd->pdev))
2515+ if (mtip_check_surprise_removal(dd))
25172516 return -EFAULT;
25182517
25192518 if (mtip_get_identify(dd->port, NULL) < 0)
@@ -2891,7 +2890,7 @@ static int mtip_hw_init(struct driver_data *dd) · 371 unmodified lines
28912890 time_before(jiffies, timeout)) {
28922891 mdelay(100);
28932892 }
2894− if (unlikely(mtip_check_surprise_removal(dd->pdev))) {
2893+ if (unlikely(mtip_check_surprise_removal(dd))) {
28952894 timetaken = jiffies - timetaken;
28962895 dev_warn(&dd->pdev->dev,
28972896 "Surprise removal detected at %u ms\n",
@@ -4098,7 +4097,7 @@ static void mtip_pci_remove(struct pci_dev *pdev) · 1200 unmodified lines
40984097 list_add(&dd->remove_list, &removing_list);
40994098 spin_unlock_irqrestore(&dev_lock, flags);
41004099
4101− mtip_check_surprise_removal(pdev);
4100+ mtip_check_surprise_removal(dd);
41024101 synchronize_irq(dd->pdev->irq);
41034102
41044103 /* Spin until workers are done */
@@ -4145,36 +4144,17 @@ static void mtip_pci_remove(struct pci_dev *pdev) · 40 unmodified lines
41454144 * 0 Success
41464145 * <0 Error
41474146 */
4148−static int mtip_pci_suspend(struct pci_dev *pdev, pm_message_t mesg)
4147+static int __maybe_unused mtip_pci_suspend(struct device *dev)
41494148 {
41504149 int rv = 0;
4151− struct driver_data *dd = pci_get_drvdata(pdev);
4152−
4153− if (!dd) {
4154− dev_err(&pdev->dev,
4155− "Driver private datastructure is NULL\n");
4156− return -EFAULT;
4157− }
4150+ struct driver_data *dd = dev_get_drvdata(dev);
41584151
41594152 set_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag);
41604153
41614154 /* Disable ports & interrupts then send standby immediate */
41624155 rv = mtip_block_suspend(dd);
4163− if (rv < 0) {
4164− dev_err(&pdev->dev,
4165− "Failed to suspend controller\n");
4166− return rv;
4167− }
4168−
4169− /*
4170− * Save the pci config space to pdev structure &
4171− * disable the device
4172− */
4173− pci_save_state(pdev);
4174− pci_disable_device(pdev);
4175−
4176− /* Move to Low power state*/
4177− pci_set_power_state(pdev, PCI_D3hot);
4156+ if (rv < 0)
4157+ dev_err(dev, "Failed to suspend controller\n");
41784158
41794159 return rv;
41804160 }
@@ -4186,32 +4166,10 @@ static int mtip_pci_suspend(struct pci_dev *pdev, pm_message_t mesg) · 5 unmodified lines
41864166 * 0 Success
41874167 * <0 Error
41884168 */
4189−static int mtip_pci_resume(struct pci_dev *pdev)
4169+static int __maybe_unused mtip_pci_resume(struct device *dev)
41904170 {
41914171 int rv = 0;
4192− struct driver_data *dd;
4193−
4194− dd = pci_get_drvdata(pdev);
4195− if (!dd) {
4196− dev_err(&pdev->dev,
4197− "Driver private datastructure is NULL\n");
4198− return -EFAULT;
4199− }
4200−
4201− /* Move the device to active State */
4202− pci_set_power_state(pdev, PCI_D0);
4203−
4204− /* Restore PCI configuration space */
4205− pci_restore_state(pdev);
4206−
4207− /* Enable the PCI device*/
4208− rv = pcim_enable_device(pdev);
4209− if (rv < 0) {
4210− dev_err(&pdev->dev,
4211− "Failed to enable card during resume\n");
4212− goto err;
4213− }
4214− pci_set_master(pdev);
4172+ struct driver_data *dd = dev_get_drvdata(dev);
42154173
42164174 /*
42174175 * Calls hbaReset, initPort, & startPort function
@@ -4219,9 +4177,8 @@ static int mtip_pci_resume(struct pci_dev *pdev) · 1 unmodified line
42194177 */
42204178 rv = mtip_block_resume(dd);
42214179 if (rv < 0)
4222− dev_err(&pdev->dev, "Unable to resume\n");
4180+ dev_err(dev, "Unable to resume\n");
42234181
4224−err:
42254182 clear_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag);
42264183
42274184 return rv;
@@ -4252,14 +4209,15 @@ static const struct pci_device_id mtip_pci_tbl[] = { · 24 unmodified lines
42524209 { 0 }
42534210 };
42544211
4212+static SIMPLE_DEV_PM_OPS(mtip_pci_pm_ops, mtip_pci_suspend, mtip_pci_resume);
4213+
42554214 /* Structure that describes the PCI driver functions. */
42564215 static struct pci_driver mtip_pci_driver = {
42574216 .name = MTIP_DRV_NAME,
42584217 .id_table = mtip_pci_tbl,
42594218 .probe = mtip_pci_probe,
42604219 .remove = mtip_pci_remove,
4261− .suspend = mtip_pci_suspend,
4262− .resume = mtip_pci_resume,
4220+ .driver.pm = &mtip_pci_pm_ops,
42634221 .shutdown = mtip_pci_shutdown,
42644222 };
42654223
drivers/block/null_blk/main.c+6−6
@@ -340,9 +340,9 @@ static int nullb_update_nr_hw_queues(struct nullb_device *dev, · 339 unmodified lines
340340 return 0;
341341
342342 /*
343− * Make sure at least one queue exists for each of submit and poll.
343+ * Make sure at least one submit queue exists.
344344 */
345− if (!submit_queues || !poll_queues)
345+ if (!submit_queues)
346346 return -EINVAL;
347347
348348 /*
@@ -1574,7 +1574,9 @@ static int null_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) · 1225 unmodified lines
15741574 cmd = blk_mq_rq_to_pdu(req);
15751575 cmd->error = null_process_cmd(cmd, req_op(req), blk_rq_pos(req),
15761576 blk_rq_sectors(req));
1577− end_cmd(cmd);
1577+ if (!blk_mq_add_to_batch(req, iob, (__force int) cmd->error,
1578+ blk_mq_end_request_batch))
1579+ end_cmd(cmd);
15781580 nr++;
15791581 }
15801582
@@ -1890,7 +1892,7 @@ static int null_init_tag_set(struct nullb *nullb, struct blk_mq_tag_set *set) · 309 unmodified lines
18901892 if (g_shared_tag_bitmap)
18911893 set->flags |= BLK_MQ_F_TAG_HCTX_SHARED;
18921894 set->driver_data = nullb;
1893− if (g_poll_queues)
1895+ if (poll_queues)
18941896 set->nr_maps = 3;
18951897 else
18961898 set->nr_maps = 1;
@@ -1917,8 +1919,6 @@ static int null_validate_conf(struct nullb_device *dev) · 20 unmodified lines
19171919
19181920 if (dev->poll_queues > g_poll_queues)
19191921 dev->poll_queues = g_poll_queues;
1920− else if (dev->poll_queues == 0)
1921− dev->poll_queues = 1;
19221922 dev->prev_poll_queues = dev->poll_queues;
19231923
19241924 dev->queue_mode = min_t(unsigned int, dev->queue_mode, NULL_Q_MQ);
drivers/block/pktcdvd.c+154−152
@@ -113,57 +113,10 @@ static sector_t get_zone(sector_t sector, struct pktcdvd_device *pd) · 112 unmodified lines
113113 return (sector + pd->offset) & ~(sector_t)(pd->settings.size - 1);
114114 }
115115
116−/*
117− * create and register a pktcdvd kernel object.
118− */
119−static struct pktcdvd_kobj* pkt_kobj_create(struct pktcdvd_device *pd,
120− const char* name,
121− struct kobject* parent,
122− struct kobj_type* ktype)
123−{
124− struct pktcdvd_kobj *p;
125− int error;
126−
127− p = kzalloc(sizeof(*p), GFP_KERNEL);
128− if (!p)
129− return NULL;
130− p->pd = pd;
131− error = kobject_init_and_add(&p->kobj, ktype, parent, "%s", name);
132− if (error) {
133− kobject_put(&p->kobj);
134− return NULL;
135− }
136− kobject_uevent(&p->kobj, KOBJ_ADD);
137− return p;
138−}
139−/*
140− * remove a pktcdvd kernel object.
141− */
142−static void pkt_kobj_remove(struct pktcdvd_kobj *p)
143−{
144− if (p)
145− kobject_put(&p->kobj);
146−}
147−/*
148− * default release function for pktcdvd kernel objects.
149− */
150−static void pkt_kobj_release(struct kobject *kobj)
151−{
152− kfree(to_pktcdvdkobj(kobj));
153−}
154−
155−
156116 /**********************************************************
157− *
158117 * sysfs interface for pktcdvd
159118 * by (C) 2006 Thomas Maier <balagi@justmail.de>
160− *
161− **********************************************************/
162−
163−#define DEF_ATTR(_obj,_name,_mode) \
164− static struct attribute _obj = { .name = _name, .mode = _mode }
165−
166−/**********************************************************
119+
167120 /sys/class/pktcdvd/pktcdvd[0-7]/
168121 stat/reset
169122 stat/packets_started
@@ -176,75 +129,94 @@ static void pkt_kobj_release(struct kobject *kobj) · 6 unmodified lines
176129 write_queue/congestion_on
177130 **********************************************************/
178131
179−DEF_ATTR(kobj_pkt_attr_st1, "reset", 0200);
180−DEF_ATTR(kobj_pkt_attr_st2, "packets_started", 0444);
181−DEF_ATTR(kobj_pkt_attr_st3, "packets_finished", 0444);
182−DEF_ATTR(kobj_pkt_attr_st4, "kb_written", 0444);
183−DEF_ATTR(kobj_pkt_attr_st5, "kb_read", 0444);
184−DEF_ATTR(kobj_pkt_attr_st6, "kb_read_gather", 0444);
185−
186−static struct attribute *kobj_pkt_attrs_stat[] = {
187− &kobj_pkt_attr_st1,
188− &kobj_pkt_attr_st2,
189− &kobj_pkt_attr_st3,
190− &kobj_pkt_attr_st4,
191− &kobj_pkt_attr_st5,
192− &kobj_pkt_attr_st6,
193− NULL
194−};
132+static ssize_t packets_started_show(struct device *dev,
133+ struct device_attribute *attr, char *buf)
134+{
135+ struct pktcdvd_device *pd = dev_get_drvdata(dev);
195136
196−DEF_ATTR(kobj_pkt_attr_wq1, "size", 0444);
197−DEF_ATTR(kobj_pkt_attr_wq2, "congestion_off", 0644);
198−DEF_ATTR(kobj_pkt_attr_wq3, "congestion_on", 0644);
137+ return sysfs_emit(buf, "%lu\n", pd->stats.pkt_started);
138+}
139+static DEVICE_ATTR_RO(packets_started);
199140
200−static struct attribute *kobj_pkt_attrs_wqueue[] = {
201− &kobj_pkt_attr_wq1,
202− &kobj_pkt_attr_wq2,
203− &kobj_pkt_attr_wq3,
204− NULL
205−};
141+static ssize_t packets_finished_show(struct device *dev,
142+ struct device_attribute *attr, char *buf)
143+{
144+ struct pktcdvd_device *pd = dev_get_drvdata(dev);
206145
207−static ssize_t kobj_pkt_show(struct kobject *kobj,
208− struct attribute *attr, char *data)
146+ return sysfs_emit(buf, "%lu\n", pd->stats.pkt_ended);
147+}
148+static DEVICE_ATTR_RO(packets_finished);
149+
150+static ssize_t kb_written_show(struct device *dev,
151+ struct device_attribute *attr, char *buf)
209152 {
210− struct pktcdvd_device *pd = to_pktcdvdkobj(kobj)->pd;
211− int n = 0;
212− int v;
213− if (strcmp(attr->name, "packets_started") == 0) {
214− n = sprintf(data, "%lu\n", pd->stats.pkt_started);
153+ struct pktcdvd_device *pd = dev_get_drvdata(dev);
215154
216− } else if (strcmp(attr->name, "packets_finished") == 0) {
217− n = sprintf(data, "%lu\n", pd->stats.pkt_ended);
155+ return sysfs_emit(buf, "%lu\n", pd->stats.secs_w >> 1);
156+}
157+static DEVICE_ATTR_RO(kb_written);
218158
219− } else if (strcmp(attr->name, "kb_written") == 0) {
220− n = sprintf(data, "%lu\n", pd->stats.secs_w >> 1);
159+static ssize_t kb_read_show(struct device *dev,
160+ struct device_attribute *attr, char *buf)
161+{
162+ struct pktcdvd_device *pd = dev_get_drvdata(dev);
221163
222− } else if (strcmp(attr->name, "kb_read") == 0) {
223− n = sprintf(data, "%lu\n", pd->stats.secs_r >> 1);
164+ return sysfs_emit(buf, "%lu\n", pd->stats.secs_r >> 1);
165+}
166+static DEVICE_ATTR_RO(kb_read);
224167
225− } else if (strcmp(attr->name, "kb_read_gather") == 0) {
226− n = sprintf(data, "%lu\n", pd->stats.secs_rg >> 1);
168+static ssize_t kb_read_gather_show(struct device *dev,
169+ struct device_attribute *attr, char *buf)
170+{
171+ struct pktcdvd_device *pd = dev_get_drvdata(dev);
227172
228− } else if (strcmp(attr->name, "size") == 0) {
229− spin_lock(&pd->lock);
230− v = pd->bio_queue_size;
231− spin_unlock(&pd->lock);
232− n = sprintf(data, "%d\n", v);
173+ return sysfs_emit(buf, "%lu\n", pd->stats.secs_rg >> 1);
174+}
175+static DEVICE_ATTR_RO(kb_read_gather);
233176
234− } else if (strcmp(attr->name, "congestion_off") == 0) {
235− spin_lock(&pd->lock);
236− v = pd->write_congestion_off;
237− spin_unlock(&pd->lock);
238− n = sprintf(data, "%d\n", v);
177+static ssize_t reset_store(struct device *dev, struct device_attribute *attr,
178+ const char *buf, size_t len)
179+{
180+ struct pktcdvd_device *pd = dev_get_drvdata(dev);
239181
240− } else if (strcmp(attr->name, "congestion_on") == 0) {
241− spin_lock(&pd->lock);
242− v = pd->write_congestion_on;
243− spin_unlock(&pd->lock);
244− n = sprintf(data, "%d\n", v);
182+ if (len > 0) {
183+ pd->stats.pkt_started = 0;
184+ pd->stats.pkt_ended = 0;
185+ pd->stats.secs_w = 0;
186+ pd->stats.secs_rg = 0;
187+ pd->stats.secs_r = 0;
245188 }
189+ return len;
190+}
191+static DEVICE_ATTR_WO(reset);
192+
193+static struct attribute *pkt_stat_attrs[] = {
194+ &dev_attr_packets_finished.attr,
195+ &dev_attr_packets_started.attr,
196+ &dev_attr_kb_read.attr,
197+ &dev_attr_kb_written.attr,
198+ &dev_attr_kb_read_gather.attr,
199+ &dev_attr_reset.attr,
200+ NULL,
201+};
202+
203+static const struct attribute_group pkt_stat_group = {
204+ .name = "stat",
205+ .attrs = pkt_stat_attrs,
206+};
207+
208+static ssize_t size_show(struct device *dev,
209+ struct device_attribute *attr, char *buf)
210+{
211+ struct pktcdvd_device *pd = dev_get_drvdata(dev);
212+ int n;
213+
214+ spin_lock(&pd->lock);
215+ n = sysfs_emit(buf, "%d\n", pd->bio_queue_size);
216+ spin_unlock(&pd->lock);
246217 return n;
247218 }
219+static DEVICE_ATTR_RO(size);
248220
249221 static void init_write_congestion_marks(int* lo, int* hi)
250222 {
@@ -263,30 +235,56 @@ static void init_write_congestion_marks(int* lo, int* hi) · 12 unmodified lines
263235 }
264236 }
265237
266−static ssize_t kobj_pkt_store(struct kobject *kobj,
267− struct attribute *attr,
268− const char *data, size_t len)
238+static ssize_t congestion_off_show(struct device *dev,
239+ struct device_attribute *attr, char *buf)
269240 {
270− struct pktcdvd_device *pd = to_pktcdvdkobj(kobj)->pd;
271− int val;
241+ struct pktcdvd_device *pd = dev_get_drvdata(dev);
242+ int n;
272243
273− if (strcmp(attr->name, "reset") == 0 && len > 0) {
274− pd->stats.pkt_started = 0;
275− pd->stats.pkt_ended = 0;
276− pd->stats.secs_w = 0;
277− pd->stats.secs_rg = 0;
278− pd->stats.secs_r = 0;
244+ spin_lock(&pd->lock);
245+ n = sysfs_emit(buf, "%d\n", pd->write_congestion_off);
246+ spin_unlock(&pd->lock);
247+ return n;
248+}
249+
250+static ssize_t congestion_off_store(struct device *dev,
251+ struct device_attribute *attr,
252+ const char *buf, size_t len)
253+{
254+ struct pktcdvd_device *pd = dev_get_drvdata(dev);
255+ int val;
279256
280− } else if (strcmp(attr->name, "congestion_off") == 0
281− && sscanf(data, "%d", &val) == 1) {
257+ if (sscanf(buf, "%d", &val) == 1) {
282258 spin_lock(&pd->lock);
283259 pd->write_congestion_off = val;
284260 init_write_congestion_marks(&pd->write_congestion_off,
285261 &pd->write_congestion_on);
286262 spin_unlock(&pd->lock);
263+ }
264+ return len;
265+}
266+static DEVICE_ATTR_RW(congestion_off);
267+
268+static ssize_t congestion_on_show(struct device *dev,
269+ struct device_attribute *attr, char *buf)
270+{
271+ struct pktcdvd_device *pd = dev_get_drvdata(dev);
272+ int n;
273+
274+ spin_lock(&pd->lock);
275+ n = sysfs_emit(buf, "%d\n", pd->write_congestion_on);
276+ spin_unlock(&pd->lock);
277+ return n;
278+}
279+
280+static ssize_t congestion_on_store(struct device *dev,
281+ struct device_attribute *attr,
282+ const char *buf, size_t len)
283+{
284+ struct pktcdvd_device *pd = dev_get_drvdata(dev);
285+ int val;
287286
288− } else if (strcmp(attr->name, "congestion_on") == 0
289− && sscanf(data, "%d", &val) == 1) {
287+ if (sscanf(buf, "%d", &val) == 1) {
290288 spin_lock(&pd->lock);
291289 pd->write_congestion_on = val;
292290 init_write_congestion_marks(&pd->write_congestion_off,
@@ -295,44 +293,39 @@ static ssize_t kobj_pkt_store(struct kobject *kobj, · 2 unmodified lines
295293 }
296294 return len;
297295 }
296+static DEVICE_ATTR_RW(congestion_on);
298297
299−static const struct sysfs_ops kobj_pkt_ops = {
300− .show = kobj_pkt_show,
301− .store = kobj_pkt_store
298+static struct attribute *pkt_wq_attrs[] = {
299+ &dev_attr_congestion_on.attr,
300+ &dev_attr_congestion_off.attr,
301+ &dev_attr_size.attr,
302+ NULL,
302303 };
303−static struct kobj_type kobj_pkt_type_stat = {
304− .release = pkt_kobj_release,
305− .sysfs_ops = &kobj_pkt_ops,
306− .default_attrs = kobj_pkt_attrs_stat
304+
305+static const struct attribute_group pkt_wq_group = {
306+ .name = "write_queue",
307+ .attrs = pkt_wq_attrs,
307308 };
308−static struct kobj_type kobj_pkt_type_wqueue = {
309− .release = pkt_kobj_release,
310− .sysfs_ops = &kobj_pkt_ops,
311− .default_attrs = kobj_pkt_attrs_wqueue
309+
310+static const struct attribute_group *pkt_groups[] = {
311+ &pkt_stat_group,
312+ &pkt_wq_group,
313+ NULL,
312314 };
313315
314316 static void pkt_sysfs_dev_new(struct pktcdvd_device *pd)
315317 {
316318 if (class_pktcdvd) {
317− pd->dev = device_create(class_pktcdvd, NULL, MKDEV(0, 0), NULL,
318− "%s", pd->name);
319+ pd->dev = device_create_with_groups(class_pktcdvd, NULL,
320+ MKDEV(0, 0), pd, pkt_groups,
321+ "%s", pd->name);
319322 if (IS_ERR(pd->dev))
320323 pd->dev = NULL;
321324 }
322− if (pd->dev) {
323− pd->kobj_stat = pkt_kobj_create(pd, "stat",
324− &pd->dev->kobj,
325− &kobj_pkt_type_stat);
326− pd->kobj_wqueue = pkt_kobj_create(pd, "write_queue",
327− &pd->dev->kobj,
328− &kobj_pkt_type_wqueue);
329− }
330325 }
331326
332327 static void pkt_sysfs_dev_remove(struct pktcdvd_device *pd)
333328 {
334− pkt_kobj_remove(pd->kobj_stat);
335− pkt_kobj_remove(pd->kobj_wqueue);
336329 if (class_pktcdvd)
337330 device_unregister(pd->dev);
338331 }
@@ -1107,7 +1100,6 @@ static int pkt_handle_queue(struct pktcdvd_device *pd) · 768 unmodified lines
11071100 sector_t zone = 0; /* Suppress gcc warning */
11081101 struct pkt_rb_node *node, *first_node;
11091102 struct rb_node *n;
1110− int wakeup;
11111103
11121104 atomic_set(&pd->scan_queue, 0);
11131105
@@ -1179,12 +1171,14 @@ try_next_bio: · 65 unmodified lines
11791171 spin_unlock(&pkt->lock);
11801172 }
11811173 /* check write congestion marks, and if bio_queue_size is
1182− below, wake up any waiters */
1183− wakeup = (pd->write_congestion_on > 0
1184− && pd->bio_queue_size <= pd->write_congestion_off);
1174+ * below, wake up any waiters
1175+ */
1176+ if (pd->congested &&
1177+ pd->bio_queue_size <= pd->write_congestion_off) {
1178+ pd->congested = false;
1179+ wake_up_var(&pd->congested);
1180+ }
11851181 spin_unlock(&pd->lock);
1186− if (wakeup)
1187− clear_bdi_congested(pd->disk->bdi, BLK_RW_ASYNC);
11881182
11891183 pkt->sleep_time = max(PACKET_WAIT_TIME, 1);
11901184 pkt_set_state(pkt, PACKET_WAITING_STATE);
@@ -2356,7 +2350,7 @@ static void pkt_make_request_write(struct request_queue *q, struct bio *bio) · 1165 unmodified lines
23562350 }
23572351 spin_unlock(&pd->cdrw.active_list_lock);
23582352
2359− /*
2353+ /*
23602354 * Test if there is enough room left in the bio work queue
23612355 * (queue size >= congestion on mark).
23622356 * If not, wait till the work queue size is below the congestion off mark.
@@ -2364,12 +2358,20 @@ static void pkt_make_request_write(struct request_queue *q, struct bio *bio) · 1 unmodified line
23642358 spin_lock(&pd->lock);
23652359 if (pd->write_congestion_on > 0
23662360 && pd->bio_queue_size >= pd->write_congestion_on) {
2367− set_bdi_congested(bio->bi_bdev->bd_disk->bdi, BLK_RW_ASYNC);
2368− do {
2361+ struct wait_bit_queue_entry wqe;
2362+
2363+ init_wait_var_entry(&wqe, &pd->congested, 0);
2364+ for (;;) {
2365+ prepare_to_wait_event(__var_waitqueue(&pd->congested),
2366+ &wqe.wq_entry,
2367+ TASK_UNINTERRUPTIBLE);
2368+ if (pd->bio_queue_size <= pd->write_congestion_off)
2369+ break;
2370+ pd->congested = true;
23692371 spin_unlock(&pd->lock);
2370− congestion_wait(BLK_RW_ASYNC, HZ);
2372+ schedule();
23712373 spin_lock(&pd->lock);
2372− } while(pd->bio_queue_size > pd->write_congestion_off);
2374+ }
23732375 }
23742376 spin_unlock(&pd->lock);
23752377
drivers/block/rnbd/rnbd-clt-sysfs.c+2−1
@@ -452,6 +452,7 @@ static struct attribute *rnbd_dev_attrs[] = { · 451 unmodified lines
452452 &rnbd_clt_nr_poll_queues.attr,
453453 NULL,
454454 };
455+ATTRIBUTE_GROUPS(rnbd_dev);
455456
456457 void rnbd_clt_remove_dev_symlink(struct rnbd_clt_dev *dev)
457458 {
@@ -474,7 +475,7 @@ void rnbd_clt_remove_dev_symlink(struct rnbd_clt_dev *dev) · 16 unmodified lines
474475
475476 static struct kobj_type rnbd_dev_ktype = {
476477 .sysfs_ops = &kobj_sysfs_ops,
477− .default_attrs = rnbd_dev_attrs,
478+ .default_groups = rnbd_dev_groups,
478479 };
479480
480481 static int rnbd_clt_add_dev_kobj(struct rnbd_clt_dev *dev)
drivers/block/rsxx/Makefile+0−3
@@ -1,3 +0,0 @@
1−# SPDX-License-Identifier: GPL-2.0-only
2−obj-$(CONFIG_BLK_DEV_RSXX) += rsxx.o
3−rsxx-objs := config.o core.o cregs.o dev.o dma.o
drivers/block/rsxx/config.c+0−197
@@ -1,197 +0,0 @@
1−// SPDX-License-Identifier: GPL-2.0-or-later
2−/*
3−* Filename: config.c
4−*
5−* Authors: Joshua Morris <josh.h.morris@us.ibm.com>
6−* Philip Kelleher <pjk1939@linux.vnet.ibm.com>
7−*
8−* (C) Copyright 2013 IBM Corporation
9−*/
10−
11−#include <linux/types.h>
12−#include <linux/crc32.h>
13−#include <linux/swab.h>
14−
15−#include "rsxx_priv.h"
16−#include "rsxx_cfg.h"
17−
18−static void initialize_config(struct rsxx_card_cfg *cfg)
19−{
20− cfg->hdr.version = RSXX_CFG_VERSION;
21−
22− cfg->data.block_size = RSXX_HW_BLK_SIZE;
23− cfg->data.stripe_size = RSXX_HW_BLK_SIZE;
24− cfg->data.vendor_id = RSXX_VENDOR_ID_IBM;
25− cfg->data.cache_order = (-1);
26− cfg->data.intr_coal.mode = RSXX_INTR_COAL_DISABLED;
27− cfg->data.intr_coal.count = 0;
28− cfg->data.intr_coal.latency = 0;
29−}
30−
31−static u32 config_data_crc32(struct rsxx_card_cfg *cfg)
32−{
33− /*
34− * Return the compliment of the CRC to ensure compatibility
35− * (i.e. this is how early rsxx drivers did it.)
36− */
37−
38− return ~crc32(~0, &cfg->data, sizeof(cfg->data));
39−}
40−
41−
42−/*----------------- Config Byte Swap Functions -------------------*/
43−static void config_hdr_be_to_cpu(struct card_cfg_hdr *hdr)
44−{
45− hdr->version = be32_to_cpu((__force __be32) hdr->version);
46− hdr->crc = be32_to_cpu((__force __be32) hdr->crc);
47−}
48−
49−static void config_hdr_cpu_to_be(struct card_cfg_hdr *hdr)
50−{
51− hdr->version = (__force u32) cpu_to_be32(hdr->version);
52− hdr->crc = (__force u32) cpu_to_be32(hdr->crc);
53−}
54−
55−static void config_data_swab(struct rsxx_card_cfg *cfg)
56−{
57− u32 *data = (u32 *) &cfg->data;
58− int i;
59−
60− for (i = 0; i < (sizeof(cfg->data) / 4); i++)
61− data[i] = swab32(data[i]);
62−}
63−
64−static void config_data_le_to_cpu(struct rsxx_card_cfg *cfg)
65−{
66− u32 *data = (u32 *) &cfg->data;
67− int i;
68−
69− for (i = 0; i < (sizeof(cfg->data) / 4); i++)
70− data[i] = le32_to_cpu((__force __le32) data[i]);
71−}
72−
73−static void config_data_cpu_to_le(struct rsxx_card_cfg *cfg)
74−{
75− u32 *data = (u32 *) &cfg->data;
76− int i;
77−
78− for (i = 0; i < (sizeof(cfg->data) / 4); i++)
79− data[i] = (__force u32) cpu_to_le32(data[i]);
80−}
81−
82−
83−/*----------------- Config Operations ------------------*/
84−static int rsxx_save_config(struct rsxx_cardinfo *card)
85−{
86− struct rsxx_card_cfg cfg;
87− int st;
88−
89− memcpy(&cfg, &card->config, sizeof(cfg));
90−
91− if (unlikely(cfg.hdr.version != RSXX_CFG_VERSION)) {
92− dev_err(CARD_TO_DEV(card),
93− "Cannot save config with invalid version %d\n",
94− cfg.hdr.version);
95− return -EINVAL;
96− }
97−
98− /* Convert data to little endian for the CRC calculation. */
99− config_data_cpu_to_le(&cfg);
100−
101− cfg.hdr.crc = config_data_crc32(&cfg);
102−
103− /*
104− * Swap the data from little endian to big endian so it can be
105− * stored.
106− */
107− config_data_swab(&cfg);
108− config_hdr_cpu_to_be(&cfg.hdr);
109−
110− st = rsxx_creg_write(card, CREG_ADD_CONFIG, sizeof(cfg), &cfg, 1);
111− if (st)
112− return st;
113−
114− return 0;
115−}
116−
117−int rsxx_load_config(struct rsxx_cardinfo *card)
118−{
119− int st;
120− u32 crc;
121−
122− st = rsxx_creg_read(card, CREG_ADD_CONFIG, sizeof(card->config),
123− &card->config, 1);
124− if (st) {
125− dev_err(CARD_TO_DEV(card),
126− "Failed reading card config.\n");
127− return st;
128− }
129−
130− config_hdr_be_to_cpu(&card->config.hdr);
131−
132− if (card->config.hdr.version == RSXX_CFG_VERSION) {
133− /*
134− * We calculate the CRC with the data in little endian, because
135− * early drivers did not take big endian CPUs into account.
136− * The data is always stored in big endian, so we need to byte
137− * swap it before calculating the CRC.
138− */
139−
140− config_data_swab(&card->config);
141−
142− /* Check the CRC */
143− crc = config_data_crc32(&card->config);
144− if (crc != card->config.hdr.crc) {
145− dev_err(CARD_TO_DEV(card),
146− "Config corruption detected!\n");
147− dev_info(CARD_TO_DEV(card),
148− "CRC (sb x%08x is x%08x)\n",
149− card->config.hdr.crc, crc);
150− return -EIO;
151− }
152−
153− /* Convert the data to CPU byteorder */
154− config_data_le_to_cpu(&card->config);
155−
156− } else if (card->config.hdr.version != 0) {
157− dev_err(CARD_TO_DEV(card),
158− "Invalid config version %d.\n",
159− card->config.hdr.version);
160− /*
161− * Config version changes require special handling from the
162− * user
163− */
164− return -EINVAL;
165− } else {
166− dev_info(CARD_TO_DEV(card),
167− "Initializing card configuration.\n");
168− initialize_config(&card->config);
169− st = rsxx_save_config(card);
170− if (st)
171− return st;
172− }
173−
174− card->config_valid = 1;
175−
176− dev_dbg(CARD_TO_DEV(card), "version: x%08x\n",
177− card->config.hdr.version);
178− dev_dbg(CARD_TO_DEV(card), "crc: x%08x\n",
179− card->config.hdr.crc);
180− dev_dbg(CARD_TO_DEV(card), "block_size: x%08x\n",
181− card->config.data.block_size);
182− dev_dbg(CARD_TO_DEV(card), "stripe_size: x%08x\n",
183− card->config.data.stripe_size);
184− dev_dbg(CARD_TO_DEV(card), "vendor_id: x%08x\n",
185− card->config.data.vendor_id);
186− dev_dbg(CARD_TO_DEV(card), "cache_order: x%08x\n",
187− card->config.data.cache_order);
188− dev_dbg(CARD_TO_DEV(card), "mode: x%08x\n",
189− card->config.data.intr_coal.mode);
190− dev_dbg(CARD_TO_DEV(card), "count: x%08x\n",
191− card->config.data.intr_coal.count);
192− dev_dbg(CARD_TO_DEV(card), "latency: x%08x\n",
193− card->config.data.intr_coal.latency);
194−
195− return 0;
196−}
197−
drivers/block/rsxx/core.c+0−1126
@@ -1,1126 +0,0 @@
1−// SPDX-License-Identifier: GPL-2.0-or-later
2−/*
3−* Filename: core.c
4−*
5−* Authors: Joshua Morris <josh.h.morris@us.ibm.com>
6−* Philip Kelleher <pjk1939@linux.vnet.ibm.com>
7−*
8−* (C) Copyright 2013 IBM Corporation
9−*/
10−
11−#include <linux/kernel.h>
12−#include <linux/init.h>
13−#include <linux/interrupt.h>
14−#include <linux/module.h>
15−#include <linux/pci.h>
16−#include <linux/reboot.h>
17−#include <linux/slab.h>
18−#include <linux/bitops.h>
19−#include <linux/delay.h>
20−#include <linux/debugfs.h>
21−#include <linux/seq_file.h>
22−
23−#include <linux/genhd.h>
24−#include <linux/idr.h>
25−
26−#include "rsxx_priv.h"
27−#include "rsxx_cfg.h"
28−
29−#define NO_LEGACY 0
30−#define SYNC_START_TIMEOUT (10 * 60) /* 10 minutes */
31−
32−MODULE_DESCRIPTION("IBM Flash Adapter 900GB Full Height Device Driver");
33−MODULE_AUTHOR("Joshua Morris/Philip Kelleher, IBM");
34−MODULE_LICENSE("GPL");
35−MODULE_VERSION(DRIVER_VERSION);
36−
37−static unsigned int force_legacy = NO_LEGACY;
38−module_param(force_legacy, uint, 0444);
39−MODULE_PARM_DESC(force_legacy, "Force the use of legacy type PCI interrupts");
40−
41−static unsigned int sync_start = 1;
42−module_param(sync_start, uint, 0444);
43−MODULE_PARM_DESC(sync_start, "On by Default: Driver load will not complete "
44− "until the card startup has completed.");
45−
46−static DEFINE_IDA(rsxx_disk_ida);
47−
48−/* --------------------Debugfs Setup ------------------- */
49−
50−static int rsxx_attr_pci_regs_show(struct seq_file *m, void *p)
51−{
52− struct rsxx_cardinfo *card = m->private;
53−
54− seq_printf(m, "HWID 0x%08x\n",
55− ioread32(card->regmap + HWID));
56− seq_printf(m, "SCRATCH 0x%08x\n",
57− ioread32(card->regmap + SCRATCH));
58− seq_printf(m, "IER 0x%08x\n",
59− ioread32(card->regmap + IER));
60− seq_printf(m, "IPR 0x%08x\n",
61− ioread32(card->regmap + IPR));
62− seq_printf(m, "CREG_CMD 0x%08x\n",
63− ioread32(card->regmap + CREG_CMD));
64− seq_printf(m, "CREG_ADD 0x%08x\n",
65− ioread32(card->regmap + CREG_ADD));
66− seq_printf(m, "CREG_CNT 0x%08x\n",
67− ioread32(card->regmap + CREG_CNT));
68− seq_printf(m, "CREG_STAT 0x%08x\n",
69− ioread32(card->regmap + CREG_STAT));
70− seq_printf(m, "CREG_DATA0 0x%08x\n",
71− ioread32(card->regmap + CREG_DATA0));
72− seq_printf(m, "CREG_DATA1 0x%08x\n",
73− ioread32(card->regmap + CREG_DATA1));
74− seq_printf(m, "CREG_DATA2 0x%08x\n",
75− ioread32(card->regmap + CREG_DATA2));
76− seq_printf(m, "CREG_DATA3 0x%08x\n",
77− ioread32(card->regmap + CREG_DATA3));
78− seq_printf(m, "CREG_DATA4 0x%08x\n",
79− ioread32(card->regmap + CREG_DATA4));
80− seq_printf(m, "CREG_DATA5 0x%08x\n",
81− ioread32(card->regmap + CREG_DATA5));
82− seq_printf(m, "CREG_DATA6 0x%08x\n",
83− ioread32(card->regmap + CREG_DATA6));
84− seq_printf(m, "CREG_DATA7 0x%08x\n",
85− ioread32(card->regmap + CREG_DATA7));
86− seq_printf(m, "INTR_COAL 0x%08x\n",
87− ioread32(card->regmap + INTR_COAL));
88− seq_printf(m, "HW_ERROR 0x%08x\n",
89− ioread32(card->regmap + HW_ERROR));
90− seq_printf(m, "DEBUG0 0x%08x\n",
91− ioread32(card->regmap + PCI_DEBUG0));
92− seq_printf(m, "DEBUG1 0x%08x\n",
93− ioread32(card->regmap + PCI_DEBUG1));
94− seq_printf(m, "DEBUG2 0x%08x\n",
95− ioread32(card->regmap + PCI_DEBUG2));
96− seq_printf(m, "DEBUG3 0x%08x\n",
97− ioread32(card->regmap + PCI_DEBUG3));
98− seq_printf(m, "DEBUG4 0x%08x\n",
99− ioread32(card->regmap + PCI_DEBUG4));
100− seq_printf(m, "DEBUG5 0x%08x\n",
101− ioread32(card->regmap + PCI_DEBUG5));
102− seq_printf(m, "DEBUG6 0x%08x\n",
103− ioread32(card->regmap + PCI_DEBUG6));
104− seq_printf(m, "DEBUG7 0x%08x\n",
105− ioread32(card->regmap + PCI_DEBUG7));
106− seq_printf(m, "RECONFIG 0x%08x\n",
107− ioread32(card->regmap + PCI_RECONFIG));
108−
109− return 0;
110−}
111−
112−static int rsxx_attr_stats_show(struct seq_file *m, void *p)
113−{
114− struct rsxx_cardinfo *card = m->private;
115− int i;
116−
117− for (i = 0; i < card->n_targets; i++) {
118− seq_printf(m, "Ctrl %d CRC Errors = %d\n",
119− i, card->ctrl[i].stats.crc_errors);
120− seq_printf(m, "Ctrl %d Hard Errors = %d\n",
121− i, card->ctrl[i].stats.hard_errors);
122− seq_printf(m, "Ctrl %d Soft Errors = %d\n",
123− i, card->ctrl[i].stats.soft_errors);
124− seq_printf(m, "Ctrl %d Writes Issued = %d\n",
125− i, card->ctrl[i].stats.writes_issued);
126− seq_printf(m, "Ctrl %d Writes Failed = %d\n",
127− i, card->ctrl[i].stats.writes_failed);
128− seq_printf(m, "Ctrl %d Reads Issued = %d\n",
129− i, card->ctrl[i].stats.reads_issued);
130− seq_printf(m, "Ctrl %d Reads Failed = %d\n",
131− i, card->ctrl[i].stats.reads_failed);
132− seq_printf(m, "Ctrl %d Reads Retried = %d\n",
133− i, card->ctrl[i].stats.reads_retried);
134− seq_printf(m, "Ctrl %d Discards Issued = %d\n",
135− i, card->ctrl[i].stats.discards_issued);
136− seq_printf(m, "Ctrl %d Discards Failed = %d\n",
137− i, card->ctrl[i].stats.discards_failed);
138− seq_printf(m, "Ctrl %d DMA SW Errors = %d\n",
139− i, card->ctrl[i].stats.dma_sw_err);
140− seq_printf(m, "Ctrl %d DMA HW Faults = %d\n",
141− i, card->ctrl[i].stats.dma_hw_fault);
142− seq_printf(m, "Ctrl %d DMAs Cancelled = %d\n",
143− i, card->ctrl[i].stats.dma_cancelled);
144− seq_printf(m, "Ctrl %d SW Queue Depth = %d\n",
145− i, card->ctrl[i].stats.sw_q_depth);
146− seq_printf(m, "Ctrl %d HW Queue Depth = %d\n",
147− i, atomic_read(&card->ctrl[i].stats.hw_q_depth));
148− }
149−
150− return 0;
151−}
152−
153−static int rsxx_attr_stats_open(struct inode *inode, struct file *file)
154−{
155− return single_open(file, rsxx_attr_stats_show, inode->i_private);
156−}
157−
158−static int rsxx_attr_pci_regs_open(struct inode *inode, struct file *file)
159−{
160− return single_open(file, rsxx_attr_pci_regs_show, inode->i_private);
161−}
162−
163−static ssize_t rsxx_cram_read(struct file *fp, char __user *ubuf,
164− size_t cnt, loff_t *ppos)
165−{
166− struct rsxx_cardinfo *card = file_inode(fp)->i_private;
167− char *buf;
168− int st;
169−
170− buf = kzalloc(cnt, GFP_KERNEL);
171− if (!buf)
172− return -ENOMEM;
173−
174− st = rsxx_creg_read(card, CREG_ADD_CRAM + (u32)*ppos, cnt, buf, 1);
175− if (!st) {
176− if (copy_to_user(ubuf, buf, cnt))
177− st = -EFAULT;
178− }
179− kfree(buf);
180− if (st)
181− return st;
182− *ppos += cnt;
183− return cnt;
184−}
185−
186−static ssize_t rsxx_cram_write(struct file *fp, const char __user *ubuf,
187− size_t cnt, loff_t *ppos)
188−{
189− struct rsxx_cardinfo *card = file_inode(fp)->i_private;
190− char *buf;
191− ssize_t st;
192−
193− buf = memdup_user(ubuf, cnt);
194− if (IS_ERR(buf))
195− return PTR_ERR(buf);
196−
197− st = rsxx_creg_write(card, CREG_ADD_CRAM + (u32)*ppos, cnt, buf, 1);
198− kfree(buf);
199− if (st)
200− return st;
201− *ppos += cnt;
202− return cnt;
203−}
204−
205−static const struct file_operations debugfs_cram_fops = {
206− .owner = THIS_MODULE,
207− .read = rsxx_cram_read,
208− .write = rsxx_cram_write,
209−};
210−
211−static const struct file_operations debugfs_stats_fops = {
212− .owner = THIS_MODULE,
213− .open = rsxx_attr_stats_open,
214− .read = seq_read,
215− .llseek = seq_lseek,
216− .release = single_release,
217−};
218−
219−static const struct file_operations debugfs_pci_regs_fops = {
220− .owner = THIS_MODULE,
221− .open = rsxx_attr_pci_regs_open,
222− .read = seq_read,
223− .llseek = seq_lseek,
224− .release = single_release,
225−};
226−
227−static void rsxx_debugfs_dev_new(struct rsxx_cardinfo *card)
228−{
229− struct dentry *debugfs_stats;
230− struct dentry *debugfs_pci_regs;
231− struct dentry *debugfs_cram;
232−
233− card->debugfs_dir = debugfs_create_dir(card->gendisk->disk_name, NULL);
234− if (IS_ERR_OR_NULL(card->debugfs_dir))
235− goto failed_debugfs_dir;
236−
237− debugfs_stats = debugfs_create_file("stats", 0444,
238− card->debugfs_dir, card,
239− &debugfs_stats_fops);
240− if (IS_ERR_OR_NULL(debugfs_stats))
241− goto failed_debugfs_stats;
242−
243− debugfs_pci_regs = debugfs_create_file("pci_regs", 0444,
244− card->debugfs_dir, card,
245− &debugfs_pci_regs_fops);
246− if (IS_ERR_OR_NULL(debugfs_pci_regs))
247− goto failed_debugfs_pci_regs;
248−
249− debugfs_cram = debugfs_create_file("cram", 0644,
250− card->debugfs_dir, card,
251− &debugfs_cram_fops);
252− if (IS_ERR_OR_NULL(debugfs_cram))
253− goto failed_debugfs_cram;
254−
255− return;
256−failed_debugfs_cram:
257− debugfs_remove(debugfs_pci_regs);
258−failed_debugfs_pci_regs:
259− debugfs_remove(debugfs_stats);
260−failed_debugfs_stats:
261− debugfs_remove(card->debugfs_dir);
262−failed_debugfs_dir:
263− card->debugfs_dir = NULL;
264−}
265−
266−/*----------------- Interrupt Control & Handling -------------------*/
267−
268−static void rsxx_mask_interrupts(struct rsxx_cardinfo *card)
269−{
270− card->isr_mask = 0;
271− card->ier_mask = 0;
272−}
273−
274−static void __enable_intr(unsigned int *mask, unsigned int intr)
275−{
276− *mask |= intr;
277−}
278−
279−static void __disable_intr(unsigned int *mask, unsigned int intr)
280−{
281− *mask &= ~intr;
282−}
283−
284−/*
285− * NOTE: Disabling the IER will disable the hardware interrupt.
286− * Disabling the ISR will disable the software handling of the ISR bit.
287− *
288− * Enable/Disable interrupt functions assume the card->irq_lock
289− * is held by the caller.
290− */
291−void rsxx_enable_ier(struct rsxx_cardinfo *card, unsigned int intr)
292−{
293− if (unlikely(card->halt) ||
294− unlikely(card->eeh_state))
295− return;
296−
297− __enable_intr(&card->ier_mask, intr);
298− iowrite32(card->ier_mask, card->regmap + IER);
299−}
300−
301−void rsxx_disable_ier(struct rsxx_cardinfo *card, unsigned int intr)
302−{
303− if (unlikely(card->eeh_state))
304− return;
305−
306− __disable_intr(&card->ier_mask, intr);
307− iowrite32(card->ier_mask, card->regmap + IER);
308−}
309−
310−void rsxx_enable_ier_and_isr(struct rsxx_cardinfo *card,
311− unsigned int intr)
312−{
313− if (unlikely(card->halt) ||
314− unlikely(card->eeh_state))
315− return;
316−
317− __enable_intr(&card->isr_mask, intr);
318− __enable_intr(&card->ier_mask, intr);
319− iowrite32(card->ier_mask, card->regmap + IER);
320−}
321−void rsxx_disable_ier_and_isr(struct rsxx_cardinfo *card,
322− unsigned int intr)
323−{
324− if (unlikely(card->eeh_state))
325− return;
326−
327− __disable_intr(&card->isr_mask, intr);
328− __disable_intr(&card->ier_mask, intr);
329− iowrite32(card->ier_mask, card->regmap + IER);
330−}
331−
332−static irqreturn_t rsxx_isr(int irq, void *pdata)
333−{
334− struct rsxx_cardinfo *card = pdata;
335− unsigned int isr;
336− int handled = 0;
337− int reread_isr;
338− int i;
339−
340− spin_lock(&card->irq_lock);
341−
342− do {
343− reread_isr = 0;
344−
345− if (unlikely(card->eeh_state))
346− break;
347−
348− isr = ioread32(card->regmap + ISR);
349− if (isr == 0xffffffff) {
350− /*
351− * A few systems seem to have an intermittent issue
352− * where PCI reads return all Fs, but retrying the read
353− * a little later will return as expected.
354− */
355− dev_info(CARD_TO_DEV(card),
356− "ISR = 0xFFFFFFFF, retrying later\n");
357− break;
358− }
359−
360− isr &= card->isr_mask;
361− if (!isr)
362− break;
363−
364− for (i = 0; i < card->n_targets; i++) {
365− if (isr & CR_INTR_DMA(i)) {
366− if (card->ier_mask & CR_INTR_DMA(i)) {
367− rsxx_disable_ier(card, CR_INTR_DMA(i));
368− reread_isr = 1;
369− }
370− queue_work(card->ctrl[i].done_wq,
371− &card->ctrl[i].dma_done_work);
372− handled++;
373− }
374− }
375−
376− if (isr & CR_INTR_CREG) {
377− queue_work(card->creg_ctrl.creg_wq,
378− &card->creg_ctrl.done_work);
379− handled++;
380− }
381−
382− if (isr & CR_INTR_EVENT) {
383− queue_work(card->event_wq, &card->event_work);
384− rsxx_disable_ier_and_isr(card, CR_INTR_EVENT);
385− handled++;
386− }
387− } while (reread_isr);
388−
389− spin_unlock(&card->irq_lock);
390−
391− return handled ? IRQ_HANDLED : IRQ_NONE;
392−}
393−
394−/*----------------- Card Event Handler -------------------*/
395−static const char *rsxx_card_state_to_str(unsigned int state)
396−{
397− static const char * const state_strings[] = {
398− "Unknown", "Shutdown", "Starting", "Formatting",
399− "Uninitialized", "Good", "Shutting Down",
400− "Fault", "Read Only Fault", "dStroying"
401− };
402−
403− return state_strings[ffs(state)];
404−}
405−
406−static void card_state_change(struct rsxx_cardinfo *card,
407− unsigned int new_state)
408−{
409− int st;
410−
411− dev_info(CARD_TO_DEV(card),
412− "card state change detected.(%s -> %s)\n",
413− rsxx_card_state_to_str(card->state),
414− rsxx_card_state_to_str(new_state));
415−
416− card->state = new_state;
417−
418− /* Don't attach DMA interfaces if the card has an invalid config */
419− if (!card->config_valid)
420− return;
421−
422− switch (new_state) {
423− case CARD_STATE_RD_ONLY_FAULT:
424− dev_crit(CARD_TO_DEV(card),
425− "Hardware has entered read-only mode!\n");
426− /*
427− * Fall through so the DMA devices can be attached and
428− * the user can attempt to pull off their data.
429− */
430− fallthrough;
431− case CARD_STATE_GOOD:
432− st = rsxx_get_card_size8(card, &card->size8);
433− if (st)
434− dev_err(CARD_TO_DEV(card),
435− "Failed attaching DMA devices\n");
436−
437− if (card->config_valid)
438− set_capacity(card->gendisk, card->size8 >> 9);
439− break;
440−
441− case CARD_STATE_FAULT:
442− dev_crit(CARD_TO_DEV(card),
443− "Hardware Fault reported!\n");
444− fallthrough;
445−
446− /* Everything else, detach DMA interface if it's attached. */
447− case CARD_STATE_SHUTDOWN:
448− case CARD_STATE_STARTING:
449− case CARD_STATE_FORMATTING:
450− case CARD_STATE_UNINITIALIZED:
451− case CARD_STATE_SHUTTING_DOWN:
452− /*
453− * dStroy is a term coined by marketing to represent the low level
454− * secure erase.
455− */
456− case CARD_STATE_DSTROYING:
457− set_capacity(card->gendisk, 0);
458− break;
459− }
460−}
461−
462−static void card_event_handler(struct work_struct *work)
463−{
464− struct rsxx_cardinfo *card;
465− unsigned int state;
466− unsigned long flags;
467− int st;
468−
469− card = container_of(work, struct rsxx_cardinfo, event_work);
470−
471− if (unlikely(card->halt))
472− return;
473−
474− /*
475− * Enable the interrupt now to avoid any weird race conditions where a
476− * state change might occur while rsxx_get_card_state() is
477− * processing a returned creg cmd.
478− */
479− spin_lock_irqsave(&card->irq_lock, flags);
480− rsxx_enable_ier_and_isr(card, CR_INTR_EVENT);
481− spin_unlock_irqrestore(&card->irq_lock, flags);
482−
483− st = rsxx_get_card_state(card, &state);
484− if (st) {
485− dev_info(CARD_TO_DEV(card),
486− "Failed reading state after event.\n");
487− return;
488− }
489−
490− if (card->state != state)
491− card_state_change(card, state);
492−
493− if (card->creg_ctrl.creg_stats.stat & CREG_STAT_LOG_PENDING)
494− rsxx_read_hw_log(card);
495−}
496−
497−/*----------------- Card Operations -------------------*/
498−static int card_shutdown(struct rsxx_cardinfo *card)
499−{
500− unsigned int state;
501− signed long start;
502− const int timeout = msecs_to_jiffies(120000);
503− int st;
504−
505− /* We can't issue a shutdown if the card is in a transition state */
506− start = jiffies;
507− do {
508− st = rsxx_get_card_state(card, &state);
509− if (st)
510− return st;
511− } while (state == CARD_STATE_STARTING &&
512− (jiffies - start < timeout));
513−
514− if (state == CARD_STATE_STARTING)
515− return -ETIMEDOUT;
516−
517− /* Only issue a shutdown if we need to */
518− if ((state != CARD_STATE_SHUTTING_DOWN) &&
519− (state != CARD_STATE_SHUTDOWN)) {
520− st = rsxx_issue_card_cmd(card, CARD_CMD_SHUTDOWN);
521− if (st)
522− return st;
523− }
524−
525− start = jiffies;
526− do {
527− st = rsxx_get_card_state(card, &state);
528− if (st)
529− return st;
530− } while (state != CARD_STATE_SHUTDOWN &&
531− (jiffies - start < timeout));
532−
533− if (state != CARD_STATE_SHUTDOWN)
534− return -ETIMEDOUT;
535−
536− return 0;
537−}
538−
539−static int rsxx_eeh_frozen(struct pci_dev *dev)
540−{
541− struct rsxx_cardinfo *card = pci_get_drvdata(dev);
542− int i;
543− int st;
544−
545− dev_warn(&dev->dev, "IBM Flash Adapter PCI: preparing for slot reset.\n");
546−
547− card->eeh_state = 1;
548− rsxx_mask_interrupts(card);
549−
550− /*
551− * We need to guarantee that the write for eeh_state and masking
552− * interrupts does not become reordered. This will prevent a possible
553− * race condition with the EEH code.
554− */
555− wmb();
556−
557− pci_disable_device(dev);
558−
559− st = rsxx_eeh_save_issued_dmas(card);
560− if (st)
561− return st;
562−
563− rsxx_eeh_save_issued_creg(card);
564−
565− for (i = 0; i < card->n_targets; i++) {
566− if (card->ctrl[i].status.buf)
567− dma_free_coherent(&card->dev->dev,
568− STATUS_BUFFER_SIZE8,
569− card->ctrl[i].status.buf,
570− card->ctrl[i].status.dma_addr);
571− if (card->ctrl[i].cmd.buf)
572− dma_free_coherent(&card->dev->dev,
573− COMMAND_BUFFER_SIZE8,
574− card->ctrl[i].cmd.buf,
575− card->ctrl[i].cmd.dma_addr);
576− }
577−
578− return 0;
579−}
580−
581−static void rsxx_eeh_failure(struct pci_dev *dev)
582−{
583− struct rsxx_cardinfo *card = pci_get_drvdata(dev);
584− int i;
585− int cnt = 0;
586−
587− dev_err(&dev->dev, "IBM Flash Adapter PCI: disabling failed card.\n");
588−
589− card->eeh_state = 1;
590− card->halt = 1;
591−
592− for (i = 0; i < card->n_targets; i++) {
593− spin_lock_bh(&card->ctrl[i].queue_lock);
594− cnt = rsxx_cleanup_dma_queue(&card->ctrl[i],
595− &card->ctrl[i].queue,
596− COMPLETE_DMA);
597− spin_unlock_bh(&card->ctrl[i].queue_lock);
598−
599− cnt += rsxx_dma_cancel(&card->ctrl[i]);
600−
601− if (cnt)
602− dev_info(CARD_TO_DEV(card),
603− "Freed %d queued DMAs on channel %d\n",
604− cnt, card->ctrl[i].id);
605− }
606−}
607−
608−static int rsxx_eeh_fifo_flush_poll(struct rsxx_cardinfo *card)
609−{
610− unsigned int status;
611− int iter = 0;
612−
613− /* We need to wait for the hardware to reset */
614− while (iter++ < 10) {
615− status = ioread32(card->regmap + PCI_RECONFIG);
616−
617− if (status & RSXX_FLUSH_BUSY) {
618− ssleep(1);
619− continue;
620− }
621−
622− if (status & RSXX_FLUSH_TIMEOUT)
623− dev_warn(CARD_TO_DEV(card), "HW: flash controller timeout\n");
624− return 0;
625− }
626−
627− /* Hardware failed resetting itself. */
628− return -1;
629−}
630−
631−static pci_ers_result_t rsxx_error_detected(struct pci_dev *dev,
632− pci_channel_state_t error)
633−{
634− int st;
635−
636− if (dev->revision < RSXX_EEH_SUPPORT)
637− return PCI_ERS_RESULT_NONE;
638−
639− if (error == pci_channel_io_perm_failure) {
640− rsxx_eeh_failure(dev);
641− return PCI_ERS_RESULT_DISCONNECT;
642− }
643−
644− st = rsxx_eeh_frozen(dev);
645− if (st) {
646− dev_err(&dev->dev, "Slot reset setup failed\n");
647− rsxx_eeh_failure(dev);
648− return PCI_ERS_RESULT_DISCONNECT;
649− }
650−
651− return PCI_ERS_RESULT_NEED_RESET;
652−}
653−
654−static pci_ers_result_t rsxx_slot_reset(struct pci_dev *dev)
655−{
656− struct rsxx_cardinfo *card = pci_get_drvdata(dev);
657− unsigned long flags;
658− int i;
659− int st;
660−
661− dev_warn(&dev->dev,
662− "IBM Flash Adapter PCI: recovering from slot reset.\n");
663−
664− st = pci_enable_device(dev);
665− if (st)
666− goto failed_hw_setup;
667−
668− pci_set_master(dev);
669−
670− st = rsxx_eeh_fifo_flush_poll(card);
671− if (st)
672− goto failed_hw_setup;
673−
674− rsxx_dma_queue_reset(card);
675−
676− for (i = 0; i < card->n_targets; i++) {
677− st = rsxx_hw_buffers_init(dev, &card->ctrl[i]);
678− if (st)
679− goto failed_hw_buffers_init;
680− }
681−
682− if (card->config_valid)
683− rsxx_dma_configure(card);
684−
685− /* Clears the ISR register from spurious interrupts */
686− st = ioread32(card->regmap + ISR);
687−
688− card->eeh_state = 0;
689−
690− spin_lock_irqsave(&card->irq_lock, flags);
691− if (card->n_targets & RSXX_MAX_TARGETS)
692− rsxx_enable_ier_and_isr(card, CR_INTR_ALL_G);
693− else
694− rsxx_enable_ier_and_isr(card, CR_INTR_ALL_C);
695− spin_unlock_irqrestore(&card->irq_lock, flags);
696−
697− rsxx_kick_creg_queue(card);
698−
699− for (i = 0; i < card->n_targets; i++) {
700− spin_lock(&card->ctrl[i].queue_lock);
701− if (list_empty(&card->ctrl[i].queue)) {
702− spin_unlock(&card->ctrl[i].queue_lock);
703− continue;
704− }
705− spin_unlock(&card->ctrl[i].queue_lock);
706−
707− queue_work(card->ctrl[i].issue_wq,
708− &card->ctrl[i].issue_dma_work);
709− }
710−
711− dev_info(&dev->dev, "IBM Flash Adapter PCI: recovery complete.\n");
712−
713− return PCI_ERS_RESULT_RECOVERED;
714−
715−failed_hw_buffers_init:
716− for (i = 0; i < card->n_targets; i++) {
717− if (card->ctrl[i].status.buf)
718− dma_free_coherent(&card->dev->dev,
719− STATUS_BUFFER_SIZE8,
720− card->ctrl[i].status.buf,
721− card->ctrl[i].status.dma_addr);
722− if (card->ctrl[i].cmd.buf)
723− dma_free_coherent(&card->dev->dev,
724− COMMAND_BUFFER_SIZE8,
725− card->ctrl[i].cmd.buf,
726− card->ctrl[i].cmd.dma_addr);
727− }
728−failed_hw_setup:
729− rsxx_eeh_failure(dev);
730− return PCI_ERS_RESULT_DISCONNECT;
731−
732−}
733−
734−/*----------------- Driver Initialization & Setup -------------------*/
735−/* Returns: 0 if the driver is compatible with the device
736− -1 if the driver is NOT compatible with the device */
737−static int rsxx_compatibility_check(struct rsxx_cardinfo *card)
738−{
739− unsigned char pci_rev;
740−
741− pci_read_config_byte(card->dev, PCI_REVISION_ID, &pci_rev);
742−
743− if (pci_rev > RS70_PCI_REV_SUPPORTED)
744− return -1;
745− return 0;
746−}
747−
748−static int rsxx_pci_probe(struct pci_dev *dev,
749− const struct pci_device_id *id)
750−{
751− struct rsxx_cardinfo *card;
752− int st;
753− unsigned int sync_timeout;
754−
755− dev_info(&dev->dev, "PCI-Flash SSD discovered\n");
756−
757− card = kzalloc(sizeof(*card), GFP_KERNEL);
758− if (!card)
759− return -ENOMEM;
760−
761− card->dev = dev;
762− pci_set_drvdata(dev, card);
763−
764− st = ida_alloc(&rsxx_disk_ida, GFP_KERNEL);
765− if (st < 0)
766− goto failed_ida_get;
767− card->disk_id = st;
768−
769− st = pci_enable_device(dev);
770− if (st)
771− goto failed_enable;
772−
773− pci_set_master(dev);
774−
775− st = dma_set_mask(&dev->dev, DMA_BIT_MASK(64));
776− if (st) {
777− dev_err(CARD_TO_DEV(card),
778− "No usable DMA configuration,aborting\n");
779− goto failed_dma_mask;
780− }
781−
782− st = pci_request_regions(dev, DRIVER_NAME);
783− if (st) {
784− dev_err(CARD_TO_DEV(card),
785− "Failed to request memory region\n");
786− goto failed_request_regions;
787− }
788−
789− if (pci_resource_len(dev, 0) == 0) {
790− dev_err(CARD_TO_DEV(card), "BAR0 has length 0!\n");
791− st = -ENOMEM;
792− goto failed_iomap;
793− }
794−
795− card->regmap = pci_iomap(dev, 0, 0);
796− if (!card->regmap) {
797− dev_err(CARD_TO_DEV(card), "Failed to map BAR0\n");
798− st = -ENOMEM;
799− goto failed_iomap;
800− }
801−
802− spin_lock_init(&card->irq_lock);
803− card->halt = 0;
804− card->eeh_state = 0;
805−
806− spin_lock_irq(&card->irq_lock);
807− rsxx_disable_ier_and_isr(card, CR_INTR_ALL);
808− spin_unlock_irq(&card->irq_lock);
809−
810− if (!force_legacy) {
811− st = pci_enable_msi(dev);
812− if (st)
813− dev_warn(CARD_TO_DEV(card),
814− "Failed to enable MSI\n");
815− }
816−
817− st = request_irq(dev->irq, rsxx_isr, IRQF_SHARED,
818− DRIVER_NAME, card);
819− if (st) {
820− dev_err(CARD_TO_DEV(card),
821− "Failed requesting IRQ%d\n", dev->irq);
822− goto failed_irq;
823− }
824−
825− /************* Setup Processor Command Interface *************/
826− st = rsxx_creg_setup(card);
827− if (st) {
828− dev_err(CARD_TO_DEV(card), "Failed to setup creg interface.\n");
829− goto failed_creg_setup;
830− }
831−
832− spin_lock_irq(&card->irq_lock);
833− rsxx_enable_ier_and_isr(card, CR_INTR_CREG);
834− spin_unlock_irq(&card->irq_lock);
835−
836− st = rsxx_compatibility_check(card);
837− if (st) {
838− dev_warn(CARD_TO_DEV(card),
839− "Incompatible driver detected. Please update the driver.\n");
840− st = -EINVAL;
841− goto failed_compatiblity_check;
842− }
843−
844− /************* Load Card Config *************/
845− st = rsxx_load_config(card);
846− if (st)
847− dev_err(CARD_TO_DEV(card),
848− "Failed loading card config\n");
849−
850− /************* Setup DMA Engine *************/
851− st = rsxx_get_num_targets(card, &card->n_targets);
852− if (st)
853− dev_info(CARD_TO_DEV(card),
854− "Failed reading the number of DMA targets\n");
855−
856− card->ctrl = kcalloc(card->n_targets, sizeof(*card->ctrl),
857− GFP_KERNEL);
858− if (!card->ctrl) {
859− st = -ENOMEM;
860− goto failed_dma_setup;
861− }
862−
863− st = rsxx_dma_setup(card);
864− if (st) {
865− dev_info(CARD_TO_DEV(card),
866− "Failed to setup DMA engine\n");
867− goto failed_dma_setup;
868− }
869−
870− /************* Setup Card Event Handler *************/
871− card->event_wq = create_singlethread_workqueue(DRIVER_NAME"_event");
872− if (!card->event_wq) {
873− dev_err(CARD_TO_DEV(card), "Failed card event setup.\n");
874− st = -ENOMEM;
875− goto failed_event_handler;
876− }
877−
878− INIT_WORK(&card->event_work, card_event_handler);
879−
880− st = rsxx_setup_dev(card);
881− if (st)
882− goto failed_create_dev;
883−
884− rsxx_get_card_state(card, &card->state);
885−
886− dev_info(CARD_TO_DEV(card),
887− "card state: %s\n",
888− rsxx_card_state_to_str(card->state));
889−
890− /*
891− * Now that the DMA Engine and devices have been setup,
892− * we can enable the event interrupt(it kicks off actions in
893− * those layers so we couldn't enable it right away.)
894− */
895− spin_lock_irq(&card->irq_lock);
896− rsxx_enable_ier_and_isr(card, CR_INTR_EVENT);
897− spin_unlock_irq(&card->irq_lock);
898−
899− if (card->state == CARD_STATE_SHUTDOWN) {
900− st = rsxx_issue_card_cmd(card, CARD_CMD_STARTUP);
901− if (st)
902− dev_crit(CARD_TO_DEV(card),
903− "Failed issuing card startup\n");
904− if (sync_start) {
905− sync_timeout = SYNC_START_TIMEOUT;
906−
907− dev_info(CARD_TO_DEV(card),
908− "Waiting for card to startup\n");
909−
910− do {
911− ssleep(1);
912− sync_timeout--;
913−
914− rsxx_get_card_state(card, &card->state);
915− } while (sync_timeout &&
916− (card->state == CARD_STATE_STARTING));
917−
918− if (card->state == CARD_STATE_STARTING) {
919− dev_warn(CARD_TO_DEV(card),
920− "Card startup timed out\n");
921− card->size8 = 0;
922− } else {
923− dev_info(CARD_TO_DEV(card),
924− "card state: %s\n",
925− rsxx_card_state_to_str(card->state));
926− st = rsxx_get_card_size8(card, &card->size8);
927− if (st)
928− card->size8 = 0;
929− }
930− }
931− } else if (card->state == CARD_STATE_GOOD ||
932− card->state == CARD_STATE_RD_ONLY_FAULT) {
933− st = rsxx_get_card_size8(card, &card->size8);
934− if (st)
935− card->size8 = 0;
936− }
937−
938− st = rsxx_attach_dev(card);
939− if (st)
940− goto failed_create_dev;
941−
942− /************* Setup Debugfs *************/
943− rsxx_debugfs_dev_new(card);
944−
945− return 0;
946−
947−failed_create_dev:
948− destroy_workqueue(card->event_wq);
949− card->event_wq = NULL;
950−failed_event_handler:
951− rsxx_dma_destroy(card);
952−failed_dma_setup:
953−failed_compatiblity_check:
954− destroy_workqueue(card->creg_ctrl.creg_wq);
955− card->creg_ctrl.creg_wq = NULL;
956−failed_creg_setup:
957− spin_lock_irq(&card->irq_lock);
958− rsxx_disable_ier_and_isr(card, CR_INTR_ALL);
959− spin_unlock_irq(&card->irq_lock);
960− free_irq(dev->irq, card);
961− if (!force_legacy)
962− pci_disable_msi(dev);
963−failed_irq:
964− pci_iounmap(dev, card->regmap);
965−failed_iomap:
966− pci_release_regions(dev);
967−failed_request_regions:
968−failed_dma_mask:
969− pci_disable_device(dev);
970−failed_enable:
971− ida_free(&rsxx_disk_ida, card->disk_id);
972−failed_ida_get:
973− kfree(card);
974−
975− return st;
976−}
977−
978−static void rsxx_pci_remove(struct pci_dev *dev)
979−{
980− struct rsxx_cardinfo *card = pci_get_drvdata(dev);
981− unsigned long flags;
982− int st;
983− int i;
984−
985− if (!card)
986− return;
987−
988− dev_info(CARD_TO_DEV(card),
989− "Removing PCI-Flash SSD.\n");
990−
991− rsxx_detach_dev(card);
992−
993− for (i = 0; i < card->n_targets; i++) {
994− spin_lock_irqsave(&card->irq_lock, flags);
995− rsxx_disable_ier_and_isr(card, CR_INTR_DMA(i));
996− spin_unlock_irqrestore(&card->irq_lock, flags);
997− }
998−
999− st = card_shutdown(card);
1000− if (st)
1001− dev_crit(CARD_TO_DEV(card), "Shutdown failed!\n");
1002−
1003− /* Sync outstanding event handlers. */
1004− spin_lock_irqsave(&card->irq_lock, flags);
1005− rsxx_disable_ier_and_isr(card, CR_INTR_EVENT);
1006− spin_unlock_irqrestore(&card->irq_lock, flags);
1007−
1008− cancel_work_sync(&card->event_work);
1009−
1010− destroy_workqueue(card->event_wq);
1011− rsxx_destroy_dev(card);
1012− rsxx_dma_destroy(card);
1013− destroy_workqueue(card->creg_ctrl.creg_wq);
1014−
1015− spin_lock_irqsave(&card->irq_lock, flags);
1016− rsxx_disable_ier_and_isr(card, CR_INTR_ALL);
1017− spin_unlock_irqrestore(&card->irq_lock, flags);
1018−
1019− /* Prevent work_structs from re-queuing themselves. */
1020− card->halt = 1;
1021−
1022− debugfs_remove_recursive(card->debugfs_dir);
1023−
1024− free_irq(dev->irq, card);
1025−
1026− if (!force_legacy)
1027− pci_disable_msi(dev);
1028−
1029− rsxx_creg_destroy(card);
1030−
1031− pci_iounmap(dev, card->regmap);
1032−
1033− pci_disable_device(dev);
1034− pci_release_regions(dev);
1035−
1036− ida_free(&rsxx_disk_ida, card->disk_id);
1037− kfree(card);
1038−}
1039−
1040−static int rsxx_pci_suspend(struct pci_dev *dev, pm_message_t state)
1041−{
1042− /* We don't support suspend at this time. */
1043− return -ENOSYS;
1044−}
1045−
1046−static void rsxx_pci_shutdown(struct pci_dev *dev)
1047−{
1048− struct rsxx_cardinfo *card = pci_get_drvdata(dev);
1049− unsigned long flags;
1050− int i;
1051−
1052− if (!card)
1053− return;
1054−
1055− dev_info(CARD_TO_DEV(card), "Shutting down PCI-Flash SSD.\n");
1056−
1057− rsxx_detach_dev(card);
1058−
1059− for (i = 0; i < card->n_targets; i++) {
1060− spin_lock_irqsave(&card->irq_lock, flags);
1061− rsxx_disable_ier_and_isr(card, CR_INTR_DMA(i));
1062− spin_unlock_irqrestore(&card->irq_lock, flags);
1063− }
1064−
1065− card_shutdown(card);
1066−}
1067−
1068−static const struct pci_error_handlers rsxx_err_handler = {
1069− .error_detected = rsxx_error_detected,
1070− .slot_reset = rsxx_slot_reset,
1071−};
1072−
1073−static const struct pci_device_id rsxx_pci_ids[] = {
1074− {PCI_DEVICE(PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_FS70_FLASH)},
1075− {PCI_DEVICE(PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_FS80_FLASH)},
1076− {0,},
1077−};
1078−
1079−MODULE_DEVICE_TABLE(pci, rsxx_pci_ids);
1080−
1081−static struct pci_driver rsxx_pci_driver = {
1082− .name = DRIVER_NAME,
1083− .id_table = rsxx_pci_ids,
1084− .probe = rsxx_pci_probe,
1085− .remove = rsxx_pci_remove,
1086− .suspend = rsxx_pci_suspend,
1087− .shutdown = rsxx_pci_shutdown,
1088− .err_handler = &rsxx_err_handler,
1089−};
1090−
1091−static int __init rsxx_core_init(void)
1092−{
1093− int st;
1094−
1095− st = rsxx_dev_init();
1096− if (st)
1097− return st;
1098−
1099− st = rsxx_dma_init();
1100− if (st)
1101− goto dma_init_failed;
1102−
1103− st = rsxx_creg_init();
1104− if (st)
1105− goto creg_init_failed;
1106−
1107− return pci_register_driver(&rsxx_pci_driver);
1108−
1109−creg_init_failed:
1110− rsxx_dma_cleanup();
1111−dma_init_failed:
1112− rsxx_dev_cleanup();
1113−
1114− return st;
1115−}
1116−
1117−static void __exit rsxx_core_cleanup(void)
1118−{
1119− pci_unregister_driver(&rsxx_pci_driver);
1120− rsxx_creg_cleanup();
1121− rsxx_dma_cleanup();
1122− rsxx_dev_cleanup();
1123−}
1124−
1125−module_init(rsxx_core_init);
1126−module_exit(rsxx_core_cleanup);
drivers/block/rsxx/cregs.c+0−789
@@ -1,789 +0,0 @@
1−// SPDX-License-Identifier: GPL-2.0-or-later
2−/*
3−* Filename: cregs.c
4−*
5−* Authors: Joshua Morris <josh.h.morris@us.ibm.com>
6−* Philip Kelleher <pjk1939@linux.vnet.ibm.com>
7−*
8−* (C) Copyright 2013 IBM Corporation
9−*/
10−
11−#include <linux/completion.h>
12−#include <linux/slab.h>
13−
14−#include "rsxx_priv.h"
15−
16−#define CREG_TIMEOUT_MSEC 10000
17−
18−typedef void (*creg_cmd_cb)(struct rsxx_cardinfo *card,
19− struct creg_cmd *cmd,
20− int st);
21−
22−struct creg_cmd {
23− struct list_head list;
24− creg_cmd_cb cb;
25− void *cb_private;
26− unsigned int op;
27− unsigned int addr;
28− int cnt8;
29− void *buf;
30− unsigned int stream;
31− unsigned int status;
32−};
33−
34−static struct kmem_cache *creg_cmd_pool;
35−
36−
37−/*------------ Private Functions --------------*/
38−
39−#if defined(__LITTLE_ENDIAN)
40−#define LITTLE_ENDIAN 1
41−#elif defined(__BIG_ENDIAN)
42−#define LITTLE_ENDIAN 0
43−#else
44−#error Unknown endianess!!! Aborting...
45−#endif
46−
47−static int copy_to_creg_data(struct rsxx_cardinfo *card,
48− int cnt8,
49− void *buf,
50− unsigned int stream)
51−{
52− int i = 0;
53− u32 *data = buf;
54−
55− if (unlikely(card->eeh_state))
56− return -EIO;
57−
58− for (i = 0; cnt8 > 0; i++, cnt8 -= 4) {
59− /*
60− * Firmware implementation makes it necessary to byte swap on
61− * little endian processors.
62− */
63− if (LITTLE_ENDIAN && stream)
64− iowrite32be(data[i], card->regmap + CREG_DATA(i));
65− else
66− iowrite32(data[i], card->regmap + CREG_DATA(i));
67− }
68−
69− return 0;
70−}
71−
72−
73−static int copy_from_creg_data(struct rsxx_cardinfo *card,
74− int cnt8,
75− void *buf,
76− unsigned int stream)
77−{
78− int i = 0;
79− u32 *data = buf;
80−
81− if (unlikely(card->eeh_state))
82− return -EIO;
83−
84− for (i = 0; cnt8 > 0; i++, cnt8 -= 4) {
85− /*
86− * Firmware implementation makes it necessary to byte swap on
87− * little endian processors.
88− */
89− if (LITTLE_ENDIAN && stream)
90− data[i] = ioread32be(card->regmap + CREG_DATA(i));
91− else
92− data[i] = ioread32(card->regmap + CREG_DATA(i));
93− }
94−
95− return 0;
96−}
97−
98−static void creg_issue_cmd(struct rsxx_cardinfo *card, struct creg_cmd *cmd)
99−{
100− int st;
101−
102− if (unlikely(card->eeh_state))
103− return;
104−
105− iowrite32(cmd->addr, card->regmap + CREG_ADD);
106− iowrite32(cmd->cnt8, card->regmap + CREG_CNT);
107−
108− if (cmd->op == CREG_OP_WRITE) {
109− if (cmd->buf) {
110− st = copy_to_creg_data(card, cmd->cnt8,
111− cmd->buf, cmd->stream);
112− if (st)
113− return;
114− }
115− }
116−
117− if (unlikely(card->eeh_state))
118− return;
119−
120− /* Setting the valid bit will kick off the command. */
121− iowrite32(cmd->op, card->regmap + CREG_CMD);
122−}
123−
124−static void creg_kick_queue(struct rsxx_cardinfo *card)
125−{
126− if (card->creg_ctrl.active || list_empty(&card->creg_ctrl.queue))
127− return;
128−
129− card->creg_ctrl.active = 1;
130− card->creg_ctrl.active_cmd = list_first_entry(&card->creg_ctrl.queue,
131− struct creg_cmd, list);
132− list_del(&card->creg_ctrl.active_cmd->list);
133− card->creg_ctrl.q_depth--;
134−
135− /*
136− * We have to set the timer before we push the new command. Otherwise,
137− * we could create a race condition that would occur if the timer
138− * was not canceled, and expired after the new command was pushed,
139− * but before the command was issued to hardware.
140− */
141− mod_timer(&card->creg_ctrl.cmd_timer,
142− jiffies + msecs_to_jiffies(CREG_TIMEOUT_MSEC));
143−
144− creg_issue_cmd(card, card->creg_ctrl.active_cmd);
145−}
146−
147−static int creg_queue_cmd(struct rsxx_cardinfo *card,
148− unsigned int op,
149− unsigned int addr,
150− unsigned int cnt8,
151− void *buf,
152− int stream,
153− creg_cmd_cb callback,
154− void *cb_private)
155−{
156− struct creg_cmd *cmd;
157−
158− /* Don't queue stuff up if we're halted. */
159− if (unlikely(card->halt))
160− return -EINVAL;
161−
162− if (card->creg_ctrl.reset)
163− return -EAGAIN;
164−
165− if (cnt8 > MAX_CREG_DATA8)
166− return -EINVAL;
167−
168− cmd = kmem_cache_alloc(creg_cmd_pool, GFP_KERNEL);
169− if (!cmd)
170− return -ENOMEM;
171−
172− INIT_LIST_HEAD(&cmd->list);
173−
174− cmd->op = op;
175− cmd->addr = addr;
176− cmd->cnt8 = cnt8;
177− cmd->buf = buf;
178− cmd->stream = stream;
179− cmd->cb = callback;
180− cmd->cb_private = cb_private;
181− cmd->status = 0;
182−
183− spin_lock_bh(&card->creg_ctrl.lock);
184− list_add_tail(&cmd->list, &card->creg_ctrl.queue);
185− card->creg_ctrl.q_depth++;
186− creg_kick_queue(card);
187− spin_unlock_bh(&card->creg_ctrl.lock);
188−
189− return 0;
190−}
191−
192−static void creg_cmd_timed_out(struct timer_list *t)
193−{
194− struct rsxx_cardinfo *card = from_timer(card, t, creg_ctrl.cmd_timer);
195− struct creg_cmd *cmd;
196−
197− spin_lock(&card->creg_ctrl.lock);
198− cmd = card->creg_ctrl.active_cmd;
199− card->creg_ctrl.active_cmd = NULL;
200− spin_unlock(&card->creg_ctrl.lock);
201−
202− if (cmd == NULL) {
203− card->creg_ctrl.creg_stats.creg_timeout++;
204− dev_warn(CARD_TO_DEV(card),
205− "No active command associated with timeout!\n");
206− return;
207− }
208−
209− if (cmd->cb)
210− cmd->cb(card, cmd, -ETIMEDOUT);
211−
212− kmem_cache_free(creg_cmd_pool, cmd);
213−
214−
215− spin_lock(&card->creg_ctrl.lock);
216− card->creg_ctrl.active = 0;
217− creg_kick_queue(card);
218− spin_unlock(&card->creg_ctrl.lock);
219−}
220−
221−
222−static void creg_cmd_done(struct work_struct *work)
223−{
224− struct rsxx_cardinfo *card;
225− struct creg_cmd *cmd;
226− int st = 0;
227−
228− card = container_of(work, struct rsxx_cardinfo,
229− creg_ctrl.done_work);
230−
231− /*
232− * The timer could not be cancelled for some reason,
233− * race to pop the active command.
234− */
235− if (del_timer_sync(&card->creg_ctrl.cmd_timer) == 0)
236− card->creg_ctrl.creg_stats.failed_cancel_timer++;
237−
238− spin_lock_bh(&card->creg_ctrl.lock);
239− cmd = card->creg_ctrl.active_cmd;
240− card->creg_ctrl.active_cmd = NULL;
241− spin_unlock_bh(&card->creg_ctrl.lock);
242−
243− if (cmd == NULL) {
244− dev_err(CARD_TO_DEV(card),
245− "Spurious creg interrupt!\n");
246− return;
247− }
248−
249− card->creg_ctrl.creg_stats.stat = ioread32(card->regmap + CREG_STAT);
250− cmd->status = card->creg_ctrl.creg_stats.stat;
251− if ((cmd->status & CREG_STAT_STATUS_MASK) == 0) {
252− dev_err(CARD_TO_DEV(card),
253− "Invalid status on creg command\n");
254− /*
255− * At this point we're probably reading garbage from HW. Don't
256− * do anything else that could mess up the system and let
257− * the sync function return an error.
258− */
259− st = -EIO;
260− goto creg_done;
261− } else if (cmd->status & CREG_STAT_ERROR) {
262− st = -EIO;
263− }
264−
265− if (cmd->op == CREG_OP_READ) {
266− unsigned int cnt8 = ioread32(card->regmap + CREG_CNT);
267−
268− /* Paranoid Sanity Checks */
269− if (!cmd->buf) {
270− dev_err(CARD_TO_DEV(card),
271− "Buffer not given for read.\n");
272− st = -EIO;
273− goto creg_done;
274− }
275− if (cnt8 != cmd->cnt8) {
276− dev_err(CARD_TO_DEV(card),
277− "count mismatch\n");
278− st = -EIO;
279− goto creg_done;
280− }
281−
282− st = copy_from_creg_data(card, cnt8, cmd->buf, cmd->stream);
283− }
284−
285−creg_done:
286− if (cmd->cb)
287− cmd->cb(card, cmd, st);
288−
289− kmem_cache_free(creg_cmd_pool, cmd);
290−
291− spin_lock_bh(&card->creg_ctrl.lock);
292− card->creg_ctrl.active = 0;
293− creg_kick_queue(card);
294− spin_unlock_bh(&card->creg_ctrl.lock);
295−}
296−
297−static void creg_reset(struct rsxx_cardinfo *card)
298−{
299− struct creg_cmd *cmd = NULL;
300− struct creg_cmd *tmp;
301− unsigned long flags;
302−
303− /*
304− * mutex_trylock is used here because if reset_lock is taken then a
305− * reset is already happening. So, we can just go ahead and return.
306− */
307− if (!mutex_trylock(&card->creg_ctrl.reset_lock))
308− return;
309−
310− card->creg_ctrl.reset = 1;
311− spin_lock_irqsave(&card->irq_lock, flags);
312− rsxx_disable_ier_and_isr(card, CR_INTR_CREG | CR_INTR_EVENT);
313− spin_unlock_irqrestore(&card->irq_lock, flags);
314−
315− dev_warn(CARD_TO_DEV(card),
316− "Resetting creg interface for recovery\n");
317−
318− /* Cancel outstanding commands */
319− spin_lock_bh(&card->creg_ctrl.lock);
320− list_for_each_entry_safe(cmd, tmp, &card->creg_ctrl.queue, list) {
321− list_del(&cmd->list);
322− card->creg_ctrl.q_depth--;
323− if (cmd->cb)
324− cmd->cb(card, cmd, -ECANCELED);
325− kmem_cache_free(creg_cmd_pool, cmd);
326− }
327−
328− cmd = card->creg_ctrl.active_cmd;
329− card->creg_ctrl.active_cmd = NULL;
330− if (cmd) {
331− if (timer_pending(&card->creg_ctrl.cmd_timer))
332− del_timer_sync(&card->creg_ctrl.cmd_timer);
333−
334− if (cmd->cb)
335− cmd->cb(card, cmd, -ECANCELED);
336− kmem_cache_free(creg_cmd_pool, cmd);
337−
338− card->creg_ctrl.active = 0;
339− }
340− spin_unlock_bh(&card->creg_ctrl.lock);
341−
342− card->creg_ctrl.reset = 0;
343− spin_lock_irqsave(&card->irq_lock, flags);
344− rsxx_enable_ier_and_isr(card, CR_INTR_CREG | CR_INTR_EVENT);
345− spin_unlock_irqrestore(&card->irq_lock, flags);
346−
347− mutex_unlock(&card->creg_ctrl.reset_lock);
348−}
349−
350−/* Used for synchronous accesses */
351−struct creg_completion {
352− struct completion *cmd_done;
353− int st;
354− u32 creg_status;
355−};
356−
357−static void creg_cmd_done_cb(struct rsxx_cardinfo *card,
358− struct creg_cmd *cmd,
359− int st)
360−{
361− struct creg_completion *cmd_completion;
362−
363− cmd_completion = cmd->cb_private;
364− BUG_ON(!cmd_completion);
365−
366− cmd_completion->st = st;
367− cmd_completion->creg_status = cmd->status;
368− complete(cmd_completion->cmd_done);
369−}
370−
371−static int __issue_creg_rw(struct rsxx_cardinfo *card,
372− unsigned int op,
373− unsigned int addr,
374− unsigned int cnt8,
375− void *buf,
376− int stream,
377− unsigned int *hw_stat)
378−{
379− DECLARE_COMPLETION_ONSTACK(cmd_done);
380− struct creg_completion completion;
381− unsigned long timeout;
382− int st;
383−
384− completion.cmd_done = &cmd_done;
385− completion.st = 0;
386− completion.creg_status = 0;
387−
388− st = creg_queue_cmd(card, op, addr, cnt8, buf, stream, creg_cmd_done_cb,
389− &completion);
390− if (st)
391− return st;
392−
393− /*
394− * This timeout is necessary for unresponsive hardware. The additional
395− * 20 seconds to used to guarantee that each cregs requests has time to
396− * complete.
397− */
398− timeout = msecs_to_jiffies(CREG_TIMEOUT_MSEC *
399− card->creg_ctrl.q_depth + 20000);
400−
401− /*
402− * The creg interface is guaranteed to complete. It has a timeout
403− * mechanism that will kick in if hardware does not respond.
404− */
405− st = wait_for_completion_timeout(completion.cmd_done, timeout);
406− if (st == 0) {
407− /*
408− * This is really bad, because the kernel timer did not
409− * expire and notify us of a timeout!
410− */
411− dev_crit(CARD_TO_DEV(card),
412− "cregs timer failed\n");
413− creg_reset(card);
414− return -EIO;
415− }
416−
417− *hw_stat = completion.creg_status;
418−
419− if (completion.st) {
420− /*
421− * This read is needed to verify that there has not been any
422− * extreme errors that might have occurred, i.e. EEH. The
423− * function iowrite32 will not detect EEH errors, so it is
424− * necessary that we recover if such an error is the reason
425− * for the timeout. This is a dummy read.
426− */
427− ioread32(card->regmap + SCRATCH);
428−
429− dev_warn(CARD_TO_DEV(card),
430− "creg command failed(%d x%08x)\n",
431− completion.st, addr);
432− return completion.st;
433− }
434−
435− return 0;
436−}
437−
438−static int issue_creg_rw(struct rsxx_cardinfo *card,
439− u32 addr,
440− unsigned int size8,
441− void *data,
442− int stream,
443− int read)
444−{
445− unsigned int hw_stat;
446− unsigned int xfer;
447− unsigned int op;
448− int st;
449−
450− op = read ? CREG_OP_READ : CREG_OP_WRITE;
451−
452− do {
453− xfer = min_t(unsigned int, size8, MAX_CREG_DATA8);
454−
455− st = __issue_creg_rw(card, op, addr, xfer,
456− data, stream, &hw_stat);
457− if (st)
458− return st;
459−
460− data = (char *)data + xfer;
461− addr += xfer;
462− size8 -= xfer;
463− } while (size8);
464−
465− return 0;
466−}
467−
468−/* ---------------------------- Public API ---------------------------------- */
469−int rsxx_creg_write(struct rsxx_cardinfo *card,
470− u32 addr,
471− unsigned int size8,
472− void *data,
473− int byte_stream)
474−{
475− return issue_creg_rw(card, addr, size8, data, byte_stream, 0);
476−}
477−
478−int rsxx_creg_read(struct rsxx_cardinfo *card,
479− u32 addr,
480− unsigned int size8,
481− void *data,
482− int byte_stream)
483−{
484− return issue_creg_rw(card, addr, size8, data, byte_stream, 1);
485−}
486−
487−int rsxx_get_card_state(struct rsxx_cardinfo *card, unsigned int *state)
488−{
489− return rsxx_creg_read(card, CREG_ADD_CARD_STATE,
490− sizeof(*state), state, 0);
491−}
492−
493−int rsxx_get_card_size8(struct rsxx_cardinfo *card, u64 *size8)
494−{
495− unsigned int size;
496− int st;
497−
498− st = rsxx_creg_read(card, CREG_ADD_CARD_SIZE,
499− sizeof(size), &size, 0);
500− if (st)
501− return st;
502−
503− *size8 = (u64)size * RSXX_HW_BLK_SIZE;
504− return 0;
505−}
506−
507−int rsxx_get_num_targets(struct rsxx_cardinfo *card,
508− unsigned int *n_targets)
509−{
510− return rsxx_creg_read(card, CREG_ADD_NUM_TARGETS,
511− sizeof(*n_targets), n_targets, 0);
512−}
513−
514−int rsxx_get_card_capabilities(struct rsxx_cardinfo *card,
515− u32 *capabilities)
516−{
517− return rsxx_creg_read(card, CREG_ADD_CAPABILITIES,
518− sizeof(*capabilities), capabilities, 0);
519−}
520−
521−int rsxx_issue_card_cmd(struct rsxx_cardinfo *card, u32 cmd)
522−{
523− return rsxx_creg_write(card, CREG_ADD_CARD_CMD,
524− sizeof(cmd), &cmd, 0);
525−}
526−
527−
528−/*----------------- HW Log Functions -------------------*/
529−static void hw_log_msg(struct rsxx_cardinfo *card, const char *str, int len)
530−{
531− static char level;
532−
533− /*
534− * New messages start with "<#>", where # is the log level. Messages
535− * that extend past the log buffer will use the previous level
536− */
537− if ((len > 3) && (str[0] == '<') && (str[2] == '>')) {
538− level = str[1];
539− str += 3; /* Skip past the log level. */
540− len -= 3;
541− }
542−
543− switch (level) {
544− case '0':
545− dev_emerg(CARD_TO_DEV(card), "HW: %.*s", len, str);
546− break;
547− case '1':
548− dev_alert(CARD_TO_DEV(card), "HW: %.*s", len, str);
549− break;
550− case '2':
551− dev_crit(CARD_TO_DEV(card), "HW: %.*s", len, str);
552− break;
553− case '3':
554− dev_err(CARD_TO_DEV(card), "HW: %.*s", len, str);
555− break;
556− case '4':
557− dev_warn(CARD_TO_DEV(card), "HW: %.*s", len, str);
558− break;
559− case '5':
560− dev_notice(CARD_TO_DEV(card), "HW: %.*s", len, str);
561− break;
562− case '6':
563− dev_info(CARD_TO_DEV(card), "HW: %.*s", len, str);
564− break;
565− case '7':
566− dev_dbg(CARD_TO_DEV(card), "HW: %.*s", len, str);
567− break;
568− default:
569− dev_info(CARD_TO_DEV(card), "HW: %.*s", len, str);
570− break;
571− }
572−}
573−
574−/*
575− * The substrncpy function copies the src string (which includes the
576− * terminating '\0' character), up to the count into the dest pointer.
577− * Returns the number of bytes copied to dest.
578− */
579−static int substrncpy(char *dest, const char *src, int count)
580−{
581− int max_cnt = count;
582−
583− while (count) {
584− count--;
585− *dest = *src;
586− if (*dest == '\0')
587− break;
588− src++;
589− dest++;
590− }
591− return max_cnt - count;
592−}
593−
594−
595−static void read_hw_log_done(struct rsxx_cardinfo *card,
596− struct creg_cmd *cmd,
597− int st)
598−{
599− char *buf;
600− char *log_str;
601− int cnt;
602− int len;
603− int off;
604−
605− buf = cmd->buf;
606− off = 0;
607−
608− /* Failed getting the log message */
609− if (st)
610− return;
611−
612− while (off < cmd->cnt8) {
613− log_str = &card->log.buf[card->log.buf_len];
614− cnt = min(cmd->cnt8 - off, LOG_BUF_SIZE8 - card->log.buf_len);
615− len = substrncpy(log_str, &buf[off], cnt);
616−
617− off += len;
618− card->log.buf_len += len;
619−
620− /*
621− * Flush the log if we've hit the end of a message or if we've
622− * run out of buffer space.
623− */
624− if ((log_str[len - 1] == '\0') ||
625− (card->log.buf_len == LOG_BUF_SIZE8)) {
626− if (card->log.buf_len != 1) /* Don't log blank lines. */
627− hw_log_msg(card, card->log.buf,
628− card->log.buf_len);
629− card->log.buf_len = 0;
630− }
631−
632− }
633−
634− if (cmd->status & CREG_STAT_LOG_PENDING)
635− rsxx_read_hw_log(card);
636−}
637−
638−int rsxx_read_hw_log(struct rsxx_cardinfo *card)
639−{
640− int st;
641−
642− st = creg_queue_cmd(card, CREG_OP_READ, CREG_ADD_LOG,
643− sizeof(card->log.tmp), card->log.tmp,
644− 1, read_hw_log_done, NULL);
645− if (st)
646− dev_err(CARD_TO_DEV(card),
647− "Failed getting log text\n");
648−
649− return st;
650−}
651−
652−/*-------------- IOCTL REG Access ------------------*/
653−static int issue_reg_cmd(struct rsxx_cardinfo *card,
654− struct rsxx_reg_access *cmd,
655− int read)
656−{
657− unsigned int op = read ? CREG_OP_READ : CREG_OP_WRITE;
658−
659− return __issue_creg_rw(card, op, cmd->addr, cmd->cnt, cmd->data,
660− cmd->stream, &cmd->stat);
661−}
662−
663−int rsxx_reg_access(struct rsxx_cardinfo *card,
664− struct rsxx_reg_access __user *ucmd,
665− int read)
666−{
667− struct rsxx_reg_access cmd;
668− int st;
669−
670− st = copy_from_user(&cmd, ucmd, sizeof(cmd));
671− if (st)
672− return -EFAULT;
673−
674− if (cmd.cnt > RSXX_MAX_REG_CNT)
675− return -EFAULT;
676−
677− st = issue_reg_cmd(card, &cmd, read);
678− if (st)
679− return st;
680−
681− st = put_user(cmd.stat, &ucmd->stat);
682− if (st)
683− return -EFAULT;
684−
685− if (read) {
686− st = copy_to_user(ucmd->data, cmd.data, cmd.cnt);
687− if (st)
688− return -EFAULT;
689− }
690−
691− return 0;
692−}
693−
694−void rsxx_eeh_save_issued_creg(struct rsxx_cardinfo *card)
695−{
696− struct creg_cmd *cmd = NULL;
697−
698− cmd = card->creg_ctrl.active_cmd;
699− card->creg_ctrl.active_cmd = NULL;
700−
701− if (cmd) {
702− del_timer_sync(&card->creg_ctrl.cmd_timer);
703−
704− spin_lock_bh(&card->creg_ctrl.lock);
705− list_add(&cmd->list, &card->creg_ctrl.queue);
706− card->creg_ctrl.q_depth++;
707− card->creg_ctrl.active = 0;
708− spin_unlock_bh(&card->creg_ctrl.lock);
709− }
710−}
711−
712−void rsxx_kick_creg_queue(struct rsxx_cardinfo *card)
713−{
714− spin_lock_bh(&card->creg_ctrl.lock);
715− if (!list_empty(&card->creg_ctrl.queue))
716− creg_kick_queue(card);
717− spin_unlock_bh(&card->creg_ctrl.lock);
718−}
719−
720−/*------------ Initialization & Setup --------------*/
721−int rsxx_creg_setup(struct rsxx_cardinfo *card)
722−{
723− card->creg_ctrl.active_cmd = NULL;
724−
725− card->creg_ctrl.creg_wq =
726− create_singlethread_workqueue(DRIVER_NAME"_creg");
727− if (!card->creg_ctrl.creg_wq)
728− return -ENOMEM;
729−
730− INIT_WORK(&card->creg_ctrl.done_work, creg_cmd_done);
731− mutex_init(&card->creg_ctrl.reset_lock);
732− INIT_LIST_HEAD(&card->creg_ctrl.queue);
733− spin_lock_init(&card->creg_ctrl.lock);
734− timer_setup(&card->creg_ctrl.cmd_timer, creg_cmd_timed_out, 0);
735−
736− return 0;
737−}
738−
739−void rsxx_creg_destroy(struct rsxx_cardinfo *card)
740−{
741− struct creg_cmd *cmd;
742− struct creg_cmd *tmp;
743− int cnt = 0;
744−
745− /* Cancel outstanding commands */
746− spin_lock_bh(&card->creg_ctrl.lock);
747− list_for_each_entry_safe(cmd, tmp, &card->creg_ctrl.queue, list) {
748− list_del(&cmd->list);
749− if (cmd->cb)
750− cmd->cb(card, cmd, -ECANCELED);
751− kmem_cache_free(creg_cmd_pool, cmd);
752− cnt++;
753− }
754−
755− if (cnt)
756− dev_info(CARD_TO_DEV(card),
757− "Canceled %d queue creg commands\n", cnt);
758−
759− cmd = card->creg_ctrl.active_cmd;
760− card->creg_ctrl.active_cmd = NULL;
761− if (cmd) {
762− if (timer_pending(&card->creg_ctrl.cmd_timer))
763− del_timer_sync(&card->creg_ctrl.cmd_timer);
764−
765− if (cmd->cb)
766− cmd->cb(card, cmd, -ECANCELED);
767− dev_info(CARD_TO_DEV(card),
768− "Canceled active creg command\n");
769− kmem_cache_free(creg_cmd_pool, cmd);
770− }
771− spin_unlock_bh(&card->creg_ctrl.lock);
772−
773− cancel_work_sync(&card->creg_ctrl.done_work);
774−}
775−
776−
777−int rsxx_creg_init(void)
778−{
779− creg_cmd_pool = KMEM_CACHE(creg_cmd, SLAB_HWCACHE_ALIGN);
780− if (!creg_cmd_pool)
781− return -ENOMEM;
782−
783− return 0;
784−}
785−
786−void rsxx_creg_cleanup(void)
787−{
788− kmem_cache_destroy(creg_cmd_pool);
789−}
drivers/block/rsxx/dev.c+0−306
@@ -1,306 +0,0 @@
1−// SPDX-License-Identifier: GPL-2.0-or-later
2−/*
3−* Filename: dev.c
4−*
5−* Authors: Joshua Morris <josh.h.morris@us.ibm.com>
6−* Philip Kelleher <pjk1939@linux.vnet.ibm.com>
7−*
8−* (C) Copyright 2013 IBM Corporation
9−*/
10−
11−#include <linux/kernel.h>
12−#include <linux/interrupt.h>
13−#include <linux/module.h>
14−#include <linux/pci.h>
15−#include <linux/slab.h>
16−
17−#include <linux/hdreg.h>
18−#include <linux/genhd.h>
19−#include <linux/blkdev.h>
20−#include <linux/bio.h>
21−
22−#include <linux/fs.h>
23−
24−#include "rsxx_priv.h"
25−
26−static unsigned int blkdev_minors = 64;
27−module_param(blkdev_minors, uint, 0444);
28−MODULE_PARM_DESC(blkdev_minors, "Number of minors(partitions)");
29−
30−/*
31− * For now I'm making this tweakable in case any applications hit this limit.
32− * If you see a "bio too big" error in the log you will need to raise this
33− * value.
34− */
35−static unsigned int blkdev_max_hw_sectors = 1024;
36−module_param(blkdev_max_hw_sectors, uint, 0444);
37−MODULE_PARM_DESC(blkdev_max_hw_sectors, "Max hw sectors for a single BIO");
38−
39−static unsigned int enable_blkdev = 1;
40−module_param(enable_blkdev , uint, 0444);
41−MODULE_PARM_DESC(enable_blkdev, "Enable block device interfaces");
42−
43−
44−struct rsxx_bio_meta {
45− struct bio *bio;
46− atomic_t pending_dmas;
47− atomic_t error;
48− unsigned long start_time;
49−};
50−
51−static struct kmem_cache *bio_meta_pool;
52−
53−static void rsxx_submit_bio(struct bio *bio);
54−
55−/*----------------- Block Device Operations -----------------*/
56−static int rsxx_blkdev_ioctl(struct block_device *bdev,
57− fmode_t mode,
58− unsigned int cmd,
59− unsigned long arg)
60−{
61− struct rsxx_cardinfo *card = bdev->bd_disk->private_data;
62−
63− switch (cmd) {
64− case RSXX_GETREG:
65− return rsxx_reg_access(card, (void __user *)arg, 1);
66− case RSXX_SETREG:
67− return rsxx_reg_access(card, (void __user *)arg, 0);
68− }
69−
70− return -ENOTTY;
71−}
72−
73−static int rsxx_getgeo(struct block_device *bdev, struct hd_geometry *geo)
74−{
75− struct rsxx_cardinfo *card = bdev->bd_disk->private_data;
76− u64 blocks = card->size8 >> 9;
77−
78− /*
79− * get geometry: Fake it. I haven't found any drivers that set
80− * geo->start, so we won't either.
81− */
82− if (card->size8) {
83− geo->heads = 64;
84− geo->sectors = 16;
85− do_div(blocks, (geo->heads * geo->sectors));
86− geo->cylinders = blocks;
87− } else {
88− geo->heads = 0;
89− geo->sectors = 0;
90− geo->cylinders = 0;
91− }
92− return 0;
93−}
94−
95−static const struct block_device_operations rsxx_fops = {
96− .owner = THIS_MODULE,
97− .submit_bio = rsxx_submit_bio,
98− .getgeo = rsxx_getgeo,
99− .ioctl = rsxx_blkdev_ioctl,
100−};
101−
102−static void bio_dma_done_cb(struct rsxx_cardinfo *card,
103− void *cb_data,
104− unsigned int error)
105−{
106− struct rsxx_bio_meta *meta = cb_data;
107−
108− if (error)
109− atomic_set(&meta->error, 1);
110−
111− if (atomic_dec_and_test(&meta->pending_dmas)) {
112− if (!card->eeh_state && card->gendisk)
113− bio_end_io_acct(meta->bio, meta->start_time);
114−
115− if (atomic_read(&meta->error))
116− bio_io_error(meta->bio);
117− else
118− bio_endio(meta->bio);
119− kmem_cache_free(bio_meta_pool, meta);
120− }
121−}
122−
123−static void rsxx_submit_bio(struct bio *bio)
124−{
125− struct rsxx_cardinfo *card = bio->bi_bdev->bd_disk->private_data;
126− struct rsxx_bio_meta *bio_meta;
127− blk_status_t st = BLK_STS_IOERR;
128−
129− blk_queue_split(&bio);
130−
131− might_sleep();
132−
133− if (!card)
134− goto req_err;
135−
136− if (bio_end_sector(bio) > get_capacity(card->gendisk))
137− goto req_err;
138−
139− if (unlikely(card->halt))
140− goto req_err;
141−
142− if (unlikely(card->dma_fault))
143− goto req_err;
144−
145− if (bio->bi_iter.bi_size == 0) {
146− dev_err(CARD_TO_DEV(card), "size zero BIO!\n");
147− goto req_err;
148− }
149−
150− bio_meta = kmem_cache_alloc(bio_meta_pool, GFP_KERNEL);
151− if (!bio_meta) {
152− st = BLK_STS_RESOURCE;
153− goto req_err;
154− }
155−
156− bio_meta->bio = bio;
157− atomic_set(&bio_meta->error, 0);
158− atomic_set(&bio_meta->pending_dmas, 0);
159−
160− if (!unlikely(card->halt))
161− bio_meta->start_time = bio_start_io_acct(bio);
162−
163− dev_dbg(CARD_TO_DEV(card), "BIO[%c]: meta: %p addr8: x%llx size: %d\n",
164− bio_data_dir(bio) ? 'W' : 'R', bio_meta,
165− (u64)bio->bi_iter.bi_sector << 9, bio->bi_iter.bi_size);
166−
167− st = rsxx_dma_queue_bio(card, bio, &bio_meta->pending_dmas,
168− bio_dma_done_cb, bio_meta);
169− if (st)
170− goto queue_err;
171−
172− return;
173−
174−queue_err:
175− kmem_cache_free(bio_meta_pool, bio_meta);
176−req_err:
177− if (st)
178− bio->bi_status = st;
179− bio_endio(bio);
180−}
181−
182−/*----------------- Device Setup -------------------*/
183−static bool rsxx_discard_supported(struct rsxx_cardinfo *card)
184−{
185− unsigned char pci_rev;
186−
187− pci_read_config_byte(card->dev, PCI_REVISION_ID, &pci_rev);
188−
189− return (pci_rev >= RSXX_DISCARD_SUPPORT);
190−}
191−
192−int rsxx_attach_dev(struct rsxx_cardinfo *card)
193−{
194− int err = 0;
195−
196− mutex_lock(&card->dev_lock);
197−
198− /* The block device requires the stripe size from the config. */
199− if (enable_blkdev) {
200− if (card->config_valid)
201− set_capacity(card->gendisk, card->size8 >> 9);
202− else
203− set_capacity(card->gendisk, 0);
204− err = device_add_disk(CARD_TO_DEV(card), card->gendisk, NULL);
205− if (err == 0)
206− card->bdev_attached = 1;
207− }
208−
209− mutex_unlock(&card->dev_lock);
210−
211− if (err)
212− blk_cleanup_disk(card->gendisk);
213−
214− return err;
215−}
216−
217−void rsxx_detach_dev(struct rsxx_cardinfo *card)
218−{
219− mutex_lock(&card->dev_lock);
220−
221− if (card->bdev_attached) {
222− del_gendisk(card->gendisk);
223− card->bdev_attached = 0;
224− }
225−
226− mutex_unlock(&card->dev_lock);
227−}
228−
229−int rsxx_setup_dev(struct rsxx_cardinfo *card)
230−{
231− unsigned short blk_size;
232−
233− mutex_init(&card->dev_lock);
234−
235− if (!enable_blkdev)
236− return 0;
237−
238− card->major = register_blkdev(0, DRIVER_NAME);
239− if (card->major < 0) {
240− dev_err(CARD_TO_DEV(card), "Failed to get major number\n");
241− return -ENOMEM;
242− }
243−
244− card->gendisk = blk_alloc_disk(blkdev_minors);
245− if (!card->gendisk) {
246− dev_err(CARD_TO_DEV(card), "Failed disk alloc\n");
247− unregister_blkdev(card->major, DRIVER_NAME);
248− return -ENOMEM;
249− }
250−
251− if (card->config_valid) {
252− blk_size = card->config.data.block_size;
253− blk_queue_dma_alignment(card->gendisk->queue, blk_size - 1);
254− blk_queue_logical_block_size(card->gendisk->queue, blk_size);
255− }
256−
257− blk_queue_max_hw_sectors(card->gendisk->queue, blkdev_max_hw_sectors);
258− blk_queue_physical_block_size(card->gendisk->queue, RSXX_HW_BLK_SIZE);
259−
260− blk_queue_flag_set(QUEUE_FLAG_NONROT, card->gendisk->queue);
261− blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, card->gendisk->queue);
262− if (rsxx_discard_supported(card)) {
263− blk_queue_flag_set(QUEUE_FLAG_DISCARD, card->gendisk->queue);
264− blk_queue_max_discard_sectors(card->gendisk->queue,
265− RSXX_HW_BLK_SIZE >> 9);
266− card->gendisk->queue->limits.discard_granularity =
267− RSXX_HW_BLK_SIZE;
268− card->gendisk->queue->limits.discard_alignment =
269− RSXX_HW_BLK_SIZE;
270− }
271−
272− snprintf(card->gendisk->disk_name, sizeof(card->gendisk->disk_name),
273− "rsxx%d", card->disk_id);
274− card->gendisk->major = card->major;
275− card->gendisk->minors = blkdev_minors;
276− card->gendisk->fops = &rsxx_fops;
277− card->gendisk->private_data = card;
278−
279− return 0;
280−}
281−
282−void rsxx_destroy_dev(struct rsxx_cardinfo *card)
283−{
284− if (!enable_blkdev)
285− return;
286−
287− blk_cleanup_disk(card->gendisk);
288− card->gendisk = NULL;
289− unregister_blkdev(card->major, DRIVER_NAME);
290−}
291−
292−int rsxx_dev_init(void)
293−{
294− bio_meta_pool = KMEM_CACHE(rsxx_bio_meta, SLAB_HWCACHE_ALIGN);
295− if (!bio_meta_pool)
296− return -ENOMEM;
297−
298− return 0;
299−}
300−
301−void rsxx_dev_cleanup(void)
302−{
303− kmem_cache_destroy(bio_meta_pool);
304−}
305−
306−
drivers/block/rsxx/dma.c+0−1085
@@ -1,1085 +0,0 @@
1−// SPDX-License-Identifier: GPL-2.0-or-later
2−/*
3−* Filename: dma.c
4−*
5−* Authors: Joshua Morris <josh.h.morris@us.ibm.com>
6−* Philip Kelleher <pjk1939@linux.vnet.ibm.com>
7−*
8−* (C) Copyright 2013 IBM Corporation
9−*/
10−
11−#include <linux/slab.h>
12−#include "rsxx_priv.h"
13−
14−struct rsxx_dma {
15− struct list_head list;
16− u8 cmd;
17− unsigned int laddr; /* Logical address */
18− struct {
19− u32 off;
20− u32 cnt;
21− } sub_page;
22− dma_addr_t dma_addr;
23− struct page *page;
24− unsigned int pg_off; /* Page Offset */
25− rsxx_dma_cb cb;
26− void *cb_data;
27−};
28−
29−/* This timeout is used to detect a stalled DMA channel */
30−#define DMA_ACTIVITY_TIMEOUT msecs_to_jiffies(10000)
31−
32−struct hw_status {
33− u8 status;
34− u8 tag;
35− __le16 count;
36− __le32 _rsvd2;
37− __le64 _rsvd3;
38−} __packed;
39−
40−enum rsxx_dma_status {
41− DMA_SW_ERR = 0x1,
42− DMA_HW_FAULT = 0x2,
43− DMA_CANCELLED = 0x4,
44−};
45−
46−struct hw_cmd {
47− u8 command;
48− u8 tag;
49− u8 _rsvd;
50− u8 sub_page; /* Bit[0:2]: 512byte offset */
51− /* Bit[4:6]: 512byte count */
52− __le32 device_addr;
53− __le64 host_addr;
54−} __packed;
55−
56−enum rsxx_hw_cmd {
57− HW_CMD_BLK_DISCARD = 0x70,
58− HW_CMD_BLK_WRITE = 0x80,
59− HW_CMD_BLK_READ = 0xC0,
60− HW_CMD_BLK_RECON_READ = 0xE0,
61−};
62−
63−enum rsxx_hw_status {
64− HW_STATUS_CRC = 0x01,
65− HW_STATUS_HARD_ERR = 0x02,
66− HW_STATUS_SOFT_ERR = 0x04,
67− HW_STATUS_FAULT = 0x08,
68−};
69−
70−static struct kmem_cache *rsxx_dma_pool;
71−
72−struct dma_tracker {
73− int next_tag;
74− struct rsxx_dma *dma;
75−};
76−
77−struct dma_tracker_list {
78− spinlock_t lock;
79− int head;
80− struct dma_tracker list[];
81−};
82−
83−
84−/*----------------- Misc Utility Functions -------------------*/
85−static unsigned int rsxx_addr8_to_laddr(u64 addr8, struct rsxx_cardinfo *card)
86−{
87− unsigned long long tgt_addr8;
88−
89− tgt_addr8 = ((addr8 >> card->_stripe.upper_shift) &
90− card->_stripe.upper_mask) |
91− ((addr8) & card->_stripe.lower_mask);
92− do_div(tgt_addr8, RSXX_HW_BLK_SIZE);
93− return tgt_addr8;
94−}
95−
96−static unsigned int rsxx_get_dma_tgt(struct rsxx_cardinfo *card, u64 addr8)
97−{
98− unsigned int tgt;
99−
100− tgt = (addr8 >> card->_stripe.target_shift) & card->_stripe.target_mask;
101−
102− return tgt;
103−}
104−
105−void rsxx_dma_queue_reset(struct rsxx_cardinfo *card)
106−{
107− /* Reset all DMA Command/Status Queues */
108− iowrite32(DMA_QUEUE_RESET, card->regmap + RESET);
109−}
110−
111−static unsigned int get_dma_size(struct rsxx_dma *dma)
112−{
113− if (dma->sub_page.cnt)
114− return dma->sub_page.cnt << 9;
115− else
116− return RSXX_HW_BLK_SIZE;
117−}
118−
119−
120−/*----------------- DMA Tracker -------------------*/
121−static void set_tracker_dma(struct dma_tracker_list *trackers,
122− int tag,
123− struct rsxx_dma *dma)
124−{
125− trackers->list[tag].dma = dma;
126−}
127−
128−static struct rsxx_dma *get_tracker_dma(struct dma_tracker_list *trackers,
129− int tag)
130−{
131− return trackers->list[tag].dma;
132−}
133−
134−static int pop_tracker(struct dma_tracker_list *trackers)
135−{
136− int tag;
137−
138− spin_lock(&trackers->lock);
139− tag = trackers->head;
140− if (tag != -1) {
141− trackers->head = trackers->list[tag].next_tag;
142− trackers->list[tag].next_tag = -1;
143− }
144− spin_unlock(&trackers->lock);
145−
146− return tag;
147−}
148−
149−static void push_tracker(struct dma_tracker_list *trackers, int tag)
150−{
151− spin_lock(&trackers->lock);
152− trackers->list[tag].next_tag = trackers->head;
153− trackers->head = tag;
154− trackers->list[tag].dma = NULL;
155− spin_unlock(&trackers->lock);
156−}
157−
158−
159−/*----------------- Interrupt Coalescing -------------*/
160−/*
161− * Interrupt Coalescing Register Format:
162− * Interrupt Timer (64ns units) [15:0]
163− * Interrupt Count [24:16]
164− * Reserved [31:25]
165−*/
166−#define INTR_COAL_LATENCY_MASK (0x0000ffff)
167−
168−#define INTR_COAL_COUNT_SHIFT 16
169−#define INTR_COAL_COUNT_BITS 9
170−#define INTR_COAL_COUNT_MASK (((1 << INTR_COAL_COUNT_BITS) - 1) << \
171− INTR_COAL_COUNT_SHIFT)
172−#define INTR_COAL_LATENCY_UNITS_NS 64
173−
174−
175−static u32 dma_intr_coal_val(u32 mode, u32 count, u32 latency)
176−{
177− u32 latency_units = latency / INTR_COAL_LATENCY_UNITS_NS;
178−
179− if (mode == RSXX_INTR_COAL_DISABLED)
180− return 0;
181−
182− return ((count << INTR_COAL_COUNT_SHIFT) & INTR_COAL_COUNT_MASK) |
183− (latency_units & INTR_COAL_LATENCY_MASK);
184−
185−}
186−
187−static void dma_intr_coal_auto_tune(struct rsxx_cardinfo *card)
188−{
189− int i;
190− u32 q_depth = 0;
191− u32 intr_coal;
192−
193− if (card->config.data.intr_coal.mode != RSXX_INTR_COAL_AUTO_TUNE ||
194− unlikely(card->eeh_state))
195− return;
196−
197− for (i = 0; i < card->n_targets; i++)
198− q_depth += atomic_read(&card->ctrl[i].stats.hw_q_depth);
199−
200− intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode,
201− q_depth / 2,
202− card->config.data.intr_coal.latency);
203− iowrite32(intr_coal, card->regmap + INTR_COAL);
204−}
205−
206−/*----------------- RSXX DMA Handling -------------------*/
207−static void rsxx_free_dma(struct rsxx_dma_ctrl *ctrl, struct rsxx_dma *dma)
208−{
209− if (dma->cmd != HW_CMD_BLK_DISCARD) {
210− if (!dma_mapping_error(&ctrl->card->dev->dev, dma->dma_addr)) {
211− dma_unmap_page(&ctrl->card->dev->dev, dma->dma_addr,
212− get_dma_size(dma),
213− dma->cmd == HW_CMD_BLK_WRITE ?
214− DMA_TO_DEVICE :
215− DMA_FROM_DEVICE);
216− }
217− }
218−
219− kmem_cache_free(rsxx_dma_pool, dma);
220−}
221−
222−static void rsxx_complete_dma(struct rsxx_dma_ctrl *ctrl,
223− struct rsxx_dma *dma,
224− unsigned int status)
225−{
226− if (status & DMA_SW_ERR)
227− ctrl->stats.dma_sw_err++;
228− if (status & DMA_HW_FAULT)
229− ctrl->stats.dma_hw_fault++;
230− if (status & DMA_CANCELLED)
231− ctrl->stats.dma_cancelled++;
232−
233− if (dma->cb)
234− dma->cb(ctrl->card, dma->cb_data, status ? 1 : 0);
235−
236− rsxx_free_dma(ctrl, dma);
237−}
238−
239−int rsxx_cleanup_dma_queue(struct rsxx_dma_ctrl *ctrl,
240− struct list_head *q, unsigned int done)
241−{
242− struct rsxx_dma *dma;
243− struct rsxx_dma *tmp;
244− int cnt = 0;
245−
246− list_for_each_entry_safe(dma, tmp, q, list) {
247− list_del(&dma->list);
248− if (done & COMPLETE_DMA)
249− rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
250− else
251− rsxx_free_dma(ctrl, dma);
252− cnt++;
253− }
254−
255− return cnt;
256−}
257−
258−static void rsxx_requeue_dma(struct rsxx_dma_ctrl *ctrl,
259− struct rsxx_dma *dma)
260−{
261− /*
262− * Requeued DMAs go to the front of the queue so they are issued
263− * first.
264− */
265− spin_lock_bh(&ctrl->queue_lock);
266− ctrl->stats.sw_q_depth++;
267− list_add(&dma->list, &ctrl->queue);
268− spin_unlock_bh(&ctrl->queue_lock);
269−}
270−
271−static void rsxx_handle_dma_error(struct rsxx_dma_ctrl *ctrl,
272− struct rsxx_dma *dma,
273− u8 hw_st)
274−{
275− unsigned int status = 0;
276− int requeue_cmd = 0;
277−
278− dev_dbg(CARD_TO_DEV(ctrl->card),
279− "Handling DMA error(cmd x%02x, laddr x%08x st:x%02x)\n",
280− dma->cmd, dma->laddr, hw_st);
281−
282− if (hw_st & HW_STATUS_CRC)
283− ctrl->stats.crc_errors++;
284− if (hw_st & HW_STATUS_HARD_ERR)
285− ctrl->stats.hard_errors++;
286− if (hw_st & HW_STATUS_SOFT_ERR)
287− ctrl->stats.soft_errors++;
288−
289− switch (dma->cmd) {
290− case HW_CMD_BLK_READ:
291− if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) {
292− if (ctrl->card->scrub_hard) {
293− dma->cmd = HW_CMD_BLK_RECON_READ;
294− requeue_cmd = 1;
295− ctrl->stats.reads_retried++;
296− } else {
297− status |= DMA_HW_FAULT;
298− ctrl->stats.reads_failed++;
299− }
300− } else if (hw_st & HW_STATUS_FAULT) {
301− status |= DMA_HW_FAULT;
302− ctrl->stats.reads_failed++;
303− }
304−
305− break;
306− case HW_CMD_BLK_RECON_READ:
307− if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) {
308− /* Data could not be reconstructed. */
309− status |= DMA_HW_FAULT;
310− ctrl->stats.reads_failed++;
311− }
312−
313− break;
314− case HW_CMD_BLK_WRITE:
315− status |= DMA_HW_FAULT;
316− ctrl->stats.writes_failed++;
317−
318− break;
319− case HW_CMD_BLK_DISCARD:
320− status |= DMA_HW_FAULT;
321− ctrl->stats.discards_failed++;
322−
323− break;
324− default:
325− dev_err(CARD_TO_DEV(ctrl->card),
326− "Unknown command in DMA!(cmd: x%02x "
327− "laddr x%08x st: x%02x\n",
328− dma->cmd, dma->laddr, hw_st);
329− status |= DMA_SW_ERR;
330−
331− break;
332− }
333−
334− if (requeue_cmd)
335− rsxx_requeue_dma(ctrl, dma);
336− else
337− rsxx_complete_dma(ctrl, dma, status);
338−}
339−
340−static void dma_engine_stalled(struct timer_list *t)
341−{
342− struct rsxx_dma_ctrl *ctrl = from_timer(ctrl, t, activity_timer);
343− int cnt;
344−
345− if (atomic_read(&ctrl->stats.hw_q_depth) == 0 ||
346− unlikely(ctrl->card->eeh_state))
347− return;
348−
349− if (ctrl->cmd.idx != ioread32(ctrl->regmap + SW_CMD_IDX)) {
350− /*
351− * The dma engine was stalled because the SW_CMD_IDX write
352− * was lost. Issue it again to recover.
353− */
354− dev_warn(CARD_TO_DEV(ctrl->card),
355− "SW_CMD_IDX write was lost, re-writing...\n");
356− iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
357− mod_timer(&ctrl->activity_timer,
358− jiffies + DMA_ACTIVITY_TIMEOUT);
359− } else {
360− dev_warn(CARD_TO_DEV(ctrl->card),
361− "DMA channel %d has stalled, faulting interface.\n",
362− ctrl->id);
363− ctrl->card->dma_fault = 1;
364−
365− /* Clean up the DMA queue */
366− spin_lock(&ctrl->queue_lock);
367− cnt = rsxx_cleanup_dma_queue(ctrl, &ctrl->queue, COMPLETE_DMA);
368− spin_unlock(&ctrl->queue_lock);
369−
370− cnt += rsxx_dma_cancel(ctrl);
371−
372− if (cnt)
373− dev_info(CARD_TO_DEV(ctrl->card),
374− "Freed %d queued DMAs on channel %d\n",
375− cnt, ctrl->id);
376− }
377−}
378−
379−static void rsxx_issue_dmas(struct rsxx_dma_ctrl *ctrl)
380−{
381− struct rsxx_dma *dma;
382− int tag;
383− int cmds_pending = 0;
384− struct hw_cmd *hw_cmd_buf;
385− int dir;
386−
387− hw_cmd_buf = ctrl->cmd.buf;
388−
389− if (unlikely(ctrl->card->halt) ||
390− unlikely(ctrl->card->eeh_state))
391− return;
392−
393− while (1) {
394− spin_lock_bh(&ctrl->queue_lock);
395− if (list_empty(&ctrl->queue)) {
396− spin_unlock_bh(&ctrl->queue_lock);
397− break;
398− }
399− spin_unlock_bh(&ctrl->queue_lock);
400−
401− tag = pop_tracker(ctrl->trackers);
402− if (tag == -1)
403− break;
404−
405− spin_lock_bh(&ctrl->queue_lock);
406− dma = list_entry(ctrl->queue.next, struct rsxx_dma, list);
407− list_del(&dma->list);
408− ctrl->stats.sw_q_depth--;
409− spin_unlock_bh(&ctrl->queue_lock);
410−
411− /*
412− * This will catch any DMAs that slipped in right before the
413− * fault, but was queued after all the other DMAs were
414− * cancelled.
415− */
416− if (unlikely(ctrl->card->dma_fault)) {
417− push_tracker(ctrl->trackers, tag);
418− rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
419− continue;
420− }
421−
422− if (dma->cmd != HW_CMD_BLK_DISCARD) {
423− if (dma->cmd == HW_CMD_BLK_WRITE)
424− dir = DMA_TO_DEVICE;
425− else
426− dir = DMA_FROM_DEVICE;
427−
428− /*
429− * The function dma_map_page is placed here because we
430− * can only, by design, issue up to 255 commands to the
431− * hardware at one time per DMA channel. So the maximum
432− * amount of mapped memory would be 255 * 4 channels *
433− * 4096 Bytes which is less than 2GB, the limit of a x8
434− * Non-HWWD PCIe slot. This way the dma_map_page
435− * function should never fail because of a lack of
436− * mappable memory.
437− */
438− dma->dma_addr = dma_map_page(&ctrl->card->dev->dev, dma->page,
439− dma->pg_off, dma->sub_page.cnt << 9, dir);
440− if (dma_mapping_error(&ctrl->card->dev->dev, dma->dma_addr)) {
441− push_tracker(ctrl->trackers, tag);
442− rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
443− continue;
444− }
445− }
446−
447− set_tracker_dma(ctrl->trackers, tag, dma);
448− hw_cmd_buf[ctrl->cmd.idx].command = dma->cmd;
449− hw_cmd_buf[ctrl->cmd.idx].tag = tag;
450− hw_cmd_buf[ctrl->cmd.idx]._rsvd = 0;
451− hw_cmd_buf[ctrl->cmd.idx].sub_page =
452− ((dma->sub_page.cnt & 0x7) << 4) |
453− (dma->sub_page.off & 0x7);
454−
455− hw_cmd_buf[ctrl->cmd.idx].device_addr =
456− cpu_to_le32(dma->laddr);
457−
458− hw_cmd_buf[ctrl->cmd.idx].host_addr =
459− cpu_to_le64(dma->dma_addr);
460−
461− dev_dbg(CARD_TO_DEV(ctrl->card),
462− "Issue DMA%d(laddr %d tag %d) to idx %d\n",
463− ctrl->id, dma->laddr, tag, ctrl->cmd.idx);
464−
465− ctrl->cmd.idx = (ctrl->cmd.idx + 1) & RSXX_CS_IDX_MASK;
466− cmds_pending++;
467−
468− if (dma->cmd == HW_CMD_BLK_WRITE)
469− ctrl->stats.writes_issued++;
470− else if (dma->cmd == HW_CMD_BLK_DISCARD)
471− ctrl->stats.discards_issued++;
472− else
473− ctrl->stats.reads_issued++;
474− }
475−
476− /* Let HW know we've queued commands. */
477− if (cmds_pending) {
478− atomic_add(cmds_pending, &ctrl->stats.hw_q_depth);
479− mod_timer(&ctrl->activity_timer,
480− jiffies + DMA_ACTIVITY_TIMEOUT);
481−
482− if (unlikely(ctrl->card->eeh_state)) {
483− del_timer_sync(&ctrl->activity_timer);
484− return;
485− }
486−
487− iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
488− }
489−}
490−
491−static void rsxx_dma_done(struct rsxx_dma_ctrl *ctrl)
492−{
493− struct rsxx_dma *dma;
494− unsigned long flags;
495− u16 count;
496− u8 status;
497− u8 tag;
498− struct hw_status *hw_st_buf;
499−
500− hw_st_buf = ctrl->status.buf;
501−
502− if (unlikely(ctrl->card->halt) ||
503− unlikely(ctrl->card->dma_fault) ||
504− unlikely(ctrl->card->eeh_state))
505− return;
506−
507− count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count);
508−
509− while (count == ctrl->e_cnt) {
510− /*
511− * The read memory-barrier is necessary to keep aggressive
512− * processors/optimizers (such as the PPC Apple G5) from
513− * reordering the following status-buffer tag & status read
514− * *before* the count read on subsequent iterations of the
515− * loop!
516− */
517− rmb();
518−
519− status = hw_st_buf[ctrl->status.idx].status;
520− tag = hw_st_buf[ctrl->status.idx].tag;
521−
522− dma = get_tracker_dma(ctrl->trackers, tag);
523− if (dma == NULL) {
524− spin_lock_irqsave(&ctrl->card->irq_lock, flags);
525− rsxx_disable_ier(ctrl->card, CR_INTR_DMA_ALL);
526− spin_unlock_irqrestore(&ctrl->card->irq_lock, flags);
527−
528− dev_err(CARD_TO_DEV(ctrl->card),
529− "No tracker for tag %d "
530− "(idx %d id %d)\n",
531− tag, ctrl->status.idx, ctrl->id);
532− return;
533− }
534−
535− dev_dbg(CARD_TO_DEV(ctrl->card),
536− "Completing DMA%d"
537− "(laddr x%x tag %d st: x%x cnt: x%04x) from idx %d.\n",
538− ctrl->id, dma->laddr, tag, status, count,
539− ctrl->status.idx);
540−
541− atomic_dec(&ctrl->stats.hw_q_depth);
542−
543− mod_timer(&ctrl->activity_timer,
544− jiffies + DMA_ACTIVITY_TIMEOUT);
545−
546− if (status)
547− rsxx_handle_dma_error(ctrl, dma, status);
548− else
549− rsxx_complete_dma(ctrl, dma, 0);
550−
551− push_tracker(ctrl->trackers, tag);
552−
553− ctrl->status.idx = (ctrl->status.idx + 1) &
554− RSXX_CS_IDX_MASK;
555− ctrl->e_cnt++;
556−
557− count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count);
558− }
559−
560− dma_intr_coal_auto_tune(ctrl->card);
561−
562− if (atomic_read(&ctrl->stats.hw_q_depth) == 0)
563− del_timer_sync(&ctrl->activity_timer);
564−
565− spin_lock_irqsave(&ctrl->card->irq_lock, flags);
566− rsxx_enable_ier(ctrl->card, CR_INTR_DMA(ctrl->id));
567− spin_unlock_irqrestore(&ctrl->card->irq_lock, flags);
568−
569− spin_lock_bh(&ctrl->queue_lock);
570− if (ctrl->stats.sw_q_depth)
571− queue_work(ctrl->issue_wq, &ctrl->issue_dma_work);
572− spin_unlock_bh(&ctrl->queue_lock);
573−}
574−
575−static void rsxx_schedule_issue(struct work_struct *work)
576−{
577− struct rsxx_dma_ctrl *ctrl;
578−
579− ctrl = container_of(work, struct rsxx_dma_ctrl, issue_dma_work);
580−
581− mutex_lock(&ctrl->work_lock);
582− rsxx_issue_dmas(ctrl);
583− mutex_unlock(&ctrl->work_lock);
584−}
585−
586−static void rsxx_schedule_done(struct work_struct *work)
587−{
588− struct rsxx_dma_ctrl *ctrl;
589−
590− ctrl = container_of(work, struct rsxx_dma_ctrl, dma_done_work);
591−
592− mutex_lock(&ctrl->work_lock);
593− rsxx_dma_done(ctrl);
594− mutex_unlock(&ctrl->work_lock);
595−}
596−
597−static blk_status_t rsxx_queue_discard(struct rsxx_cardinfo *card,
598− struct list_head *q,
599− unsigned int laddr,
600− rsxx_dma_cb cb,
601− void *cb_data)
602−{
603− struct rsxx_dma *dma;
604−
605− dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL);
606− if (!dma)
607− return BLK_STS_RESOURCE;
608−
609− dma->cmd = HW_CMD_BLK_DISCARD;
610− dma->laddr = laddr;
611− dma->dma_addr = 0;
612− dma->sub_page.off = 0;
613− dma->sub_page.cnt = 0;
614− dma->page = NULL;
615− dma->pg_off = 0;
616− dma->cb = cb;
617− dma->cb_data = cb_data;
618−
619− dev_dbg(CARD_TO_DEV(card), "Queuing[D] laddr %x\n", dma->laddr);
620−
621− list_add_tail(&dma->list, q);
622−
623− return 0;
624−}
625−
626−static blk_status_t rsxx_queue_dma(struct rsxx_cardinfo *card,
627− struct list_head *q,
628− int dir,
629− unsigned int dma_off,
630− unsigned int dma_len,
631− unsigned int laddr,
632− struct page *page,
633− unsigned int pg_off,
634− rsxx_dma_cb cb,
635− void *cb_data)
636−{
637− struct rsxx_dma *dma;
638−
639− dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL);
640− if (!dma)
641− return BLK_STS_RESOURCE;
642−
643− dma->cmd = dir ? HW_CMD_BLK_WRITE : HW_CMD_BLK_READ;
644− dma->laddr = laddr;
645− dma->sub_page.off = (dma_off >> 9);
646− dma->sub_page.cnt = (dma_len >> 9);
647− dma->page = page;
648− dma->pg_off = pg_off;
649− dma->cb = cb;
650− dma->cb_data = cb_data;
651−
652− dev_dbg(CARD_TO_DEV(card),
653− "Queuing[%c] laddr %x off %d cnt %d page %p pg_off %d\n",
654− dir ? 'W' : 'R', dma->laddr, dma->sub_page.off,
655− dma->sub_page.cnt, dma->page, dma->pg_off);
656−
657− /* Queue the DMA */
658− list_add_tail(&dma->list, q);
659−
660− return 0;
661−}
662−
663−blk_status_t rsxx_dma_queue_bio(struct rsxx_cardinfo *card,
664− struct bio *bio,
665− atomic_t *n_dmas,
666− rsxx_dma_cb cb,
667− void *cb_data)
668−{
669− struct list_head dma_list[RSXX_MAX_TARGETS];
670− struct bio_vec bvec;
671− struct bvec_iter iter;
672− unsigned long long addr8;
673− unsigned int laddr;
674− unsigned int bv_len;
675− unsigned int bv_off;
676− unsigned int dma_off;
677− unsigned int dma_len;
678− int dma_cnt[RSXX_MAX_TARGETS];
679− int tgt;
680− blk_status_t st;
681− int i;
682−
683− addr8 = bio->bi_iter.bi_sector << 9; /* sectors are 512 bytes */
684− atomic_set(n_dmas, 0);
685−
686− for (i = 0; i < card->n_targets; i++) {
687− INIT_LIST_HEAD(&dma_list[i]);
688− dma_cnt[i] = 0;
689− }
690−
691− if (bio_op(bio) == REQ_OP_DISCARD) {
692− bv_len = bio->bi_iter.bi_size;
693−
694− while (bv_len > 0) {
695− tgt = rsxx_get_dma_tgt(card, addr8);
696− laddr = rsxx_addr8_to_laddr(addr8, card);
697−
698− st = rsxx_queue_discard(card, &dma_list[tgt], laddr,
699− cb, cb_data);
700− if (st)
701− goto bvec_err;
702−
703− dma_cnt[tgt]++;
704− atomic_inc(n_dmas);
705− addr8 += RSXX_HW_BLK_SIZE;
706− bv_len -= RSXX_HW_BLK_SIZE;
707− }
708− } else {
709− bio_for_each_segment(bvec, bio, iter) {
710− bv_len = bvec.bv_len;
711− bv_off = bvec.bv_offset;
712−
713− while (bv_len > 0) {
714− tgt = rsxx_get_dma_tgt(card, addr8);
715− laddr = rsxx_addr8_to_laddr(addr8, card);
716− dma_off = addr8 & RSXX_HW_BLK_MASK;
717− dma_len = min(bv_len,
718− RSXX_HW_BLK_SIZE - dma_off);
719−
720− st = rsxx_queue_dma(card, &dma_list[tgt],
721− bio_data_dir(bio),
722− dma_off, dma_len,
723− laddr, bvec.bv_page,
724− bv_off, cb, cb_data);
725− if (st)
726− goto bvec_err;
727−
728− dma_cnt[tgt]++;
729− atomic_inc(n_dmas);
730− addr8 += dma_len;
731− bv_off += dma_len;
732− bv_len -= dma_len;
733− }
734− }
735− }
736−
737− for (i = 0; i < card->n_targets; i++) {
738− if (!list_empty(&dma_list[i])) {
739− spin_lock_bh(&card->ctrl[i].queue_lock);
740− card->ctrl[i].stats.sw_q_depth += dma_cnt[i];
741− list_splice_tail(&dma_list[i], &card->ctrl[i].queue);
742− spin_unlock_bh(&card->ctrl[i].queue_lock);
743−
744− queue_work(card->ctrl[i].issue_wq,
745− &card->ctrl[i].issue_dma_work);
746− }
747− }
748−
749− return 0;
750−
751−bvec_err:
752− for (i = 0; i < card->n_targets; i++)
753− rsxx_cleanup_dma_queue(&card->ctrl[i], &dma_list[i],
754− FREE_DMA);
755− return st;
756−}
757−
758−
759−/*----------------- DMA Engine Initialization & Setup -------------------*/
760−int rsxx_hw_buffers_init(struct pci_dev *dev, struct rsxx_dma_ctrl *ctrl)
761−{
762− ctrl->status.buf = dma_alloc_coherent(&dev->dev, STATUS_BUFFER_SIZE8,
763− &ctrl->status.dma_addr, GFP_KERNEL);
764− ctrl->cmd.buf = dma_alloc_coherent(&dev->dev, COMMAND_BUFFER_SIZE8,
765− &ctrl->cmd.dma_addr, GFP_KERNEL);
766− if (ctrl->status.buf == NULL || ctrl->cmd.buf == NULL)
767− return -ENOMEM;
768−
769− memset(ctrl->status.buf, 0xac, STATUS_BUFFER_SIZE8);
770− iowrite32(lower_32_bits(ctrl->status.dma_addr),
771− ctrl->regmap + SB_ADD_LO);
772− iowrite32(upper_32_bits(ctrl->status.dma_addr),
773− ctrl->regmap + SB_ADD_HI);
774−
775− memset(ctrl->cmd.buf, 0x83, COMMAND_BUFFER_SIZE8);
776− iowrite32(lower_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_LO);
777− iowrite32(upper_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_HI);
778−
779− ctrl->status.idx = ioread32(ctrl->regmap + HW_STATUS_CNT);
780− if (ctrl->status.idx > RSXX_MAX_OUTSTANDING_CMDS) {
781− dev_crit(&dev->dev, "Failed reading status cnt x%x\n",
782− ctrl->status.idx);
783− return -EINVAL;
784− }
785− iowrite32(ctrl->status.idx, ctrl->regmap + HW_STATUS_CNT);
786− iowrite32(ctrl->status.idx, ctrl->regmap + SW_STATUS_CNT);
787−
788− ctrl->cmd.idx = ioread32(ctrl->regmap + HW_CMD_IDX);
789− if (ctrl->cmd.idx > RSXX_MAX_OUTSTANDING_CMDS) {
790− dev_crit(&dev->dev, "Failed reading cmd cnt x%x\n",
791− ctrl->status.idx);
792− return -EINVAL;
793− }
794− iowrite32(ctrl->cmd.idx, ctrl->regmap + HW_CMD_IDX);
795− iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
796−
797− return 0;
798−}
799−
800−static int rsxx_dma_ctrl_init(struct pci_dev *dev,
801− struct rsxx_dma_ctrl *ctrl)
802−{
803− int i;
804− int st;
805−
806− memset(&ctrl->stats, 0, sizeof(ctrl->stats));
807−
808− ctrl->trackers = vmalloc(struct_size(ctrl->trackers, list,
809− RSXX_MAX_OUTSTANDING_CMDS));
810− if (!ctrl->trackers)
811− return -ENOMEM;
812−
813− ctrl->trackers->head = 0;
814− for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) {
815− ctrl->trackers->list[i].next_tag = i + 1;
816− ctrl->trackers->list[i].dma = NULL;
817− }
818− ctrl->trackers->list[RSXX_MAX_OUTSTANDING_CMDS-1].next_tag = -1;
819− spin_lock_init(&ctrl->trackers->lock);
820−
821− spin_lock_init(&ctrl->queue_lock);
822− mutex_init(&ctrl->work_lock);
823− INIT_LIST_HEAD(&ctrl->queue);
824−
825− timer_setup(&ctrl->activity_timer, dma_engine_stalled, 0);
826−
827− ctrl->issue_wq = alloc_ordered_workqueue(DRIVER_NAME"_issue", 0);
828− if (!ctrl->issue_wq)
829− return -ENOMEM;
830−
831− ctrl->done_wq = alloc_ordered_workqueue(DRIVER_NAME"_done", 0);
832− if (!ctrl->done_wq)
833− return -ENOMEM;
834−
835− INIT_WORK(&ctrl->issue_dma_work, rsxx_schedule_issue);
836− INIT_WORK(&ctrl->dma_done_work, rsxx_schedule_done);
837−
838− st = rsxx_hw_buffers_init(dev, ctrl);
839− if (st)
840− return st;
841−
842− return 0;
843−}
844−
845−static int rsxx_dma_stripe_setup(struct rsxx_cardinfo *card,
846− unsigned int stripe_size8)
847−{
848− if (!is_power_of_2(stripe_size8)) {
849− dev_err(CARD_TO_DEV(card),
850− "stripe_size is NOT a power of 2!\n");
851− return -EINVAL;
852− }
853−
854− card->_stripe.lower_mask = stripe_size8 - 1;
855−
856− card->_stripe.upper_mask = ~(card->_stripe.lower_mask);
857− card->_stripe.upper_shift = ffs(card->n_targets) - 1;
858−
859− card->_stripe.target_mask = card->n_targets - 1;
860− card->_stripe.target_shift = ffs(stripe_size8) - 1;
861−
862− dev_dbg(CARD_TO_DEV(card), "_stripe.lower_mask = x%016llx\n",
863− card->_stripe.lower_mask);
864− dev_dbg(CARD_TO_DEV(card), "_stripe.upper_shift = x%016llx\n",
865− card->_stripe.upper_shift);
866− dev_dbg(CARD_TO_DEV(card), "_stripe.upper_mask = x%016llx\n",
867− card->_stripe.upper_mask);
868− dev_dbg(CARD_TO_DEV(card), "_stripe.target_mask = x%016llx\n",
869− card->_stripe.target_mask);
870− dev_dbg(CARD_TO_DEV(card), "_stripe.target_shift = x%016llx\n",
871− card->_stripe.target_shift);
872−
873− return 0;
874−}
875−
876−int rsxx_dma_configure(struct rsxx_cardinfo *card)
877−{
878− u32 intr_coal;
879−
880− intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode,
881− card->config.data.intr_coal.count,
882− card->config.data.intr_coal.latency);
883− iowrite32(intr_coal, card->regmap + INTR_COAL);
884−
885− return rsxx_dma_stripe_setup(card, card->config.data.stripe_size);
886−}
887−
888−int rsxx_dma_setup(struct rsxx_cardinfo *card)
889−{
890− unsigned long flags;
891− int st;
892− int i;
893−
894− dev_info(CARD_TO_DEV(card),
895− "Initializing %d DMA targets\n",
896− card->n_targets);
897−
898− /* Regmap is divided up into 4K chunks. One for each DMA channel */
899− for (i = 0; i < card->n_targets; i++)
900− card->ctrl[i].regmap = card->regmap + (i * 4096);
901−
902− card->dma_fault = 0;
903−
904− /* Reset the DMA queues */
905− rsxx_dma_queue_reset(card);
906−
907− /************* Setup DMA Control *************/
908− for (i = 0; i < card->n_targets; i++) {
909− st = rsxx_dma_ctrl_init(card->dev, &card->ctrl[i]);
910− if (st)
911− goto failed_dma_setup;
912−
913− card->ctrl[i].card = card;
914− card->ctrl[i].id = i;
915− }
916−
917− card->scrub_hard = 1;
918−
919− if (card->config_valid)
920− rsxx_dma_configure(card);
921−
922− /* Enable the interrupts after all setup has completed. */
923− for (i = 0; i < card->n_targets; i++) {
924− spin_lock_irqsave(&card->irq_lock, flags);
925− rsxx_enable_ier_and_isr(card, CR_INTR_DMA(i));
926− spin_unlock_irqrestore(&card->irq_lock, flags);
927− }
928−
929− return 0;
930−
931−failed_dma_setup:
932− for (i = 0; i < card->n_targets; i++) {
933− struct rsxx_dma_ctrl *ctrl = &card->ctrl[i];
934−
935− if (ctrl->issue_wq) {
936− destroy_workqueue(ctrl->issue_wq);
937− ctrl->issue_wq = NULL;
938− }
939−
940− if (ctrl->done_wq) {
941− destroy_workqueue(ctrl->done_wq);
942− ctrl->done_wq = NULL;
943− }
944−
945− vfree(ctrl->trackers);
946−
947− if (ctrl->status.buf)
948− dma_free_coherent(&card->dev->dev, STATUS_BUFFER_SIZE8,
949− ctrl->status.buf,
950− ctrl->status.dma_addr);
951− if (ctrl->cmd.buf)
952− dma_free_coherent(&card->dev->dev, COMMAND_BUFFER_SIZE8,
953− ctrl->cmd.buf, ctrl->cmd.dma_addr);
954− }
955−
956− return st;
957−}
958−
959−int rsxx_dma_cancel(struct rsxx_dma_ctrl *ctrl)
960−{
961− struct rsxx_dma *dma;
962− int i;
963− int cnt = 0;
964−
965− /* Clean up issued DMAs */
966− for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) {
967− dma = get_tracker_dma(ctrl->trackers, i);
968− if (dma) {
969− atomic_dec(&ctrl->stats.hw_q_depth);
970− rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
971− push_tracker(ctrl->trackers, i);
972− cnt++;
973− }
974− }
975−
976− return cnt;
977−}
978−
979−void rsxx_dma_destroy(struct rsxx_cardinfo *card)
980−{
981− struct rsxx_dma_ctrl *ctrl;
982− int i;
983−
984− for (i = 0; i < card->n_targets; i++) {
985− ctrl = &card->ctrl[i];
986−
987− if (ctrl->issue_wq) {
988− destroy_workqueue(ctrl->issue_wq);
989− ctrl->issue_wq = NULL;
990− }
991−
992− if (ctrl->done_wq) {
993− destroy_workqueue(ctrl->done_wq);
994− ctrl->done_wq = NULL;
995− }
996−
997− if (timer_pending(&ctrl->activity_timer))
998− del_timer_sync(&ctrl->activity_timer);
999−
1000− /* Clean up the DMA queue */
1001− spin_lock_bh(&ctrl->queue_lock);
1002− rsxx_cleanup_dma_queue(ctrl, &ctrl->queue, COMPLETE_DMA);
1003− spin_unlock_bh(&ctrl->queue_lock);
1004−
1005− rsxx_dma_cancel(ctrl);
1006−
1007− vfree(ctrl->trackers);
1008−
1009− dma_free_coherent(&card->dev->dev, STATUS_BUFFER_SIZE8,
1010− ctrl->status.buf, ctrl->status.dma_addr);
1011− dma_free_coherent(&card->dev->dev, COMMAND_BUFFER_SIZE8,
1012− ctrl->cmd.buf, ctrl->cmd.dma_addr);
1013− }
1014−}
1015−
1016−int rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo *card)
1017−{
1018− int i;
1019− int j;
1020− int cnt;
1021− struct rsxx_dma *dma;
1022− struct list_head *issued_dmas;
1023−
1024− issued_dmas = kcalloc(card->n_targets, sizeof(*issued_dmas),
1025− GFP_KERNEL);
1026− if (!issued_dmas)
1027− return -ENOMEM;
1028−
1029− for (i = 0; i < card->n_targets; i++) {
1030− INIT_LIST_HEAD(&issued_dmas[i]);
1031− cnt = 0;
1032− for (j = 0; j < RSXX_MAX_OUTSTANDING_CMDS; j++) {
1033− dma = get_tracker_dma(card->ctrl[i].trackers, j);
1034− if (dma == NULL)
1035− continue;
1036−
1037− if (dma->cmd == HW_CMD_BLK_WRITE)
1038− card->ctrl[i].stats.writes_issued--;
1039− else if (dma->cmd == HW_CMD_BLK_DISCARD)
1040− card->ctrl[i].stats.discards_issued--;
1041− else
1042− card->ctrl[i].stats.reads_issued--;
1043−
1044− if (dma->cmd != HW_CMD_BLK_DISCARD) {
1045− dma_unmap_page(&card->dev->dev, dma->dma_addr,
1046− get_dma_size(dma),
1047− dma->cmd == HW_CMD_BLK_WRITE ?
1048− DMA_TO_DEVICE :
1049− DMA_FROM_DEVICE);
1050− }
1051−
1052− list_add_tail(&dma->list, &issued_dmas[i]);
1053− push_tracker(card->ctrl[i].trackers, j);
1054− cnt++;
1055− }
1056−
1057− spin_lock_bh(&card->ctrl[i].queue_lock);
1058− list_splice(&issued_dmas[i], &card->ctrl[i].queue);
1059−
1060− atomic_sub(cnt, &card->ctrl[i].stats.hw_q_depth);
1061− card->ctrl[i].stats.sw_q_depth += cnt;
1062− card->ctrl[i].e_cnt = 0;
1063− spin_unlock_bh(&card->ctrl[i].queue_lock);
1064− }
1065−
1066− kfree(issued_dmas);
1067−
1068− return 0;
1069−}
1070−
1071−int rsxx_dma_init(void)
1072−{
1073− rsxx_dma_pool = KMEM_CACHE(rsxx_dma, SLAB_HWCACHE_ALIGN);
1074− if (!rsxx_dma_pool)
1075− return -ENOMEM;
1076−
1077− return 0;
1078−}
1079−
1080−
1081−void rsxx_dma_cleanup(void)
1082−{
1083− kmem_cache_destroy(rsxx_dma_pool);
1084−}
1085−
drivers/block/rsxx/rsxx.h+0−33
@@ -1,33 +0,0 @@
1−/* SPDX-License-Identifier: GPL-2.0-or-later */
2−/*
3−* Filename: rsxx.h
4−*
5−* Authors: Joshua Morris <josh.h.morris@us.ibm.com>
6−* Philip Kelleher <pjk1939@linux.vnet.ibm.com>
7−*
8−* (C) Copyright 2013 IBM Corporation
9−*/
10−
11−#ifndef __RSXX_H__
12−#define __RSXX_H__
13−
14−/*----------------- IOCTL Definitions -------------------*/
15−
16−#define RSXX_MAX_DATA 8
17−
18−struct rsxx_reg_access {
19− __u32 addr;
20− __u32 cnt;
21− __u32 stat;
22− __u32 stream;
23− __u32 data[RSXX_MAX_DATA];
24−};
25−
26−#define RSXX_MAX_REG_CNT (RSXX_MAX_DATA * (sizeof(__u32)))
27−
28−#define RSXX_IOC_MAGIC 'r'
29−
30−#define RSXX_GETREG _IOWR(RSXX_IOC_MAGIC, 0x20, struct rsxx_reg_access)
31−#define RSXX_SETREG _IOWR(RSXX_IOC_MAGIC, 0x21, struct rsxx_reg_access)
32−
33−#endif /* __RSXX_H_ */
drivers/block/rsxx/rsxx_cfg.h+0−58
@@ -1,58 +0,0 @@
1−/* SPDX-License-Identifier: GPL-2.0-or-later */
2−/*
3−* Filename: rsXX_cfg.h
4−*
5−* Authors: Joshua Morris <josh.h.morris@us.ibm.com>
6−* Philip Kelleher <pjk1939@linux.vnet.ibm.com>
7−*
8−* (C) Copyright 2013 IBM Corporation
9−*/
10−
11−#ifndef __RSXX_CFG_H__
12−#define __RSXX_CFG_H__
13−
14−/* NOTE: Config values will be saved in network byte order (i.e. Big endian) */
15−#include <linux/types.h>
16−
17−/*
18− * The card config version must match the driver's expected version. If it does
19− * not, the DMA interfaces will not be attached and the user will need to
20− * initialize/upgrade the card configuration using the card config utility.
21− */
22−#define RSXX_CFG_VERSION 4
23−
24−struct card_cfg_hdr {
25− __u32 version;
26− __u32 crc;
27−};
28−
29−struct card_cfg_data {
30− __u32 block_size;
31− __u32 stripe_size;
32− __u32 vendor_id;
33− __u32 cache_order;
34− struct {
35− __u32 mode; /* Disabled, manual, auto-tune... */
36− __u32 count; /* Number of intr to coalesce */
37− __u32 latency;/* Max wait time (in ns) */
38− } intr_coal;
39−};
40−
41−struct rsxx_card_cfg {
42− struct card_cfg_hdr hdr;
43− struct card_cfg_data data;
44−};
45−
46−/* Vendor ID Values */
47−#define RSXX_VENDOR_ID_IBM 0
48−#define RSXX_VENDOR_ID_DSI 1
49−#define RSXX_VENDOR_COUNT 2
50−
51−/* Interrupt Coalescing Values */
52−#define RSXX_INTR_COAL_DISABLED 0
53−#define RSXX_INTR_COAL_EXPLICIT 1
54−#define RSXX_INTR_COAL_AUTO_TUNE 2
55−
56−
57−#endif /* __RSXX_CFG_H__ */
58−
drivers/block/rsxx/rsxx_priv.h+0−418
@@ -1,418 +0,0 @@
1−/* SPDX-License-Identifier: GPL-2.0-or-later */
2−/*
3−* Filename: rsxx_priv.h
4−*
5−* Authors: Joshua Morris <josh.h.morris@us.ibm.com>
6−* Philip Kelleher <pjk1939@linux.vnet.ibm.com>
7−*
8−* (C) Copyright 2013 IBM Corporation
9−*/
10−
11−#ifndef __RSXX_PRIV_H__
12−#define __RSXX_PRIV_H__
13−
14−#include <linux/semaphore.h>
15−
16−#include <linux/fs.h>
17−#include <linux/interrupt.h>
18−#include <linux/mutex.h>
19−#include <linux/pci.h>
20−#include <linux/spinlock.h>
21−#include <linux/sysfs.h>
22−#include <linux/workqueue.h>
23−#include <linux/bio.h>
24−#include <linux/vmalloc.h>
25−#include <linux/timer.h>
26−#include <linux/ioctl.h>
27−#include <linux/delay.h>
28−
29−#include "rsxx.h"
30−#include "rsxx_cfg.h"
31−
32−struct proc_cmd;
33−
34−#define PCI_DEVICE_ID_FS70_FLASH 0x04A9
35−#define PCI_DEVICE_ID_FS80_FLASH 0x04AA
36−
37−#define RS70_PCI_REV_SUPPORTED 4
38−
39−#define DRIVER_NAME "rsxx"
40−#define DRIVER_VERSION "4.0.3.2516"
41−
42−/* Block size is 4096 */
43−#define RSXX_HW_BLK_SHIFT 12
44−#define RSXX_HW_BLK_SIZE (1 << RSXX_HW_BLK_SHIFT)
45−#define RSXX_HW_BLK_MASK (RSXX_HW_BLK_SIZE - 1)
46−
47−#define MAX_CREG_DATA8 32
48−#define LOG_BUF_SIZE8 128
49−
50−#define RSXX_MAX_OUTSTANDING_CMDS 255
51−#define RSXX_CS_IDX_MASK 0xff
52−
53−#define STATUS_BUFFER_SIZE8 4096
54−#define COMMAND_BUFFER_SIZE8 4096
55−
56−#define RSXX_MAX_TARGETS 8
57−
58−struct dma_tracker_list;
59−
60−/* DMA Command/Status Buffer structure */
61−struct rsxx_cs_buffer {
62− dma_addr_t dma_addr;
63− void *buf;
64− u32 idx;
65−};
66−
67−struct rsxx_dma_stats {
68− u32 crc_errors;
69− u32 hard_errors;
70− u32 soft_errors;
71− u32 writes_issued;
72− u32 writes_failed;
73− u32 reads_issued;
74− u32 reads_failed;
75− u32 reads_retried;
76− u32 discards_issued;
77− u32 discards_failed;
78− u32 done_rescheduled;
79− u32 issue_rescheduled;
80− u32 dma_sw_err;
81− u32 dma_hw_fault;
82− u32 dma_cancelled;
83− u32 sw_q_depth; /* Number of DMAs on the SW queue. */
84− atomic_t hw_q_depth; /* Number of DMAs queued to HW. */
85−};
86−
87−struct rsxx_dma_ctrl {
88− struct rsxx_cardinfo *card;
89− int id;
90− void __iomem *regmap;
91− struct rsxx_cs_buffer status;
92− struct rsxx_cs_buffer cmd;
93− u16 e_cnt;
94− spinlock_t queue_lock;
95− struct list_head queue;
96− struct workqueue_struct *issue_wq;
97− struct work_struct issue_dma_work;
98− struct workqueue_struct *done_wq;
99− struct work_struct dma_done_work;
100− struct timer_list activity_timer;
101− struct dma_tracker_list *trackers;
102− struct rsxx_dma_stats stats;
103− struct mutex work_lock;
104−};
105−
106−struct rsxx_cardinfo {
107− struct pci_dev *dev;
108− unsigned int halt;
109− unsigned int eeh_state;
110−
111− void __iomem *regmap;
112− spinlock_t irq_lock;
113− unsigned int isr_mask;
114− unsigned int ier_mask;
115−
116− struct rsxx_card_cfg config;
117− int config_valid;
118−
119− /* Embedded CPU Communication */
120− struct {
121− spinlock_t lock;
122− bool active;
123− struct creg_cmd *active_cmd;
124− struct workqueue_struct *creg_wq;
125− struct work_struct done_work;
126− struct list_head queue;
127− unsigned int q_depth;
128− /* Cache the creg status to prevent ioreads */
129− struct {
130− u32 stat;
131− u32 failed_cancel_timer;
132− u32 creg_timeout;
133− } creg_stats;
134− struct timer_list cmd_timer;
135− struct mutex reset_lock;
136− int reset;
137− } creg_ctrl;
138−
139− struct {
140− char tmp[MAX_CREG_DATA8];
141− char buf[LOG_BUF_SIZE8]; /* terminated */
142− int buf_len;
143− } log;
144−
145− struct workqueue_struct *event_wq;
146− struct work_struct event_work;
147− unsigned int state;
148− u64 size8;
149−
150− /* Lock the device attach/detach function */
151− struct mutex dev_lock;
152−
153− /* Block Device Variables */
154− bool bdev_attached;
155− int disk_id;
156− int major;
157− struct gendisk *gendisk;
158− struct {
159− /* Used to convert a byte address to a device address. */
160− u64 lower_mask;
161− u64 upper_shift;
162− u64 upper_mask;
163− u64 target_mask;
164− u64 target_shift;
165− } _stripe;
166− unsigned int dma_fault;
167−
168− int scrub_hard;
169−
170− int n_targets;
171− struct rsxx_dma_ctrl *ctrl;
172−
173− struct dentry *debugfs_dir;
174−};
175−
176−enum rsxx_pci_regmap {
177− HWID = 0x00, /* Hardware Identification Register */
178− SCRATCH = 0x04, /* Scratch/Debug Register */
179− RESET = 0x08, /* Reset Register */
180− ISR = 0x10, /* Interrupt Status Register */
181− IER = 0x14, /* Interrupt Enable Register */
182− IPR = 0x18, /* Interrupt Poll Register */
183− CB_ADD_LO = 0x20, /* Command Host Buffer Address [31:0] */
184− CB_ADD_HI = 0x24, /* Command Host Buffer Address [63:32]*/
185− HW_CMD_IDX = 0x28, /* Hardware Processed Command Index */
186− SW_CMD_IDX = 0x2C, /* Software Processed Command Index */
187− SB_ADD_LO = 0x30, /* Status Host Buffer Address [31:0] */
188− SB_ADD_HI = 0x34, /* Status Host Buffer Address [63:32] */
189− HW_STATUS_CNT = 0x38, /* Hardware Status Counter */
190− SW_STATUS_CNT = 0x3C, /* Deprecated */
191− CREG_CMD = 0x40, /* CPU Command Register */
192− CREG_ADD = 0x44, /* CPU Address Register */
193− CREG_CNT = 0x48, /* CPU Count Register */
194− CREG_STAT = 0x4C, /* CPU Status Register */
195− CREG_DATA0 = 0x50, /* CPU Data Registers */
196− CREG_DATA1 = 0x54,
197− CREG_DATA2 = 0x58,
198− CREG_DATA3 = 0x5C,
199− CREG_DATA4 = 0x60,
200− CREG_DATA5 = 0x64,
201− CREG_DATA6 = 0x68,
202− CREG_DATA7 = 0x6c,
203− INTR_COAL = 0x70, /* Interrupt Coalescing Register */
204− HW_ERROR = 0x74, /* Card Error Register */
205− PCI_DEBUG0 = 0x78, /* PCI Debug Registers */
206− PCI_DEBUG1 = 0x7C,
207− PCI_DEBUG2 = 0x80,
208− PCI_DEBUG3 = 0x84,
209− PCI_DEBUG4 = 0x88,
210− PCI_DEBUG5 = 0x8C,
211− PCI_DEBUG6 = 0x90,
212− PCI_DEBUG7 = 0x94,
213− PCI_POWER_THROTTLE = 0x98,
214− PERF_CTRL = 0x9c,
215− PERF_TIMER_LO = 0xa0,
216− PERF_TIMER_HI = 0xa4,
217− PERF_RD512_LO = 0xa8,
218− PERF_RD512_HI = 0xac,
219− PERF_WR512_LO = 0xb0,
220− PERF_WR512_HI = 0xb4,
221− PCI_RECONFIG = 0xb8,
222−};
223−
224−enum rsxx_intr {
225− CR_INTR_DMA0 = 0x00000001,
226− CR_INTR_CREG = 0x00000002,
227− CR_INTR_DMA1 = 0x00000004,
228− CR_INTR_EVENT = 0x00000008,
229− CR_INTR_DMA2 = 0x00000010,
230− CR_INTR_DMA3 = 0x00000020,
231− CR_INTR_DMA4 = 0x00000040,
232− CR_INTR_DMA5 = 0x00000080,
233− CR_INTR_DMA6 = 0x00000100,
234− CR_INTR_DMA7 = 0x00000200,
235− CR_INTR_ALL_C = 0x0000003f,
236− CR_INTR_ALL_G = 0x000003ff,
237− CR_INTR_DMA_ALL = 0x000003f5,
238− CR_INTR_ALL = 0xffffffff,
239−};
240−
241−static inline int CR_INTR_DMA(int N)
242−{
243− static const unsigned int _CR_INTR_DMA[] = {
244− CR_INTR_DMA0, CR_INTR_DMA1, CR_INTR_DMA2, CR_INTR_DMA3,
245− CR_INTR_DMA4, CR_INTR_DMA5, CR_INTR_DMA6, CR_INTR_DMA7
246− };
247− return _CR_INTR_DMA[N];
248−}
249−enum rsxx_pci_reset {
250− DMA_QUEUE_RESET = 0x00000001,
251−};
252−
253−enum rsxx_hw_fifo_flush {
254− RSXX_FLUSH_BUSY = 0x00000002,
255− RSXX_FLUSH_TIMEOUT = 0x00000004,
256−};
257−
258−enum rsxx_pci_revision {
259− RSXX_DISCARD_SUPPORT = 2,
260− RSXX_EEH_SUPPORT = 3,
261−};
262−
263−enum rsxx_creg_cmd {
264− CREG_CMD_TAG_MASK = 0x0000FF00,
265− CREG_OP_WRITE = 0x000000C0,
266− CREG_OP_READ = 0x000000E0,
267−};
268−
269−enum rsxx_creg_addr {
270− CREG_ADD_CARD_CMD = 0x80001000,
271− CREG_ADD_CARD_STATE = 0x80001004,
272− CREG_ADD_CARD_SIZE = 0x8000100c,
273− CREG_ADD_CAPABILITIES = 0x80001050,
274− CREG_ADD_LOG = 0x80002000,
275− CREG_ADD_NUM_TARGETS = 0x80003000,
276− CREG_ADD_CRAM = 0xA0000000,
277− CREG_ADD_CONFIG = 0xB0000000,
278−};
279−
280−enum rsxx_creg_card_cmd {
281− CARD_CMD_STARTUP = 1,
282− CARD_CMD_SHUTDOWN = 2,
283− CARD_CMD_LOW_LEVEL_FORMAT = 3,
284− CARD_CMD_FPGA_RECONFIG_BR = 4,
285− CARD_CMD_FPGA_RECONFIG_MAIN = 5,
286− CARD_CMD_BACKUP = 6,
287− CARD_CMD_RESET = 7,
288− CARD_CMD_deprecated = 8,
289− CARD_CMD_UNINITIALIZE = 9,
290− CARD_CMD_DSTROY_EMERGENCY = 10,
291− CARD_CMD_DSTROY_NORMAL = 11,
292− CARD_CMD_DSTROY_EXTENDED = 12,
293− CARD_CMD_DSTROY_ABORT = 13,
294−};
295−
296−enum rsxx_card_state {
297− CARD_STATE_SHUTDOWN = 0x00000001,
298− CARD_STATE_STARTING = 0x00000002,
299− CARD_STATE_FORMATTING = 0x00000004,
300− CARD_STATE_UNINITIALIZED = 0x00000008,
301− CARD_STATE_GOOD = 0x00000010,
302− CARD_STATE_SHUTTING_DOWN = 0x00000020,
303− CARD_STATE_FAULT = 0x00000040,
304− CARD_STATE_RD_ONLY_FAULT = 0x00000080,
305− CARD_STATE_DSTROYING = 0x00000100,
306−};
307−
308−enum rsxx_led {
309− LED_DEFAULT = 0x0,
310− LED_IDENTIFY = 0x1,
311− LED_SOAK = 0x2,
312−};
313−
314−enum rsxx_creg_flash_lock {
315− CREG_FLASH_LOCK = 1,
316− CREG_FLASH_UNLOCK = 2,
317−};
318−
319−enum rsxx_card_capabilities {
320− CARD_CAP_SUBPAGE_WRITES = 0x00000080,
321−};
322−
323−enum rsxx_creg_stat {
324− CREG_STAT_STATUS_MASK = 0x00000003,
325− CREG_STAT_SUCCESS = 0x1,
326− CREG_STAT_ERROR = 0x2,
327− CREG_STAT_CHAR_PENDING = 0x00000004, /* Character I/O pending bit */
328− CREG_STAT_LOG_PENDING = 0x00000008, /* HW log message pending bit */
329− CREG_STAT_TAG_MASK = 0x0000ff00,
330−};
331−
332−enum rsxx_dma_finish {
333− FREE_DMA = 0x0,
334− COMPLETE_DMA = 0x1,
335−};
336−
337−static inline unsigned int CREG_DATA(int N)
338−{
339− return CREG_DATA0 + (N << 2);
340−}
341−
342−/*----------------- Convenient Log Wrappers -------------------*/
343−#define CARD_TO_DEV(__CARD) (&(__CARD)->dev->dev)
344−
345−/***** config.c *****/
346−int rsxx_load_config(struct rsxx_cardinfo *card);
347−
348−/***** core.c *****/
349−void rsxx_enable_ier(struct rsxx_cardinfo *card, unsigned int intr);
350−void rsxx_disable_ier(struct rsxx_cardinfo *card, unsigned int intr);
351−void rsxx_enable_ier_and_isr(struct rsxx_cardinfo *card,
352− unsigned int intr);
353−void rsxx_disable_ier_and_isr(struct rsxx_cardinfo *card,
354− unsigned int intr);
355−
356−/***** dev.c *****/
357−int rsxx_attach_dev(struct rsxx_cardinfo *card);
358−void rsxx_detach_dev(struct rsxx_cardinfo *card);
359−int rsxx_setup_dev(struct rsxx_cardinfo *card);
360−void rsxx_destroy_dev(struct rsxx_cardinfo *card);
361−int rsxx_dev_init(void);
362−void rsxx_dev_cleanup(void);
363−
364−/***** dma.c ****/
365−typedef void (*rsxx_dma_cb)(struct rsxx_cardinfo *card,
366− void *cb_data,
367− unsigned int status);
368−int rsxx_dma_setup(struct rsxx_cardinfo *card);
369−void rsxx_dma_destroy(struct rsxx_cardinfo *card);
370−int rsxx_dma_init(void);
371−int rsxx_cleanup_dma_queue(struct rsxx_dma_ctrl *ctrl,
372− struct list_head *q,
373− unsigned int done);
374−int rsxx_dma_cancel(struct rsxx_dma_ctrl *ctrl);
375−void rsxx_dma_cleanup(void);
376−void rsxx_dma_queue_reset(struct rsxx_cardinfo *card);
377−int rsxx_dma_configure(struct rsxx_cardinfo *card);
378−blk_status_t rsxx_dma_queue_bio(struct rsxx_cardinfo *card,
379− struct bio *bio,
380− atomic_t *n_dmas,
381− rsxx_dma_cb cb,
382− void *cb_data);
383−int rsxx_hw_buffers_init(struct pci_dev *dev, struct rsxx_dma_ctrl *ctrl);
384−int rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo *card);
385−int rsxx_eeh_remap_dmas(struct rsxx_cardinfo *card);
386−
387−/***** cregs.c *****/
388−int rsxx_creg_write(struct rsxx_cardinfo *card, u32 addr,
389− unsigned int size8,
390− void *data,
391− int byte_stream);
392−int rsxx_creg_read(struct rsxx_cardinfo *card,
393− u32 addr,
394− unsigned int size8,
395− void *data,
396− int byte_stream);
397−int rsxx_read_hw_log(struct rsxx_cardinfo *card);
398−int rsxx_get_card_state(struct rsxx_cardinfo *card,
399− unsigned int *state);
400−int rsxx_get_card_size8(struct rsxx_cardinfo *card, u64 *size8);
401−int rsxx_get_num_targets(struct rsxx_cardinfo *card,
402− unsigned int *n_targets);
403−int rsxx_get_card_capabilities(struct rsxx_cardinfo *card,
404− u32 *capabilities);
405−int rsxx_issue_card_cmd(struct rsxx_cardinfo *card, u32 cmd);
406−int rsxx_creg_setup(struct rsxx_cardinfo *card);
407−void rsxx_creg_destroy(struct rsxx_cardinfo *card);
408−int rsxx_creg_init(void);
409−void rsxx_creg_cleanup(void);
410−int rsxx_reg_access(struct rsxx_cardinfo *card,
411− struct rsxx_reg_access __user *ucmd,
412− int read);
413−void rsxx_eeh_save_issued_creg(struct rsxx_cardinfo *card);
414−void rsxx_kick_creg_queue(struct rsxx_cardinfo *card);
415−
416−
417−
418−#endif /* __DRIVERS_BLOCK_RSXX_H__ */
drivers/md/md-cluster.c+1−1
@@ -574,7 +574,7 @@ static int process_recvd_msg(struct mddev *mddev, struct cluster_msg *msg) · 573 unmodified lines
574574 int ret = 0;
575575
576576 if (WARN(mddev->cluster_info->slot_number - 1 == le32_to_cpu(msg->slot),
577− "node %d received it's own msg\n", le32_to_cpu(msg->slot)))
577+ "node %d received its own msg\n", le32_to_cpu(msg->slot)))
578578 return -1;
579579 switch (le32_to_cpu(msg->type)) {
580580 case METADATA_UPDATED:
drivers/md/md.c+41−12
@@ -418,6 +418,12 @@ check_suspended: · 417 unmodified lines
418418 rcu_read_lock();
419419 if (is_suspended(mddev, bio)) {
420420 DEFINE_WAIT(__wait);
421+ /* Bail out if REQ_NOWAIT is set for the bio */
422+ if (bio->bi_opf & REQ_NOWAIT) {
423+ rcu_read_unlock();
424+ bio_wouldblock_error(bio);
425+ return;
426+ }
421427 for (;;) {
422428 prepare_to_wait(&mddev->sb_wait, &__wait,
423429 TASK_UNINTERRUPTIBLE);
@@ -3603,6 +3609,7 @@ static struct attribute *rdev_default_attrs[] = { · 3179 unmodified lines
36033609 &rdev_ppl_size.attr,
36043610 NULL,
36053611 };
3612+ATTRIBUTE_GROUPS(rdev_default);
36063613 static ssize_t
36073614 rdev_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
36083615 {
@@ -3652,7 +3659,7 @@ static const struct sysfs_ops rdev_sysfs_ops = { · 43 unmodified lines
36523659 static struct kobj_type rdev_ktype = {
36533660 .release = rdev_free,
36543661 .sysfs_ops = &rdev_sysfs_ops,
3655− .default_attrs = rdev_default_attrs,
3662+ .default_groups = rdev_default_groups,
36563663 };
36573664
36583665 int md_rdev_init(struct md_rdev *rdev)
@@ -5788,6 +5795,7 @@ int md_run(struct mddev *mddev) · 2129 unmodified lines
57885795 int err;
57895796 struct md_rdev *rdev;
57905797 struct md_personality *pers;
5798+ bool nowait = true;
57915799
57925800 if (list_empty(&mddev->disks))
57935801 /* cannot run an array with no devices.. */
@@ -5858,8 +5866,13 @@ int md_run(struct mddev *mddev) · 64 unmodified lines
58585866 }
58595867 }
58605868 sysfs_notify_dirent_safe(rdev->sysfs_state);
5869+ nowait = nowait && blk_queue_nowait(bdev_get_queue(rdev->bdev));
58615870 }
58625871
5872+ /* Set the NOWAIT flags if all underlying devices support it */
5873+ if (nowait)
5874+ blk_queue_flag_set(QUEUE_FLAG_NOWAIT, mddev->queue);
5875+
58635876 if (!bioset_initialized(&mddev->bio_set)) {
58645877 err = bioset_init(&mddev->bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
58655878 if (err)
@@ -5870,13 +5883,6 @@ int md_run(struct mddev *mddev) · 4 unmodified lines
58705883 if (err)
58715884 goto exit_bio_set;
58725885 }
5873− if (mddev->level != 1 && mddev->level != 10 &&
5874− !bioset_initialized(&mddev->io_acct_set)) {
5875− err = bioset_init(&mddev->io_acct_set, BIO_POOL_SIZE,
5876− offsetof(struct md_io_acct, bio_clone), 0);
5877− if (err)
5878− goto exit_sync_set;
5879− }
58805886
58815887 spin_lock(&pers_lock);
58825888 pers = find_pers(mddev->level, mddev->clevel);
@@ -6053,9 +6059,6 @@ bitmap_abort: · 170 unmodified lines
60536059 module_put(pers->owner);
60546060 md_bitmap_destroy(mddev);
60556061 abort:
6056− if (mddev->level != 1 && mddev->level != 10)
6057− bioset_exit(&mddev->io_acct_set);
6058−exit_sync_set:
60596062 bioset_exit(&mddev->sync_set);
60606063 exit_bio_set:
60616064 bioset_exit(&mddev->bio_set);
@@ -7004,6 +7007,15 @@ static int hot_add_disk(struct mddev *mddev, dev_t dev) · 942 unmodified lines
70047007 set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
70057008 if (!mddev->thread)
70067009 md_update_sb(mddev, 1);
7010+ /*
7011+ * If the new disk does not support REQ_NOWAIT,
7012+ * disable on the whole MD.
7013+ */
7014+ if (!blk_queue_nowait(bdev_get_queue(rdev->bdev))) {
7015+ pr_info("%s: Disabling nowait because %s does not support nowait\n",
7016+ mdname(mddev), bdevname(rdev->bdev, b));
7017+ blk_queue_flag_clear(QUEUE_FLAG_NOWAIT, mddev->queue);
7018+ }
70077019 /*
70087020 * Kick recovery, maybe this spare has to be added to the
70097021 * array immediately.
@@ -8402,7 +8414,7 @@ int md_setup_cluster(struct mddev *mddev, int nodes) · 1392 unmodified lines
84028414 spin_lock(&pers_lock);
84038415 /* ensure module won't be unloaded */
84048416 if (!md_cluster_ops || !try_module_get(md_cluster_mod)) {
8405− pr_warn("can't find md-cluster module or get it's reference.\n");
8417+ pr_warn("can't find md-cluster module or get its reference.\n");
84068418 spin_unlock(&pers_lock);
84078419 return -ENOENT;
84088420 }
@@ -8589,6 +8601,23 @@ void md_submit_discard_bio(struct mddev *mddev, struct md_rdev *rdev, · 180 unmodified lines
85898601 }
85908602 EXPORT_SYMBOL_GPL(md_submit_discard_bio);
85918603
8604+int acct_bioset_init(struct mddev *mddev)
8605+{
8606+ int err = 0;
8607+
8608+ if (!bioset_initialized(&mddev->io_acct_set))
8609+ err = bioset_init(&mddev->io_acct_set, BIO_POOL_SIZE,
8610+ offsetof(struct md_io_acct, bio_clone), 0);
8611+ return err;
8612+}
8613+EXPORT_SYMBOL_GPL(acct_bioset_init);
8614+
8615+void acct_bioset_exit(struct mddev *mddev)
8616+{
8617+ bioset_exit(&mddev->io_acct_set);
8618+}
8619+EXPORT_SYMBOL_GPL(acct_bioset_exit);
8620+
85928621 static void md_end_io_acct(struct bio *bio)
85938622 {
85948623 struct md_io_acct *md_io_acct = bio->bi_private;
drivers/md/md.h+2−0
@@ -721,6 +721,8 @@ extern void md_error(struct mddev *mddev, struct md_rdev *rdev); · 720 unmodified lines
721721 extern void md_finish_reshape(struct mddev *mddev);
722722 void md_submit_discard_bio(struct mddev *mddev, struct md_rdev *rdev,
723723 struct bio *bio, sector_t start, sector_t size);
724+int acct_bioset_init(struct mddev *mddev);
725+void acct_bioset_exit(struct mddev *mddev);
724726 void md_account_bio(struct mddev *mddev, struct bio **bio);
725727
726728 extern bool __must_check md_flush_request(struct mddev *mddev, struct bio *bio);
drivers/md/raid0.c+28−10
@@ -356,7 +356,21 @@ static sector_t raid0_size(struct mddev *mddev, sector_t sectors, int raid_disks · 355 unmodified lines
356356 return array_sectors;
357357 }
358358
359−static void raid0_free(struct mddev *mddev, void *priv);
359+static void free_conf(struct mddev *mddev, struct r0conf *conf)
360+{
361+ kfree(conf->strip_zone);
362+ kfree(conf->devlist);
363+ kfree(conf);
364+ mddev->private = NULL;
365+}
366+
367+static void raid0_free(struct mddev *mddev, void *priv)
368+{
369+ struct r0conf *conf = priv;
370+
371+ free_conf(mddev, conf);
372+ acct_bioset_exit(mddev);
373+}
360374
361375 static int raid0_run(struct mddev *mddev)
362376 {
@@ -370,11 +384,16 @@ static int raid0_run(struct mddev *mddev) · 7 unmodified lines
370384 if (md_check_no_bitmap(mddev))
371385 return -EINVAL;
372386
387+ if (acct_bioset_init(mddev)) {
388+ pr_err("md/raid0:%s: alloc acct bioset failed.\n", mdname(mddev));
389+ return -ENOMEM;
390+ }
391+
373392 /* if private is not null, we are here after takeover */
374393 if (mddev->private == NULL) {
375394 ret = create_strip_zones(mddev, &conf);
376395 if (ret < 0)
377− return ret;
396+ goto exit_acct_set;
378397 mddev->private = conf;
379398 }
380399 conf = mddev->private;
@@ -413,17 +432,16 @@ static int raid0_run(struct mddev *mddev) · 32 unmodified lines
413432 dump_zones(mddev);
414433
415434 ret = md_integrity_register(mddev);
435+ if (ret)
436+ goto free;
416437
417438 return ret;
418−}
419439
420−static void raid0_free(struct mddev *mddev, void *priv)
421−{
422− struct r0conf *conf = priv;
423−
424− kfree(conf->strip_zone);
425− kfree(conf->devlist);
426− kfree(conf);
440+free:
441+ free_conf(mddev, conf);
442+exit_acct_set:
443+ acct_bioset_exit(mddev);
444+ return ret;
427445 }
428446
429447 static void raid0_handle_discard(struct mddev *mddev, struct bio *bio)
drivers/md/raid1-10.c+0−6
@@ -22,12 +22,6 @@ · 21 unmodified lines
2222
2323 #define BIO_SPECIAL(bio) ((unsigned long)bio <= 2)
2424
25−/* When there are this many requests queue to be written by
26− * the raid thread, we become 'congested' to provide back-pressure
27− * for writeback.
28− */
29−static int max_queued_requests = 1024;
30−
3125 /* for managing resync I/O pages */
3226 struct resync_pages {
3327 void *raid_bio;
drivers/md/raid1.c+56−27
@@ -929,8 +929,10 @@ static void lower_barrier(struct r1conf *conf, sector_t sector_nr) · 928 unmodified lines
929929 wake_up(&conf->wait_barrier);
930930 }
931931
932−static void _wait_barrier(struct r1conf *conf, int idx)
932+static bool _wait_barrier(struct r1conf *conf, int idx, bool nowait)
933933 {
934+ bool ret = true;
935+
934936 /*
935937 * We need to increase conf->nr_pending[idx] very early here,
936938 * then raise_barrier() can be blocked when it waits for
@@ -961,7 +963,7 @@ static void _wait_barrier(struct r1conf *conf, int idx) · 24 unmodified lines
961963 */
962964 if (!READ_ONCE(conf->array_frozen) &&
963965 !atomic_read(&conf->barrier[idx]))
964− return;
966+ return ret;
965967
966968 /*
967969 * After holding conf->resync_lock, conf->nr_pending[idx]
@@ -979,18 +981,27 @@ static void _wait_barrier(struct r1conf *conf, int idx) · 11 unmodified lines
979981 */
980982 wake_up(&conf->wait_barrier);
981983 /* Wait for the barrier in same barrier unit bucket to drop. */
982− wait_event_lock_irq(conf->wait_barrier,
983− !conf->array_frozen &&
984− !atomic_read(&conf->barrier[idx]),
985− conf->resync_lock);
986− atomic_inc(&conf->nr_pending[idx]);
984+
985+ /* Return false when nowait flag is set */
986+ if (nowait) {
987+ ret = false;
988+ } else {
989+ wait_event_lock_irq(conf->wait_barrier,
990+ !conf->array_frozen &&
991+ !atomic_read(&conf->barrier[idx]),
992+ conf->resync_lock);
993+ atomic_inc(&conf->nr_pending[idx]);
994+ }
995+
987996 atomic_dec(&conf->nr_waiting[idx]);
988997 spin_unlock_irq(&conf->resync_lock);
998+ return ret;
989999 }
9901000
991−static void wait_read_barrier(struct r1conf *conf, sector_t sector_nr)
1001+static bool wait_read_barrier(struct r1conf *conf, sector_t sector_nr, bool nowait)
9921002 {
9931003 int idx = sector_to_idx(sector_nr);
1004+ bool ret = true;
9941005
9951006 /*
9961007 * Very similar to _wait_barrier(). The difference is, for read
@@ -1002,7 +1013,7 @@ static void wait_read_barrier(struct r1conf *conf, sector_t sector_nr) · 5 unmodified lines
10021013 atomic_inc(&conf->nr_pending[idx]);
10031014
10041015 if (!READ_ONCE(conf->array_frozen))
1005− return;
1016+ return ret;
10061017
10071018 spin_lock_irq(&conf->resync_lock);
10081019 atomic_inc(&conf->nr_waiting[idx]);
@@ -1013,19 +1024,28 @@ static void wait_read_barrier(struct r1conf *conf, sector_t sector_nr) · 4 unmodified lines
10131024 */
10141025 wake_up(&conf->wait_barrier);
10151026 /* Wait for array to be unfrozen */
1016− wait_event_lock_irq(conf->wait_barrier,
1017− !conf->array_frozen,
1018− conf->resync_lock);
1019− atomic_inc(&conf->nr_pending[idx]);
1027+
1028+ /* Return false when nowait flag is set */
1029+ if (nowait) {
1030+ /* Return false when nowait flag is set */
1031+ ret = false;
1032+ } else {
1033+ wait_event_lock_irq(conf->wait_barrier,
1034+ !conf->array_frozen,
1035+ conf->resync_lock);
1036+ atomic_inc(&conf->nr_pending[idx]);
1037+ }
1038+
10201039 atomic_dec(&conf->nr_waiting[idx]);
10211040 spin_unlock_irq(&conf->resync_lock);
1041+ return ret;
10221042 }
10231043
1024−static void wait_barrier(struct r1conf *conf, sector_t sector_nr)
1044+static bool wait_barrier(struct r1conf *conf, sector_t sector_nr, bool nowait)
10251045 {
10261046 int idx = sector_to_idx(sector_nr);
10271047
1028− _wait_barrier(conf, idx);
1048+ return _wait_barrier(conf, idx, nowait);
10291049 }
10301050
10311051 static void _allow_barrier(struct r1conf *conf, int idx)
@@ -1236,7 +1256,11 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, · 204 unmodified lines
12361256 * Still need barrier for READ in case that whole
12371257 * array is frozen.
12381258 */
1239− wait_read_barrier(conf, bio->bi_iter.bi_sector);
1259+ if (!wait_read_barrier(conf, bio->bi_iter.bi_sector,
1260+ bio->bi_opf & REQ_NOWAIT)) {
1261+ bio_wouldblock_error(bio);
1262+ return;
1263+ }
12401264
12411265 if (!r1_bio)
12421266 r1_bio = alloc_r1bio(mddev, bio);
@@ -1336,6 +1360,10 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, · 93 unmodified lines
13361360 bio->bi_iter.bi_sector, bio_end_sector(bio))) {
13371361
13381362 DEFINE_WAIT(w);
1363+ if (bio->bi_opf & REQ_NOWAIT) {
1364+ bio_wouldblock_error(bio);
1365+ return;
1366+ }
13391367 for (;;) {
13401368 prepare_to_wait(&conf->wait_barrier,
13411369 &w, TASK_IDLE);
@@ -1353,17 +1381,15 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, · 11 unmodified lines
13531381 * thread has put up a bar for new requests.
13541382 * Continue immediately if no resync is active currently.
13551383 */
1356− wait_barrier(conf, bio->bi_iter.bi_sector);
1384+ if (!wait_barrier(conf, bio->bi_iter.bi_sector,
1385+ bio->bi_opf & REQ_NOWAIT)) {
1386+ bio_wouldblock_error(bio);
1387+ return;
1388+ }
13571389
13581390 r1_bio = alloc_r1bio(mddev, bio);
13591391 r1_bio->sectors = max_write_sectors;
13601392
1361− if (conf->pending_count >= max_queued_requests) {
1362− md_wakeup_thread(mddev->thread);
1363− raid1_log(mddev, "wait queued");
1364− wait_event(conf->wait_barrier,
1365− conf->pending_count < max_queued_requests);
1366− }
13671393 /* first select target devices under rcu_lock and
13681394 * inc refcount on their rdev. Record them by setting
13691395 * bios[x] to bio
@@ -1458,9 +1484,14 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, · 88 unmodified lines
14581484 rdev_dec_pending(conf->mirrors[j].rdev, mddev);
14591485 r1_bio->state = 0;
14601486 allow_barrier(conf, bio->bi_iter.bi_sector);
1487+
1488+ if (bio->bi_opf & REQ_NOWAIT) {
1489+ bio_wouldblock_error(bio);
1490+ return;
1491+ }
14611492 raid1_log(mddev, "wait rdev %d blocked", blocked_rdev->raid_disk);
14621493 md_wait_for_blocked_rdev(blocked_rdev, mddev);
1463− wait_barrier(conf, bio->bi_iter.bi_sector);
1494+ wait_barrier(conf, bio->bi_iter.bi_sector, false);
14641495 goto retry_write;
14651496 }
14661497
@@ -1688,7 +1719,7 @@ static void close_sync(struct r1conf *conf) · 221 unmodified lines
16881719 int idx;
16891720
16901721 for (idx = 0; idx < BARRIER_BUCKETS_NR; idx++) {
1691− _wait_barrier(conf, idx);
1722+ _wait_barrier(conf, idx, false);
16921723 _allow_barrier(conf, idx);
16931724 }
16941725
@@ -3410,5 +3441,3 @@ MODULE_DESCRIPTION("RAID1 (mirroring) personality for MD"); · 1715 unmodified lines
34103441 MODULE_ALIAS("md-personality-3"); /* RAID1 */
34113442 MODULE_ALIAS("md-raid1");
34123443 MODULE_ALIAS("md-level-1");
3413−
3414−module_param(max_queued_requests, int, S_IRUGO|S_IWUSR);
drivers/md/raid10.c+67−40
@@ -952,8 +952,10 @@ static void lower_barrier(struct r10conf *conf) · 951 unmodified lines
952952 wake_up(&conf->wait_barrier);
953953 }
954954
955−static void wait_barrier(struct r10conf *conf)
955+static bool wait_barrier(struct r10conf *conf, bool nowait)
956956 {
957+ bool ret = true;
958+
957959 spin_lock_irq(&conf->resync_lock);
958960 if (conf->barrier) {
959961 struct bio_list *bio_list = current->bio_list;
@@ -967,27 +969,35 @@ static void wait_barrier(struct r10conf *conf) · 7 unmodified lines
967969 * that queue to get the nr_pending
968970 * count down.
969971 */
970− raid10_log(conf->mddev, "wait barrier");
971− wait_event_lock_irq(conf->wait_barrier,
972− !conf->barrier ||
973− (atomic_read(&conf->nr_pending) &&
974− bio_list &&
975− (!bio_list_empty(&bio_list[0]) ||
976− !bio_list_empty(&bio_list[1]))) ||
977− /* move on if recovery thread is
978− * blocked by us
979− */
980− (conf->mddev->thread->tsk == current &&
981− test_bit(MD_RECOVERY_RUNNING,
982− &conf->mddev->recovery) &&
983− conf->nr_queued > 0),
984− conf->resync_lock);
972+ /* Return false when nowait flag is set */
973+ if (nowait) {
974+ ret = false;
975+ } else {
976+ raid10_log(conf->mddev, "wait barrier");
977+ wait_event_lock_irq(conf->wait_barrier,
978+ !conf->barrier ||
979+ (atomic_read(&conf->nr_pending) &&
980+ bio_list &&
981+ (!bio_list_empty(&bio_list[0]) ||
982+ !bio_list_empty(&bio_list[1]))) ||
983+ /* move on if recovery thread is
984+ * blocked by us
985+ */
986+ (conf->mddev->thread->tsk == current &&
987+ test_bit(MD_RECOVERY_RUNNING,
988+ &conf->mddev->recovery) &&
989+ conf->nr_queued > 0),
990+ conf->resync_lock);
991+ }
985992 conf->nr_waiting--;
986993 if (!conf->nr_waiting)
987994 wake_up(&conf->wait_barrier);
988995 }
989− atomic_inc(&conf->nr_pending);
996+ /* Only increment nr_pending when we wait */
997+ if (ret)
998+ atomic_inc(&conf->nr_pending);
990999 spin_unlock_irq(&conf->resync_lock);
1000+ return ret;
9911001 }
9921002
9931003 static void allow_barrier(struct r10conf *conf)
@@ -1098,21 +1108,30 @@ static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule) · 104 unmodified lines
10981108 * currently.
10991109 * 2. If IO spans the reshape position. Need to wait for reshape to pass.
11001110 */
1101−static void regular_request_wait(struct mddev *mddev, struct r10conf *conf,
1111+static bool regular_request_wait(struct mddev *mddev, struct r10conf *conf,
11021112 struct bio *bio, sector_t sectors)
11031113 {
1104− wait_barrier(conf);
1114+ /* Bail out if REQ_NOWAIT is set for the bio */
1115+ if (!wait_barrier(conf, bio->bi_opf & REQ_NOWAIT)) {
1116+ bio_wouldblock_error(bio);
1117+ return false;
1118+ }
11051119 while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
11061120 bio->bi_iter.bi_sector < conf->reshape_progress &&
11071121 bio->bi_iter.bi_sector + sectors > conf->reshape_progress) {
1108− raid10_log(conf->mddev, "wait reshape");
11091122 allow_barrier(conf);
1123+ if (bio->bi_opf & REQ_NOWAIT) {
1124+ bio_wouldblock_error(bio);
1125+ return false;
1126+ }
1127+ raid10_log(conf->mddev, "wait reshape");
11101128 wait_event(conf->wait_barrier,
11111129 conf->reshape_progress <= bio->bi_iter.bi_sector ||
11121130 conf->reshape_progress >= bio->bi_iter.bi_sector +
11131131 sectors);
1114− wait_barrier(conf);
1132+ wait_barrier(conf, false);
11151133 }
1134+ return true;
11161135 }
11171136
11181137 static void raid10_read_request(struct mddev *mddev, struct bio *bio,
@@ -1157,7 +1176,8 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, · 38 unmodified lines
11571176 rcu_read_unlock();
11581177 }
11591178
1160− regular_request_wait(mddev, conf, bio, r10_bio->sectors);
1179+ if (!regular_request_wait(mddev, conf, bio, r10_bio->sectors))
1180+ return;
11611181 rdev = read_balance(conf, r10_bio, &max_sectors);
11621182 if (!rdev) {
11631183 if (err_rdev) {
@@ -1179,7 +1199,7 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, · 15 unmodified lines
11791199 bio_chain(split, bio);
11801200 allow_barrier(conf);
11811201 submit_bio_noacct(bio);
1182− wait_barrier(conf);
1202+ wait_barrier(conf, false);
11831203 bio = split;
11841204 r10_bio->master_bio = bio;
11851205 r10_bio->sectors = max_sectors;
@@ -1338,7 +1358,7 @@ retry_wait: · 152 unmodified lines
13381358 raid10_log(conf->mddev, "%s wait rdev %d blocked",
13391359 __func__, blocked_rdev->raid_disk);
13401360 md_wait_for_blocked_rdev(blocked_rdev, mddev);
1341− wait_barrier(conf);
1361+ wait_barrier(conf, false);
13421362 goto retry_wait;
13431363 }
13441364 }
@@ -1356,6 +1376,11 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio, · 11 unmodified lines
13561376 bio->bi_iter.bi_sector,
13571377 bio_end_sector(bio)))) {
13581378 DEFINE_WAIT(w);
1379+ /* Bail out if REQ_NOWAIT is set for the bio */
1380+ if (bio->bi_opf & REQ_NOWAIT) {
1381+ bio_wouldblock_error(bio);
1382+ return;
1383+ }
13591384 for (;;) {
13601385 prepare_to_wait(&conf->wait_barrier,
13611386 &w, TASK_IDLE);
@@ -1368,7 +1393,8 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio, · 6 unmodified lines
13681393 }
13691394
13701395 sectors = r10_bio->sectors;
1371− regular_request_wait(mddev, conf, bio, sectors);
1396+ if (!regular_request_wait(mddev, conf, bio, sectors))
1397+ return;
13721398 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
13731399 (mddev->reshape_backwards
13741400 ? (bio->bi_iter.bi_sector < conf->reshape_safe &&
@@ -1380,6 +1406,11 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio, · 5 unmodified lines
13801406 set_mask_bits(&mddev->sb_flags, 0,
13811407 BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
13821408 md_wakeup_thread(mddev->thread);
1409+ if (bio->bi_opf & REQ_NOWAIT) {
1410+ allow_barrier(conf);
1411+ bio_wouldblock_error(bio);
1412+ return;
1413+ }
13831414 raid10_log(conf->mddev, "wait reshape metadata");
13841415 wait_event(mddev->sb_wait,
13851416 !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags));
@@ -1387,12 +1418,6 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio, · 1 unmodified line
13871418 conf->reshape_safe = mddev->reshape_position;
13881419 }
13891420
1390− if (conf->pending_count >= max_queued_requests) {
1391− md_wakeup_thread(mddev->thread);
1392− raid10_log(mddev, "wait queued");
1393− wait_event(conf->wait_barrier,
1394− conf->pending_count < max_queued_requests);
1395− }
13961421 /* first select target devices under rcu_lock and
13971422 * inc refcount on their rdev. Record them by setting
13981423 * bios[x] to bio
@@ -1482,7 +1507,7 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio, · 83 unmodified lines
14821507 bio_chain(split, bio);
14831508 allow_barrier(conf);
14841509 submit_bio_noacct(bio);
1485− wait_barrier(conf);
1510+ wait_barrier(conf, false);
14861511 bio = split;
14871512 r10_bio->master_bio = bio;
14881513 }
@@ -1607,7 +1632,11 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) · 118 unmodified lines
16071632 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
16081633 return -EAGAIN;
16091634
1610− wait_barrier(conf);
1635+ if (WARN_ON_ONCE(bio->bi_opf & REQ_NOWAIT)) {
1636+ bio_wouldblock_error(bio);
1637+ return 0;
1638+ }
1639+ wait_barrier(conf, false);
16111640
16121641 /*
16131642 * Check reshape again to avoid reshape happens after checking
@@ -1649,7 +1678,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) · 35 unmodified lines
16491678 allow_barrier(conf);
16501679 /* Resend the fist split part */
16511680 submit_bio_noacct(split);
1652− wait_barrier(conf);
1681+ wait_barrier(conf, false);
16531682 }
16541683 div_u64_rem(bio_end, stripe_size, &remainder);
16551684 if (remainder) {
@@ -1660,7 +1689,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) · 4 unmodified lines
16601689 /* Resend the second split part */
16611690 submit_bio_noacct(bio);
16621691 bio = split;
1663− wait_barrier(conf);
1692+ wait_barrier(conf, false);
16641693 }
16651694
16661695 bio_start = bio->bi_iter.bi_sector;
@@ -1816,7 +1845,7 @@ retry_discard: · 149 unmodified lines
18161845 end_disk_offset += geo->stride;
18171846 atomic_inc(&first_r10bio->remaining);
18181847 raid_end_discard_bio(r10_bio);
1819− wait_barrier(conf);
1848+ wait_barrier(conf, false);
18201849 goto retry_discard;
18211850 }
18221851
@@ -2011,7 +2040,7 @@ static void print_conf(struct r10conf *conf) · 188 unmodified lines
20112040
20122041 static void close_sync(struct r10conf *conf)
20132042 {
2014− wait_barrier(conf);
2043+ wait_barrier(conf, false);
20152044 allow_barrier(conf);
20162045
20172046 mempool_exit(&conf->r10buf_pool);
@@ -4819,7 +4848,7 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, · 2801 unmodified lines
48194848 if (need_flush ||
48204849 time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
48214850 /* Need to update reshape_position in metadata */
4822− wait_barrier(conf);
4851+ wait_barrier(conf, false);
48234852 mddev->reshape_position = conf->reshape_progress;
48244853 if (mddev->reshape_backwards)
48254854 mddev->curr_resync_completed = raid10_size(mddev, 0, 0)
@@ -5242,5 +5271,3 @@ MODULE_DESCRIPTION("RAID10 (striped mirror) personality for MD"); · 416 unmodified lines
52425271 MODULE_ALIAS("md-personality-9"); /* RAID10 */
52435272 MODULE_ALIAS("md-raid10");
52445273 MODULE_ALIAS("md-level-10");
5245−
5246−module_param(max_queued_requests, int, S_IRUGO|S_IWUSR);
drivers/md/raid5.c+51−16
@@ -2215,10 +2215,9 @@ static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request) · 2214 unmodified lines
22152215 struct r5conf *conf = sh->raid_conf;
22162216 int level = conf->level;
22172217 struct raid5_percpu *percpu;
2218− unsigned long cpu;
22192218
2220− cpu = get_cpu();
2221− percpu = per_cpu_ptr(conf->percpu, cpu);
2219+ local_lock(&conf->percpu->lock);
2220+ percpu = this_cpu_ptr(conf->percpu);
22222221 if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {
22232222 ops_run_biofill(sh);
22242223 overlap_clear++;
@@ -2271,13 +2270,14 @@ static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request) · 46 unmodified lines
22712270 BUG();
22722271 }
22732272
2274− if (overlap_clear && !sh->batch_head)
2273+ if (overlap_clear && !sh->batch_head) {
22752274 for (i = disks; i--; ) {
22762275 struct r5dev *dev = &sh->dev[i];
22772276 if (test_and_clear_bit(R5_Overlap, &dev->flags))
22782277 wake_up(&sh->raid_conf->wait_for_overlap);
22792278 }
2280− put_cpu();
2279+ }
2280+ local_unlock(&conf->percpu->lock);
22812281 }
22822282
22832283 static void free_stripe(struct kmem_cache *sc, struct stripe_head *sh)
@@ -5686,6 +5686,10 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi) · 3402 unmodified lines
56865686 struct stripe_head *sh;
56875687 int stripe_sectors;
56885688
5689+ /* We need to handle this when io_uring supports discard/trim */
5690+ if (WARN_ON_ONCE(bi->bi_opf & REQ_NOWAIT))
5691+ return;
5692+
56895693 if (mddev->reshape_position != MaxSector)
56905694 /* Skip discard while reshape is happening */
56915695 return;
@@ -5819,6 +5823,17 @@ static bool raid5_make_request(struct mddev *mddev, struct bio * bi) · 127 unmodified lines
58195823 last_sector = bio_end_sector(bi);
58205824 bi->bi_next = NULL;
58215825
5826+ /* Bail out if conflicts with reshape and REQ_NOWAIT is set */
5827+ if ((bi->bi_opf & REQ_NOWAIT) &&
5828+ (conf->reshape_progress != MaxSector) &&
5829+ (mddev->reshape_backwards
5830+ ? (logical_sector > conf->reshape_progress && logical_sector <= conf->reshape_safe)
5831+ : (logical_sector >= conf->reshape_safe && logical_sector < conf->reshape_progress))) {
5832+ bio_wouldblock_error(bi);
5833+ if (rw == WRITE)
5834+ md_write_end(mddev);
5835+ return true;
5836+ }
58225837 md_account_bio(mddev, &bi);
58235838 prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE);
58245839 for (; logical_sector < last_sector; logical_sector += RAID5_STRIPE_SECTORS(conf)) {
@@ -7052,6 +7067,7 @@ static int alloc_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu · 1227 unmodified lines
70527067 return -ENOMEM;
70537068 }
70547069
7070+ local_lock_init(&percpu->lock);
70557071 return 0;
70567072 }
70577073
@@ -7446,12 +7462,19 @@ static int raid5_run(struct mddev *mddev) · 388 unmodified lines
74467462 struct md_rdev *rdev;
74477463 struct md_rdev *journal_dev = NULL;
74487464 sector_t reshape_offset = 0;
7449− int i;
7465+ int i, ret = 0;
74507466 long long min_offset_diff = 0;
74517467 int first = 1;
74527468
7453− if (mddev_init_writes_pending(mddev) < 0)
7469+ if (acct_bioset_init(mddev)) {
7470+ pr_err("md/raid456:%s: alloc acct bioset failed.\n", mdname(mddev));
74547471 return -ENOMEM;
7472+ }
7473+
7474+ if (mddev_init_writes_pending(mddev) < 0) {
7475+ ret = -ENOMEM;
7476+ goto exit_acct_set;
7477+ }
74557478
74567479 if (mddev->recovery_cp != MaxSector)
74577480 pr_notice("md/raid:%s: not clean -- starting background reconstruction\n",
@@ -7482,7 +7505,8 @@ static int raid5_run(struct mddev *mddev) · 24 unmodified lines
74827505 (mddev->bitmap_info.offset || mddev->bitmap_info.file)) {
74837506 pr_notice("md/raid:%s: array cannot have both journal and bitmap\n",
74847507 mdname(mddev));
7485− return -EINVAL;
7508+ ret = -EINVAL;
7509+ goto exit_acct_set;
74867510 }
74877511
74887512 if (mddev->reshape_position != MaxSector) {
@@ -7507,13 +7531,15 @@ static int raid5_run(struct mddev *mddev) · 18 unmodified lines
75077531 if (journal_dev) {
75087532 pr_warn("md/raid:%s: don't support reshape with journal - aborting.\n",
75097533 mdname(mddev));
7510− return -EINVAL;
7534+ ret = -EINVAL;
7535+ goto exit_acct_set;
75117536 }
75127537
75137538 if (mddev->new_level != mddev->level) {
75147539 pr_warn("md/raid:%s: unsupported reshape required - aborting.\n",
75157540 mdname(mddev));
7516− return -EINVAL;
7541+ ret = -EINVAL;
7542+ goto exit_acct_set;
75177543 }
75187544 old_disks = mddev->raid_disks - mddev->delta_disks;
75197545 /* reshape_position must be on a new-stripe boundary, and one
@@ -7529,7 +7555,8 @@ static int raid5_run(struct mddev *mddev) · 9 unmodified lines
75297555 if (sector_div(here_new, chunk_sectors * new_data_disks)) {
75307556 pr_warn("md/raid:%s: reshape_position not on a stripe boundary\n",
75317557 mdname(mddev));
7532− return -EINVAL;
7558+ ret = -EINVAL;
7559+ goto exit_acct_set;
75337560 }
75347561 reshape_offset = here_new * chunk_sectors;
75357562 /* here_new is the stripe we will write to */
@@ -7551,7 +7578,8 @@ static int raid5_run(struct mddev *mddev) · 15 unmodified lines
75517578 else if (mddev->ro == 0) {
75527579 pr_warn("md/raid:%s: in-place reshape must be started in read-only mode - aborting\n",
75537580 mdname(mddev));
7554− return -EINVAL;
7581+ ret = -EINVAL;
7582+ goto exit_acct_set;
75557583 }
75567584 } else if (mddev->reshape_backwards
75577585 ? (here_new * chunk_sectors + min_offset_diff <=
@@ -7561,7 +7589,8 @@ static int raid5_run(struct mddev *mddev) · 3 unmodified lines
75617589 /* Reading from the same stripe as writing to - bad */
75627590 pr_warn("md/raid:%s: reshape_position too early for auto-recovery - aborting.\n",
75637591 mdname(mddev));
7564− return -EINVAL;
7592+ ret = -EINVAL;
7593+ goto exit_acct_set;
75657594 }
75667595 pr_debug("md/raid:%s: reshape will continue\n", mdname(mddev));
75677596 /* OK, we should be able to continue; */
@@ -7585,8 +7614,10 @@ static int raid5_run(struct mddev *mddev) · 17 unmodified lines
75857614 else
75867615 conf = mddev->private;
75877616
7588− if (IS_ERR(conf))
7589− return PTR_ERR(conf);
7617+ if (IS_ERR(conf)) {
7618+ ret = PTR_ERR(conf);
7619+ goto exit_acct_set;
7620+ }
75907621
75917622 if (test_bit(MD_HAS_JOURNAL, &mddev->flags)) {
75927623 if (!journal_dev) {
@@ -7783,7 +7814,10 @@ abort: · 190 unmodified lines
77837814 free_conf(conf);
77847815 mddev->private = NULL;
77857816 pr_warn("md/raid:%s: failed to run raid set.\n", mdname(mddev));
7786− return -EIO;
7817+ ret = -EIO;
7818+exit_acct_set:
7819+ acct_bioset_exit(mddev);
7820+ return ret;
77877821 }
77887822
77897823 static void raid5_free(struct mddev *mddev, void *priv)
@@ -7791,6 +7825,7 @@ static void raid5_free(struct mddev *mddev, void *priv) · 1 unmodified line
77917825 struct r5conf *conf = priv;
77927826
77937827 free_conf(conf);
7828+ acct_bioset_exit(mddev);
77947829 mddev->to_remove = &raid5_attrs_group;
77957830 }
77967831
drivers/md/raid5.h+3−1
@@ -4,6 +4,7 @@ · 3 unmodified lines
44
55 #include <linux/raid/xor.h>
66 #include <linux/dmaengine.h>
7+#include <linux/local_lock.h>
78
89 /*
910 *
@@ -640,7 +641,8 @@ struct r5conf { · 630 unmodified lines
640641 * lists and performing address
641642 * conversions
642643 */
643− int scribble_obj_size;
644+ int scribble_obj_size;
645+ local_lock_t lock;
644646 } __percpu *percpu;
645647 int scribble_disks;
646648 int scribble_sectors;
drivers/nvme/host/core.c+1−6
@@ -991,7 +991,6 @@ EXPORT_SYMBOL_GPL(nvme_cleanup_cmd); · 990 unmodified lines
991991 blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req)
992992 {
993993 struct nvme_command *cmd = nvme_req(req)->cmd;
994− struct nvme_ctrl *ctrl = nvme_req(req)->ctrl;
995994 blk_status_t ret = BLK_STS_OK;
996995
997996 if (!(req->rq_flags & RQF_DONTPREP))
@@ -1038,8 +1037,6 @@ blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req) · 40 unmodified lines
10381037 return BLK_STS_IOERR;
10391038 }
10401039
1041− if (!(ctrl->quirks & NVME_QUIRK_SKIP_CID_GEN))
1042− nvme_req(req)->genctr++;
10431040 cmd->common.command_id = nvme_cid(req);
10441041 trace_nvme_setup_cmd(req, cmd);
10451042 return ret;
@@ -2762,9 +2759,7 @@ static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) · 1716 unmodified lines
27622759 return -EINVAL;
27632760 }
27642761 subsys->awupf = le16_to_cpu(id->awupf);
2765−#ifdef CONFIG_NVME_MULTIPATH
2766− subsys->iopolicy = NVME_IOPOLICY_NUMA;
2767−#endif
2762+ nvme_mpath_default_iopolicy(subsys);
27682763
27692764 subsys->dev.class = nvme_subsys_class;
27702765 subsys->dev.release = nvme_release_subsystem;
drivers/nvme/host/fabrics.c+21−1
@@ -1069,6 +1069,26 @@ out_unlock: · 1068 unmodified lines
10691069 return ret ? ret : count;
10701070 }
10711071
1072+static void __nvmf_concat_opt_tokens(struct seq_file *seq_file)
1073+{
1074+ const struct match_token *tok;
1075+ int idx;
1076+
1077+ /*
1078+ * Add dummy entries for instance and cntlid to
1079+ * signal an invalid/non-existing controller
1080+ */
1081+ seq_puts(seq_file, "instance=-1,cntlid=-1");
1082+ for (idx = 0; idx < ARRAY_SIZE(opt_tokens); idx++) {
1083+ tok = &opt_tokens[idx];
1084+ if (tok->token == NVMF_OPT_ERR)
1085+ continue;
1086+ seq_puts(seq_file, ",");
1087+ seq_puts(seq_file, tok->pattern);
1088+ }
1089+ seq_puts(seq_file, "\n");
1090+}
1091+
10721092 static int nvmf_dev_show(struct seq_file *seq_file, void *private)
10731093 {
10741094 struct nvme_ctrl *ctrl;
@@ -1077,7 +1097,7 @@ static int nvmf_dev_show(struct seq_file *seq_file, void *private) · 2 unmodified lines
10771097 mutex_lock(&nvmf_dev_mutex);
10781098 ctrl = seq_file->private;
10791099 if (!ctrl) {
1080− ret = -EINVAL;
1100+ __nvmf_concat_opt_tokens(seq_file);
10811101 goto out_unlock;
10821102 }
10831103
drivers/nvme/host/multipath.c+36−5
@@ -13,6 +13,42 @@ module_param(multipath, bool, 0444); · 12 unmodified lines
1313 MODULE_PARM_DESC(multipath,
1414 "turn on native support for multiple controllers per subsystem");
1515
16+static const char *nvme_iopolicy_names[] = {
17+ [NVME_IOPOLICY_NUMA] = "numa",
18+ [NVME_IOPOLICY_RR] = "round-robin",
19+};
20+
21+static int iopolicy = NVME_IOPOLICY_NUMA;
22+
23+static int nvme_set_iopolicy(const char *val, const struct kernel_param *kp)
24+{
25+ if (!val)
26+ return -EINVAL;
27+ if (!strncmp(val, "numa", 4))
28+ iopolicy = NVME_IOPOLICY_NUMA;
29+ else if (!strncmp(val, "round-robin", 11))
30+ iopolicy = NVME_IOPOLICY_RR;
31+ else
32+ return -EINVAL;
33+
34+ return 0;
35+}
36+
37+static int nvme_get_iopolicy(char *buf, const struct kernel_param *kp)
38+{
39+ return sprintf(buf, "%s\n", nvme_iopolicy_names[iopolicy]);
40+}
41+
42+module_param_call(iopolicy, nvme_set_iopolicy, nvme_get_iopolicy,
43+ &iopolicy, 0644);
44+MODULE_PARM_DESC(iopolicy,
45+ "Default multipath I/O policy; 'numa' (default) or 'round-robin'");
46+
47+void nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys)
48+{
49+ subsys->iopolicy = iopolicy;
50+}
51+
1652 void nvme_mpath_unfreeze(struct nvme_subsystem *subsys)
1753 {
1854 struct nvme_ns_head *h;
@@ -706,11 +742,6 @@ void nvme_mpath_stop(struct nvme_ctrl *ctrl) · 687 unmodified lines
706742 struct device_attribute subsys_attr_##_name = \
707743 __ATTR(_name, _mode, _show, _store)
708744
709−static const char *nvme_iopolicy_names[] = {
710− [NVME_IOPOLICY_NUMA] = "numa",
711− [NVME_IOPOLICY_RR] = "round-robin",
712−};
713−
714745 static ssize_t nvme_subsys_iopolicy_show(struct device *dev,
715746 struct device_attribute *attr, char *buf)
716747 {
drivers/nvme/host/nvme.h+8−0
@@ -614,6 +614,10 @@ static inline bool nvme_try_complete_req(struct request *req, __le16 status, · 613 unmodified lines
614614 union nvme_result result)
615615 {
616616 struct nvme_request *rq = nvme_req(req);
617+ struct nvme_ctrl *ctrl = rq->ctrl;
618+
619+ if (!(ctrl->quirks & NVME_QUIRK_SKIP_CID_GEN))
620+ rq->genctr++;
617621
618622 rq->status = le16_to_cpu(status) >> 1;
619623 rq->result = result;
@@ -763,6 +767,7 @@ static inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl) · 143 unmodified lines
763767 void nvme_mpath_unfreeze(struct nvme_subsystem *subsys);
764768 void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys);
765769 void nvme_mpath_start_freeze(struct nvme_subsystem *subsys);
770+void nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys);
766771 bool nvme_mpath_set_disk_name(struct nvme_ns *ns, char *disk_name, int *flags);
767772 void nvme_failover_req(struct request *req);
768773 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl);
@@ -860,6 +865,9 @@ static inline void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys) · 91 unmodified lines
860865 static inline void nvme_mpath_start_freeze(struct nvme_subsystem *subsys)
861866 {
862867 }
868+static inline void nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys)
869+{
870+}
863871 #endif /* CONFIG_NVME_MULTIPATH */
864872
865873 int nvme_revalidate_zones(struct nvme_ns *ns);
include/linux/pktcdvd.h+2−10
@@ -152,14 +152,6 @@ struct packet_stacked_data · 151 unmodified lines
152152 };
153153 #define PSD_POOL_SIZE 64
154154
155−struct pktcdvd_kobj
156−{
157− struct kobject kobj;
158− struct pktcdvd_device *pd;
159−};
160−#define to_pktcdvdkobj(_k) \
161− ((struct pktcdvd_kobj*)container_of(_k,struct pktcdvd_kobj,kobj))
162−
163155 struct pktcdvd_device
164156 {
165157 struct block_device *bdev; /* dev attached */
@@ -183,6 +175,8 @@ struct pktcdvd_device · 17 unmodified lines
183175 spinlock_t lock; /* Serialize access to bio_queue */
184176 struct rb_root bio_queue; /* Work queue of bios we need to handle */
185177 int bio_queue_size; /* Number of nodes in bio_queue */
178+ bool congested; /* Someone is waiting for bio_queue_size
179+ * to drop. */
186180 sector_t current_sector; /* Keep track of where the elevator is */
187181 atomic_t scan_queue; /* Set to non-zero when pkt_handle_queue */
188182 /* needs to be run. */
@@ -195,8 +189,6 @@ struct pktcdvd_device · 6 unmodified lines
195189 int write_congestion_on;
196190
197191 struct device *dev; /* sysfs pktcdvd[0-7] dev */
198− struct pktcdvd_kobj *kobj_stat; /* sysfs pktcdvd[0-7]/stat/ */
199− struct pktcdvd_kobj *kobj_wqueue; /* sysfs pktcdvd[0-7]/write_queue/ */
200192
201193 struct dentry *dfs_d_root; /* debugfs: devname directory */
202194 struct dentry *dfs_f_info; /* debugfs: info file */
include/linux/raid/pq.h+1−1
@@ -81,7 +81,7 @@ struct raid6_calls { · 80 unmodified lines
8181 void (*xor_syndrome)(int, int, int, size_t, void **);
8282 int (*valid)(void); /* Returns 1 if this routine set is usable */
8383 const char *name; /* Name of this routine set */
84− int prefer; /* Has special performance attribute */
84+ int priority; /* Relative priority ranking if non-zero */
8585 };
8686
8787 /* Selected algorithm */
lib/raid6/algos.c+38−40
@@ -145,13 +145,13 @@ static inline const struct raid6_recov_calls *raid6_choose_recov(void) · 144 unmodified lines
145145 static inline const struct raid6_calls *raid6_choose_gen(
146146 void *(*const dptrs)[RAID6_TEST_DISKS], const int disks)
147147 {
148− unsigned long perf, bestgenperf, bestxorperf, j0, j1;
148+ unsigned long perf, bestgenperf, j0, j1;
149149 int start = (disks>>1)-1, stop = disks-3; /* work on the second half of the disks */
150150 const struct raid6_calls *const *algo;
151151 const struct raid6_calls *best;
152152
153− for (bestgenperf = 0, bestxorperf = 0, best = NULL, algo = raid6_algos; *algo; algo++) {
154− if (!best || (*algo)->prefer >= best->prefer) {
153+ for (bestgenperf = 0, best = NULL, algo = raid6_algos; *algo; algo++) {
154+ if (!best || (*algo)->priority >= best->priority) {
155155 if ((*algo)->valid && !(*algo)->valid())
156156 continue;
157157
@@ -180,50 +180,48 @@ static inline const struct raid6_calls *raid6_choose_gen( · 22 unmodified lines
180180 pr_info("raid6: %-8s gen() %5ld MB/s\n", (*algo)->name,
181181 (perf * HZ * (disks-2)) >>
182182 (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2));
183+ }
184+ }
183185
184− if (!(*algo)->xor_syndrome)
185− continue;
186+ if (!best) {
187+ pr_err("raid6: Yikes! No algorithm found!\n");
188+ goto out;
189+ }
186190
187− perf = 0;
191+ raid6_call = *best;
188192
189− preempt_disable();
190− j0 = jiffies;
191− while ((j1 = jiffies) == j0)
192− cpu_relax();
193− while (time_before(jiffies,
194− j1 + (1<<RAID6_TIME_JIFFIES_LG2))) {
195− (*algo)->xor_syndrome(disks, start, stop,
196− PAGE_SIZE, *dptrs);
197− perf++;
198− }
199− preempt_enable();
200−
201− if (best == *algo)
202− bestxorperf = perf;
193+ if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) {
194+ pr_info("raid6: skipped pq benchmark and selected %s\n",
195+ best->name);
196+ goto out;
197+ }
203198
204− pr_info("raid6: %-8s xor() %5ld MB/s\n", (*algo)->name,
205− (perf * HZ * (disks-2)) >>
206− (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1));
199+ pr_info("raid6: using algorithm %s gen() %ld MB/s\n",
200+ best->name,
201+ (bestgenperf * HZ * (disks - 2)) >>
202+ (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2));
203+
204+ if (best->xor_syndrome) {
205+ perf = 0;
206+
207+ preempt_disable();
208+ j0 = jiffies;
209+ while ((j1 = jiffies) == j0)
210+ cpu_relax();
211+ while (time_before(jiffies,
212+ j1 + (1 << RAID6_TIME_JIFFIES_LG2))) {
213+ best->xor_syndrome(disks, start, stop,
214+ PAGE_SIZE, *dptrs);
215+ perf++;
207216 }
208− }
217+ preempt_enable();
209218
210− if (best) {
211− if (IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) {
212− pr_info("raid6: using algorithm %s gen() %ld MB/s\n",
213− best->name,
214− (bestgenperf * HZ * (disks-2)) >>
215− (20 - PAGE_SHIFT+RAID6_TIME_JIFFIES_LG2));
216− if (best->xor_syndrome)
217− pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n",
218− (bestxorperf * HZ * (disks-2)) >>
219− (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1));
220− } else
221− pr_info("raid6: skip pq benchmark and using algorithm %s\n",
222− best->name);
223− raid6_call = *best;
224− } else
225− pr_err("raid6: Yikes! No algorithm found!\n");
219+ pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n",
220+ (perf * HZ * (disks - 2)) >>
221+ (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1));
222+ }
226223
224+out:
227225 return best;
228226 }
229227
lib/raid6/avx2.c+4−4
@@ -132,7 +132,7 @@ const struct raid6_calls raid6_avx2x1 = { · 131 unmodified lines
132132 raid6_avx21_xor_syndrome,
133133 raid6_have_avx2,
134134 "avx2x1",
135− 1 /* Has cache hints */
135+ .priority = 2 /* Prefer AVX2 over priority 1 (SSE2 and others) */
136136 };
137137
138138 /*
@@ -262,7 +262,7 @@ const struct raid6_calls raid6_avx2x2 = { · 123 unmodified lines
262262 raid6_avx22_xor_syndrome,
263263 raid6_have_avx2,
264264 "avx2x2",
265− 1 /* Has cache hints */
265+ .priority = 2 /* Prefer AVX2 over priority 1 (SSE2 and others) */
266266 };
267267
268268 #ifdef CONFIG_X86_64
@@ -465,6 +465,6 @@ const struct raid6_calls raid6_avx2x4 = { · 196 unmodified lines
465465 raid6_avx24_xor_syndrome,
466466 raid6_have_avx2,
467467 "avx2x4",
468− 1 /* Has cache hints */
468+ .priority = 2 /* Prefer AVX2 over priority 1 (SSE2 and others) */
469469 };
470−#endif
470+#endif /* CONFIG_X86_64 */
lib/raid6/avx512.c+3−3
@@ -162,7 +162,7 @@ const struct raid6_calls raid6_avx512x1 = { · 161 unmodified lines
162162 raid6_avx5121_xor_syndrome,
163163 raid6_have_avx512,
164164 "avx512x1",
165− 1 /* Has cache hints */
165+ .priority = 2 /* Prefer AVX512 over priority 1 (SSE2 and others) */
166166 };
167167
168168 /*
@@ -319,7 +319,7 @@ const struct raid6_calls raid6_avx512x2 = { · 150 unmodified lines
319319 raid6_avx5122_xor_syndrome,
320320 raid6_have_avx512,
321321 "avx512x2",
322− 1 /* Has cache hints */
322+ .priority = 2 /* Prefer AVX512 over priority 1 (SSE2 and others) */
323323 };
324324
325325 #ifdef CONFIG_X86_64
@@ -557,7 +557,7 @@ const struct raid6_calls raid6_avx512x4 = { · 231 unmodified lines
557557 raid6_avx5124_xor_syndrome,
558558 raid6_have_avx512,
559559 "avx512x4",
560− 1 /* Has cache hints */
560+ .priority = 2 /* Prefer AVX512 over priority 1 (SSE2 and others) */
561561 };
562562 #endif
563563

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t