lael-labs/linux

drivers/block/Kconfig+0−11

@@ -392,17 +392,6 @@ config BLK_DEV_RBD · 391 unmodified lines
392		392
393	If unsure, say N.	393	If unsure, say N.
394		394
395	config BLK_DEV_RSXX
396	tristate "IBM Flash Adapter 900GB Full Height PCIe Device Driver"
397	depends on PCI
398	select CRC32
399	help
400	Device driver for IBM's high speed PCIe SSD
401	storage device: Flash Adapter 900GB Full Height.
402
403	To compile this driver as a module, choose M here: the
404	module will be called rsxx.
405
406	source "drivers/block/rnbd/Kconfig"	395	source "drivers/block/rnbd/Kconfig"
407		396
408	endif # BLK_DEV	397	endif # BLK_DEV

drivers/block/Makefile+0−1

@@ -34,7 +34,6 @@ obj-$(CONFIG_BLK_DEV_DRBD) += drbd/ · 33 unmodified lines
34	obj-$(CONFIG_BLK_DEV_RBD) += rbd.o	34	obj-$(CONFIG_BLK_DEV_RBD) += rbd.o
35	obj-$(CONFIG_BLK_DEV_PCIESSD_MTIP32XX) += mtip32xx/	35	obj-$(CONFIG_BLK_DEV_PCIESSD_MTIP32XX) += mtip32xx/
36		36
37	obj-$(CONFIG_BLK_DEV_RSXX) += rsxx/
38	obj-$(CONFIG_ZRAM) += zram/	37	obj-$(CONFIG_ZRAM) += zram/
39	obj-$(CONFIG_BLK_DEV_RNBD) += rnbd/	38	obj-$(CONFIG_BLK_DEV_RNBD) += rnbd/
40		39

drivers/block/drbd/drbd_main.c+2−1

@@ -729,7 +729,8 @@ int drbd_send_sync_param(struct drbd_peer_device *peer_device) · 728 unmodified lines
729	cmd = apv >= 89 ? P_SYNC_PARAM89 : P_SYNC_PARAM;	729	cmd = apv >= 89 ? P_SYNC_PARAM89 : P_SYNC_PARAM;
730		730
731	/* initialize verify_alg and csums_alg */	731	/* initialize verify_alg and csums_alg */
732	memset(p->verify_alg, 0, 2 * SHARED_SECRET_MAX);	732	BUILD_BUG_ON(sizeof(p->algs) != 2 * SHARED_SECRET_MAX);
		733	memset(&p->algs, 0, sizeof(p->algs));
733		734
734	if (get_ldev(peer_device->device)) {	735	if (get_ldev(peer_device->device)) {
735	dc = rcu_dereference(peer_device->device->ldev->disk_conf);	736	dc = rcu_dereference(peer_device->device->ldev->disk_conf);

drivers/block/drbd/drbd_protocol.h+4−2

 struct p_rs_param_95 {
 	u32 resync_rate;
-	char verify_alg[SHARED_SECRET_MAX];
+	struct_group(algs,
-	char csums_alg[SHARED_SECRET_MAX];
+		char verify_alg[SHARED_SECRET_MAX];
+		char csums_alg[SHARED_SECRET_MAX];
+	);
 	u32 c_plan_ahead;
 	u32 c_delay_target;
 	u32 c_fill_target;

drivers/block/drbd/drbd_receiver.c+2−1

@@ -3921,7 +3921,8 @@ static int receive_SyncParam(struct drbd_connection *connection, struct packet_i · 3920 unmodified lines
3921		3921
3922	/* initialize verify_alg and csums_alg */	3922	/* initialize verify_alg and csums_alg */
3923	p = pi->data;	3923	p = pi->data;
3924	memset(p->verify_alg, 0, 2 * SHARED_SECRET_MAX);	3924	BUILD_BUG_ON(sizeof(p->algs) != 2 * SHARED_SECRET_MAX);
		3925	memset(&p->algs, 0, sizeof(p->algs));
3925		3926
3926	err = drbd_recv_all(peer_device->connection, p, header_size);	3927	err = drbd_recv_all(peer_device->connection, p, header_size);
3927	if (err)	3928	if (err)

drivers/block/floppy.c+4−2

 static void cancel_activity(void)
 {
 	do_floppy = NULL;
-	cancel_delayed_work_sync(&fd_timer);
+	cancel_delayed_work(&fd_timer);
 	cancel_work_sync(&floppy_work);
 }
 	}
 }
+#define MAX_LEN (1UL << MAX_ORDER << PAGE_SHIFT)
 static int raw_cmd_copyin(int cmd, void __user *param,
 				 struct floppy_raw_cmd **rcmd)
 {
 	ptr->resultcode = 0;
 	if (ptr->flags & (FD_RAW_READ | FD_RAW_WRITE)) {
-		if (ptr->length <= 0)
+		if (ptr->length <= 0 || ptr->length >= MAX_LEN)
 			return -EINVAL;
 		ptr->kernel_data = (char *)fd_dma_mem_alloc(ptr->length);
 		fallback_on_nodma_alloc(&ptr->kernel_data, ptr->length);

drivers/block/loop.c+52−56

@@ -1082,13 +1082,10 @@ out_putf: · 1081 unmodified lines
1082	return error;	1082	return error;
1083	}	1083	}
1084		1084
1085	static int __loop_clr_fd(struct loop_device *lo, bool release)	1085	static void __loop_clr_fd(struct loop_device *lo)
1086	{	1086	{
1087	struct file *filp = NULL;	1087	struct file *filp;
1088	gfp_t gfp = lo->old_gfp_mask;	1088	gfp_t gfp = lo->old_gfp_mask;
1089	int err = 0;
1090	bool partscan = false;
1091	int lo_number;
1092	struct loop_worker pos, worker;	1089	struct loop_worker pos, worker;
1093		1090
1094	/*	1091	/*
@@ -1103,17 +1100,14 @@ static int __loop_clr_fd(struct loop_device *lo, bool release) · 8 unmodified lines
1103	* became visible.	1100	* became visible.
1104	*/	1101	*/
1105		1102
		1103	/*
		1104	* Since this function is called upon "ioctl(LOOP_CLR_FD)" xor "close()
		1105	* after ioctl(LOOP_CLR_FD)", it is a sign of something going wrong if
		1106	* lo->lo_state has changed while waiting for lo->lo_mutex.
		1107	*/
1106	mutex_lock(&lo->lo_mutex);	1108	mutex_lock(&lo->lo_mutex);
1107	if (WARN_ON_ONCE(lo->lo_state != Lo_rundown)) {	1109	BUG_ON(lo->lo_state != Lo_rundown);
1108	err = -ENXIO;	1110	mutex_unlock(&lo->lo_mutex);
1109	goto out_unlock;
1110	}
1111
1112	filp = lo->lo_backing_file;
1113	if (filp == NULL) {
1114	err = -EINVAL;
1115	goto out_unlock;
1116	}
1117		1111
1118	if (test_bit(QUEUE_FLAG_WC, &lo->lo_queue->queue_flags))	1112	if (test_bit(QUEUE_FLAG_WC, &lo->lo_queue->queue_flags))
1119	blk_queue_write_cache(lo->lo_queue, false, false);	1113	blk_queue_write_cache(lo->lo_queue, false, false);
@@ -1134,6 +1128,7 @@ static int __loop_clr_fd(struct loop_device *lo, bool release) · 14 unmodified lines
1134	del_timer_sync(&lo->timer);	1128	del_timer_sync(&lo->timer);
1135		1129
1136	spin_lock_irq(&lo->lo_lock);	1130	spin_lock_irq(&lo->lo_lock);
		1131	filp = lo->lo_backing_file;
1137	lo->lo_backing_file = NULL;	1132	lo->lo_backing_file = NULL;
1138	spin_unlock_irq(&lo->lo_lock);	1133	spin_unlock_irq(&lo->lo_lock);
1139		1134
@@ -1149,60 +1144,59 @@ static int __loop_clr_fd(struct loop_device *lo, bool release) · 9 unmodified lines
1149	/* let user-space know about this change */	1144	/* let user-space know about this change */
1150	kobject_uevent(&disk_to_dev(lo->lo_disk)->kobj, KOBJ_CHANGE);	1145	kobject_uevent(&disk_to_dev(lo->lo_disk)->kobj, KOBJ_CHANGE);
1151	mapping_set_gfp_mask(filp->f_mapping, gfp);	1146	mapping_set_gfp_mask(filp->f_mapping, gfp);
1152	/* This is safe: open() is still holding a reference. */
1153	module_put(THIS_MODULE);
1154	blk_mq_unfreeze_queue(lo->lo_queue);	1147	blk_mq_unfreeze_queue(lo->lo_queue);
1155		1148
1156	partscan = lo->lo_flags & LO_FLAGS_PARTSCAN;
1157	lo_number = lo->lo_number;
1158	disk_force_media_change(lo->lo_disk, DISK_EVENT_MEDIA_CHANGE);	1149	disk_force_media_change(lo->lo_disk, DISK_EVENT_MEDIA_CHANGE);
1159	out_unlock:	1150
1160	mutex_unlock(&lo->lo_mutex);	1151	if (lo->lo_flags & LO_FLAGS_PARTSCAN) {
1161	if (partscan) {	1152	int err;
1162	/*	1153
1163	* open_mutex has been held already in release path, so don't	1154	mutex_lock(&lo->lo_disk->open_mutex);
1164	* acquire it if this function is called in such case.
1165	*
1166	* If the reread partition isn't from release path, lo_refcnt
1167	* must be at least one and it can only become zero when the
1168	* current holder is released.
1169	*/
1170	if (!release)
1171	mutex_lock(&lo->lo_disk->open_mutex);
1172	err = bdev_disk_changed(lo->lo_disk, false);	1155	err = bdev_disk_changed(lo->lo_disk, false);
1173	if (!release)	1156	mutex_unlock(&lo->lo_disk->open_mutex);
1174	mutex_unlock(&lo->lo_disk->open_mutex);
1175	if (err)	1157	if (err)
1176	pr_warn("%s: partition scan of loop%d failed (rc=%d)\n",	1158	pr_warn("%s: partition scan of loop%d failed (rc=%d)\n",
1177	__func__, lo_number, err);	1159	__func__, lo->lo_number, err);
1178	/* Device is gone, no point in returning error */	1160	/* Device is gone, no point in returning error */
1179	err = 0;
1180	}	1161	}
1181		1162
1182	/*
1183	* lo->lo_state is set to Lo_unbound here after above partscan has
1184	* finished.
1185	*
1186	* There cannot be anybody else entering __loop_clr_fd() as
1187	* lo->lo_backing_file is already cleared and Lo_rundown state
1188	* protects us from all the other places trying to change the 'lo'
1189	* device.
1190	*/
1191	mutex_lock(&lo->lo_mutex);
1192	lo->lo_flags = 0;	1163	lo->lo_flags = 0;
1193	if (!part_shift)	1164	if (!part_shift)
1194	lo->lo_disk->flags \|= GENHD_FL_NO_PART;	1165	lo->lo_disk->flags \|= GENHD_FL_NO_PART;
		1166
		1167	fput(filp);
		1168	}
		1169
		1170	static void loop_rundown_completed(struct loop_device *lo)
		1171	{
		1172	mutex_lock(&lo->lo_mutex);
1195	lo->lo_state = Lo_unbound;	1173	lo->lo_state = Lo_unbound;
1196	mutex_unlock(&lo->lo_mutex);	1174	mutex_unlock(&lo->lo_mutex);
		1175	module_put(THIS_MODULE);
		1176	}
1197		1177
1198	/*	1178	static void loop_rundown_workfn(struct work_struct *work)
1199	* Need not hold lo_mutex to fput backing file. Calling fput holding	1179	{
1200	* lo_mutex triggers a circular lock dependency possibility warning as	1180	struct loop_device *lo = container_of(work, struct loop_device,
1201	* fput can take open_mutex which is usually taken before lo_mutex.	1181	rundown_work);
1202	*/	1182	struct block_device *bdev = lo->lo_device;
1203	if (filp)	1183	struct gendisk *disk = lo->lo_disk;
1204	fput(filp);	1184
1205	return err;	1185	__loop_clr_fd(lo);
		1186	kobject_put(&bdev->bd_device.kobj);
		1187	module_put(disk->fops->owner);
		1188	loop_rundown_completed(lo);
		1189	}
		1190
		1191	static void loop_schedule_rundown(struct loop_device *lo)
		1192	{
		1193	struct block_device *bdev = lo->lo_device;
		1194	struct gendisk *disk = lo->lo_disk;
		1195
		1196	__module_get(disk->fops->owner);
		1197	kobject_get(&bdev->bd_device.kobj);
		1198	INIT_WORK(&lo->rundown_work, loop_rundown_workfn);
		1199	queue_work(system_long_wq, &lo->rundown_work);
1206	}	1200	}
1207		1201
1208	static int loop_clr_fd(struct loop_device *lo)	1202	static int loop_clr_fd(struct loop_device *lo)
@@ -1234,7 +1228,9 @@ static int loop_clr_fd(struct loop_device *lo) · 25 unmodified lines
1234	lo->lo_state = Lo_rundown;	1228	lo->lo_state = Lo_rundown;
1235	mutex_unlock(&lo->lo_mutex);	1229	mutex_unlock(&lo->lo_mutex);
1236		1230
1237	return __loop_clr_fd(lo, false);	1231	__loop_clr_fd(lo);
		1232	loop_rundown_completed(lo);
		1233	return 0;
1238	}	1234	}
1239		1235
1240	static int	1236	static int
@@ -1758,7 +1754,7 @@ static void lo_release(struct gendisk *disk, fmode_t mode) · 517 unmodified lines
1758	* In autoclear mode, stop the loop thread	1754	* In autoclear mode, stop the loop thread
1759	* and remove configuration after last close.	1755	* and remove configuration after last close.
1760	*/	1756	*/
1761	__loop_clr_fd(lo, true);	1757	loop_schedule_rundown(lo);
1762	return;	1758	return;
1763	} else if (lo->lo_state == Lo_bound) {	1759	} else if (lo->lo_state == Lo_bound) {
1764	/*	1760	/*

drivers/block/loop.h+1−0

@@ -56,6 +56,7 @@ struct loop_device { · 55 unmodified lines
56	struct gendisk *lo_disk;	56	struct gendisk *lo_disk;
57	struct mutex lo_mutex;	57	struct mutex lo_mutex;
58	bool idr_visible;	58	bool idr_visible;
		59	struct work_struct rundown_work;
59	};	60	};
60		61
61	struct loop_cmd {	62	struct loop_cmd {

drivers/block/mtip32xx/mtip32xx.c+22−64

  * return value
  *	 true if device removed, else false
  */
-static bool mtip_check_surprise_removal(struct pci_dev *pdev)
+static bool mtip_check_surprise_removal(struct driver_data *dd)
 {
 	u16 vendor_id = 0;
-	struct driver_data *dd = pci_get_drvdata(pdev);
 	if (dd->sr)
 		return true;
        /* Read the vendorID from the configuration space */
-	pci_read_config_word(pdev, 0x00, &vendor_id);
+	pci_read_config_word(dd->pdev, 0x00, &vendor_id);
 	if (vendor_id == 0xFFFF) {
 		dd->sr = true;
 		if (dd->queue)
 {
 	int rv = 0;
-	if (mtip_check_surprise_removal(dd->pdev))
+	if (mtip_check_surprise_removal(dd))
 		return 0;
 	if (mtip_hba_reset(dd) < 0)
 		dev_warn(&dd->pdev->dev,
 			"Port stat errors %x unhandled\n",
 			(port_stat & ~PORT_IRQ_HANDLED));
-		if (mtip_check_surprise_removal(dd->pdev))
+		if (mtip_check_surprise_removal(dd))
 			return;
 	}
 	if (likely(port_stat & (PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR))) {
 		/* Acknowledge the interrupt status on the port.*/
 		port_stat = readl(port->mmio + PORT_IRQ_STAT);
 		if (unlikely(port_stat == 0xFFFFFFFF)) {
-			mtip_check_surprise_removal(dd->pdev);
+			mtip_check_surprise_removal(dd);
 			return IRQ_HANDLED;
 		}
 		writel(port_stat, port->mmio + PORT_IRQ_STAT);
 		}
 		if (unlikely(port_stat & PORT_IRQ_ERR)) {
-			if (unlikely(mtip_check_surprise_removal(dd->pdev))) {
+			if (unlikely(mtip_check_surprise_removal(dd))) {
 				/* don't proceed further */
 				return IRQ_HANDLED;
 			}
 		msleep(100);
-		if (mtip_check_surprise_removal(port->dd->pdev))
+		if (mtip_check_surprise_removal(port->dd))
 			goto err_fault;
 		active = mtip_commands_active(port);
 		return -EFAULT;
 	}
-	if (mtip_check_surprise_removal(dd->pdev))
+	if (mtip_check_surprise_removal(dd))
 		return -EFAULT;
 	rq = blk_mq_alloc_request(dd->queue, REQ_OP_DRV_IN, BLK_MQ_REQ_RESERVED);
 				fis->command, int_cmd->status);
 		rv = -EIO;
-		if (mtip_check_surprise_removal(dd->pdev) ||
+		if (mtip_check_surprise_removal(dd) ||
 			test_bit(MTIP_DDF_REMOVE_PENDING_BIT,
 					&dd->dd_flag)) {
 			dev_err(&dd->pdev->dev,
 		if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT,
 				&dd->dd_flag)))
 			return -EFAULT;
-		if (mtip_check_surprise_removal(dd->pdev))
+		if (mtip_check_surprise_removal(dd))
 			return -EFAULT;
 		if (mtip_get_identify(dd->port, NULL) < 0)
 		 time_before(jiffies, timeout)) {
 		mdelay(100);
 	}
-	if (unlikely(mtip_check_surprise_removal(dd->pdev))) {
+	if (unlikely(mtip_check_surprise_removal(dd))) {
 		timetaken = jiffies - timetaken;
 		dev_warn(&dd->pdev->dev,
 			"Surprise removal detected at %u ms\n",
 	list_add(&dd->remove_list, &removing_list);
 	spin_unlock_irqrestore(&dev_lock, flags);
-	mtip_check_surprise_removal(pdev);
+	mtip_check_surprise_removal(dd);
 	synchronize_irq(dd->pdev->irq);
 	/* Spin until workers are done */
  *	0  Success
  *	<0 Error
  */
-static int mtip_pci_suspend(struct pci_dev *pdev, pm_message_t mesg)
+static int __maybe_unused mtip_pci_suspend(struct device *dev)
 {
 	int rv = 0;
-	struct driver_data *dd = pci_get_drvdata(pdev);
+	struct driver_data *dd = dev_get_drvdata(dev);
-	if (!dd) {
-		dev_err(&pdev->dev,
-			"Driver private datastructure is NULL\n");
-		return -EFAULT;
-	}
 	set_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag);
 	/* Disable ports & interrupts then send standby immediate */
 	rv = mtip_block_suspend(dd);
-	if (rv < 0) {
+	if (rv < 0)
-		dev_err(&pdev->dev,
+		dev_err(dev, "Failed to suspend controller\n");
-			"Failed to suspend controller\n");
-		return rv;
-	}
-	/*
-	 * Save the pci config space to pdev structure &
-	 * disable the device
-	 */
-	pci_save_state(pdev);
-	pci_disable_device(pdev);
-	/* Move to Low power state*/
-	pci_set_power_state(pdev, PCI_D3hot);
 	return rv;
 }
  *      0  Success
  *      <0 Error
  */
-static int mtip_pci_resume(struct pci_dev *pdev)
+static int __maybe_unused mtip_pci_resume(struct device *dev)
 {
 	int rv = 0;
-	struct driver_data *dd;
+	struct driver_data *dd = dev_get_drvdata(dev);
-	dd = pci_get_drvdata(pdev);
-	if (!dd) {
-		dev_err(&pdev->dev,
-			"Driver private datastructure is NULL\n");
-		return -EFAULT;
-	}
-	/* Move the device to active State */
-	pci_set_power_state(pdev, PCI_D0);
-	/* Restore PCI configuration space */
-	pci_restore_state(pdev);
-	/* Enable the PCI device*/
-	rv = pcim_enable_device(pdev);
-	if (rv < 0) {
-		dev_err(&pdev->dev,
-			"Failed to enable card during resume\n");
-		goto err;
-	}
-	pci_set_master(pdev);
 	/*
 	 * Calls hbaReset, initPort, & startPort function
 	 */
 	rv = mtip_block_resume(dd);
 	if (rv < 0)
-		dev_err(&pdev->dev, "Unable to resume\n");
+		dev_err(dev, "Unable to resume\n");
-err:
 	clear_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag);
 	return rv;
 	{ 0 }
 };
+static SIMPLE_DEV_PM_OPS(mtip_pci_pm_ops, mtip_pci_suspend, mtip_pci_resume);
 /* Structure that describes the PCI driver functions. */
 static struct pci_driver mtip_pci_driver = {
 	.name			= MTIP_DRV_NAME,
 	.id_table		= mtip_pci_tbl,
 	.probe			= mtip_pci_probe,
 	.remove			= mtip_pci_remove,
-	.suspend		= mtip_pci_suspend,
+	.driver.pm		= &mtip_pci_pm_ops,
-	.resume			= mtip_pci_resume,
 	.shutdown		= mtip_pci_shutdown,
 };

drivers/block/null_blk/main.c+6−6

 		return 0;
 	/*
-	 * Make sure at least one queue exists for each of submit and poll.
+	 * Make sure at least one submit queue exists.
 	 */
-	if (!submit_queues || !poll_queues)
+	if (!submit_queues)
 		return -EINVAL;
 	/*
 		cmd = blk_mq_rq_to_pdu(req);
 		cmd->error = null_process_cmd(cmd, req_op(req), blk_rq_pos(req),
 						blk_rq_sectors(req));
-		end_cmd(cmd);
+		if (!blk_mq_add_to_batch(req, iob, (__force int) cmd->error,
+					blk_mq_end_request_batch))
+			end_cmd(cmd);
 		nr++;
 	}
 	if (g_shared_tag_bitmap)
 		set->flags |= BLK_MQ_F_TAG_HCTX_SHARED;
 	set->driver_data = nullb;
-	if (g_poll_queues)
+	if (poll_queues)
 		set->nr_maps = 3;
 	else
 		set->nr_maps = 1;
 	if (dev->poll_queues > g_poll_queues)
 		dev->poll_queues = g_poll_queues;
-	else if (dev->poll_queues == 0)
-		dev->poll_queues = 1;
 	dev->prev_poll_queues = dev->poll_queues;
 	dev->queue_mode = min_t(unsigned int, dev->queue_mode, NULL_Q_MQ);

drivers/block/pktcdvd.c+154−152

@@ -113,57 +113,10 @@ static sector_t get_zone(sector_t sector, struct pktcdvd_device *pd) · 112 unmodified lines
113	return (sector + pd->offset) & ~(sector_t)(pd->settings.size - 1);	113	return (sector + pd->offset) & ~(sector_t)(pd->settings.size - 1);
114	}	114	}
115		115
116	/*
117	* create and register a pktcdvd kernel object.
118	*/
119	static struct pktcdvd_kobj* pkt_kobj_create(struct pktcdvd_device *pd,
120	const char* name,
121	struct kobject* parent,
122	struct kobj_type* ktype)
123	{
124	struct pktcdvd_kobj *p;
125	int error;
126
127	p = kzalloc(sizeof(*p), GFP_KERNEL);
128	if (!p)
129	return NULL;
130	p->pd = pd;
131	error = kobject_init_and_add(&p->kobj, ktype, parent, "%s", name);
132	if (error) {
133	kobject_put(&p->kobj);
134	return NULL;
135	}
136	kobject_uevent(&p->kobj, KOBJ_ADD);
137	return p;
138	}
139	/*
140	* remove a pktcdvd kernel object.
141	*/
142	static void pkt_kobj_remove(struct pktcdvd_kobj *p)
143	{
144	if (p)
145	kobject_put(&p->kobj);
146	}
147	/*
148	* default release function for pktcdvd kernel objects.
149	*/
150	static void pkt_kobj_release(struct kobject *kobj)
151	{
152	kfree(to_pktcdvdkobj(kobj));
153	}
154
155
156	/**********************************************************	116	/**********************************************************
157	*
158	* sysfs interface for pktcdvd	117	* sysfs interface for pktcdvd
159	* by (C) 2006 Thomas Maier <balagi@justmail.de>	118	* by (C) 2006 Thomas Maier <balagi@justmail.de>
160	*	119
161	**********************************************************/
162
163	#define DEF_ATTR(_obj,_name,_mode) \
164	static struct attribute _obj = { .name = _name, .mode = _mode }
165
166	/**********************************************************
167	/sys/class/pktcdvd/pktcdvd[0-7]/	120	/sys/class/pktcdvd/pktcdvd[0-7]/
168	stat/reset	121	stat/reset
169	stat/packets_started	122	stat/packets_started
@@ -176,75 +129,94 @@ static void pkt_kobj_release(struct kobject *kobj) · 6 unmodified lines
176	write_queue/congestion_on	129	write_queue/congestion_on
177	**********************************************************/	130	**********************************************************/
178		131
179	DEF_ATTR(kobj_pkt_attr_st1, "reset", 0200);	132	static ssize_t packets_started_show(struct device *dev,
180	DEF_ATTR(kobj_pkt_attr_st2, "packets_started", 0444);	133	struct device_attribute attr, char buf)
181	DEF_ATTR(kobj_pkt_attr_st3, "packets_finished", 0444);	134	{
182	DEF_ATTR(kobj_pkt_attr_st4, "kb_written", 0444);	135	struct pktcdvd_device *pd = dev_get_drvdata(dev);
183	DEF_ATTR(kobj_pkt_attr_st5, "kb_read", 0444);
184	DEF_ATTR(kobj_pkt_attr_st6, "kb_read_gather", 0444);
185
186	static struct attribute *kobj_pkt_attrs_stat[] = {
187	&kobj_pkt_attr_st1,
188	&kobj_pkt_attr_st2,
189	&kobj_pkt_attr_st3,
190	&kobj_pkt_attr_st4,
191	&kobj_pkt_attr_st5,
192	&kobj_pkt_attr_st6,
193	NULL
194	};
195		136
196	DEF_ATTR(kobj_pkt_attr_wq1, "size", 0444);	137	return sysfs_emit(buf, "%lu\n", pd->stats.pkt_started);
197	DEF_ATTR(kobj_pkt_attr_wq2, "congestion_off", 0644);	138	}
198	DEF_ATTR(kobj_pkt_attr_wq3, "congestion_on", 0644);	139	static DEVICE_ATTR_RO(packets_started);
199		140
200	static struct attribute *kobj_pkt_attrs_wqueue[] = {	141	static ssize_t packets_finished_show(struct device *dev,
201	&kobj_pkt_attr_wq1,	142	struct device_attribute attr, char buf)
202	&kobj_pkt_attr_wq2,	143	{
203	&kobj_pkt_attr_wq3,	144	struct pktcdvd_device *pd = dev_get_drvdata(dev);
204	NULL
205	};
206		145
207	static ssize_t kobj_pkt_show(struct kobject *kobj,	146	return sysfs_emit(buf, "%lu\n", pd->stats.pkt_ended);
208	struct attribute attr, char data)	147	}
		148	static DEVICE_ATTR_RO(packets_finished);
		149
		150	static ssize_t kb_written_show(struct device *dev,
		151	struct device_attribute attr, char buf)
209	{	152	{
210	struct pktcdvd_device *pd = to_pktcdvdkobj(kobj)->pd;	153	struct pktcdvd_device *pd = dev_get_drvdata(dev);
211	int n = 0;
212	int v;
213	if (strcmp(attr->name, "packets_started") == 0) {
214	n = sprintf(data, "%lu\n", pd->stats.pkt_started);
215		154
216	} else if (strcmp(attr->name, "packets_finished") == 0) {	155	return sysfs_emit(buf, "%lu\n", pd->stats.secs_w >> 1);
217	n = sprintf(data, "%lu\n", pd->stats.pkt_ended);	156	}
		157	static DEVICE_ATTR_RO(kb_written);
218		158
219	} else if (strcmp(attr->name, "kb_written") == 0) {	159	static ssize_t kb_read_show(struct device *dev,
220	n = sprintf(data, "%lu\n", pd->stats.secs_w >> 1);	160	struct device_attribute attr, char buf)
		161	{
		162	struct pktcdvd_device *pd = dev_get_drvdata(dev);
221		163
222	} else if (strcmp(attr->name, "kb_read") == 0) {	164	return sysfs_emit(buf, "%lu\n", pd->stats.secs_r >> 1);
223	n = sprintf(data, "%lu\n", pd->stats.secs_r >> 1);	165	}
		166	static DEVICE_ATTR_RO(kb_read);
224		167
225	} else if (strcmp(attr->name, "kb_read_gather") == 0) {	168	static ssize_t kb_read_gather_show(struct device *dev,
226	n = sprintf(data, "%lu\n", pd->stats.secs_rg >> 1);	169	struct device_attribute attr, char buf)
		170	{
		171	struct pktcdvd_device *pd = dev_get_drvdata(dev);
227		172
228	} else if (strcmp(attr->name, "size") == 0) {	173	return sysfs_emit(buf, "%lu\n", pd->stats.secs_rg >> 1);
229	spin_lock(&pd->lock);	174	}
230	v = pd->bio_queue_size;	175	static DEVICE_ATTR_RO(kb_read_gather);
231	spin_unlock(&pd->lock);
232	n = sprintf(data, "%d\n", v);
233		176
234	} else if (strcmp(attr->name, "congestion_off") == 0) {	177	static ssize_t reset_store(struct device dev, struct device_attribute attr,
235	spin_lock(&pd->lock);	178	const char *buf, size_t len)
236	v = pd->write_congestion_off;	179	{
237	spin_unlock(&pd->lock);	180	struct pktcdvd_device *pd = dev_get_drvdata(dev);
238	n = sprintf(data, "%d\n", v);
239		181
240	} else if (strcmp(attr->name, "congestion_on") == 0) {	182	if (len > 0) {
241	spin_lock(&pd->lock);	183	pd->stats.pkt_started = 0;
242	v = pd->write_congestion_on;	184	pd->stats.pkt_ended = 0;
243	spin_unlock(&pd->lock);	185	pd->stats.secs_w = 0;
244	n = sprintf(data, "%d\n", v);	186	pd->stats.secs_rg = 0;
		187	pd->stats.secs_r = 0;
245	}	188	}
		189	return len;
		190	}
		191	static DEVICE_ATTR_WO(reset);
		192
		193	static struct attribute *pkt_stat_attrs[] = {
		194	&dev_attr_packets_finished.attr,
		195	&dev_attr_packets_started.attr,
		196	&dev_attr_kb_read.attr,
		197	&dev_attr_kb_written.attr,
		198	&dev_attr_kb_read_gather.attr,
		199	&dev_attr_reset.attr,
		200	NULL,
		201	};
		202
		203	static const struct attribute_group pkt_stat_group = {
		204	.name = "stat",
		205	.attrs = pkt_stat_attrs,
		206	};
		207
		208	static ssize_t size_show(struct device *dev,
		209	struct device_attribute attr, char buf)
		210	{
		211	struct pktcdvd_device *pd = dev_get_drvdata(dev);
		212	int n;
		213
		214	spin_lock(&pd->lock);
		215	n = sysfs_emit(buf, "%d\n", pd->bio_queue_size);
		216	spin_unlock(&pd->lock);
246	return n;	217	return n;
247	}	218	}
		219	static DEVICE_ATTR_RO(size);
248		220
249	static void init_write_congestion_marks(int* lo, int* hi)	221	static void init_write_congestion_marks(int* lo, int* hi)
250	{	222	{
@@ -263,30 +235,56 @@ static void init_write_congestion_marks(int* lo, int* hi) · 12 unmodified lines
263	}	235	}
264	}	236	}
265		237
266	static ssize_t kobj_pkt_store(struct kobject *kobj,	238	static ssize_t congestion_off_show(struct device *dev,
267	struct attribute *attr,	239	struct device_attribute attr, char buf)
268	const char *data, size_t len)
269	{	240	{
270	struct pktcdvd_device *pd = to_pktcdvdkobj(kobj)->pd;	241	struct pktcdvd_device *pd = dev_get_drvdata(dev);
271	int val;	242	int n;
272		243
273	if (strcmp(attr->name, "reset") == 0 && len > 0) {	244	spin_lock(&pd->lock);
274	pd->stats.pkt_started = 0;	245	n = sysfs_emit(buf, "%d\n", pd->write_congestion_off);
275	pd->stats.pkt_ended = 0;	246	spin_unlock(&pd->lock);
276	pd->stats.secs_w = 0;	247	return n;
277	pd->stats.secs_rg = 0;	248	}
278	pd->stats.secs_r = 0;	249
		250	static ssize_t congestion_off_store(struct device *dev,
		251	struct device_attribute *attr,
		252	const char *buf, size_t len)
		253	{
		254	struct pktcdvd_device *pd = dev_get_drvdata(dev);
		255	int val;
279		256
280	} else if (strcmp(attr->name, "congestion_off") == 0	257	if (sscanf(buf, "%d", &val) == 1) {
281	&& sscanf(data, "%d", &val) == 1) {
282	spin_lock(&pd->lock);	258	spin_lock(&pd->lock);
283	pd->write_congestion_off = val;	259	pd->write_congestion_off = val;
284	init_write_congestion_marks(&pd->write_congestion_off,	260	init_write_congestion_marks(&pd->write_congestion_off,
285	&pd->write_congestion_on);	261	&pd->write_congestion_on);
286	spin_unlock(&pd->lock);	262	spin_unlock(&pd->lock);
		263	}
		264	return len;
		265	}
		266	static DEVICE_ATTR_RW(congestion_off);
		267
		268	static ssize_t congestion_on_show(struct device *dev,
		269	struct device_attribute attr, char buf)
		270	{
		271	struct pktcdvd_device *pd = dev_get_drvdata(dev);
		272	int n;
		273
		274	spin_lock(&pd->lock);
		275	n = sysfs_emit(buf, "%d\n", pd->write_congestion_on);
		276	spin_unlock(&pd->lock);
		277	return n;
		278	}
		279
		280	static ssize_t congestion_on_store(struct device *dev,
		281	struct device_attribute *attr,
		282	const char *buf, size_t len)
		283	{
		284	struct pktcdvd_device *pd = dev_get_drvdata(dev);
		285	int val;
287		286
288	} else if (strcmp(attr->name, "congestion_on") == 0	287	if (sscanf(buf, "%d", &val) == 1) {
289	&& sscanf(data, "%d", &val) == 1) {
290	spin_lock(&pd->lock);	288	spin_lock(&pd->lock);
291	pd->write_congestion_on = val;	289	pd->write_congestion_on = val;
292	init_write_congestion_marks(&pd->write_congestion_off,	290	init_write_congestion_marks(&pd->write_congestion_off,
@@ -295,44 +293,39 @@ static ssize_t kobj_pkt_store(struct kobject *kobj, · 2 unmodified lines
295	}	293	}
296	return len;	294	return len;
297	}	295	}
		296	static DEVICE_ATTR_RW(congestion_on);
298		297
299	static const struct sysfs_ops kobj_pkt_ops = {	298	static struct attribute *pkt_wq_attrs[] = {
300	.show = kobj_pkt_show,	299	&dev_attr_congestion_on.attr,
301	.store = kobj_pkt_store	300	&dev_attr_congestion_off.attr,
		301	&dev_attr_size.attr,
		302	NULL,
302	};	303	};
303	static struct kobj_type kobj_pkt_type_stat = {	304
304	.release = pkt_kobj_release,	305	static const struct attribute_group pkt_wq_group = {
305	.sysfs_ops = &kobj_pkt_ops,	306	.name = "write_queue",
306	.default_attrs = kobj_pkt_attrs_stat	307	.attrs = pkt_wq_attrs,
307	};	308	};
308	static struct kobj_type kobj_pkt_type_wqueue = {	309
309	.release = pkt_kobj_release,	310	static const struct attribute_group *pkt_groups[] = {
310	.sysfs_ops = &kobj_pkt_ops,	311	&pkt_stat_group,
311	.default_attrs = kobj_pkt_attrs_wqueue	312	&pkt_wq_group,
		313	NULL,
312	};	314	};
313		315
314	static void pkt_sysfs_dev_new(struct pktcdvd_device *pd)	316	static void pkt_sysfs_dev_new(struct pktcdvd_device *pd)
315	{	317	{
316	if (class_pktcdvd) {	318	if (class_pktcdvd) {
317	pd->dev = device_create(class_pktcdvd, NULL, MKDEV(0, 0), NULL,	319	pd->dev = device_create_with_groups(class_pktcdvd, NULL,
318	"%s", pd->name);	320	MKDEV(0, 0), pd, pkt_groups,
		321	"%s", pd->name);
319	if (IS_ERR(pd->dev))	322	if (IS_ERR(pd->dev))
320	pd->dev = NULL;	323	pd->dev = NULL;
321	}	324	}
322	if (pd->dev) {
323	pd->kobj_stat = pkt_kobj_create(pd, "stat",
324	&pd->dev->kobj,
325	&kobj_pkt_type_stat);
326	pd->kobj_wqueue = pkt_kobj_create(pd, "write_queue",
327	&pd->dev->kobj,
328	&kobj_pkt_type_wqueue);
329	}
330	}	325	}
331		326
332	static void pkt_sysfs_dev_remove(struct pktcdvd_device *pd)	327	static void pkt_sysfs_dev_remove(struct pktcdvd_device *pd)
333	{	328	{
334	pkt_kobj_remove(pd->kobj_stat);
335	pkt_kobj_remove(pd->kobj_wqueue);
336	if (class_pktcdvd)	329	if (class_pktcdvd)
337	device_unregister(pd->dev);	330	device_unregister(pd->dev);
338	}	331	}
@@ -1107,7 +1100,6 @@ static int pkt_handle_queue(struct pktcdvd_device *pd) · 768 unmodified lines
1107	sector_t zone = 0; /* Suppress gcc warning */	1100	sector_t zone = 0; /* Suppress gcc warning */
1108	struct pkt_rb_node node, first_node;	1101	struct pkt_rb_node node, first_node;
1109	struct rb_node *n;	1102	struct rb_node *n;
1110	int wakeup;
1111		1103
1112	atomic_set(&pd->scan_queue, 0);	1104	atomic_set(&pd->scan_queue, 0);
1113		1105
@@ -1179,12 +1171,14 @@ try_next_bio: · 65 unmodified lines
1179	spin_unlock(&pkt->lock);	1171	spin_unlock(&pkt->lock);
1180	}	1172	}
1181	/* check write congestion marks, and if bio_queue_size is	1173	/* check write congestion marks, and if bio_queue_size is
1182	below, wake up any waiters */	1174	* below, wake up any waiters
1183	wakeup = (pd->write_congestion_on > 0	1175	*/
1184	&& pd->bio_queue_size <= pd->write_congestion_off);	1176	if (pd->congested &&
		1177	pd->bio_queue_size <= pd->write_congestion_off) {
		1178	pd->congested = false;
		1179	wake_up_var(&pd->congested);
		1180	}
1185	spin_unlock(&pd->lock);	1181	spin_unlock(&pd->lock);
1186	if (wakeup)
1187	clear_bdi_congested(pd->disk->bdi, BLK_RW_ASYNC);
1188		1182
1189	pkt->sleep_time = max(PACKET_WAIT_TIME, 1);	1183	pkt->sleep_time = max(PACKET_WAIT_TIME, 1);
1190	pkt_set_state(pkt, PACKET_WAITING_STATE);	1184	pkt_set_state(pkt, PACKET_WAITING_STATE);
@@ -2356,7 +2350,7 @@ static void pkt_make_request_write(struct request_queue q, struct bio bio) · 1165 unmodified lines
2356	}	2350	}
2357	spin_unlock(&pd->cdrw.active_list_lock);	2351	spin_unlock(&pd->cdrw.active_list_lock);
2358		2352
2359	/*	2353	/*
2360	* Test if there is enough room left in the bio work queue	2354	* Test if there is enough room left in the bio work queue
2361	* (queue size >= congestion on mark).	2355	* (queue size >= congestion on mark).
2362	* If not, wait till the work queue size is below the congestion off mark.	2356	* If not, wait till the work queue size is below the congestion off mark.
@@ -2364,12 +2358,20 @@ static void pkt_make_request_write(struct request_queue q, struct bio bio) · 1 unmodified line
2364	spin_lock(&pd->lock);	2358	spin_lock(&pd->lock);
2365	if (pd->write_congestion_on > 0	2359	if (pd->write_congestion_on > 0
2366	&& pd->bio_queue_size >= pd->write_congestion_on) {	2360	&& pd->bio_queue_size >= pd->write_congestion_on) {
2367	set_bdi_congested(bio->bi_bdev->bd_disk->bdi, BLK_RW_ASYNC);	2361	struct wait_bit_queue_entry wqe;
2368	do {	2362
		2363	init_wait_var_entry(&wqe, &pd->congested, 0);
		2364	for (;;) {
		2365	prepare_to_wait_event(__var_waitqueue(&pd->congested),
		2366	&wqe.wq_entry,
		2367	TASK_UNINTERRUPTIBLE);
		2368	if (pd->bio_queue_size <= pd->write_congestion_off)
		2369	break;
		2370	pd->congested = true;
2369	spin_unlock(&pd->lock);	2371	spin_unlock(&pd->lock);
2370	congestion_wait(BLK_RW_ASYNC, HZ);	2372	schedule();
2371	spin_lock(&pd->lock);	2373	spin_lock(&pd->lock);
2372	} while(pd->bio_queue_size > pd->write_congestion_off);	2374	}
2373	}	2375	}
2374	spin_unlock(&pd->lock);	2376	spin_unlock(&pd->lock);
2375		2377

drivers/block/rnbd/rnbd-clt-sysfs.c+2−1

 	&rnbd_clt_nr_poll_queues.attr,
 	NULL,
 };
+ATTRIBUTE_GROUPS(rnbd_dev);
 void rnbd_clt_remove_dev_symlink(struct rnbd_clt_dev *dev)
 {
 static struct kobj_type rnbd_dev_ktype = {
 	.sysfs_ops      = &kobj_sysfs_ops,
-	.default_attrs  = rnbd_dev_attrs,
+	.default_groups = rnbd_dev_groups,
 };
 static int rnbd_clt_add_dev_kobj(struct rnbd_clt_dev *dev)

drivers/block/rsxx/Makefile+0−3

@@ -1,3 +0,0 @@
1	# SPDX-License-Identifier: GPL-2.0-only
2	obj-$(CONFIG_BLK_DEV_RSXX) += rsxx.o
3	rsxx-objs := config.o core.o cregs.o dev.o dma.o

drivers/block/rsxx/config.c+0−197

@@ -1,197 +0,0 @@
1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* Filename: config.c
4	*
5	* Authors: Joshua Morris <josh.h.morris@us.ibm.com>
6	* Philip Kelleher <pjk1939@linux.vnet.ibm.com>
7	*
8	* (C) Copyright 2013 IBM Corporation
9	*/
10
11	#include <linux/types.h>
12	#include <linux/crc32.h>
13	#include <linux/swab.h>
14
15	#include "rsxx_priv.h"
16	#include "rsxx_cfg.h"
17
18	static void initialize_config(struct rsxx_card_cfg *cfg)
19	{
20	cfg->hdr.version = RSXX_CFG_VERSION;
21
22	cfg->data.block_size = RSXX_HW_BLK_SIZE;
23	cfg->data.stripe_size = RSXX_HW_BLK_SIZE;
24	cfg->data.vendor_id = RSXX_VENDOR_ID_IBM;
25	cfg->data.cache_order = (-1);
26	cfg->data.intr_coal.mode = RSXX_INTR_COAL_DISABLED;
27	cfg->data.intr_coal.count = 0;
28	cfg->data.intr_coal.latency = 0;
29	}
30
31	static u32 config_data_crc32(struct rsxx_card_cfg *cfg)
32	{
33	/*
34	* Return the compliment of the CRC to ensure compatibility
35	* (i.e. this is how early rsxx drivers did it.)
36	*/
37
38	return ~crc32(~0, &cfg->data, sizeof(cfg->data));
39	}
40
41
42	/----------------- Config Byte Swap Functions -------------------/
43	static void config_hdr_be_to_cpu(struct card_cfg_hdr *hdr)
44	{
45	hdr->version = be32_to_cpu((__force __be32) hdr->version);
46	hdr->crc = be32_to_cpu((__force __be32) hdr->crc);
47	}
48
49	static void config_hdr_cpu_to_be(struct card_cfg_hdr *hdr)
50	{
51	hdr->version = (__force u32) cpu_to_be32(hdr->version);
52	hdr->crc = (__force u32) cpu_to_be32(hdr->crc);
53	}
54
55	static void config_data_swab(struct rsxx_card_cfg *cfg)
56	{
57	u32 data = (u32 ) &cfg->data;
58	int i;
59
60	for (i = 0; i < (sizeof(cfg->data) / 4); i++)
61	data[i] = swab32(data[i]);
62	}
63
64	static void config_data_le_to_cpu(struct rsxx_card_cfg *cfg)
65	{
66	u32 data = (u32 ) &cfg->data;
67	int i;
68
69	for (i = 0; i < (sizeof(cfg->data) / 4); i++)
70	data[i] = le32_to_cpu((__force __le32) data[i]);
71	}
72
73	static void config_data_cpu_to_le(struct rsxx_card_cfg *cfg)
74	{
75	u32 data = (u32 ) &cfg->data;
76	int i;
77
78	for (i = 0; i < (sizeof(cfg->data) / 4); i++)
79	data[i] = (__force u32) cpu_to_le32(data[i]);
80	}
81
82
83	/----------------- Config Operations ------------------/
84	static int rsxx_save_config(struct rsxx_cardinfo *card)
85	{
86	struct rsxx_card_cfg cfg;
87	int st;
88
89	memcpy(&cfg, &card->config, sizeof(cfg));
90
91	if (unlikely(cfg.hdr.version != RSXX_CFG_VERSION)) {
92	dev_err(CARD_TO_DEV(card),
93	"Cannot save config with invalid version %d\n",
94	cfg.hdr.version);
95	return -EINVAL;
96	}
97
98	/* Convert data to little endian for the CRC calculation. */
99	config_data_cpu_to_le(&cfg);
100
101	cfg.hdr.crc = config_data_crc32(&cfg);
102
103	/*
104	* Swap the data from little endian to big endian so it can be
105	* stored.
106	*/
107	config_data_swab(&cfg);
108	config_hdr_cpu_to_be(&cfg.hdr);
109
110	st = rsxx_creg_write(card, CREG_ADD_CONFIG, sizeof(cfg), &cfg, 1);
111	if (st)
112	return st;
113
114	return 0;
115	}
116
117	int rsxx_load_config(struct rsxx_cardinfo *card)
118	{
119	int st;
120	u32 crc;
121
122	st = rsxx_creg_read(card, CREG_ADD_CONFIG, sizeof(card->config),
123	&card->config, 1);
124	if (st) {
125	dev_err(CARD_TO_DEV(card),
126	"Failed reading card config.\n");
127	return st;
128	}
129
130	config_hdr_be_to_cpu(&card->config.hdr);
131
132	if (card->config.hdr.version == RSXX_CFG_VERSION) {
133	/*
134	* We calculate the CRC with the data in little endian, because
135	* early drivers did not take big endian CPUs into account.
136	* The data is always stored in big endian, so we need to byte
137	* swap it before calculating the CRC.
138	*/
139
140	config_data_swab(&card->config);
141
142	/* Check the CRC */
143	crc = config_data_crc32(&card->config);
144	if (crc != card->config.hdr.crc) {
145	dev_err(CARD_TO_DEV(card),
146	"Config corruption detected!\n");
147	dev_info(CARD_TO_DEV(card),
148	"CRC (sb x%08x is x%08x)\n",
149	card->config.hdr.crc, crc);
150	return -EIO;
151	}
152
153	/* Convert the data to CPU byteorder */
154	config_data_le_to_cpu(&card->config);
155
156	} else if (card->config.hdr.version != 0) {
157	dev_err(CARD_TO_DEV(card),
158	"Invalid config version %d.\n",
159	card->config.hdr.version);
160	/*
161	* Config version changes require special handling from the
162	* user
163	*/
164	return -EINVAL;
165	} else {
166	dev_info(CARD_TO_DEV(card),
167	"Initializing card configuration.\n");
168	initialize_config(&card->config);
169	st = rsxx_save_config(card);
170	if (st)
171	return st;
172	}
173
174	card->config_valid = 1;
175
176	dev_dbg(CARD_TO_DEV(card), "version: x%08x\n",
177	card->config.hdr.version);
178	dev_dbg(CARD_TO_DEV(card), "crc: x%08x\n",
179	card->config.hdr.crc);
180	dev_dbg(CARD_TO_DEV(card), "block_size: x%08x\n",
181	card->config.data.block_size);
182	dev_dbg(CARD_TO_DEV(card), "stripe_size: x%08x\n",
183	card->config.data.stripe_size);
184	dev_dbg(CARD_TO_DEV(card), "vendor_id: x%08x\n",
185	card->config.data.vendor_id);
186	dev_dbg(CARD_TO_DEV(card), "cache_order: x%08x\n",
187	card->config.data.cache_order);
188	dev_dbg(CARD_TO_DEV(card), "mode: x%08x\n",
189	card->config.data.intr_coal.mode);
190	dev_dbg(CARD_TO_DEV(card), "count: x%08x\n",
191	card->config.data.intr_coal.count);
192	dev_dbg(CARD_TO_DEV(card), "latency: x%08x\n",
193	card->config.data.intr_coal.latency);
194
195	return 0;
196	}
197

drivers/block/rsxx/core.c+0−1126

@@ -1,1126 +0,0 @@
1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* Filename: core.c
4	*
5	* Authors: Joshua Morris <josh.h.morris@us.ibm.com>
6	* Philip Kelleher <pjk1939@linux.vnet.ibm.com>
7	*
8	* (C) Copyright 2013 IBM Corporation
9	*/
10
11	#include <linux/kernel.h>
12	#include <linux/init.h>
13	#include <linux/interrupt.h>
14	#include <linux/module.h>
15	#include <linux/pci.h>
16	#include <linux/reboot.h>
17	#include <linux/slab.h>
18	#include <linux/bitops.h>
19	#include <linux/delay.h>
20	#include <linux/debugfs.h>
21	#include <linux/seq_file.h>
22
23	#include <linux/genhd.h>
24	#include <linux/idr.h>
25
26	#include "rsxx_priv.h"
27	#include "rsxx_cfg.h"
28
29	#define NO_LEGACY 0
30	#define SYNC_START_TIMEOUT (10 * 60) /* 10 minutes */
31
32	MODULE_DESCRIPTION("IBM Flash Adapter 900GB Full Height Device Driver");
33	MODULE_AUTHOR("Joshua Morris/Philip Kelleher, IBM");
34	MODULE_LICENSE("GPL");
35	MODULE_VERSION(DRIVER_VERSION);
36
37	static unsigned int force_legacy = NO_LEGACY;
38	module_param(force_legacy, uint, 0444);
39	MODULE_PARM_DESC(force_legacy, "Force the use of legacy type PCI interrupts");
40
41	static unsigned int sync_start = 1;
42	module_param(sync_start, uint, 0444);
43	MODULE_PARM_DESC(sync_start, "On by Default: Driver load will not complete "
44	"until the card startup has completed.");
45
46	static DEFINE_IDA(rsxx_disk_ida);
47
48	/* --------------------Debugfs Setup ------------------- */
49
50	static int rsxx_attr_pci_regs_show(struct seq_file m, void p)
51	{
52	struct rsxx_cardinfo *card = m->private;
53
54	seq_printf(m, "HWID 0x%08x\n",
55	ioread32(card->regmap + HWID));
56	seq_printf(m, "SCRATCH 0x%08x\n",
57	ioread32(card->regmap + SCRATCH));
58	seq_printf(m, "IER 0x%08x\n",
59	ioread32(card->regmap + IER));
60	seq_printf(m, "IPR 0x%08x\n",
61	ioread32(card->regmap + IPR));
62	seq_printf(m, "CREG_CMD 0x%08x\n",
63	ioread32(card->regmap + CREG_CMD));
64	seq_printf(m, "CREG_ADD 0x%08x\n",
65	ioread32(card->regmap + CREG_ADD));
66	seq_printf(m, "CREG_CNT 0x%08x\n",
67	ioread32(card->regmap + CREG_CNT));
68	seq_printf(m, "CREG_STAT 0x%08x\n",
69	ioread32(card->regmap + CREG_STAT));
70	seq_printf(m, "CREG_DATA0 0x%08x\n",
71	ioread32(card->regmap + CREG_DATA0));
72	seq_printf(m, "CREG_DATA1 0x%08x\n",
73	ioread32(card->regmap + CREG_DATA1));
74	seq_printf(m, "CREG_DATA2 0x%08x\n",
75	ioread32(card->regmap + CREG_DATA2));
76	seq_printf(m, "CREG_DATA3 0x%08x\n",
77	ioread32(card->regmap + CREG_DATA3));
78	seq_printf(m, "CREG_DATA4 0x%08x\n",
79	ioread32(card->regmap + CREG_DATA4));
80	seq_printf(m, "CREG_DATA5 0x%08x\n",
81	ioread32(card->regmap + CREG_DATA5));
82	seq_printf(m, "CREG_DATA6 0x%08x\n",
83	ioread32(card->regmap + CREG_DATA6));
84	seq_printf(m, "CREG_DATA7 0x%08x\n",
85	ioread32(card->regmap + CREG_DATA7));
86	seq_printf(m, "INTR_COAL 0x%08x\n",
87	ioread32(card->regmap + INTR_COAL));
88	seq_printf(m, "HW_ERROR 0x%08x\n",
89	ioread32(card->regmap + HW_ERROR));
90	seq_printf(m, "DEBUG0 0x%08x\n",
91	ioread32(card->regmap + PCI_DEBUG0));
92	seq_printf(m, "DEBUG1 0x%08x\n",
93	ioread32(card->regmap + PCI_DEBUG1));
94	seq_printf(m, "DEBUG2 0x%08x\n",
95	ioread32(card->regmap + PCI_DEBUG2));
96	seq_printf(m, "DEBUG3 0x%08x\n",
97	ioread32(card->regmap + PCI_DEBUG3));
98	seq_printf(m, "DEBUG4 0x%08x\n",
99	ioread32(card->regmap + PCI_DEBUG4));
100	seq_printf(m, "DEBUG5 0x%08x\n",
101	ioread32(card->regmap + PCI_DEBUG5));
102	seq_printf(m, "DEBUG6 0x%08x\n",
103	ioread32(card->regmap + PCI_DEBUG6));
104	seq_printf(m, "DEBUG7 0x%08x\n",
105	ioread32(card->regmap + PCI_DEBUG7));
106	seq_printf(m, "RECONFIG 0x%08x\n",
107	ioread32(card->regmap + PCI_RECONFIG));
108
109	return 0;
110	}
111
112	static int rsxx_attr_stats_show(struct seq_file m, void p)
113	{
114	struct rsxx_cardinfo *card = m->private;
115	int i;
116
117	for (i = 0; i < card->n_targets; i++) {
118	seq_printf(m, "Ctrl %d CRC Errors = %d\n",
119	i, card->ctrl[i].stats.crc_errors);
120	seq_printf(m, "Ctrl %d Hard Errors = %d\n",
121	i, card->ctrl[i].stats.hard_errors);
122	seq_printf(m, "Ctrl %d Soft Errors = %d\n",
123	i, card->ctrl[i].stats.soft_errors);
124	seq_printf(m, "Ctrl %d Writes Issued = %d\n",
125	i, card->ctrl[i].stats.writes_issued);
126	seq_printf(m, "Ctrl %d Writes Failed = %d\n",
127	i, card->ctrl[i].stats.writes_failed);
128	seq_printf(m, "Ctrl %d Reads Issued = %d\n",
129	i, card->ctrl[i].stats.reads_issued);
130	seq_printf(m, "Ctrl %d Reads Failed = %d\n",
131	i, card->ctrl[i].stats.reads_failed);
132	seq_printf(m, "Ctrl %d Reads Retried = %d\n",
133	i, card->ctrl[i].stats.reads_retried);
134	seq_printf(m, "Ctrl %d Discards Issued = %d\n",
135	i, card->ctrl[i].stats.discards_issued);
136	seq_printf(m, "Ctrl %d Discards Failed = %d\n",
137	i, card->ctrl[i].stats.discards_failed);
138	seq_printf(m, "Ctrl %d DMA SW Errors = %d\n",
139	i, card->ctrl[i].stats.dma_sw_err);
140	seq_printf(m, "Ctrl %d DMA HW Faults = %d\n",
141	i, card->ctrl[i].stats.dma_hw_fault);
142	seq_printf(m, "Ctrl %d DMAs Cancelled = %d\n",
143	i, card->ctrl[i].stats.dma_cancelled);
144	seq_printf(m, "Ctrl %d SW Queue Depth = %d\n",
145	i, card->ctrl[i].stats.sw_q_depth);
146	seq_printf(m, "Ctrl %d HW Queue Depth = %d\n",
147	i, atomic_read(&card->ctrl[i].stats.hw_q_depth));
148	}
149
150	return 0;
151	}
152
153	static int rsxx_attr_stats_open(struct inode inode, struct file file)
154	{
155	return single_open(file, rsxx_attr_stats_show, inode->i_private);
156	}
157
158	static int rsxx_attr_pci_regs_open(struct inode inode, struct file file)
159	{
160	return single_open(file, rsxx_attr_pci_regs_show, inode->i_private);
161	}
162
163	static ssize_t rsxx_cram_read(struct file fp, char __user ubuf,
164	size_t cnt, loff_t *ppos)
165	{
166	struct rsxx_cardinfo *card = file_inode(fp)->i_private;
167	char *buf;
168	int st;
169
170	buf = kzalloc(cnt, GFP_KERNEL);
171	if (!buf)
172	return -ENOMEM;
173
174	st = rsxx_creg_read(card, CREG_ADD_CRAM + (u32)*ppos, cnt, buf, 1);
175	if (!st) {
176	if (copy_to_user(ubuf, buf, cnt))
177	st = -EFAULT;
178	}
179	kfree(buf);
180	if (st)
181	return st;
182	*ppos += cnt;
183	return cnt;
184	}
185
186	static ssize_t rsxx_cram_write(struct file fp, const char __user ubuf,
187	size_t cnt, loff_t *ppos)
188	{
189	struct rsxx_cardinfo *card = file_inode(fp)->i_private;
190	char *buf;
191	ssize_t st;
192
193	buf = memdup_user(ubuf, cnt);
194	if (IS_ERR(buf))
195	return PTR_ERR(buf);
196
197	st = rsxx_creg_write(card, CREG_ADD_CRAM + (u32)*ppos, cnt, buf, 1);
198	kfree(buf);
199	if (st)
200	return st;
201	*ppos += cnt;
202	return cnt;
203	}
204
205	static const struct file_operations debugfs_cram_fops = {
206	.owner = THIS_MODULE,
207	.read = rsxx_cram_read,
208	.write = rsxx_cram_write,
209	};
210
211	static const struct file_operations debugfs_stats_fops = {
212	.owner = THIS_MODULE,
213	.open = rsxx_attr_stats_open,
214	.read = seq_read,
215	.llseek = seq_lseek,
216	.release = single_release,
217	};
218
219	static const struct file_operations debugfs_pci_regs_fops = {
220	.owner = THIS_MODULE,
221	.open = rsxx_attr_pci_regs_open,
222	.read = seq_read,
223	.llseek = seq_lseek,
224	.release = single_release,
225	};
226
227	static void rsxx_debugfs_dev_new(struct rsxx_cardinfo *card)
228	{
229	struct dentry *debugfs_stats;
230	struct dentry *debugfs_pci_regs;
231	struct dentry *debugfs_cram;
232
233	card->debugfs_dir = debugfs_create_dir(card->gendisk->disk_name, NULL);
234	if (IS_ERR_OR_NULL(card->debugfs_dir))
235	goto failed_debugfs_dir;
236
237	debugfs_stats = debugfs_create_file("stats", 0444,
238	card->debugfs_dir, card,
239	&debugfs_stats_fops);
240	if (IS_ERR_OR_NULL(debugfs_stats))
241	goto failed_debugfs_stats;
242
243	debugfs_pci_regs = debugfs_create_file("pci_regs", 0444,
244	card->debugfs_dir, card,
245	&debugfs_pci_regs_fops);
246	if (IS_ERR_OR_NULL(debugfs_pci_regs))
247	goto failed_debugfs_pci_regs;
248
249	debugfs_cram = debugfs_create_file("cram", 0644,
250	card->debugfs_dir, card,
251	&debugfs_cram_fops);
252	if (IS_ERR_OR_NULL(debugfs_cram))
253	goto failed_debugfs_cram;
254
255	return;
256	failed_debugfs_cram:
257	debugfs_remove(debugfs_pci_regs);
258	failed_debugfs_pci_regs:
259	debugfs_remove(debugfs_stats);
260	failed_debugfs_stats:
261	debugfs_remove(card->debugfs_dir);
262	failed_debugfs_dir:
263	card->debugfs_dir = NULL;
264	}
265
266	/----------------- Interrupt Control & Handling -------------------/
267
268	static void rsxx_mask_interrupts(struct rsxx_cardinfo *card)
269	{
270	card->isr_mask = 0;
271	card->ier_mask = 0;
272	}
273
274	static void __enable_intr(unsigned int *mask, unsigned int intr)
275	{
276	*mask \|= intr;
277	}
278
279	static void __disable_intr(unsigned int *mask, unsigned int intr)
280	{
281	*mask &= ~intr;
282	}
283
284	/*
285	* NOTE: Disabling the IER will disable the hardware interrupt.
286	* Disabling the ISR will disable the software handling of the ISR bit.
287	*
288	* Enable/Disable interrupt functions assume the card->irq_lock
289	* is held by the caller.
290	*/
291	void rsxx_enable_ier(struct rsxx_cardinfo *card, unsigned int intr)
292	{
293	if (unlikely(card->halt) \|\|
294	unlikely(card->eeh_state))
295	return;
296
297	__enable_intr(&card->ier_mask, intr);
298	iowrite32(card->ier_mask, card->regmap + IER);
299	}
300
301	void rsxx_disable_ier(struct rsxx_cardinfo *card, unsigned int intr)
302	{
303	if (unlikely(card->eeh_state))
304	return;
305
306	__disable_intr(&card->ier_mask, intr);
307	iowrite32(card->ier_mask, card->regmap + IER);
308	}
309
310	void rsxx_enable_ier_and_isr(struct rsxx_cardinfo *card,
311	unsigned int intr)
312	{
313	if (unlikely(card->halt) \|\|
314	unlikely(card->eeh_state))
315	return;
316
317	__enable_intr(&card->isr_mask, intr);
318	__enable_intr(&card->ier_mask, intr);
319	iowrite32(card->ier_mask, card->regmap + IER);
320	}
321	void rsxx_disable_ier_and_isr(struct rsxx_cardinfo *card,
322	unsigned int intr)
323	{
324	if (unlikely(card->eeh_state))
325	return;
326
327	__disable_intr(&card->isr_mask, intr);
328	__disable_intr(&card->ier_mask, intr);
329	iowrite32(card->ier_mask, card->regmap + IER);
330	}
331
332	static irqreturn_t rsxx_isr(int irq, void *pdata)
333	{
334	struct rsxx_cardinfo *card = pdata;
335	unsigned int isr;
336	int handled = 0;
337	int reread_isr;
338	int i;
339
340	spin_lock(&card->irq_lock);
341
342	do {
343	reread_isr = 0;
344
345	if (unlikely(card->eeh_state))
346	break;
347
348	isr = ioread32(card->regmap + ISR);
349	if (isr == 0xffffffff) {
350	/*
351	* A few systems seem to have an intermittent issue
352	* where PCI reads return all Fs, but retrying the read
353	* a little later will return as expected.
354	*/
355	dev_info(CARD_TO_DEV(card),
356	"ISR = 0xFFFFFFFF, retrying later\n");
357	break;
358	}
359
360	isr &= card->isr_mask;
361	if (!isr)
362	break;
363
364	for (i = 0; i < card->n_targets; i++) {
365	if (isr & CR_INTR_DMA(i)) {
366	if (card->ier_mask & CR_INTR_DMA(i)) {
367	rsxx_disable_ier(card, CR_INTR_DMA(i));
368	reread_isr = 1;
369	}
370	queue_work(card->ctrl[i].done_wq,
371	&card->ctrl[i].dma_done_work);
372	handled++;
373	}
374	}
375
376	if (isr & CR_INTR_CREG) {
377	queue_work(card->creg_ctrl.creg_wq,
378	&card->creg_ctrl.done_work);
379	handled++;
380	}
381
382	if (isr & CR_INTR_EVENT) {
383	queue_work(card->event_wq, &card->event_work);
384	rsxx_disable_ier_and_isr(card, CR_INTR_EVENT);
385	handled++;
386	}
387	} while (reread_isr);
388
389	spin_unlock(&card->irq_lock);
390
391	return handled ? IRQ_HANDLED : IRQ_NONE;
392	}
393
394	/----------------- Card Event Handler -------------------/
395	static const char *rsxx_card_state_to_str(unsigned int state)
396	{
397	static const char * const state_strings[] = {
398	"Unknown", "Shutdown", "Starting", "Formatting",
399	"Uninitialized", "Good", "Shutting Down",
400	"Fault", "Read Only Fault", "dStroying"
401	};
402
403	return state_strings[ffs(state)];
404	}
405
406	static void card_state_change(struct rsxx_cardinfo *card,
407	unsigned int new_state)
408	{
409	int st;
410
411	dev_info(CARD_TO_DEV(card),
412	"card state change detected.(%s -> %s)\n",
413	rsxx_card_state_to_str(card->state),
414	rsxx_card_state_to_str(new_state));
415
416	card->state = new_state;
417
418	/* Don't attach DMA interfaces if the card has an invalid config */
419	if (!card->config_valid)
420	return;
421
422	switch (new_state) {
423	case CARD_STATE_RD_ONLY_FAULT:
424	dev_crit(CARD_TO_DEV(card),
425	"Hardware has entered read-only mode!\n");
426	/*
427	* Fall through so the DMA devices can be attached and
428	* the user can attempt to pull off their data.
429	*/
430	fallthrough;
431	case CARD_STATE_GOOD:
432	st = rsxx_get_card_size8(card, &card->size8);
433	if (st)
434	dev_err(CARD_TO_DEV(card),
435	"Failed attaching DMA devices\n");
436
437	if (card->config_valid)
438	set_capacity(card->gendisk, card->size8 >> 9);
439	break;
440
441	case CARD_STATE_FAULT:
442	dev_crit(CARD_TO_DEV(card),
443	"Hardware Fault reported!\n");
444	fallthrough;
445
446	/* Everything else, detach DMA interface if it's attached. */
447	case CARD_STATE_SHUTDOWN:
448	case CARD_STATE_STARTING:
449	case CARD_STATE_FORMATTING:
450	case CARD_STATE_UNINITIALIZED:
451	case CARD_STATE_SHUTTING_DOWN:
452	/*
453	* dStroy is a term coined by marketing to represent the low level
454	* secure erase.
455	*/
456	case CARD_STATE_DSTROYING:
457	set_capacity(card->gendisk, 0);
458	break;
459	}
460	}
461
462	static void card_event_handler(struct work_struct *work)
463	{
464	struct rsxx_cardinfo *card;
465	unsigned int state;
466	unsigned long flags;
467	int st;
468
469	card = container_of(work, struct rsxx_cardinfo, event_work);
470
471	if (unlikely(card->halt))
472	return;
473
474	/*
475	* Enable the interrupt now to avoid any weird race conditions where a
476	* state change might occur while rsxx_get_card_state() is
477	* processing a returned creg cmd.
478	*/
479	spin_lock_irqsave(&card->irq_lock, flags);
480	rsxx_enable_ier_and_isr(card, CR_INTR_EVENT);
481	spin_unlock_irqrestore(&card->irq_lock, flags);
482
483	st = rsxx_get_card_state(card, &state);
484	if (st) {
485	dev_info(CARD_TO_DEV(card),
486	"Failed reading state after event.\n");
487	return;
488	}
489
490	if (card->state != state)
491	card_state_change(card, state);
492
493	if (card->creg_ctrl.creg_stats.stat & CREG_STAT_LOG_PENDING)
494	rsxx_read_hw_log(card);
495	}
496
497	/----------------- Card Operations -------------------/
498	static int card_shutdown(struct rsxx_cardinfo *card)
499	{
500	unsigned int state;
501	signed long start;
502	const int timeout = msecs_to_jiffies(120000);
503	int st;
504
505	/* We can't issue a shutdown if the card is in a transition state */
506	start = jiffies;
507	do {
508	st = rsxx_get_card_state(card, &state);
509	if (st)
510	return st;
511	} while (state == CARD_STATE_STARTING &&
512	(jiffies - start < timeout));
513
514	if (state == CARD_STATE_STARTING)
515	return -ETIMEDOUT;
516
517	/* Only issue a shutdown if we need to */
518	if ((state != CARD_STATE_SHUTTING_DOWN) &&
519	(state != CARD_STATE_SHUTDOWN)) {
520	st = rsxx_issue_card_cmd(card, CARD_CMD_SHUTDOWN);
521	if (st)
522	return st;
523	}
524
525	start = jiffies;
526	do {
527	st = rsxx_get_card_state(card, &state);
528	if (st)
529	return st;
530	} while (state != CARD_STATE_SHUTDOWN &&
531	(jiffies - start < timeout));
532
533	if (state != CARD_STATE_SHUTDOWN)
534	return -ETIMEDOUT;
535
536	return 0;
537	}
538
539	static int rsxx_eeh_frozen(struct pci_dev *dev)
540	{
541	struct rsxx_cardinfo *card = pci_get_drvdata(dev);
542	int i;
543	int st;
544
545	dev_warn(&dev->dev, "IBM Flash Adapter PCI: preparing for slot reset.\n");
546
547	card->eeh_state = 1;
548	rsxx_mask_interrupts(card);
549
550	/*
551	* We need to guarantee that the write for eeh_state and masking
552	* interrupts does not become reordered. This will prevent a possible
553	* race condition with the EEH code.
554	*/
555	wmb();
556
557	pci_disable_device(dev);
558
559	st = rsxx_eeh_save_issued_dmas(card);
560	if (st)
561	return st;
562
563	rsxx_eeh_save_issued_creg(card);
564
565	for (i = 0; i < card->n_targets; i++) {
566	if (card->ctrl[i].status.buf)
567	dma_free_coherent(&card->dev->dev,
568	STATUS_BUFFER_SIZE8,
569	card->ctrl[i].status.buf,
570	card->ctrl[i].status.dma_addr);
571	if (card->ctrl[i].cmd.buf)
572	dma_free_coherent(&card->dev->dev,
573	COMMAND_BUFFER_SIZE8,
574	card->ctrl[i].cmd.buf,
575	card->ctrl[i].cmd.dma_addr);
576	}
577
578	return 0;
579	}
580
581	static void rsxx_eeh_failure(struct pci_dev *dev)
582	{
583	struct rsxx_cardinfo *card = pci_get_drvdata(dev);
584	int i;
585	int cnt = 0;
586
587	dev_err(&dev->dev, "IBM Flash Adapter PCI: disabling failed card.\n");
588
589	card->eeh_state = 1;
590	card->halt = 1;
591
592	for (i = 0; i < card->n_targets; i++) {
593	spin_lock_bh(&card->ctrl[i].queue_lock);
594	cnt = rsxx_cleanup_dma_queue(&card->ctrl[i],
595	&card->ctrl[i].queue,
596	COMPLETE_DMA);
597	spin_unlock_bh(&card->ctrl[i].queue_lock);
598
599	cnt += rsxx_dma_cancel(&card->ctrl[i]);
600
601	if (cnt)
602	dev_info(CARD_TO_DEV(card),
603	"Freed %d queued DMAs on channel %d\n",
604	cnt, card->ctrl[i].id);
605	}
606	}
607
608	static int rsxx_eeh_fifo_flush_poll(struct rsxx_cardinfo *card)
609	{
610	unsigned int status;
611	int iter = 0;
612
613	/* We need to wait for the hardware to reset */
614	while (iter++ < 10) {
615	status = ioread32(card->regmap + PCI_RECONFIG);
616
617	if (status & RSXX_FLUSH_BUSY) {
618	ssleep(1);
619	continue;
620	}
621
622	if (status & RSXX_FLUSH_TIMEOUT)
623	dev_warn(CARD_TO_DEV(card), "HW: flash controller timeout\n");
624	return 0;
625	}
626
627	/* Hardware failed resetting itself. */
628	return -1;
629	}
630
631	static pci_ers_result_t rsxx_error_detected(struct pci_dev *dev,
632	pci_channel_state_t error)
633	{
634	int st;
635
636	if (dev->revision < RSXX_EEH_SUPPORT)
637	return PCI_ERS_RESULT_NONE;
638
639	if (error == pci_channel_io_perm_failure) {
640	rsxx_eeh_failure(dev);
641	return PCI_ERS_RESULT_DISCONNECT;
642	}
643
644	st = rsxx_eeh_frozen(dev);
645	if (st) {
646	dev_err(&dev->dev, "Slot reset setup failed\n");
647	rsxx_eeh_failure(dev);
648	return PCI_ERS_RESULT_DISCONNECT;
649	}
650
651	return PCI_ERS_RESULT_NEED_RESET;
652	}
653
654	static pci_ers_result_t rsxx_slot_reset(struct pci_dev *dev)
655	{
656	struct rsxx_cardinfo *card = pci_get_drvdata(dev);
657	unsigned long flags;
658	int i;
659	int st;
660
661	dev_warn(&dev->dev,
662	"IBM Flash Adapter PCI: recovering from slot reset.\n");
663
664	st = pci_enable_device(dev);
665	if (st)
666	goto failed_hw_setup;
667
668	pci_set_master(dev);
669
670	st = rsxx_eeh_fifo_flush_poll(card);
671	if (st)
672	goto failed_hw_setup;
673
674	rsxx_dma_queue_reset(card);
675
676	for (i = 0; i < card->n_targets; i++) {
677	st = rsxx_hw_buffers_init(dev, &card->ctrl[i]);
678	if (st)
679	goto failed_hw_buffers_init;
680	}
681
682	if (card->config_valid)
683	rsxx_dma_configure(card);
684
685	/* Clears the ISR register from spurious interrupts */
686	st = ioread32(card->regmap + ISR);
687
688	card->eeh_state = 0;
689
690	spin_lock_irqsave(&card->irq_lock, flags);
691	if (card->n_targets & RSXX_MAX_TARGETS)
692	rsxx_enable_ier_and_isr(card, CR_INTR_ALL_G);
693	else
694	rsxx_enable_ier_and_isr(card, CR_INTR_ALL_C);
695	spin_unlock_irqrestore(&card->irq_lock, flags);
696
697	rsxx_kick_creg_queue(card);
698
699	for (i = 0; i < card->n_targets; i++) {
700	spin_lock(&card->ctrl[i].queue_lock);
701	if (list_empty(&card->ctrl[i].queue)) {
702	spin_unlock(&card->ctrl[i].queue_lock);
703	continue;
704	}
705	spin_unlock(&card->ctrl[i].queue_lock);
706
707	queue_work(card->ctrl[i].issue_wq,
708	&card->ctrl[i].issue_dma_work);
709	}
710
711	dev_info(&dev->dev, "IBM Flash Adapter PCI: recovery complete.\n");
712
713	return PCI_ERS_RESULT_RECOVERED;
714
715	failed_hw_buffers_init:
716	for (i = 0; i < card->n_targets; i++) {
717	if (card->ctrl[i].status.buf)
718	dma_free_coherent(&card->dev->dev,
719	STATUS_BUFFER_SIZE8,
720	card->ctrl[i].status.buf,
721	card->ctrl[i].status.dma_addr);
722	if (card->ctrl[i].cmd.buf)
723	dma_free_coherent(&card->dev->dev,
724	COMMAND_BUFFER_SIZE8,
725	card->ctrl[i].cmd.buf,
726	card->ctrl[i].cmd.dma_addr);
727	}
728	failed_hw_setup:
729	rsxx_eeh_failure(dev);
730	return PCI_ERS_RESULT_DISCONNECT;
731
732	}
733
734	/----------------- Driver Initialization & Setup -------------------/
735	/* Returns: 0 if the driver is compatible with the device
736	-1 if the driver is NOT compatible with the device */
737	static int rsxx_compatibility_check(struct rsxx_cardinfo *card)
738	{
739	unsigned char pci_rev;
740
741	pci_read_config_byte(card->dev, PCI_REVISION_ID, &pci_rev);
742
743	if (pci_rev > RS70_PCI_REV_SUPPORTED)
744	return -1;
745	return 0;
746	}
747
748	static int rsxx_pci_probe(struct pci_dev *dev,
749	const struct pci_device_id *id)
750	{
751	struct rsxx_cardinfo *card;
752	int st;
753	unsigned int sync_timeout;
754
755	dev_info(&dev->dev, "PCI-Flash SSD discovered\n");
756
757	card = kzalloc(sizeof(*card), GFP_KERNEL);
758	if (!card)
759	return -ENOMEM;
760
761	card->dev = dev;
762	pci_set_drvdata(dev, card);
763
764	st = ida_alloc(&rsxx_disk_ida, GFP_KERNEL);
765	if (st < 0)
766	goto failed_ida_get;
767	card->disk_id = st;
768
769	st = pci_enable_device(dev);
770	if (st)
771	goto failed_enable;
772
773	pci_set_master(dev);
774
775	st = dma_set_mask(&dev->dev, DMA_BIT_MASK(64));
776	if (st) {
777	dev_err(CARD_TO_DEV(card),
778	"No usable DMA configuration,aborting\n");
779	goto failed_dma_mask;
780	}
781
782	st = pci_request_regions(dev, DRIVER_NAME);
783	if (st) {
784	dev_err(CARD_TO_DEV(card),
785	"Failed to request memory region\n");
786	goto failed_request_regions;
787	}
788
789	if (pci_resource_len(dev, 0) == 0) {
790	dev_err(CARD_TO_DEV(card), "BAR0 has length 0!\n");
791	st = -ENOMEM;
792	goto failed_iomap;
793	}
794
795	card->regmap = pci_iomap(dev, 0, 0);
796	if (!card->regmap) {
797	dev_err(CARD_TO_DEV(card), "Failed to map BAR0\n");
798	st = -ENOMEM;
799	goto failed_iomap;
800	}
801
802	spin_lock_init(&card->irq_lock);
803	card->halt = 0;
804	card->eeh_state = 0;
805
806	spin_lock_irq(&card->irq_lock);
807	rsxx_disable_ier_and_isr(card, CR_INTR_ALL);
808	spin_unlock_irq(&card->irq_lock);
809
810	if (!force_legacy) {
811	st = pci_enable_msi(dev);
812	if (st)
813	dev_warn(CARD_TO_DEV(card),
814	"Failed to enable MSI\n");
815	}
816
817	st = request_irq(dev->irq, rsxx_isr, IRQF_SHARED,
818	DRIVER_NAME, card);
819	if (st) {
820	dev_err(CARD_TO_DEV(card),
821	"Failed requesting IRQ%d\n", dev->irq);
822	goto failed_irq;
823	}
824
825	/*********** Setup Processor Command Interface ***********/
826	st = rsxx_creg_setup(card);
827	if (st) {
828	dev_err(CARD_TO_DEV(card), "Failed to setup creg interface.\n");
829	goto failed_creg_setup;
830	}
831
832	spin_lock_irq(&card->irq_lock);
833	rsxx_enable_ier_and_isr(card, CR_INTR_CREG);
834	spin_unlock_irq(&card->irq_lock);
835
836	st = rsxx_compatibility_check(card);
837	if (st) {
838	dev_warn(CARD_TO_DEV(card),
839	"Incompatible driver detected. Please update the driver.\n");
840	st = -EINVAL;
841	goto failed_compatiblity_check;
842	}
843
844	/*********** Load Card Config ***********/
845	st = rsxx_load_config(card);
846	if (st)
847	dev_err(CARD_TO_DEV(card),
848	"Failed loading card config\n");
849
850	/*********** Setup DMA Engine ***********/
851	st = rsxx_get_num_targets(card, &card->n_targets);
852	if (st)
853	dev_info(CARD_TO_DEV(card),
854	"Failed reading the number of DMA targets\n");
855
856	card->ctrl = kcalloc(card->n_targets, sizeof(*card->ctrl),
857	GFP_KERNEL);
858	if (!card->ctrl) {
859	st = -ENOMEM;
860	goto failed_dma_setup;
861	}
862
863	st = rsxx_dma_setup(card);
864	if (st) {
865	dev_info(CARD_TO_DEV(card),
866	"Failed to setup DMA engine\n");
867	goto failed_dma_setup;
868	}
869
870	/*********** Setup Card Event Handler ***********/
871	card->event_wq = create_singlethread_workqueue(DRIVER_NAME"_event");
872	if (!card->event_wq) {
873	dev_err(CARD_TO_DEV(card), "Failed card event setup.\n");
874	st = -ENOMEM;
875	goto failed_event_handler;
876	}
877
878	INIT_WORK(&card->event_work, card_event_handler);
879
880	st = rsxx_setup_dev(card);
881	if (st)
882	goto failed_create_dev;
883
884	rsxx_get_card_state(card, &card->state);
885
886	dev_info(CARD_TO_DEV(card),
887	"card state: %s\n",
888	rsxx_card_state_to_str(card->state));
889
890	/*
891	* Now that the DMA Engine and devices have been setup,
892	* we can enable the event interrupt(it kicks off actions in
893	* those layers so we couldn't enable it right away.)
894	*/
895	spin_lock_irq(&card->irq_lock);
896	rsxx_enable_ier_and_isr(card, CR_INTR_EVENT);
897	spin_unlock_irq(&card->irq_lock);
898
899	if (card->state == CARD_STATE_SHUTDOWN) {
900	st = rsxx_issue_card_cmd(card, CARD_CMD_STARTUP);
901	if (st)
902	dev_crit(CARD_TO_DEV(card),
903	"Failed issuing card startup\n");
904	if (sync_start) {
905	sync_timeout = SYNC_START_TIMEOUT;
906
907	dev_info(CARD_TO_DEV(card),
908	"Waiting for card to startup\n");
909
910	do {
911	ssleep(1);
912	sync_timeout--;
913
914	rsxx_get_card_state(card, &card->state);
915	} while (sync_timeout &&
916	(card->state == CARD_STATE_STARTING));
917
918	if (card->state == CARD_STATE_STARTING) {
919	dev_warn(CARD_TO_DEV(card),
920	"Card startup timed out\n");
921	card->size8 = 0;
922	} else {
923	dev_info(CARD_TO_DEV(card),
924	"card state: %s\n",
925	rsxx_card_state_to_str(card->state));
926	st = rsxx_get_card_size8(card, &card->size8);
927	if (st)
928	card->size8 = 0;
929	}
930	}
931	} else if (card->state == CARD_STATE_GOOD \|\|
932	card->state == CARD_STATE_RD_ONLY_FAULT) {
933	st = rsxx_get_card_size8(card, &card->size8);
934	if (st)
935	card->size8 = 0;
936	}
937
938	st = rsxx_attach_dev(card);
939	if (st)
940	goto failed_create_dev;
941
942	/*********** Setup Debugfs ***********/
943	rsxx_debugfs_dev_new(card);
944
945	return 0;
946
947	failed_create_dev:
948	destroy_workqueue(card->event_wq);
949	card->event_wq = NULL;
950	failed_event_handler:
951	rsxx_dma_destroy(card);
952	failed_dma_setup:
953	failed_compatiblity_check:
954	destroy_workqueue(card->creg_ctrl.creg_wq);
955	card->creg_ctrl.creg_wq = NULL;
956	failed_creg_setup:
957	spin_lock_irq(&card->irq_lock);
958	rsxx_disable_ier_and_isr(card, CR_INTR_ALL);
959	spin_unlock_irq(&card->irq_lock);
960	free_irq(dev->irq, card);
961	if (!force_legacy)
962	pci_disable_msi(dev);
963	failed_irq:
964	pci_iounmap(dev, card->regmap);
965	failed_iomap:
966	pci_release_regions(dev);
967	failed_request_regions:
968	failed_dma_mask:
969	pci_disable_device(dev);
970	failed_enable:
971	ida_free(&rsxx_disk_ida, card->disk_id);
972	failed_ida_get:
973	kfree(card);
974
975	return st;
976	}
977
978	static void rsxx_pci_remove(struct pci_dev *dev)
979	{
980	struct rsxx_cardinfo *card = pci_get_drvdata(dev);
981	unsigned long flags;
982	int st;
983	int i;
984
985	if (!card)
986	return;
987
988	dev_info(CARD_TO_DEV(card),
989	"Removing PCI-Flash SSD.\n");
990
991	rsxx_detach_dev(card);
992
993	for (i = 0; i < card->n_targets; i++) {
994	spin_lock_irqsave(&card->irq_lock, flags);
995	rsxx_disable_ier_and_isr(card, CR_INTR_DMA(i));
996	spin_unlock_irqrestore(&card->irq_lock, flags);
997	}
998
999	st = card_shutdown(card);
1000	if (st)
1001	dev_crit(CARD_TO_DEV(card), "Shutdown failed!\n");
1002
1003	/* Sync outstanding event handlers. */
1004	spin_lock_irqsave(&card->irq_lock, flags);
1005	rsxx_disable_ier_and_isr(card, CR_INTR_EVENT);
1006	spin_unlock_irqrestore(&card->irq_lock, flags);
1007
1008	cancel_work_sync(&card->event_work);
1009
1010	destroy_workqueue(card->event_wq);
1011	rsxx_destroy_dev(card);
1012	rsxx_dma_destroy(card);
1013	destroy_workqueue(card->creg_ctrl.creg_wq);
1014
1015	spin_lock_irqsave(&card->irq_lock, flags);
1016	rsxx_disable_ier_and_isr(card, CR_INTR_ALL);
1017	spin_unlock_irqrestore(&card->irq_lock, flags);
1018
1019	/* Prevent work_structs from re-queuing themselves. */
1020	card->halt = 1;
1021
1022	debugfs_remove_recursive(card->debugfs_dir);
1023
1024	free_irq(dev->irq, card);
1025
1026	if (!force_legacy)
1027	pci_disable_msi(dev);
1028
1029	rsxx_creg_destroy(card);
1030
1031	pci_iounmap(dev, card->regmap);
1032
1033	pci_disable_device(dev);
1034	pci_release_regions(dev);
1035
1036	ida_free(&rsxx_disk_ida, card->disk_id);
1037	kfree(card);
1038	}
1039
1040	static int rsxx_pci_suspend(struct pci_dev *dev, pm_message_t state)
1041	{
1042	/* We don't support suspend at this time. */
1043	return -ENOSYS;
1044	}
1045
1046	static void rsxx_pci_shutdown(struct pci_dev *dev)
1047	{
1048	struct rsxx_cardinfo *card = pci_get_drvdata(dev);
1049	unsigned long flags;
1050	int i;
1051
1052	if (!card)
1053	return;
1054
1055	dev_info(CARD_TO_DEV(card), "Shutting down PCI-Flash SSD.\n");
1056
1057	rsxx_detach_dev(card);
1058
1059	for (i = 0; i < card->n_targets; i++) {
1060	spin_lock_irqsave(&card->irq_lock, flags);
1061	rsxx_disable_ier_and_isr(card, CR_INTR_DMA(i));
1062	spin_unlock_irqrestore(&card->irq_lock, flags);
1063	}
1064
1065	card_shutdown(card);
1066	}
1067
1068	static const struct pci_error_handlers rsxx_err_handler = {
1069	.error_detected = rsxx_error_detected,
1070	.slot_reset = rsxx_slot_reset,
1071	};
1072
1073	static const struct pci_device_id rsxx_pci_ids[] = {
1074	{PCI_DEVICE(PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_FS70_FLASH)},
1075	{PCI_DEVICE(PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_FS80_FLASH)},
1076	{0,},
1077	};
1078
1079	MODULE_DEVICE_TABLE(pci, rsxx_pci_ids);
1080
1081	static struct pci_driver rsxx_pci_driver = {
1082	.name = DRIVER_NAME,
1083	.id_table = rsxx_pci_ids,
1084	.probe = rsxx_pci_probe,
1085	.remove = rsxx_pci_remove,
1086	.suspend = rsxx_pci_suspend,
1087	.shutdown = rsxx_pci_shutdown,
1088	.err_handler = &rsxx_err_handler,
1089	};
1090
1091	static int __init rsxx_core_init(void)
1092	{
1093	int st;
1094
1095	st = rsxx_dev_init();
1096	if (st)
1097	return st;
1098
1099	st = rsxx_dma_init();
1100	if (st)
1101	goto dma_init_failed;
1102
1103	st = rsxx_creg_init();
1104	if (st)
1105	goto creg_init_failed;
1106
1107	return pci_register_driver(&rsxx_pci_driver);
1108
1109	creg_init_failed:
1110	rsxx_dma_cleanup();
1111	dma_init_failed:
1112	rsxx_dev_cleanup();
1113
1114	return st;
1115	}
1116
1117	static void __exit rsxx_core_cleanup(void)
1118	{
1119	pci_unregister_driver(&rsxx_pci_driver);
1120	rsxx_creg_cleanup();
1121	rsxx_dma_cleanup();
1122	rsxx_dev_cleanup();
1123	}
1124
1125	module_init(rsxx_core_init);
1126	module_exit(rsxx_core_cleanup);

drivers/block/rsxx/cregs.c+0−789

@@ -1,789 +0,0 @@
1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* Filename: cregs.c
4	*
5	* Authors: Joshua Morris <josh.h.morris@us.ibm.com>
6	* Philip Kelleher <pjk1939@linux.vnet.ibm.com>
7	*
8	* (C) Copyright 2013 IBM Corporation
9	*/
10
11	#include <linux/completion.h>
12	#include <linux/slab.h>
13
14	#include "rsxx_priv.h"
15
16	#define CREG_TIMEOUT_MSEC 10000
17
18	typedef void (creg_cmd_cb)(struct rsxx_cardinfo card,
19	struct creg_cmd *cmd,
20	int st);
21
22	struct creg_cmd {
23	struct list_head list;
24	creg_cmd_cb cb;
25	void *cb_private;
26	unsigned int op;
27	unsigned int addr;
28	int cnt8;
29	void *buf;
30	unsigned int stream;
31	unsigned int status;
32	};
33
34	static struct kmem_cache *creg_cmd_pool;
35
36
37	/------------ Private Functions --------------/
38
39	#if defined(__LITTLE_ENDIAN)
40	#define LITTLE_ENDIAN 1
41	#elif defined(__BIG_ENDIAN)
42	#define LITTLE_ENDIAN 0
43	#else
44	#error Unknown endianess!!! Aborting...
45	#endif
46
47	static int copy_to_creg_data(struct rsxx_cardinfo *card,
48	int cnt8,
49	void *buf,
50	unsigned int stream)
51	{
52	int i = 0;
53	u32 *data = buf;
54
55	if (unlikely(card->eeh_state))
56	return -EIO;
57
58	for (i = 0; cnt8 > 0; i++, cnt8 -= 4) {
59	/*
60	* Firmware implementation makes it necessary to byte swap on
61	* little endian processors.
62	*/
63	if (LITTLE_ENDIAN && stream)
64	iowrite32be(data[i], card->regmap + CREG_DATA(i));
65	else
66	iowrite32(data[i], card->regmap + CREG_DATA(i));
67	}
68
69	return 0;
70	}
71
72
73	static int copy_from_creg_data(struct rsxx_cardinfo *card,
74	int cnt8,
75	void *buf,
76	unsigned int stream)
77	{
78	int i = 0;
79	u32 *data = buf;
80
81	if (unlikely(card->eeh_state))
82	return -EIO;
83
84	for (i = 0; cnt8 > 0; i++, cnt8 -= 4) {
85	/*
86	* Firmware implementation makes it necessary to byte swap on
87	* little endian processors.
88	*/
89	if (LITTLE_ENDIAN && stream)
90	data[i] = ioread32be(card->regmap + CREG_DATA(i));
91	else
92	data[i] = ioread32(card->regmap + CREG_DATA(i));
93	}
94
95	return 0;
96	}
97
98	static void creg_issue_cmd(struct rsxx_cardinfo card, struct creg_cmd cmd)
99	{
100	int st;
101
102	if (unlikely(card->eeh_state))
103	return;
104
105	iowrite32(cmd->addr, card->regmap + CREG_ADD);
106	iowrite32(cmd->cnt8, card->regmap + CREG_CNT);
107
108	if (cmd->op == CREG_OP_WRITE) {
109	if (cmd->buf) {
110	st = copy_to_creg_data(card, cmd->cnt8,
111	cmd->buf, cmd->stream);
112	if (st)
113	return;
114	}
115	}
116
117	if (unlikely(card->eeh_state))
118	return;
119
120	/* Setting the valid bit will kick off the command. */
121	iowrite32(cmd->op, card->regmap + CREG_CMD);
122	}
123
124	static void creg_kick_queue(struct rsxx_cardinfo *card)
125	{
126	if (card->creg_ctrl.active \|\| list_empty(&card->creg_ctrl.queue))
127	return;
128
129	card->creg_ctrl.active = 1;
130	card->creg_ctrl.active_cmd = list_first_entry(&card->creg_ctrl.queue,
131	struct creg_cmd, list);
132	list_del(&card->creg_ctrl.active_cmd->list);
133	card->creg_ctrl.q_depth--;
134
135	/*
136	* We have to set the timer before we push the new command. Otherwise,
137	* we could create a race condition that would occur if the timer
138	* was not canceled, and expired after the new command was pushed,
139	* but before the command was issued to hardware.
140	*/
141	mod_timer(&card->creg_ctrl.cmd_timer,
142	jiffies + msecs_to_jiffies(CREG_TIMEOUT_MSEC));
143
144	creg_issue_cmd(card, card->creg_ctrl.active_cmd);
145	}
146
147	static int creg_queue_cmd(struct rsxx_cardinfo *card,
148	unsigned int op,
149	unsigned int addr,
150	unsigned int cnt8,
151	void *buf,
152	int stream,
153	creg_cmd_cb callback,
154	void *cb_private)
155	{
156	struct creg_cmd *cmd;
157
158	/* Don't queue stuff up if we're halted. */
159	if (unlikely(card->halt))
160	return -EINVAL;
161
162	if (card->creg_ctrl.reset)
163	return -EAGAIN;
164
165	if (cnt8 > MAX_CREG_DATA8)
166	return -EINVAL;
167
168	cmd = kmem_cache_alloc(creg_cmd_pool, GFP_KERNEL);
169	if (!cmd)
170	return -ENOMEM;
171
172	INIT_LIST_HEAD(&cmd->list);
173
174	cmd->op = op;
175	cmd->addr = addr;
176	cmd->cnt8 = cnt8;
177	cmd->buf = buf;
178	cmd->stream = stream;
179	cmd->cb = callback;
180	cmd->cb_private = cb_private;
181	cmd->status = 0;
182
183	spin_lock_bh(&card->creg_ctrl.lock);
184	list_add_tail(&cmd->list, &card->creg_ctrl.queue);
185	card->creg_ctrl.q_depth++;
186	creg_kick_queue(card);
187	spin_unlock_bh(&card->creg_ctrl.lock);
188
189	return 0;
190	}
191
192	static void creg_cmd_timed_out(struct timer_list *t)
193	{
194	struct rsxx_cardinfo *card = from_timer(card, t, creg_ctrl.cmd_timer);
195	struct creg_cmd *cmd;
196
197	spin_lock(&card->creg_ctrl.lock);
198	cmd = card->creg_ctrl.active_cmd;
199	card->creg_ctrl.active_cmd = NULL;
200	spin_unlock(&card->creg_ctrl.lock);
201
202	if (cmd == NULL) {
203	card->creg_ctrl.creg_stats.creg_timeout++;
204	dev_warn(CARD_TO_DEV(card),
205	"No active command associated with timeout!\n");
206	return;
207	}
208
209	if (cmd->cb)
210	cmd->cb(card, cmd, -ETIMEDOUT);
211
212	kmem_cache_free(creg_cmd_pool, cmd);
213
214
215	spin_lock(&card->creg_ctrl.lock);
216	card->creg_ctrl.active = 0;
217	creg_kick_queue(card);
218	spin_unlock(&card->creg_ctrl.lock);
219	}
220
221
222	static void creg_cmd_done(struct work_struct *work)
223	{
224	struct rsxx_cardinfo *card;
225	struct creg_cmd *cmd;
226	int st = 0;
227
228	card = container_of(work, struct rsxx_cardinfo,
229	creg_ctrl.done_work);
230
231	/*
232	* The timer could not be cancelled for some reason,
233	* race to pop the active command.
234	*/
235	if (del_timer_sync(&card->creg_ctrl.cmd_timer) == 0)
236	card->creg_ctrl.creg_stats.failed_cancel_timer++;
237
238	spin_lock_bh(&card->creg_ctrl.lock);
239	cmd = card->creg_ctrl.active_cmd;
240	card->creg_ctrl.active_cmd = NULL;
241	spin_unlock_bh(&card->creg_ctrl.lock);
242
243	if (cmd == NULL) {
244	dev_err(CARD_TO_DEV(card),
245	"Spurious creg interrupt!\n");
246	return;
247	}
248
249	card->creg_ctrl.creg_stats.stat = ioread32(card->regmap + CREG_STAT);
250	cmd->status = card->creg_ctrl.creg_stats.stat;
251	if ((cmd->status & CREG_STAT_STATUS_MASK) == 0) {
252	dev_err(CARD_TO_DEV(card),
253	"Invalid status on creg command\n");
254	/*
255	* At this point we're probably reading garbage from HW. Don't
256	* do anything else that could mess up the system and let
257	* the sync function return an error.
258	*/
259	st = -EIO;
260	goto creg_done;
261	} else if (cmd->status & CREG_STAT_ERROR) {
262	st = -EIO;
263	}
264
265	if (cmd->op == CREG_OP_READ) {
266	unsigned int cnt8 = ioread32(card->regmap + CREG_CNT);
267
268	/* Paranoid Sanity Checks */
269	if (!cmd->buf) {
270	dev_err(CARD_TO_DEV(card),
271	"Buffer not given for read.\n");
272	st = -EIO;
273	goto creg_done;
274	}
275	if (cnt8 != cmd->cnt8) {
276	dev_err(CARD_TO_DEV(card),
277	"count mismatch\n");
278	st = -EIO;
279	goto creg_done;
280	}
281
282	st = copy_from_creg_data(card, cnt8, cmd->buf, cmd->stream);
283	}
284
285	creg_done:
286	if (cmd->cb)
287	cmd->cb(card, cmd, st);
288
289	kmem_cache_free(creg_cmd_pool, cmd);
290
291	spin_lock_bh(&card->creg_ctrl.lock);
292	card->creg_ctrl.active = 0;
293	creg_kick_queue(card);
294	spin_unlock_bh(&card->creg_ctrl.lock);
295	}
296
297	static void creg_reset(struct rsxx_cardinfo *card)
298	{
299	struct creg_cmd *cmd = NULL;
300	struct creg_cmd *tmp;
301	unsigned long flags;
302
303	/*
304	* mutex_trylock is used here because if reset_lock is taken then a
305	* reset is already happening. So, we can just go ahead and return.
306	*/
307	if (!mutex_trylock(&card->creg_ctrl.reset_lock))
308	return;
309
310	card->creg_ctrl.reset = 1;
311	spin_lock_irqsave(&card->irq_lock, flags);
312	rsxx_disable_ier_and_isr(card, CR_INTR_CREG \| CR_INTR_EVENT);
313	spin_unlock_irqrestore(&card->irq_lock, flags);
314
315	dev_warn(CARD_TO_DEV(card),
316	"Resetting creg interface for recovery\n");
317
318	/* Cancel outstanding commands */
319	spin_lock_bh(&card->creg_ctrl.lock);
320	list_for_each_entry_safe(cmd, tmp, &card->creg_ctrl.queue, list) {
321	list_del(&cmd->list);
322	card->creg_ctrl.q_depth--;
323	if (cmd->cb)
324	cmd->cb(card, cmd, -ECANCELED);
325	kmem_cache_free(creg_cmd_pool, cmd);
326	}
327
328	cmd = card->creg_ctrl.active_cmd;
329	card->creg_ctrl.active_cmd = NULL;
330	if (cmd) {
331	if (timer_pending(&card->creg_ctrl.cmd_timer))
332	del_timer_sync(&card->creg_ctrl.cmd_timer);
333
334	if (cmd->cb)
335	cmd->cb(card, cmd, -ECANCELED);
336	kmem_cache_free(creg_cmd_pool, cmd);
337
338	card->creg_ctrl.active = 0;
339	}
340	spin_unlock_bh(&card->creg_ctrl.lock);
341
342	card->creg_ctrl.reset = 0;
343	spin_lock_irqsave(&card->irq_lock, flags);
344	rsxx_enable_ier_and_isr(card, CR_INTR_CREG \| CR_INTR_EVENT);
345	spin_unlock_irqrestore(&card->irq_lock, flags);
346
347	mutex_unlock(&card->creg_ctrl.reset_lock);
348	}
349
350	/* Used for synchronous accesses */
351	struct creg_completion {
352	struct completion *cmd_done;
353	int st;
354	u32 creg_status;
355	};
356
357	static void creg_cmd_done_cb(struct rsxx_cardinfo *card,
358	struct creg_cmd *cmd,
359	int st)
360	{
361	struct creg_completion *cmd_completion;
362
363	cmd_completion = cmd->cb_private;
364	BUG_ON(!cmd_completion);
365
366	cmd_completion->st = st;
367	cmd_completion->creg_status = cmd->status;
368	complete(cmd_completion->cmd_done);
369	}
370
371	static int __issue_creg_rw(struct rsxx_cardinfo *card,
372	unsigned int op,
373	unsigned int addr,
374	unsigned int cnt8,
375	void *buf,
376	int stream,
377	unsigned int *hw_stat)
378	{
379	DECLARE_COMPLETION_ONSTACK(cmd_done);
380	struct creg_completion completion;
381	unsigned long timeout;
382	int st;
383
384	completion.cmd_done = &cmd_done;
385	completion.st = 0;
386	completion.creg_status = 0;
387
388	st = creg_queue_cmd(card, op, addr, cnt8, buf, stream, creg_cmd_done_cb,
389	&completion);
390	if (st)
391	return st;
392
393	/*
394	* This timeout is necessary for unresponsive hardware. The additional
395	* 20 seconds to used to guarantee that each cregs requests has time to
396	* complete.
397	*/
398	timeout = msecs_to_jiffies(CREG_TIMEOUT_MSEC *
399	card->creg_ctrl.q_depth + 20000);
400
401	/*
402	* The creg interface is guaranteed to complete. It has a timeout
403	* mechanism that will kick in if hardware does not respond.
404	*/
405	st = wait_for_completion_timeout(completion.cmd_done, timeout);
406	if (st == 0) {
407	/*
408	* This is really bad, because the kernel timer did not
409	* expire and notify us of a timeout!
410	*/
411	dev_crit(CARD_TO_DEV(card),
412	"cregs timer failed\n");
413	creg_reset(card);
414	return -EIO;
415	}
416
417	*hw_stat = completion.creg_status;
418
419	if (completion.st) {
420	/*
421	* This read is needed to verify that there has not been any
422	* extreme errors that might have occurred, i.e. EEH. The
423	* function iowrite32 will not detect EEH errors, so it is
424	* necessary that we recover if such an error is the reason
425	* for the timeout. This is a dummy read.
426	*/
427	ioread32(card->regmap + SCRATCH);
428
429	dev_warn(CARD_TO_DEV(card),
430	"creg command failed(%d x%08x)\n",
431	completion.st, addr);
432	return completion.st;
433	}
434
435	return 0;
436	}
437
438	static int issue_creg_rw(struct rsxx_cardinfo *card,
439	u32 addr,
440	unsigned int size8,
441	void *data,
442	int stream,
443	int read)
444	{
445	unsigned int hw_stat;
446	unsigned int xfer;
447	unsigned int op;
448	int st;
449
450	op = read ? CREG_OP_READ : CREG_OP_WRITE;
451
452	do {
453	xfer = min_t(unsigned int, size8, MAX_CREG_DATA8);
454
455	st = __issue_creg_rw(card, op, addr, xfer,
456	data, stream, &hw_stat);
457	if (st)
458	return st;
459
460	data = (char *)data + xfer;
461	addr += xfer;
462	size8 -= xfer;
463	} while (size8);
464
465	return 0;
466	}
467
468	/* ---------------------------- Public API ---------------------------------- */
469	int rsxx_creg_write(struct rsxx_cardinfo *card,
470	u32 addr,
471	unsigned int size8,
472	void *data,
473	int byte_stream)
474	{
475	return issue_creg_rw(card, addr, size8, data, byte_stream, 0);
476	}
477
478	int rsxx_creg_read(struct rsxx_cardinfo *card,
479	u32 addr,
480	unsigned int size8,
481	void *data,
482	int byte_stream)
483	{
484	return issue_creg_rw(card, addr, size8, data, byte_stream, 1);
485	}
486
487	int rsxx_get_card_state(struct rsxx_cardinfo card, unsigned int state)
488	{
489	return rsxx_creg_read(card, CREG_ADD_CARD_STATE,
490	sizeof(*state), state, 0);
491	}
492
493	int rsxx_get_card_size8(struct rsxx_cardinfo card, u64 size8)
494	{
495	unsigned int size;
496	int st;
497
498	st = rsxx_creg_read(card, CREG_ADD_CARD_SIZE,
499	sizeof(size), &size, 0);
500	if (st)
501	return st;
502
503	size8 = (u64)size RSXX_HW_BLK_SIZE;
504	return 0;
505	}
506
507	int rsxx_get_num_targets(struct rsxx_cardinfo *card,
508	unsigned int *n_targets)
509	{
510	return rsxx_creg_read(card, CREG_ADD_NUM_TARGETS,
511	sizeof(*n_targets), n_targets, 0);
512	}
513
514	int rsxx_get_card_capabilities(struct rsxx_cardinfo *card,
515	u32 *capabilities)
516	{
517	return rsxx_creg_read(card, CREG_ADD_CAPABILITIES,
518	sizeof(*capabilities), capabilities, 0);
519	}
520
521	int rsxx_issue_card_cmd(struct rsxx_cardinfo *card, u32 cmd)
522	{
523	return rsxx_creg_write(card, CREG_ADD_CARD_CMD,
524	sizeof(cmd), &cmd, 0);
525	}
526
527
528	/----------------- HW Log Functions -------------------/
529	static void hw_log_msg(struct rsxx_cardinfo card, const char str, int len)
530	{
531	static char level;
532
533	/*
534	* New messages start with "<#>", where # is the log level. Messages
535	* that extend past the log buffer will use the previous level
536	*/
537	if ((len > 3) && (str[0] == '<') && (str[2] == '>')) {
538	level = str[1];
539	str += 3; /* Skip past the log level. */
540	len -= 3;
541	}
542
543	switch (level) {
544	case '0':
545	dev_emerg(CARD_TO_DEV(card), "HW: %.*s", len, str);
546	break;
547	case '1':
548	dev_alert(CARD_TO_DEV(card), "HW: %.*s", len, str);
549	break;
550	case '2':
551	dev_crit(CARD_TO_DEV(card), "HW: %.*s", len, str);
552	break;
553	case '3':
554	dev_err(CARD_TO_DEV(card), "HW: %.*s", len, str);
555	break;
556	case '4':
557	dev_warn(CARD_TO_DEV(card), "HW: %.*s", len, str);
558	break;
559	case '5':
560	dev_notice(CARD_TO_DEV(card), "HW: %.*s", len, str);
561	break;
562	case '6':
563	dev_info(CARD_TO_DEV(card), "HW: %.*s", len, str);
564	break;
565	case '7':
566	dev_dbg(CARD_TO_DEV(card), "HW: %.*s", len, str);
567	break;
568	default:
569	dev_info(CARD_TO_DEV(card), "HW: %.*s", len, str);
570	break;
571	}
572	}
573
574	/*
575	* The substrncpy function copies the src string (which includes the
576	* terminating '\0' character), up to the count into the dest pointer.
577	* Returns the number of bytes copied to dest.
578	*/
579	static int substrncpy(char dest, const char src, int count)
580	{
581	int max_cnt = count;
582
583	while (count) {
584	count--;
585	dest = src;
586	if (*dest == '\0')
587	break;
588	src++;
589	dest++;
590	}
591	return max_cnt - count;
592	}
593
594
595	static void read_hw_log_done(struct rsxx_cardinfo *card,
596	struct creg_cmd *cmd,
597	int st)
598	{
599	char *buf;
600	char *log_str;
601	int cnt;
602	int len;
603	int off;
604
605	buf = cmd->buf;
606	off = 0;
607
608	/* Failed getting the log message */
609	if (st)
610	return;
611
612	while (off < cmd->cnt8) {
613	log_str = &card->log.buf[card->log.buf_len];
614	cnt = min(cmd->cnt8 - off, LOG_BUF_SIZE8 - card->log.buf_len);
615	len = substrncpy(log_str, &buf[off], cnt);
616
617	off += len;
618	card->log.buf_len += len;
619
620	/*
621	* Flush the log if we've hit the end of a message or if we've
622	* run out of buffer space.
623	*/
624	if ((log_str[len - 1] == '\0') \|\|
625	(card->log.buf_len == LOG_BUF_SIZE8)) {
626	if (card->log.buf_len != 1) /* Don't log blank lines. */
627	hw_log_msg(card, card->log.buf,
628	card->log.buf_len);
629	card->log.buf_len = 0;
630	}
631
632	}
633
634	if (cmd->status & CREG_STAT_LOG_PENDING)
635	rsxx_read_hw_log(card);
636	}
637
638	int rsxx_read_hw_log(struct rsxx_cardinfo *card)
639	{
640	int st;
641
642	st = creg_queue_cmd(card, CREG_OP_READ, CREG_ADD_LOG,
643	sizeof(card->log.tmp), card->log.tmp,
644	1, read_hw_log_done, NULL);
645	if (st)
646	dev_err(CARD_TO_DEV(card),
647	"Failed getting log text\n");
648
649	return st;
650	}
651
652	/-------------- IOCTL REG Access ------------------/
653	static int issue_reg_cmd(struct rsxx_cardinfo *card,
654	struct rsxx_reg_access *cmd,
655	int read)
656	{
657	unsigned int op = read ? CREG_OP_READ : CREG_OP_WRITE;
658
659	return __issue_creg_rw(card, op, cmd->addr, cmd->cnt, cmd->data,
660	cmd->stream, &cmd->stat);
661	}
662
663	int rsxx_reg_access(struct rsxx_cardinfo *card,
664	struct rsxx_reg_access __user *ucmd,
665	int read)
666	{
667	struct rsxx_reg_access cmd;
668	int st;
669
670	st = copy_from_user(&cmd, ucmd, sizeof(cmd));
671	if (st)
672	return -EFAULT;
673
674	if (cmd.cnt > RSXX_MAX_REG_CNT)
675	return -EFAULT;
676
677	st = issue_reg_cmd(card, &cmd, read);
678	if (st)
679	return st;
680
681	st = put_user(cmd.stat, &ucmd->stat);
682	if (st)
683	return -EFAULT;
684
685	if (read) {
686	st = copy_to_user(ucmd->data, cmd.data, cmd.cnt);
687	if (st)
688	return -EFAULT;
689	}
690
691	return 0;
692	}
693
694	void rsxx_eeh_save_issued_creg(struct rsxx_cardinfo *card)
695	{
696	struct creg_cmd *cmd = NULL;
697
698	cmd = card->creg_ctrl.active_cmd;
699	card->creg_ctrl.active_cmd = NULL;
700
701	if (cmd) {
702	del_timer_sync(&card->creg_ctrl.cmd_timer);
703
704	spin_lock_bh(&card->creg_ctrl.lock);
705	list_add(&cmd->list, &card->creg_ctrl.queue);
706	card->creg_ctrl.q_depth++;
707	card->creg_ctrl.active = 0;
708	spin_unlock_bh(&card->creg_ctrl.lock);
709	}
710	}
711
712	void rsxx_kick_creg_queue(struct rsxx_cardinfo *card)
713	{
714	spin_lock_bh(&card->creg_ctrl.lock);
715	if (!list_empty(&card->creg_ctrl.queue))
716	creg_kick_queue(card);
717	spin_unlock_bh(&card->creg_ctrl.lock);
718	}
719
720	/------------ Initialization & Setup --------------/
721	int rsxx_creg_setup(struct rsxx_cardinfo *card)
722	{
723	card->creg_ctrl.active_cmd = NULL;
724
725	card->creg_ctrl.creg_wq =
726	create_singlethread_workqueue(DRIVER_NAME"_creg");
727	if (!card->creg_ctrl.creg_wq)
728	return -ENOMEM;
729
730	INIT_WORK(&card->creg_ctrl.done_work, creg_cmd_done);
731	mutex_init(&card->creg_ctrl.reset_lock);
732	INIT_LIST_HEAD(&card->creg_ctrl.queue);
733	spin_lock_init(&card->creg_ctrl.lock);
734	timer_setup(&card->creg_ctrl.cmd_timer, creg_cmd_timed_out, 0);
735
736	return 0;
737	}
738
739	void rsxx_creg_destroy(struct rsxx_cardinfo *card)
740	{
741	struct creg_cmd *cmd;
742	struct creg_cmd *tmp;
743	int cnt = 0;
744
745	/* Cancel outstanding commands */
746	spin_lock_bh(&card->creg_ctrl.lock);
747	list_for_each_entry_safe(cmd, tmp, &card->creg_ctrl.queue, list) {
748	list_del(&cmd->list);
749	if (cmd->cb)
750	cmd->cb(card, cmd, -ECANCELED);
751	kmem_cache_free(creg_cmd_pool, cmd);
752	cnt++;
753	}
754
755	if (cnt)
756	dev_info(CARD_TO_DEV(card),
757	"Canceled %d queue creg commands\n", cnt);
758
759	cmd = card->creg_ctrl.active_cmd;
760	card->creg_ctrl.active_cmd = NULL;
761	if (cmd) {
762	if (timer_pending(&card->creg_ctrl.cmd_timer))
763	del_timer_sync(&card->creg_ctrl.cmd_timer);
764
765	if (cmd->cb)
766	cmd->cb(card, cmd, -ECANCELED);
767	dev_info(CARD_TO_DEV(card),
768	"Canceled active creg command\n");
769	kmem_cache_free(creg_cmd_pool, cmd);
770	}
771	spin_unlock_bh(&card->creg_ctrl.lock);
772
773	cancel_work_sync(&card->creg_ctrl.done_work);
774	}
775
776
777	int rsxx_creg_init(void)
778	{
779	creg_cmd_pool = KMEM_CACHE(creg_cmd, SLAB_HWCACHE_ALIGN);
780	if (!creg_cmd_pool)
781	return -ENOMEM;
782
783	return 0;
784	}
785
786	void rsxx_creg_cleanup(void)
787	{
788	kmem_cache_destroy(creg_cmd_pool);
789	}

drivers/block/rsxx/dev.c+0−306

@@ -1,306 +0,0 @@
1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* Filename: dev.c
4	*
5	* Authors: Joshua Morris <josh.h.morris@us.ibm.com>
6	* Philip Kelleher <pjk1939@linux.vnet.ibm.com>
7	*
8	* (C) Copyright 2013 IBM Corporation
9	*/
10
11	#include <linux/kernel.h>
12	#include <linux/interrupt.h>
13	#include <linux/module.h>
14	#include <linux/pci.h>
15	#include <linux/slab.h>
16
17	#include <linux/hdreg.h>
18	#include <linux/genhd.h>
19	#include <linux/blkdev.h>
20	#include <linux/bio.h>
21
22	#include <linux/fs.h>
23
24	#include "rsxx_priv.h"
25
26	static unsigned int blkdev_minors = 64;
27	module_param(blkdev_minors, uint, 0444);
28	MODULE_PARM_DESC(blkdev_minors, "Number of minors(partitions)");
29
30	/*
31	* For now I'm making this tweakable in case any applications hit this limit.
32	* If you see a "bio too big" error in the log you will need to raise this
33	* value.
34	*/
35	static unsigned int blkdev_max_hw_sectors = 1024;
36	module_param(blkdev_max_hw_sectors, uint, 0444);
37	MODULE_PARM_DESC(blkdev_max_hw_sectors, "Max hw sectors for a single BIO");
38
39	static unsigned int enable_blkdev = 1;
40	module_param(enable_blkdev , uint, 0444);
41	MODULE_PARM_DESC(enable_blkdev, "Enable block device interfaces");
42
43
44	struct rsxx_bio_meta {
45	struct bio *bio;
46	atomic_t pending_dmas;
47	atomic_t error;
48	unsigned long start_time;
49	};
50
51	static struct kmem_cache *bio_meta_pool;
52
53	static void rsxx_submit_bio(struct bio *bio);
54
55	/----------------- Block Device Operations -----------------/
56	static int rsxx_blkdev_ioctl(struct block_device *bdev,
57	fmode_t mode,
58	unsigned int cmd,
59	unsigned long arg)
60	{
61	struct rsxx_cardinfo *card = bdev->bd_disk->private_data;
62
63	switch (cmd) {
64	case RSXX_GETREG:
65	return rsxx_reg_access(card, (void __user *)arg, 1);
66	case RSXX_SETREG:
67	return rsxx_reg_access(card, (void __user *)arg, 0);
68	}
69
70	return -ENOTTY;
71	}
72
73	static int rsxx_getgeo(struct block_device bdev, struct hd_geometry geo)
74	{
75	struct rsxx_cardinfo *card = bdev->bd_disk->private_data;
76	u64 blocks = card->size8 >> 9;
77
78	/*
79	* get geometry: Fake it. I haven't found any drivers that set
80	* geo->start, so we won't either.
81	*/
82	if (card->size8) {
83	geo->heads = 64;
84	geo->sectors = 16;
85	do_div(blocks, (geo->heads * geo->sectors));
86	geo->cylinders = blocks;
87	} else {
88	geo->heads = 0;
89	geo->sectors = 0;
90	geo->cylinders = 0;
91	}
92	return 0;
93	}
94
95	static const struct block_device_operations rsxx_fops = {
96	.owner = THIS_MODULE,
97	.submit_bio = rsxx_submit_bio,
98	.getgeo = rsxx_getgeo,
99	.ioctl = rsxx_blkdev_ioctl,
100	};
101
102	static void bio_dma_done_cb(struct rsxx_cardinfo *card,
103	void *cb_data,
104	unsigned int error)
105	{
106	struct rsxx_bio_meta *meta = cb_data;
107
108	if (error)
109	atomic_set(&meta->error, 1);
110
111	if (atomic_dec_and_test(&meta->pending_dmas)) {
112	if (!card->eeh_state && card->gendisk)
113	bio_end_io_acct(meta->bio, meta->start_time);
114
115	if (atomic_read(&meta->error))
116	bio_io_error(meta->bio);
117	else
118	bio_endio(meta->bio);
119	kmem_cache_free(bio_meta_pool, meta);
120	}
121	}
122
123	static void rsxx_submit_bio(struct bio *bio)
124	{
125	struct rsxx_cardinfo *card = bio->bi_bdev->bd_disk->private_data;
126	struct rsxx_bio_meta *bio_meta;
127	blk_status_t st = BLK_STS_IOERR;
128
129	blk_queue_split(&bio);
130
131	might_sleep();
132
133	if (!card)
134	goto req_err;
135
136	if (bio_end_sector(bio) > get_capacity(card->gendisk))
137	goto req_err;
138
139	if (unlikely(card->halt))
140	goto req_err;
141
142	if (unlikely(card->dma_fault))
143	goto req_err;
144
145	if (bio->bi_iter.bi_size == 0) {
146	dev_err(CARD_TO_DEV(card), "size zero BIO!\n");
147	goto req_err;
148	}
149
150	bio_meta = kmem_cache_alloc(bio_meta_pool, GFP_KERNEL);
151	if (!bio_meta) {
152	st = BLK_STS_RESOURCE;
153	goto req_err;
154	}
155
156	bio_meta->bio = bio;
157	atomic_set(&bio_meta->error, 0);
158	atomic_set(&bio_meta->pending_dmas, 0);
159
160	if (!unlikely(card->halt))
161	bio_meta->start_time = bio_start_io_acct(bio);
162
163	dev_dbg(CARD_TO_DEV(card), "BIO[%c]: meta: %p addr8: x%llx size: %d\n",
164	bio_data_dir(bio) ? 'W' : 'R', bio_meta,
165	(u64)bio->bi_iter.bi_sector << 9, bio->bi_iter.bi_size);
166
167	st = rsxx_dma_queue_bio(card, bio, &bio_meta->pending_dmas,
168	bio_dma_done_cb, bio_meta);
169	if (st)
170	goto queue_err;
171
172	return;
173
174	queue_err:
175	kmem_cache_free(bio_meta_pool, bio_meta);
176	req_err:
177	if (st)
178	bio->bi_status = st;
179	bio_endio(bio);
180	}
181
182	/----------------- Device Setup -------------------/
183	static bool rsxx_discard_supported(struct rsxx_cardinfo *card)
184	{
185	unsigned char pci_rev;
186
187	pci_read_config_byte(card->dev, PCI_REVISION_ID, &pci_rev);
188
189	return (pci_rev >= RSXX_DISCARD_SUPPORT);
190	}
191
192	int rsxx_attach_dev(struct rsxx_cardinfo *card)
193	{
194	int err = 0;
195
196	mutex_lock(&card->dev_lock);
197
198	/* The block device requires the stripe size from the config. */
199	if (enable_blkdev) {
200	if (card->config_valid)
201	set_capacity(card->gendisk, card->size8 >> 9);
202	else
203	set_capacity(card->gendisk, 0);
204	err = device_add_disk(CARD_TO_DEV(card), card->gendisk, NULL);
205	if (err == 0)
206	card->bdev_attached = 1;
207	}
208
209	mutex_unlock(&card->dev_lock);
210
211	if (err)
212	blk_cleanup_disk(card->gendisk);
213
214	return err;
215	}
216
217	void rsxx_detach_dev(struct rsxx_cardinfo *card)
218	{
219	mutex_lock(&card->dev_lock);
220
221	if (card->bdev_attached) {
222	del_gendisk(card->gendisk);
223	card->bdev_attached = 0;
224	}
225
226	mutex_unlock(&card->dev_lock);
227	}
228
229	int rsxx_setup_dev(struct rsxx_cardinfo *card)
230	{
231	unsigned short blk_size;
232
233	mutex_init(&card->dev_lock);
234
235	if (!enable_blkdev)
236	return 0;
237
238	card->major = register_blkdev(0, DRIVER_NAME);
239	if (card->major < 0) {
240	dev_err(CARD_TO_DEV(card), "Failed to get major number\n");
241	return -ENOMEM;
242	}
243
244	card->gendisk = blk_alloc_disk(blkdev_minors);
245	if (!card->gendisk) {
246	dev_err(CARD_TO_DEV(card), "Failed disk alloc\n");
247	unregister_blkdev(card->major, DRIVER_NAME);
248	return -ENOMEM;
249	}
250
251	if (card->config_valid) {
252	blk_size = card->config.data.block_size;
253	blk_queue_dma_alignment(card->gendisk->queue, blk_size - 1);
254	blk_queue_logical_block_size(card->gendisk->queue, blk_size);
255	}
256
257	blk_queue_max_hw_sectors(card->gendisk->queue, blkdev_max_hw_sectors);
258	blk_queue_physical_block_size(card->gendisk->queue, RSXX_HW_BLK_SIZE);
259
260	blk_queue_flag_set(QUEUE_FLAG_NONROT, card->gendisk->queue);
261	blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, card->gendisk->queue);
262	if (rsxx_discard_supported(card)) {
263	blk_queue_flag_set(QUEUE_FLAG_DISCARD, card->gendisk->queue);
264	blk_queue_max_discard_sectors(card->gendisk->queue,
265	RSXX_HW_BLK_SIZE >> 9);
266	card->gendisk->queue->limits.discard_granularity =
267	RSXX_HW_BLK_SIZE;
268	card->gendisk->queue->limits.discard_alignment =
269	RSXX_HW_BLK_SIZE;
270	}
271
272	snprintf(card->gendisk->disk_name, sizeof(card->gendisk->disk_name),
273	"rsxx%d", card->disk_id);
274	card->gendisk->major = card->major;
275	card->gendisk->minors = blkdev_minors;
276	card->gendisk->fops = &rsxx_fops;
277	card->gendisk->private_data = card;
278
279	return 0;
280	}
281
282	void rsxx_destroy_dev(struct rsxx_cardinfo *card)
283	{
284	if (!enable_blkdev)
285	return;
286
287	blk_cleanup_disk(card->gendisk);
288	card->gendisk = NULL;
289	unregister_blkdev(card->major, DRIVER_NAME);
290	}
291
292	int rsxx_dev_init(void)
293	{
294	bio_meta_pool = KMEM_CACHE(rsxx_bio_meta, SLAB_HWCACHE_ALIGN);
295	if (!bio_meta_pool)
296	return -ENOMEM;
297
298	return 0;
299	}
300
301	void rsxx_dev_cleanup(void)
302	{
303	kmem_cache_destroy(bio_meta_pool);
304	}
305
306

drivers/block/rsxx/dma.c+0−1085

@@ -1,1085 +0,0 @@
1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* Filename: dma.c
4	*
5	* Authors: Joshua Morris <josh.h.morris@us.ibm.com>
6	* Philip Kelleher <pjk1939@linux.vnet.ibm.com>
7	*
8	* (C) Copyright 2013 IBM Corporation
9	*/
10
11	#include <linux/slab.h>
12	#include "rsxx_priv.h"
13
14	struct rsxx_dma {
15	struct list_head list;
16	u8 cmd;
17	unsigned int laddr; /* Logical address */
18	struct {
19	u32 off;
20	u32 cnt;
21	} sub_page;
22	dma_addr_t dma_addr;
23	struct page *page;
24	unsigned int pg_off; /* Page Offset */
25	rsxx_dma_cb cb;
26	void *cb_data;
27	};
28
29	/* This timeout is used to detect a stalled DMA channel */
30	#define DMA_ACTIVITY_TIMEOUT msecs_to_jiffies(10000)
31
32	struct hw_status {
33	u8 status;
34	u8 tag;
35	__le16 count;
36	__le32 _rsvd2;
37	__le64 _rsvd3;
38	} __packed;
39
40	enum rsxx_dma_status {
41	DMA_SW_ERR = 0x1,
42	DMA_HW_FAULT = 0x2,
43	DMA_CANCELLED = 0x4,
44	};
45
46	struct hw_cmd {
47	u8 command;
48	u8 tag;
49	u8 _rsvd;
50	u8 sub_page; /* Bit[0:2]: 512byte offset */
51	/* Bit[4:6]: 512byte count */
52	__le32 device_addr;
53	__le64 host_addr;
54	} __packed;
55
56	enum rsxx_hw_cmd {
57	HW_CMD_BLK_DISCARD = 0x70,
58	HW_CMD_BLK_WRITE = 0x80,
59	HW_CMD_BLK_READ = 0xC0,
60	HW_CMD_BLK_RECON_READ = 0xE0,
61	};
62
63	enum rsxx_hw_status {
64	HW_STATUS_CRC = 0x01,
65	HW_STATUS_HARD_ERR = 0x02,
66	HW_STATUS_SOFT_ERR = 0x04,
67	HW_STATUS_FAULT = 0x08,
68	};
69
70	static struct kmem_cache *rsxx_dma_pool;
71
72	struct dma_tracker {
73	int next_tag;
74	struct rsxx_dma *dma;
75	};
76
77	struct dma_tracker_list {
78	spinlock_t lock;
79	int head;
80	struct dma_tracker list[];
81	};
82
83
84	/----------------- Misc Utility Functions -------------------/
85	static unsigned int rsxx_addr8_to_laddr(u64 addr8, struct rsxx_cardinfo *card)
86	{
87	unsigned long long tgt_addr8;
88
89	tgt_addr8 = ((addr8 >> card->_stripe.upper_shift) &
90	card->_stripe.upper_mask) \|
91	((addr8) & card->_stripe.lower_mask);
92	do_div(tgt_addr8, RSXX_HW_BLK_SIZE);
93	return tgt_addr8;
94	}
95
96	static unsigned int rsxx_get_dma_tgt(struct rsxx_cardinfo *card, u64 addr8)
97	{
98	unsigned int tgt;
99
100	tgt = (addr8 >> card->_stripe.target_shift) & card->_stripe.target_mask;
101
102	return tgt;
103	}
104
105	void rsxx_dma_queue_reset(struct rsxx_cardinfo *card)
106	{
107	/* Reset all DMA Command/Status Queues */
108	iowrite32(DMA_QUEUE_RESET, card->regmap + RESET);
109	}
110
111	static unsigned int get_dma_size(struct rsxx_dma *dma)
112	{
113	if (dma->sub_page.cnt)
114	return dma->sub_page.cnt << 9;
115	else
116	return RSXX_HW_BLK_SIZE;
117	}
118
119
120	/----------------- DMA Tracker -------------------/
121	static void set_tracker_dma(struct dma_tracker_list *trackers,
122	int tag,
123	struct rsxx_dma *dma)
124	{
125	trackers->list[tag].dma = dma;
126	}
127
128	static struct rsxx_dma get_tracker_dma(struct dma_tracker_list trackers,
129	int tag)
130	{
131	return trackers->list[tag].dma;
132	}
133
134	static int pop_tracker(struct dma_tracker_list *trackers)
135	{
136	int tag;
137
138	spin_lock(&trackers->lock);
139	tag = trackers->head;
140	if (tag != -1) {
141	trackers->head = trackers->list[tag].next_tag;
142	trackers->list[tag].next_tag = -1;
143	}
144	spin_unlock(&trackers->lock);
145
146	return tag;
147	}
148
149	static void push_tracker(struct dma_tracker_list *trackers, int tag)
150	{
151	spin_lock(&trackers->lock);
152	trackers->list[tag].next_tag = trackers->head;
153	trackers->head = tag;
154	trackers->list[tag].dma = NULL;
155	spin_unlock(&trackers->lock);
156	}
157
158
159	/----------------- Interrupt Coalescing -------------/
160	/*
161	* Interrupt Coalescing Register Format:
162	* Interrupt Timer (64ns units) [15:0]
163	* Interrupt Count [24:16]
164	* Reserved [31:25]
165	*/
166	#define INTR_COAL_LATENCY_MASK (0x0000ffff)
167
168	#define INTR_COAL_COUNT_SHIFT 16
169	#define INTR_COAL_COUNT_BITS 9
170	#define INTR_COAL_COUNT_MASK (((1 << INTR_COAL_COUNT_BITS) - 1) << \
171	INTR_COAL_COUNT_SHIFT)
172	#define INTR_COAL_LATENCY_UNITS_NS 64
173
174
175	static u32 dma_intr_coal_val(u32 mode, u32 count, u32 latency)
176	{
177	u32 latency_units = latency / INTR_COAL_LATENCY_UNITS_NS;
178
179	if (mode == RSXX_INTR_COAL_DISABLED)
180	return 0;
181
182	return ((count << INTR_COAL_COUNT_SHIFT) & INTR_COAL_COUNT_MASK) \|
183	(latency_units & INTR_COAL_LATENCY_MASK);
184
185	}
186
187	static void dma_intr_coal_auto_tune(struct rsxx_cardinfo *card)
188	{
189	int i;
190	u32 q_depth = 0;
191	u32 intr_coal;
192
193	if (card->config.data.intr_coal.mode != RSXX_INTR_COAL_AUTO_TUNE \|\|
194	unlikely(card->eeh_state))
195	return;
196
197	for (i = 0; i < card->n_targets; i++)
198	q_depth += atomic_read(&card->ctrl[i].stats.hw_q_depth);
199
200	intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode,
201	q_depth / 2,
202	card->config.data.intr_coal.latency);
203	iowrite32(intr_coal, card->regmap + INTR_COAL);
204	}
205
206	/----------------- RSXX DMA Handling -------------------/
207	static void rsxx_free_dma(struct rsxx_dma_ctrl ctrl, struct rsxx_dma dma)
208	{
209	if (dma->cmd != HW_CMD_BLK_DISCARD) {
210	if (!dma_mapping_error(&ctrl->card->dev->dev, dma->dma_addr)) {
211	dma_unmap_page(&ctrl->card->dev->dev, dma->dma_addr,
212	get_dma_size(dma),
213	dma->cmd == HW_CMD_BLK_WRITE ?
214	DMA_TO_DEVICE :
215	DMA_FROM_DEVICE);
216	}
217	}
218
219	kmem_cache_free(rsxx_dma_pool, dma);
220	}
221
222	static void rsxx_complete_dma(struct rsxx_dma_ctrl *ctrl,
223	struct rsxx_dma *dma,
224	unsigned int status)
225	{
226	if (status & DMA_SW_ERR)
227	ctrl->stats.dma_sw_err++;
228	if (status & DMA_HW_FAULT)
229	ctrl->stats.dma_hw_fault++;
230	if (status & DMA_CANCELLED)
231	ctrl->stats.dma_cancelled++;
232
233	if (dma->cb)
234	dma->cb(ctrl->card, dma->cb_data, status ? 1 : 0);
235
236	rsxx_free_dma(ctrl, dma);
237	}
238
239	int rsxx_cleanup_dma_queue(struct rsxx_dma_ctrl *ctrl,
240	struct list_head *q, unsigned int done)
241	{
242	struct rsxx_dma *dma;
243	struct rsxx_dma *tmp;
244	int cnt = 0;
245
246	list_for_each_entry_safe(dma, tmp, q, list) {
247	list_del(&dma->list);
248	if (done & COMPLETE_DMA)
249	rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
250	else
251	rsxx_free_dma(ctrl, dma);
252	cnt++;
253	}
254
255	return cnt;
256	}
257
258	static void rsxx_requeue_dma(struct rsxx_dma_ctrl *ctrl,
259	struct rsxx_dma *dma)
260	{
261	/*
262	* Requeued DMAs go to the front of the queue so they are issued
263	* first.
264	*/
265	spin_lock_bh(&ctrl->queue_lock);
266	ctrl->stats.sw_q_depth++;
267	list_add(&dma->list, &ctrl->queue);
268	spin_unlock_bh(&ctrl->queue_lock);
269	}
270
271	static void rsxx_handle_dma_error(struct rsxx_dma_ctrl *ctrl,
272	struct rsxx_dma *dma,
273	u8 hw_st)
274	{
275	unsigned int status = 0;
276	int requeue_cmd = 0;
277
278	dev_dbg(CARD_TO_DEV(ctrl->card),
279	"Handling DMA error(cmd x%02x, laddr x%08x st:x%02x)\n",
280	dma->cmd, dma->laddr, hw_st);
281
282	if (hw_st & HW_STATUS_CRC)
283	ctrl->stats.crc_errors++;
284	if (hw_st & HW_STATUS_HARD_ERR)
285	ctrl->stats.hard_errors++;
286	if (hw_st & HW_STATUS_SOFT_ERR)
287	ctrl->stats.soft_errors++;
288
289	switch (dma->cmd) {
290	case HW_CMD_BLK_READ:
291	if (hw_st & (HW_STATUS_CRC \| HW_STATUS_HARD_ERR)) {
292	if (ctrl->card->scrub_hard) {
293	dma->cmd = HW_CMD_BLK_RECON_READ;
294	requeue_cmd = 1;
295	ctrl->stats.reads_retried++;
296	} else {
297	status \|= DMA_HW_FAULT;
298	ctrl->stats.reads_failed++;
299	}
300	} else if (hw_st & HW_STATUS_FAULT) {
301	status \|= DMA_HW_FAULT;
302	ctrl->stats.reads_failed++;
303	}
304
305	break;
306	case HW_CMD_BLK_RECON_READ:
307	if (hw_st & (HW_STATUS_CRC \| HW_STATUS_HARD_ERR)) {
308	/* Data could not be reconstructed. */
309	status \|= DMA_HW_FAULT;
310	ctrl->stats.reads_failed++;
311	}
312
313	break;
314	case HW_CMD_BLK_WRITE:
315	status \|= DMA_HW_FAULT;
316	ctrl->stats.writes_failed++;
317
318	break;
319	case HW_CMD_BLK_DISCARD:
320	status \|= DMA_HW_FAULT;
321	ctrl->stats.discards_failed++;
322
323	break;
324	default:
325	dev_err(CARD_TO_DEV(ctrl->card),
326	"Unknown command in DMA!(cmd: x%02x "
327	"laddr x%08x st: x%02x\n",
328	dma->cmd, dma->laddr, hw_st);
329	status \|= DMA_SW_ERR;
330
331	break;
332	}
333
334	if (requeue_cmd)
335	rsxx_requeue_dma(ctrl, dma);
336	else
337	rsxx_complete_dma(ctrl, dma, status);
338	}
339
340	static void dma_engine_stalled(struct timer_list *t)
341	{
342	struct rsxx_dma_ctrl *ctrl = from_timer(ctrl, t, activity_timer);
343	int cnt;
344
345	if (atomic_read(&ctrl->stats.hw_q_depth) == 0 \|\|
346	unlikely(ctrl->card->eeh_state))
347	return;
348
349	if (ctrl->cmd.idx != ioread32(ctrl->regmap + SW_CMD_IDX)) {
350	/*
351	* The dma engine was stalled because the SW_CMD_IDX write
352	* was lost. Issue it again to recover.
353	*/
354	dev_warn(CARD_TO_DEV(ctrl->card),
355	"SW_CMD_IDX write was lost, re-writing...\n");
356	iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
357	mod_timer(&ctrl->activity_timer,
358	jiffies + DMA_ACTIVITY_TIMEOUT);
359	} else {
360	dev_warn(CARD_TO_DEV(ctrl->card),
361	"DMA channel %d has stalled, faulting interface.\n",
362	ctrl->id);
363	ctrl->card->dma_fault = 1;
364
365	/* Clean up the DMA queue */
366	spin_lock(&ctrl->queue_lock);
367	cnt = rsxx_cleanup_dma_queue(ctrl, &ctrl->queue, COMPLETE_DMA);
368	spin_unlock(&ctrl->queue_lock);
369
370	cnt += rsxx_dma_cancel(ctrl);
371
372	if (cnt)
373	dev_info(CARD_TO_DEV(ctrl->card),
374	"Freed %d queued DMAs on channel %d\n",
375	cnt, ctrl->id);
376	}
377	}
378
379	static void rsxx_issue_dmas(struct rsxx_dma_ctrl *ctrl)
380	{
381	struct rsxx_dma *dma;
382	int tag;
383	int cmds_pending = 0;
384	struct hw_cmd *hw_cmd_buf;
385	int dir;
386
387	hw_cmd_buf = ctrl->cmd.buf;
388
389	if (unlikely(ctrl->card->halt) \|\|
390	unlikely(ctrl->card->eeh_state))
391	return;
392
393	while (1) {
394	spin_lock_bh(&ctrl->queue_lock);
395	if (list_empty(&ctrl->queue)) {
396	spin_unlock_bh(&ctrl->queue_lock);
397	break;
398	}
399	spin_unlock_bh(&ctrl->queue_lock);
400
401	tag = pop_tracker(ctrl->trackers);
402	if (tag == -1)
403	break;
404
405	spin_lock_bh(&ctrl->queue_lock);
406	dma = list_entry(ctrl->queue.next, struct rsxx_dma, list);
407	list_del(&dma->list);
408	ctrl->stats.sw_q_depth--;
409	spin_unlock_bh(&ctrl->queue_lock);
410
411	/*
412	* This will catch any DMAs that slipped in right before the
413	* fault, but was queued after all the other DMAs were
414	* cancelled.
415	*/
416	if (unlikely(ctrl->card->dma_fault)) {
417	push_tracker(ctrl->trackers, tag);
418	rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
419	continue;
420	}
421
422	if (dma->cmd != HW_CMD_BLK_DISCARD) {
423	if (dma->cmd == HW_CMD_BLK_WRITE)
424	dir = DMA_TO_DEVICE;
425	else
426	dir = DMA_FROM_DEVICE;
427
428	/*
429	* The function dma_map_page is placed here because we
430	* can only, by design, issue up to 255 commands to the
431	* hardware at one time per DMA channel. So the maximum
432	* amount of mapped memory would be 255 * 4 channels *
433	* 4096 Bytes which is less than 2GB, the limit of a x8
434	* Non-HWWD PCIe slot. This way the dma_map_page
435	* function should never fail because of a lack of
436	* mappable memory.
437	*/
438	dma->dma_addr = dma_map_page(&ctrl->card->dev->dev, dma->page,
439	dma->pg_off, dma->sub_page.cnt << 9, dir);
440	if (dma_mapping_error(&ctrl->card->dev->dev, dma->dma_addr)) {
441	push_tracker(ctrl->trackers, tag);
442	rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
443	continue;
444	}
445	}
446
447	set_tracker_dma(ctrl->trackers, tag, dma);
448	hw_cmd_buf[ctrl->cmd.idx].command = dma->cmd;
449	hw_cmd_buf[ctrl->cmd.idx].tag = tag;
450	hw_cmd_buf[ctrl->cmd.idx]._rsvd = 0;
451	hw_cmd_buf[ctrl->cmd.idx].sub_page =
452	((dma->sub_page.cnt & 0x7) << 4) \|
453	(dma->sub_page.off & 0x7);
454
455	hw_cmd_buf[ctrl->cmd.idx].device_addr =
456	cpu_to_le32(dma->laddr);
457
458	hw_cmd_buf[ctrl->cmd.idx].host_addr =
459	cpu_to_le64(dma->dma_addr);
460
461	dev_dbg(CARD_TO_DEV(ctrl->card),
462	"Issue DMA%d(laddr %d tag %d) to idx %d\n",
463	ctrl->id, dma->laddr, tag, ctrl->cmd.idx);
464
465	ctrl->cmd.idx = (ctrl->cmd.idx + 1) & RSXX_CS_IDX_MASK;
466	cmds_pending++;
467
468	if (dma->cmd == HW_CMD_BLK_WRITE)
469	ctrl->stats.writes_issued++;
470	else if (dma->cmd == HW_CMD_BLK_DISCARD)
471	ctrl->stats.discards_issued++;
472	else
473	ctrl->stats.reads_issued++;
474	}
475
476	/* Let HW know we've queued commands. */
477	if (cmds_pending) {
478	atomic_add(cmds_pending, &ctrl->stats.hw_q_depth);
479	mod_timer(&ctrl->activity_timer,
480	jiffies + DMA_ACTIVITY_TIMEOUT);
481
482	if (unlikely(ctrl->card->eeh_state)) {
483	del_timer_sync(&ctrl->activity_timer);
484	return;
485	}
486
487	iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
488	}
489	}
490
491	static void rsxx_dma_done(struct rsxx_dma_ctrl *ctrl)
492	{
493	struct rsxx_dma *dma;
494	unsigned long flags;
495	u16 count;
496	u8 status;
497	u8 tag;
498	struct hw_status *hw_st_buf;
499
500	hw_st_buf = ctrl->status.buf;
501
502	if (unlikely(ctrl->card->halt) \|\|
503	unlikely(ctrl->card->dma_fault) \|\|
504	unlikely(ctrl->card->eeh_state))
505	return;
506
507	count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count);
508
509	while (count == ctrl->e_cnt) {
510	/*
511	* The read memory-barrier is necessary to keep aggressive
512	* processors/optimizers (such as the PPC Apple G5) from
513	* reordering the following status-buffer tag & status read
514	* before the count read on subsequent iterations of the
515	* loop!
516	*/
517	rmb();
518
519	status = hw_st_buf[ctrl->status.idx].status;
520	tag = hw_st_buf[ctrl->status.idx].tag;
521
522	dma = get_tracker_dma(ctrl->trackers, tag);
523	if (dma == NULL) {
524	spin_lock_irqsave(&ctrl->card->irq_lock, flags);
525	rsxx_disable_ier(ctrl->card, CR_INTR_DMA_ALL);
526	spin_unlock_irqrestore(&ctrl->card->irq_lock, flags);
527
528	dev_err(CARD_TO_DEV(ctrl->card),
529	"No tracker for tag %d "
530	"(idx %d id %d)\n",
531	tag, ctrl->status.idx, ctrl->id);
532	return;
533	}
534
535	dev_dbg(CARD_TO_DEV(ctrl->card),
536	"Completing DMA%d"
537	"(laddr x%x tag %d st: x%x cnt: x%04x) from idx %d.\n",
538	ctrl->id, dma->laddr, tag, status, count,
539	ctrl->status.idx);
540
541	atomic_dec(&ctrl->stats.hw_q_depth);
542
543	mod_timer(&ctrl->activity_timer,
544	jiffies + DMA_ACTIVITY_TIMEOUT);
545
546	if (status)
547	rsxx_handle_dma_error(ctrl, dma, status);
548	else
549	rsxx_complete_dma(ctrl, dma, 0);
550
551	push_tracker(ctrl->trackers, tag);
552
553	ctrl->status.idx = (ctrl->status.idx + 1) &
554	RSXX_CS_IDX_MASK;
555	ctrl->e_cnt++;
556
557	count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count);
558	}
559
560	dma_intr_coal_auto_tune(ctrl->card);
561
562	if (atomic_read(&ctrl->stats.hw_q_depth) == 0)
563	del_timer_sync(&ctrl->activity_timer);
564
565	spin_lock_irqsave(&ctrl->card->irq_lock, flags);
566	rsxx_enable_ier(ctrl->card, CR_INTR_DMA(ctrl->id));
567	spin_unlock_irqrestore(&ctrl->card->irq_lock, flags);
568
569	spin_lock_bh(&ctrl->queue_lock);
570	if (ctrl->stats.sw_q_depth)
571	queue_work(ctrl->issue_wq, &ctrl->issue_dma_work);
572	spin_unlock_bh(&ctrl->queue_lock);
573	}
574
575	static void rsxx_schedule_issue(struct work_struct *work)
576	{
577	struct rsxx_dma_ctrl *ctrl;
578
579	ctrl = container_of(work, struct rsxx_dma_ctrl, issue_dma_work);
580
581	mutex_lock(&ctrl->work_lock);
582	rsxx_issue_dmas(ctrl);
583	mutex_unlock(&ctrl->work_lock);
584	}
585
586	static void rsxx_schedule_done(struct work_struct *work)
587	{
588	struct rsxx_dma_ctrl *ctrl;
589
590	ctrl = container_of(work, struct rsxx_dma_ctrl, dma_done_work);
591
592	mutex_lock(&ctrl->work_lock);
593	rsxx_dma_done(ctrl);
594	mutex_unlock(&ctrl->work_lock);
595	}
596
597	static blk_status_t rsxx_queue_discard(struct rsxx_cardinfo *card,
598	struct list_head *q,
599	unsigned int laddr,
600	rsxx_dma_cb cb,
601	void *cb_data)
602	{
603	struct rsxx_dma *dma;
604
605	dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL);
606	if (!dma)
607	return BLK_STS_RESOURCE;
608
609	dma->cmd = HW_CMD_BLK_DISCARD;
610	dma->laddr = laddr;
611	dma->dma_addr = 0;
612	dma->sub_page.off = 0;
613	dma->sub_page.cnt = 0;
614	dma->page = NULL;
615	dma->pg_off = 0;
616	dma->cb = cb;
617	dma->cb_data = cb_data;
618
619	dev_dbg(CARD_TO_DEV(card), "Queuing[D] laddr %x\n", dma->laddr);
620
621	list_add_tail(&dma->list, q);
622
623	return 0;
624	}
625
626	static blk_status_t rsxx_queue_dma(struct rsxx_cardinfo *card,
627	struct list_head *q,
628	int dir,
629	unsigned int dma_off,
630	unsigned int dma_len,
631	unsigned int laddr,
632	struct page *page,
633	unsigned int pg_off,
634	rsxx_dma_cb cb,
635	void *cb_data)
636	{
637	struct rsxx_dma *dma;
638
639	dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL);
640	if (!dma)
641	return BLK_STS_RESOURCE;
642
643	dma->cmd = dir ? HW_CMD_BLK_WRITE : HW_CMD_BLK_READ;
644	dma->laddr = laddr;
645	dma->sub_page.off = (dma_off >> 9);
646	dma->sub_page.cnt = (dma_len >> 9);
647	dma->page = page;
648	dma->pg_off = pg_off;
649	dma->cb = cb;
650	dma->cb_data = cb_data;
651
652	dev_dbg(CARD_TO_DEV(card),
653	"Queuing[%c] laddr %x off %d cnt %d page %p pg_off %d\n",
654	dir ? 'W' : 'R', dma->laddr, dma->sub_page.off,
655	dma->sub_page.cnt, dma->page, dma->pg_off);
656
657	/* Queue the DMA */
658	list_add_tail(&dma->list, q);
659
660	return 0;
661	}
662
663	blk_status_t rsxx_dma_queue_bio(struct rsxx_cardinfo *card,
664	struct bio *bio,
665	atomic_t *n_dmas,
666	rsxx_dma_cb cb,
667	void *cb_data)
668	{
669	struct list_head dma_list[RSXX_MAX_TARGETS];
670	struct bio_vec bvec;
671	struct bvec_iter iter;
672	unsigned long long addr8;
673	unsigned int laddr;
674	unsigned int bv_len;
675	unsigned int bv_off;
676	unsigned int dma_off;
677	unsigned int dma_len;
678	int dma_cnt[RSXX_MAX_TARGETS];
679	int tgt;
680	blk_status_t st;
681	int i;
682
683	addr8 = bio->bi_iter.bi_sector << 9; /* sectors are 512 bytes */
684	atomic_set(n_dmas, 0);
685
686	for (i = 0; i < card->n_targets; i++) {
687	INIT_LIST_HEAD(&dma_list[i]);
688	dma_cnt[i] = 0;
689	}
690
691	if (bio_op(bio) == REQ_OP_DISCARD) {
692	bv_len = bio->bi_iter.bi_size;
693
694	while (bv_len > 0) {
695	tgt = rsxx_get_dma_tgt(card, addr8);
696	laddr = rsxx_addr8_to_laddr(addr8, card);
697
698	st = rsxx_queue_discard(card, &dma_list[tgt], laddr,
699	cb, cb_data);
700	if (st)
701	goto bvec_err;
702
703	dma_cnt[tgt]++;
704	atomic_inc(n_dmas);
705	addr8 += RSXX_HW_BLK_SIZE;
706	bv_len -= RSXX_HW_BLK_SIZE;
707	}
708	} else {
709	bio_for_each_segment(bvec, bio, iter) {
710	bv_len = bvec.bv_len;
711	bv_off = bvec.bv_offset;
712
713	while (bv_len > 0) {
714	tgt = rsxx_get_dma_tgt(card, addr8);
715	laddr = rsxx_addr8_to_laddr(addr8, card);
716	dma_off = addr8 & RSXX_HW_BLK_MASK;
717	dma_len = min(bv_len,
718	RSXX_HW_BLK_SIZE - dma_off);
719
720	st = rsxx_queue_dma(card, &dma_list[tgt],
721	bio_data_dir(bio),
722	dma_off, dma_len,
723	laddr, bvec.bv_page,
724	bv_off, cb, cb_data);
725	if (st)
726	goto bvec_err;
727
728	dma_cnt[tgt]++;
729	atomic_inc(n_dmas);
730	addr8 += dma_len;
731	bv_off += dma_len;
732	bv_len -= dma_len;
733	}
734	}
735	}
736
737	for (i = 0; i < card->n_targets; i++) {
738	if (!list_empty(&dma_list[i])) {
739	spin_lock_bh(&card->ctrl[i].queue_lock);
740	card->ctrl[i].stats.sw_q_depth += dma_cnt[i];
741	list_splice_tail(&dma_list[i], &card->ctrl[i].queue);
742	spin_unlock_bh(&card->ctrl[i].queue_lock);
743
744	queue_work(card->ctrl[i].issue_wq,
745	&card->ctrl[i].issue_dma_work);
746	}
747	}
748
749	return 0;
750
751	bvec_err:
752	for (i = 0; i < card->n_targets; i++)
753	rsxx_cleanup_dma_queue(&card->ctrl[i], &dma_list[i],
754	FREE_DMA);
755	return st;
756	}
757
758
759	/----------------- DMA Engine Initialization & Setup -------------------/
760	int rsxx_hw_buffers_init(struct pci_dev dev, struct rsxx_dma_ctrl ctrl)
761	{
762	ctrl->status.buf = dma_alloc_coherent(&dev->dev, STATUS_BUFFER_SIZE8,
763	&ctrl->status.dma_addr, GFP_KERNEL);
764	ctrl->cmd.buf = dma_alloc_coherent(&dev->dev, COMMAND_BUFFER_SIZE8,
765	&ctrl->cmd.dma_addr, GFP_KERNEL);
766	if (ctrl->status.buf == NULL \|\| ctrl->cmd.buf == NULL)
767	return -ENOMEM;
768
769	memset(ctrl->status.buf, 0xac, STATUS_BUFFER_SIZE8);
770	iowrite32(lower_32_bits(ctrl->status.dma_addr),
771	ctrl->regmap + SB_ADD_LO);
772	iowrite32(upper_32_bits(ctrl->status.dma_addr),
773	ctrl->regmap + SB_ADD_HI);
774
775	memset(ctrl->cmd.buf, 0x83, COMMAND_BUFFER_SIZE8);
776	iowrite32(lower_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_LO);
777	iowrite32(upper_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_HI);
778
779	ctrl->status.idx = ioread32(ctrl->regmap + HW_STATUS_CNT);
780	if (ctrl->status.idx > RSXX_MAX_OUTSTANDING_CMDS) {
781	dev_crit(&dev->dev, "Failed reading status cnt x%x\n",
782	ctrl->status.idx);
783	return -EINVAL;
784	}
785	iowrite32(ctrl->status.idx, ctrl->regmap + HW_STATUS_CNT);
786	iowrite32(ctrl->status.idx, ctrl->regmap + SW_STATUS_CNT);
787
788	ctrl->cmd.idx = ioread32(ctrl->regmap + HW_CMD_IDX);
789	if (ctrl->cmd.idx > RSXX_MAX_OUTSTANDING_CMDS) {
790	dev_crit(&dev->dev, "Failed reading cmd cnt x%x\n",
791	ctrl->status.idx);
792	return -EINVAL;
793	}
794	iowrite32(ctrl->cmd.idx, ctrl->regmap + HW_CMD_IDX);
795	iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
796
797	return 0;
798	}
799
800	static int rsxx_dma_ctrl_init(struct pci_dev *dev,
801	struct rsxx_dma_ctrl *ctrl)
802	{
803	int i;
804	int st;
805
806	memset(&ctrl->stats, 0, sizeof(ctrl->stats));
807
808	ctrl->trackers = vmalloc(struct_size(ctrl->trackers, list,
809	RSXX_MAX_OUTSTANDING_CMDS));
810	if (!ctrl->trackers)
811	return -ENOMEM;
812
813	ctrl->trackers->head = 0;
814	for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) {
815	ctrl->trackers->list[i].next_tag = i + 1;
816	ctrl->trackers->list[i].dma = NULL;
817	}
818	ctrl->trackers->list[RSXX_MAX_OUTSTANDING_CMDS-1].next_tag = -1;
819	spin_lock_init(&ctrl->trackers->lock);
820
821	spin_lock_init(&ctrl->queue_lock);
822	mutex_init(&ctrl->work_lock);
823	INIT_LIST_HEAD(&ctrl->queue);
824
825	timer_setup(&ctrl->activity_timer, dma_engine_stalled, 0);
826
827	ctrl->issue_wq = alloc_ordered_workqueue(DRIVER_NAME"_issue", 0);
828	if (!ctrl->issue_wq)
829	return -ENOMEM;
830
831	ctrl->done_wq = alloc_ordered_workqueue(DRIVER_NAME"_done", 0);
832	if (!ctrl->done_wq)
833	return -ENOMEM;
834
835	INIT_WORK(&ctrl->issue_dma_work, rsxx_schedule_issue);
836	INIT_WORK(&ctrl->dma_done_work, rsxx_schedule_done);
837
838	st = rsxx_hw_buffers_init(dev, ctrl);
839	if (st)
840	return st;
841
842	return 0;
843	}
844
845	static int rsxx_dma_stripe_setup(struct rsxx_cardinfo *card,
846	unsigned int stripe_size8)
847	{
848	if (!is_power_of_2(stripe_size8)) {
849	dev_err(CARD_TO_DEV(card),
850	"stripe_size is NOT a power of 2!\n");
851	return -EINVAL;
852	}
853
854	card->_stripe.lower_mask = stripe_size8 - 1;
855
856	card->_stripe.upper_mask = ~(card->_stripe.lower_mask);
857	card->_stripe.upper_shift = ffs(card->n_targets) - 1;
858
859	card->_stripe.target_mask = card->n_targets - 1;
860	card->_stripe.target_shift = ffs(stripe_size8) - 1;
861
862	dev_dbg(CARD_TO_DEV(card), "_stripe.lower_mask = x%016llx\n",
863	card->_stripe.lower_mask);
864	dev_dbg(CARD_TO_DEV(card), "_stripe.upper_shift = x%016llx\n",
865	card->_stripe.upper_shift);
866	dev_dbg(CARD_TO_DEV(card), "_stripe.upper_mask = x%016llx\n",
867	card->_stripe.upper_mask);
868	dev_dbg(CARD_TO_DEV(card), "_stripe.target_mask = x%016llx\n",
869	card->_stripe.target_mask);
870	dev_dbg(CARD_TO_DEV(card), "_stripe.target_shift = x%016llx\n",
871	card->_stripe.target_shift);
872
873	return 0;
874	}
875
876	int rsxx_dma_configure(struct rsxx_cardinfo *card)
877	{
878	u32 intr_coal;
879
880	intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode,
881	card->config.data.intr_coal.count,
882	card->config.data.intr_coal.latency);
883	iowrite32(intr_coal, card->regmap + INTR_COAL);
884
885	return rsxx_dma_stripe_setup(card, card->config.data.stripe_size);
886	}
887
888	int rsxx_dma_setup(struct rsxx_cardinfo *card)
889	{
890	unsigned long flags;
891	int st;
892	int i;
893
894	dev_info(CARD_TO_DEV(card),
895	"Initializing %d DMA targets\n",
896	card->n_targets);
897
898	/* Regmap is divided up into 4K chunks. One for each DMA channel */
899	for (i = 0; i < card->n_targets; i++)
900	card->ctrl[i].regmap = card->regmap + (i * 4096);
901
902	card->dma_fault = 0;
903
904	/* Reset the DMA queues */
905	rsxx_dma_queue_reset(card);
906
907	/*********** Setup DMA Control ***********/
908	for (i = 0; i < card->n_targets; i++) {
909	st = rsxx_dma_ctrl_init(card->dev, &card->ctrl[i]);
910	if (st)
911	goto failed_dma_setup;
912
913	card->ctrl[i].card = card;
914	card->ctrl[i].id = i;
915	}
916
917	card->scrub_hard = 1;
918
919	if (card->config_valid)
920	rsxx_dma_configure(card);
921
922	/* Enable the interrupts after all setup has completed. */
923	for (i = 0; i < card->n_targets; i++) {
924	spin_lock_irqsave(&card->irq_lock, flags);
925	rsxx_enable_ier_and_isr(card, CR_INTR_DMA(i));
926	spin_unlock_irqrestore(&card->irq_lock, flags);
927	}
928
929	return 0;
930
931	failed_dma_setup:
932	for (i = 0; i < card->n_targets; i++) {
933	struct rsxx_dma_ctrl *ctrl = &card->ctrl[i];
934
935	if (ctrl->issue_wq) {
936	destroy_workqueue(ctrl->issue_wq);
937	ctrl->issue_wq = NULL;
938	}
939
940	if (ctrl->done_wq) {
941	destroy_workqueue(ctrl->done_wq);
942	ctrl->done_wq = NULL;
943	}
944
945	vfree(ctrl->trackers);
946
947	if (ctrl->status.buf)
948	dma_free_coherent(&card->dev->dev, STATUS_BUFFER_SIZE8,
949	ctrl->status.buf,
950	ctrl->status.dma_addr);
951	if (ctrl->cmd.buf)
952	dma_free_coherent(&card->dev->dev, COMMAND_BUFFER_SIZE8,
953	ctrl->cmd.buf, ctrl->cmd.dma_addr);
954	}
955
956	return st;
957	}
958
959	int rsxx_dma_cancel(struct rsxx_dma_ctrl *ctrl)
960	{
961	struct rsxx_dma *dma;
962	int i;
963	int cnt = 0;
964
965	/* Clean up issued DMAs */
966	for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) {
967	dma = get_tracker_dma(ctrl->trackers, i);
968	if (dma) {
969	atomic_dec(&ctrl->stats.hw_q_depth);
970	rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
971	push_tracker(ctrl->trackers, i);
972	cnt++;
973	}
974	}
975
976	return cnt;
977	}
978
979	void rsxx_dma_destroy(struct rsxx_cardinfo *card)
980	{
981	struct rsxx_dma_ctrl *ctrl;
982	int i;
983
984	for (i = 0; i < card->n_targets; i++) {
985	ctrl = &card->ctrl[i];
986
987	if (ctrl->issue_wq) {
988	destroy_workqueue(ctrl->issue_wq);
989	ctrl->issue_wq = NULL;
990	}
991
992	if (ctrl->done_wq) {
993	destroy_workqueue(ctrl->done_wq);
994	ctrl->done_wq = NULL;
995	}
996
997	if (timer_pending(&ctrl->activity_timer))
998	del_timer_sync(&ctrl->activity_timer);
999
1000	/* Clean up the DMA queue */
1001	spin_lock_bh(&ctrl->queue_lock);
1002	rsxx_cleanup_dma_queue(ctrl, &ctrl->queue, COMPLETE_DMA);
1003	spin_unlock_bh(&ctrl->queue_lock);
1004
1005	rsxx_dma_cancel(ctrl);
1006
1007	vfree(ctrl->trackers);
1008
1009	dma_free_coherent(&card->dev->dev, STATUS_BUFFER_SIZE8,
1010	ctrl->status.buf, ctrl->status.dma_addr);
1011	dma_free_coherent(&card->dev->dev, COMMAND_BUFFER_SIZE8,
1012	ctrl->cmd.buf, ctrl->cmd.dma_addr);
1013	}
1014	}
1015
1016	int rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo *card)
1017	{
1018	int i;
1019	int j;
1020	int cnt;
1021	struct rsxx_dma *dma;
1022	struct list_head *issued_dmas;
1023
1024	issued_dmas = kcalloc(card->n_targets, sizeof(*issued_dmas),
1025	GFP_KERNEL);
1026	if (!issued_dmas)
1027	return -ENOMEM;
1028
1029	for (i = 0; i < card->n_targets; i++) {
1030	INIT_LIST_HEAD(&issued_dmas[i]);
1031	cnt = 0;
1032	for (j = 0; j < RSXX_MAX_OUTSTANDING_CMDS; j++) {
1033	dma = get_tracker_dma(card->ctrl[i].trackers, j);
1034	if (dma == NULL)
1035	continue;
1036
1037	if (dma->cmd == HW_CMD_BLK_WRITE)
1038	card->ctrl[i].stats.writes_issued--;
1039	else if (dma->cmd == HW_CMD_BLK_DISCARD)
1040	card->ctrl[i].stats.discards_issued--;
1041	else
1042	card->ctrl[i].stats.reads_issued--;
1043
1044	if (dma->cmd != HW_CMD_BLK_DISCARD) {
1045	dma_unmap_page(&card->dev->dev, dma->dma_addr,
1046	get_dma_size(dma),
1047	dma->cmd == HW_CMD_BLK_WRITE ?
1048	DMA_TO_DEVICE :
1049	DMA_FROM_DEVICE);
1050	}
1051
1052	list_add_tail(&dma->list, &issued_dmas[i]);
1053	push_tracker(card->ctrl[i].trackers, j);
1054	cnt++;
1055	}
1056
1057	spin_lock_bh(&card->ctrl[i].queue_lock);
1058	list_splice(&issued_dmas[i], &card->ctrl[i].queue);
1059
1060	atomic_sub(cnt, &card->ctrl[i].stats.hw_q_depth);
1061	card->ctrl[i].stats.sw_q_depth += cnt;
1062	card->ctrl[i].e_cnt = 0;
1063	spin_unlock_bh(&card->ctrl[i].queue_lock);
1064	}
1065
1066	kfree(issued_dmas);
1067
1068	return 0;
1069	}
1070
1071	int rsxx_dma_init(void)
1072	{
1073	rsxx_dma_pool = KMEM_CACHE(rsxx_dma, SLAB_HWCACHE_ALIGN);
1074	if (!rsxx_dma_pool)
1075	return -ENOMEM;
1076
1077	return 0;
1078	}
1079
1080
1081	void rsxx_dma_cleanup(void)
1082	{
1083	kmem_cache_destroy(rsxx_dma_pool);
1084	}
1085

drivers/block/rsxx/rsxx.h+0−33

@@ -1,33 +0,0 @@
1	/* SPDX-License-Identifier: GPL-2.0-or-later */
2	/*
3	* Filename: rsxx.h
4	*
5	* Authors: Joshua Morris <josh.h.morris@us.ibm.com>
6	* Philip Kelleher <pjk1939@linux.vnet.ibm.com>
7	*
8	* (C) Copyright 2013 IBM Corporation
9	*/
10
11	#ifndef __RSXX_H__
12	#define __RSXX_H__
13
14	/----------------- IOCTL Definitions -------------------/
15
16	#define RSXX_MAX_DATA 8
17
18	struct rsxx_reg_access {
19	__u32 addr;
20	__u32 cnt;
21	__u32 stat;
22	__u32 stream;
23	__u32 data[RSXX_MAX_DATA];
24	};
25
26	#define RSXX_MAX_REG_CNT (RSXX_MAX_DATA * (sizeof(__u32)))
27
28	#define RSXX_IOC_MAGIC 'r'
29
30	#define RSXX_GETREG _IOWR(RSXX_IOC_MAGIC, 0x20, struct rsxx_reg_access)
31	#define RSXX_SETREG _IOWR(RSXX_IOC_MAGIC, 0x21, struct rsxx_reg_access)
32
33	#endif /* __RSXX_H_ */

drivers/block/rsxx/rsxx_cfg.h+0−58

@@ -1,58 +0,0 @@
1	/* SPDX-License-Identifier: GPL-2.0-or-later */
2	/*
3	* Filename: rsXX_cfg.h
4	*
5	* Authors: Joshua Morris <josh.h.morris@us.ibm.com>
6	* Philip Kelleher <pjk1939@linux.vnet.ibm.com>
7	*
8	* (C) Copyright 2013 IBM Corporation
9	*/
10
11	#ifndef __RSXX_CFG_H__
12	#define __RSXX_CFG_H__
13
14	/* NOTE: Config values will be saved in network byte order (i.e. Big endian) */
15	#include <linux/types.h>
16
17	/*
18	* The card config version must match the driver's expected version. If it does
19	* not, the DMA interfaces will not be attached and the user will need to
20	* initialize/upgrade the card configuration using the card config utility.
21	*/
22	#define RSXX_CFG_VERSION 4
23
24	struct card_cfg_hdr {
25	__u32 version;
26	__u32 crc;
27	};
28
29	struct card_cfg_data {
30	__u32 block_size;
31	__u32 stripe_size;
32	__u32 vendor_id;
33	__u32 cache_order;
34	struct {
35	__u32 mode; /* Disabled, manual, auto-tune... */
36	__u32 count; /* Number of intr to coalesce */
37	__u32 latency;/* Max wait time (in ns) */
38	} intr_coal;
39	};
40
41	struct rsxx_card_cfg {
42	struct card_cfg_hdr hdr;
43	struct card_cfg_data data;
44	};
45
46	/* Vendor ID Values */
47	#define RSXX_VENDOR_ID_IBM 0
48	#define RSXX_VENDOR_ID_DSI 1
49	#define RSXX_VENDOR_COUNT 2
50
51	/* Interrupt Coalescing Values */
52	#define RSXX_INTR_COAL_DISABLED 0
53	#define RSXX_INTR_COAL_EXPLICIT 1
54	#define RSXX_INTR_COAL_AUTO_TUNE 2
55
56
57	#endif /* __RSXX_CFG_H__ */
58

drivers/block/rsxx/rsxx_priv.h+0−418

@@ -1,418 +0,0 @@
1	/* SPDX-License-Identifier: GPL-2.0-or-later */
2	/*
3	* Filename: rsxx_priv.h
4	*
5	* Authors: Joshua Morris <josh.h.morris@us.ibm.com>
6	* Philip Kelleher <pjk1939@linux.vnet.ibm.com>
7	*
8	* (C) Copyright 2013 IBM Corporation
9	*/
10
11	#ifndef __RSXX_PRIV_H__
12	#define __RSXX_PRIV_H__
13
14	#include <linux/semaphore.h>
15
16	#include <linux/fs.h>
17	#include <linux/interrupt.h>
18	#include <linux/mutex.h>
19	#include <linux/pci.h>
20	#include <linux/spinlock.h>
21	#include <linux/sysfs.h>
22	#include <linux/workqueue.h>
23	#include <linux/bio.h>
24	#include <linux/vmalloc.h>
25	#include <linux/timer.h>
26	#include <linux/ioctl.h>
27	#include <linux/delay.h>
28
29	#include "rsxx.h"
30	#include "rsxx_cfg.h"
31
32	struct proc_cmd;
33
34	#define PCI_DEVICE_ID_FS70_FLASH 0x04A9
35	#define PCI_DEVICE_ID_FS80_FLASH 0x04AA
36
37	#define RS70_PCI_REV_SUPPORTED 4
38
39	#define DRIVER_NAME "rsxx"
40	#define DRIVER_VERSION "4.0.3.2516"
41
42	/* Block size is 4096 */
43	#define RSXX_HW_BLK_SHIFT 12
44	#define RSXX_HW_BLK_SIZE (1 << RSXX_HW_BLK_SHIFT)
45	#define RSXX_HW_BLK_MASK (RSXX_HW_BLK_SIZE - 1)
46
47	#define MAX_CREG_DATA8 32
48	#define LOG_BUF_SIZE8 128
49
50	#define RSXX_MAX_OUTSTANDING_CMDS 255
51	#define RSXX_CS_IDX_MASK 0xff
52
53	#define STATUS_BUFFER_SIZE8 4096
54	#define COMMAND_BUFFER_SIZE8 4096
55
56	#define RSXX_MAX_TARGETS 8
57
58	struct dma_tracker_list;
59
60	/* DMA Command/Status Buffer structure */
61	struct rsxx_cs_buffer {
62	dma_addr_t dma_addr;
63	void *buf;
64	u32 idx;
65	};
66
67	struct rsxx_dma_stats {
68	u32 crc_errors;
69	u32 hard_errors;
70	u32 soft_errors;
71	u32 writes_issued;
72	u32 writes_failed;
73	u32 reads_issued;
74	u32 reads_failed;
75	u32 reads_retried;
76	u32 discards_issued;
77	u32 discards_failed;
78	u32 done_rescheduled;
79	u32 issue_rescheduled;
80	u32 dma_sw_err;
81	u32 dma_hw_fault;
82	u32 dma_cancelled;
83	u32 sw_q_depth; /* Number of DMAs on the SW queue. */
84	atomic_t hw_q_depth; /* Number of DMAs queued to HW. */
85	};
86
87	struct rsxx_dma_ctrl {
88	struct rsxx_cardinfo *card;
89	int id;
90	void __iomem *regmap;
91	struct rsxx_cs_buffer status;
92	struct rsxx_cs_buffer cmd;
93	u16 e_cnt;
94	spinlock_t queue_lock;
95	struct list_head queue;
96	struct workqueue_struct *issue_wq;
97	struct work_struct issue_dma_work;
98	struct workqueue_struct *done_wq;
99	struct work_struct dma_done_work;
100	struct timer_list activity_timer;
101	struct dma_tracker_list *trackers;
102	struct rsxx_dma_stats stats;
103	struct mutex work_lock;
104	};
105
106	struct rsxx_cardinfo {
107	struct pci_dev *dev;
108	unsigned int halt;
109	unsigned int eeh_state;
110
111	void __iomem *regmap;
112	spinlock_t irq_lock;
113	unsigned int isr_mask;
114	unsigned int ier_mask;
115
116	struct rsxx_card_cfg config;
117	int config_valid;
118
119	/* Embedded CPU Communication */
120	struct {
121	spinlock_t lock;
122	bool active;
123	struct creg_cmd *active_cmd;
124	struct workqueue_struct *creg_wq;
125	struct work_struct done_work;
126	struct list_head queue;
127	unsigned int q_depth;
128	/* Cache the creg status to prevent ioreads */
129	struct {
130	u32 stat;
131	u32 failed_cancel_timer;
132	u32 creg_timeout;
133	} creg_stats;
134	struct timer_list cmd_timer;
135	struct mutex reset_lock;
136	int reset;
137	} creg_ctrl;
138
139	struct {
140	char tmp[MAX_CREG_DATA8];
141	char buf[LOG_BUF_SIZE8]; /* terminated */
142	int buf_len;
143	} log;
144
145	struct workqueue_struct *event_wq;
146	struct work_struct event_work;
147	unsigned int state;
148	u64 size8;
149
150	/* Lock the device attach/detach function */
151	struct mutex dev_lock;
152
153	/* Block Device Variables */
154	bool bdev_attached;
155	int disk_id;
156	int major;
157	struct gendisk *gendisk;
158	struct {
159	/* Used to convert a byte address to a device address. */
160	u64 lower_mask;
161	u64 upper_shift;
162	u64 upper_mask;
163	u64 target_mask;
164	u64 target_shift;
165	} _stripe;
166	unsigned int dma_fault;
167
168	int scrub_hard;
169
170	int n_targets;
171	struct rsxx_dma_ctrl *ctrl;
172
173	struct dentry *debugfs_dir;
174	};
175
176	enum rsxx_pci_regmap {
177	HWID = 0x00, /* Hardware Identification Register */
178	SCRATCH = 0x04, /* Scratch/Debug Register */
179	RESET = 0x08, /* Reset Register */
180	ISR = 0x10, /* Interrupt Status Register */
181	IER = 0x14, /* Interrupt Enable Register */
182	IPR = 0x18, /* Interrupt Poll Register */
183	CB_ADD_LO = 0x20, /* Command Host Buffer Address [31:0] */
184	CB_ADD_HI = 0x24, /* Command Host Buffer Address [63:32]*/
185	HW_CMD_IDX = 0x28, /* Hardware Processed Command Index */
186	SW_CMD_IDX = 0x2C, /* Software Processed Command Index */
187	SB_ADD_LO = 0x30, /* Status Host Buffer Address [31:0] */
188	SB_ADD_HI = 0x34, /* Status Host Buffer Address [63:32] */
189	HW_STATUS_CNT = 0x38, /* Hardware Status Counter */
190	SW_STATUS_CNT = 0x3C, /* Deprecated */
191	CREG_CMD = 0x40, /* CPU Command Register */
192	CREG_ADD = 0x44, /* CPU Address Register */
193	CREG_CNT = 0x48, /* CPU Count Register */
194	CREG_STAT = 0x4C, /* CPU Status Register */
195	CREG_DATA0 = 0x50, /* CPU Data Registers */
196	CREG_DATA1 = 0x54,
197	CREG_DATA2 = 0x58,
198	CREG_DATA3 = 0x5C,
199	CREG_DATA4 = 0x60,
200	CREG_DATA5 = 0x64,
201	CREG_DATA6 = 0x68,
202	CREG_DATA7 = 0x6c,
203	INTR_COAL = 0x70, /* Interrupt Coalescing Register */
204	HW_ERROR = 0x74, /* Card Error Register */
205	PCI_DEBUG0 = 0x78, /* PCI Debug Registers */
206	PCI_DEBUG1 = 0x7C,
207	PCI_DEBUG2 = 0x80,
208	PCI_DEBUG3 = 0x84,
209	PCI_DEBUG4 = 0x88,
210	PCI_DEBUG5 = 0x8C,
211	PCI_DEBUG6 = 0x90,
212	PCI_DEBUG7 = 0x94,
213	PCI_POWER_THROTTLE = 0x98,
214	PERF_CTRL = 0x9c,
215	PERF_TIMER_LO = 0xa0,
216	PERF_TIMER_HI = 0xa4,
217	PERF_RD512_LO = 0xa8,
218	PERF_RD512_HI = 0xac,
219	PERF_WR512_LO = 0xb0,
220	PERF_WR512_HI = 0xb4,
221	PCI_RECONFIG = 0xb8,
222	};
223
224	enum rsxx_intr {
225	CR_INTR_DMA0 = 0x00000001,
226	CR_INTR_CREG = 0x00000002,
227	CR_INTR_DMA1 = 0x00000004,
228	CR_INTR_EVENT = 0x00000008,
229	CR_INTR_DMA2 = 0x00000010,
230	CR_INTR_DMA3 = 0x00000020,
231	CR_INTR_DMA4 = 0x00000040,
232	CR_INTR_DMA5 = 0x00000080,
233	CR_INTR_DMA6 = 0x00000100,
234	CR_INTR_DMA7 = 0x00000200,
235	CR_INTR_ALL_C = 0x0000003f,
236	CR_INTR_ALL_G = 0x000003ff,
237	CR_INTR_DMA_ALL = 0x000003f5,
238	CR_INTR_ALL = 0xffffffff,
239	};
240
241	static inline int CR_INTR_DMA(int N)
242	{
243	static const unsigned int _CR_INTR_DMA[] = {
244	CR_INTR_DMA0, CR_INTR_DMA1, CR_INTR_DMA2, CR_INTR_DMA3,
245	CR_INTR_DMA4, CR_INTR_DMA5, CR_INTR_DMA6, CR_INTR_DMA7
246	};
247	return _CR_INTR_DMA[N];
248	}
249	enum rsxx_pci_reset {
250	DMA_QUEUE_RESET = 0x00000001,
251	};
252
253	enum rsxx_hw_fifo_flush {
254	RSXX_FLUSH_BUSY = 0x00000002,
255	RSXX_FLUSH_TIMEOUT = 0x00000004,
256	};
257
258	enum rsxx_pci_revision {
259	RSXX_DISCARD_SUPPORT = 2,
260	RSXX_EEH_SUPPORT = 3,
261	};
262
263	enum rsxx_creg_cmd {
264	CREG_CMD_TAG_MASK = 0x0000FF00,
265	CREG_OP_WRITE = 0x000000C0,
266	CREG_OP_READ = 0x000000E0,
267	};
268
269	enum rsxx_creg_addr {
270	CREG_ADD_CARD_CMD = 0x80001000,
271	CREG_ADD_CARD_STATE = 0x80001004,
272	CREG_ADD_CARD_SIZE = 0x8000100c,
273	CREG_ADD_CAPABILITIES = 0x80001050,
274	CREG_ADD_LOG = 0x80002000,
275	CREG_ADD_NUM_TARGETS = 0x80003000,
276	CREG_ADD_CRAM = 0xA0000000,
277	CREG_ADD_CONFIG = 0xB0000000,
278	};
279
280	enum rsxx_creg_card_cmd {
281	CARD_CMD_STARTUP = 1,
282	CARD_CMD_SHUTDOWN = 2,
283	CARD_CMD_LOW_LEVEL_FORMAT = 3,
284	CARD_CMD_FPGA_RECONFIG_BR = 4,
285	CARD_CMD_FPGA_RECONFIG_MAIN = 5,
286	CARD_CMD_BACKUP = 6,
287	CARD_CMD_RESET = 7,
288	CARD_CMD_deprecated = 8,
289	CARD_CMD_UNINITIALIZE = 9,
290	CARD_CMD_DSTROY_EMERGENCY = 10,
291	CARD_CMD_DSTROY_NORMAL = 11,
292	CARD_CMD_DSTROY_EXTENDED = 12,
293	CARD_CMD_DSTROY_ABORT = 13,
294	};
295
296	enum rsxx_card_state {
297	CARD_STATE_SHUTDOWN = 0x00000001,
298	CARD_STATE_STARTING = 0x00000002,
299	CARD_STATE_FORMATTING = 0x00000004,
300	CARD_STATE_UNINITIALIZED = 0x00000008,
301	CARD_STATE_GOOD = 0x00000010,
302	CARD_STATE_SHUTTING_DOWN = 0x00000020,
303	CARD_STATE_FAULT = 0x00000040,
304	CARD_STATE_RD_ONLY_FAULT = 0x00000080,
305	CARD_STATE_DSTROYING = 0x00000100,
306	};
307
308	enum rsxx_led {
309	LED_DEFAULT = 0x0,
310	LED_IDENTIFY = 0x1,
311	LED_SOAK = 0x2,
312	};
313
314	enum rsxx_creg_flash_lock {
315	CREG_FLASH_LOCK = 1,
316	CREG_FLASH_UNLOCK = 2,
317	};
318
319	enum rsxx_card_capabilities {
320	CARD_CAP_SUBPAGE_WRITES = 0x00000080,
321	};
322
323	enum rsxx_creg_stat {
324	CREG_STAT_STATUS_MASK = 0x00000003,
325	CREG_STAT_SUCCESS = 0x1,
326	CREG_STAT_ERROR = 0x2,
327	CREG_STAT_CHAR_PENDING = 0x00000004, /* Character I/O pending bit */
328	CREG_STAT_LOG_PENDING = 0x00000008, /* HW log message pending bit */
329	CREG_STAT_TAG_MASK = 0x0000ff00,
330	};
331
332	enum rsxx_dma_finish {
333	FREE_DMA = 0x0,
334	COMPLETE_DMA = 0x1,
335	};
336
337	static inline unsigned int CREG_DATA(int N)
338	{
339	return CREG_DATA0 + (N << 2);
340	}
341
342	/----------------- Convenient Log Wrappers -------------------/
343	#define CARD_TO_DEV(__CARD) (&(__CARD)->dev->dev)
344
345	/*** config.c ***/
346	int rsxx_load_config(struct rsxx_cardinfo *card);
347
348	/*** core.c ***/
349	void rsxx_enable_ier(struct rsxx_cardinfo *card, unsigned int intr);
350	void rsxx_disable_ier(struct rsxx_cardinfo *card, unsigned int intr);
351	void rsxx_enable_ier_and_isr(struct rsxx_cardinfo *card,
352	unsigned int intr);
353	void rsxx_disable_ier_and_isr(struct rsxx_cardinfo *card,
354	unsigned int intr);
355
356	/*** dev.c ***/
357	int rsxx_attach_dev(struct rsxx_cardinfo *card);
358	void rsxx_detach_dev(struct rsxx_cardinfo *card);
359	int rsxx_setup_dev(struct rsxx_cardinfo *card);
360	void rsxx_destroy_dev(struct rsxx_cardinfo *card);
361	int rsxx_dev_init(void);
362	void rsxx_dev_cleanup(void);
363
364	/*** dma.c **/
365	typedef void (rsxx_dma_cb)(struct rsxx_cardinfo card,
366	void *cb_data,
367	unsigned int status);
368	int rsxx_dma_setup(struct rsxx_cardinfo *card);
369	void rsxx_dma_destroy(struct rsxx_cardinfo *card);
370	int rsxx_dma_init(void);
371	int rsxx_cleanup_dma_queue(struct rsxx_dma_ctrl *ctrl,
372	struct list_head *q,
373	unsigned int done);
374	int rsxx_dma_cancel(struct rsxx_dma_ctrl *ctrl);
375	void rsxx_dma_cleanup(void);
376	void rsxx_dma_queue_reset(struct rsxx_cardinfo *card);
377	int rsxx_dma_configure(struct rsxx_cardinfo *card);
378	blk_status_t rsxx_dma_queue_bio(struct rsxx_cardinfo *card,
379	struct bio *bio,
380	atomic_t *n_dmas,
381	rsxx_dma_cb cb,
382	void *cb_data);
383	int rsxx_hw_buffers_init(struct pci_dev dev, struct rsxx_dma_ctrl ctrl);
384	int rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo *card);
385	int rsxx_eeh_remap_dmas(struct rsxx_cardinfo *card);
386
387	/*** cregs.c ***/
388	int rsxx_creg_write(struct rsxx_cardinfo *card, u32 addr,
389	unsigned int size8,
390	void *data,
391	int byte_stream);
392	int rsxx_creg_read(struct rsxx_cardinfo *card,
393	u32 addr,
394	unsigned int size8,
395	void *data,
396	int byte_stream);
397	int rsxx_read_hw_log(struct rsxx_cardinfo *card);
398	int rsxx_get_card_state(struct rsxx_cardinfo *card,
399	unsigned int *state);
400	int rsxx_get_card_size8(struct rsxx_cardinfo card, u64 size8);
401	int rsxx_get_num_targets(struct rsxx_cardinfo *card,
402	unsigned int *n_targets);
403	int rsxx_get_card_capabilities(struct rsxx_cardinfo *card,
404	u32 *capabilities);
405	int rsxx_issue_card_cmd(struct rsxx_cardinfo *card, u32 cmd);
406	int rsxx_creg_setup(struct rsxx_cardinfo *card);
407	void rsxx_creg_destroy(struct rsxx_cardinfo *card);
408	int rsxx_creg_init(void);
409	void rsxx_creg_cleanup(void);
410	int rsxx_reg_access(struct rsxx_cardinfo *card,
411	struct rsxx_reg_access __user *ucmd,
412	int read);
413	void rsxx_eeh_save_issued_creg(struct rsxx_cardinfo *card);
414	void rsxx_kick_creg_queue(struct rsxx_cardinfo *card);
415
416
417
418	#endif /* __DRIVERS_BLOCK_RSXX_H__ */

drivers/md/md-cluster.c+1−1

@@ -574,7 +574,7 @@ static int process_recvd_msg(struct mddev mddev, struct cluster_msg msg) · 573 unmodified lines
574	int ret = 0;	574	int ret = 0;
575		575
576	if (WARN(mddev->cluster_info->slot_number - 1 == le32_to_cpu(msg->slot),	576	if (WARN(mddev->cluster_info->slot_number - 1 == le32_to_cpu(msg->slot),
577	"node %d received it's own msg\n", le32_to_cpu(msg->slot)))	577	"node %d received its own msg\n", le32_to_cpu(msg->slot)))
578	return -1;	578	return -1;
579	switch (le32_to_cpu(msg->type)) {	579	switch (le32_to_cpu(msg->type)) {
580	case METADATA_UPDATED:	580	case METADATA_UPDATED:

drivers/md/md.c+41−12

 	rcu_read_lock();
 	if (is_suspended(mddev, bio)) {
 		DEFINE_WAIT(__wait);
+		/* Bail out if REQ_NOWAIT is set for the bio */
+		if (bio->bi_opf & REQ_NOWAIT) {
+			rcu_read_unlock();
+			bio_wouldblock_error(bio);
+			return;
+		}
 		for (;;) {
 			prepare_to_wait(&mddev->sb_wait, &__wait,
 					TASK_UNINTERRUPTIBLE);
 	&rdev_ppl_size.attr,
 	NULL,
 };
+ATTRIBUTE_GROUPS(rdev_default);
 static ssize_t
 rdev_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
 {
 static struct kobj_type rdev_ktype = {
 	.release	= rdev_free,
 	.sysfs_ops	= &rdev_sysfs_ops,
-	.default_attrs	= rdev_default_attrs,
+	.default_groups	= rdev_default_groups,
 };
 int md_rdev_init(struct md_rdev *rdev)
 	int err;
 	struct md_rdev *rdev;
 	struct md_personality *pers;
+	bool nowait = true;
 	if (list_empty(&mddev->disks))
 		/* cannot run an array with no devices.. */
 			}
 		}
 		sysfs_notify_dirent_safe(rdev->sysfs_state);
+		nowait = nowait && blk_queue_nowait(bdev_get_queue(rdev->bdev));
 	}
+	/* Set the NOWAIT flags if all underlying devices support it */
+	if (nowait)
+		blk_queue_flag_set(QUEUE_FLAG_NOWAIT, mddev->queue);
 	if (!bioset_initialized(&mddev->bio_set)) {
 		err = bioset_init(&mddev->bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
 		if (err)
 		if (err)
 			goto exit_bio_set;
 	}
-	if (mddev->level != 1 && mddev->level != 10 &&
-	    !bioset_initialized(&mddev->io_acct_set)) {
-		err = bioset_init(&mddev->io_acct_set, BIO_POOL_SIZE,
-				  offsetof(struct md_io_acct, bio_clone), 0);
-		if (err)
-			goto exit_sync_set;
-	}
 	spin_lock(&pers_lock);
 	pers = find_pers(mddev->level, mddev->clevel);
 	module_put(pers->owner);
 	md_bitmap_destroy(mddev);
 abort:
-	if (mddev->level != 1 && mddev->level != 10)
-		bioset_exit(&mddev->io_acct_set);
-exit_sync_set:
 	bioset_exit(&mddev->sync_set);
 exit_bio_set:
 	bioset_exit(&mddev->bio_set);
 	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
 	if (!mddev->thread)
 		md_update_sb(mddev, 1);
+	/*
+	 * If the new disk does not support REQ_NOWAIT,
+	 * disable on the whole MD.
+	 */
+	if (!blk_queue_nowait(bdev_get_queue(rdev->bdev))) {
+		pr_info("%s: Disabling nowait because %s does not support nowait\n",
+			mdname(mddev), bdevname(rdev->bdev, b));
+		blk_queue_flag_clear(QUEUE_FLAG_NOWAIT, mddev->queue);
+	}
 	/*
 	 * Kick recovery, maybe this spare has to be added to the
 	 * array immediately.
 	spin_lock(&pers_lock);
 	/* ensure module won't be unloaded */
 	if (!md_cluster_ops || !try_module_get(md_cluster_mod)) {
-		pr_warn("can't find md-cluster module or get it's reference.\n");
+		pr_warn("can't find md-cluster module or get its reference.\n");
 		spin_unlock(&pers_lock);
 		return -ENOENT;
 	}
 }
 EXPORT_SYMBOL_GPL(md_submit_discard_bio);
+int acct_bioset_init(struct mddev *mddev)
+{
+	int err = 0;
+	if (!bioset_initialized(&mddev->io_acct_set))
+		err = bioset_init(&mddev->io_acct_set, BIO_POOL_SIZE,
+			offsetof(struct md_io_acct, bio_clone), 0);
+	return err;
+}
+EXPORT_SYMBOL_GPL(acct_bioset_init);
+void acct_bioset_exit(struct mddev *mddev)
+{
+	bioset_exit(&mddev->io_acct_set);
+}
+EXPORT_SYMBOL_GPL(acct_bioset_exit);
 static void md_end_io_acct(struct bio *bio)
 {
 	struct md_io_acct *md_io_acct = bio->bi_private;

drivers/md/md.h+2−0

@@ -721,6 +721,8 @@ extern void md_error(struct mddev mddev, struct md_rdev rdev); · 720 unmodified lines
721	extern void md_finish_reshape(struct mddev *mddev);	721	extern void md_finish_reshape(struct mddev *mddev);
722	void md_submit_discard_bio(struct mddev mddev, struct md_rdev rdev,	722	void md_submit_discard_bio(struct mddev mddev, struct md_rdev rdev,
723	struct bio *bio, sector_t start, sector_t size);	723	struct bio *bio, sector_t start, sector_t size);
		724	int acct_bioset_init(struct mddev *mddev);
		725	void acct_bioset_exit(struct mddev *mddev);
724	void md_account_bio(struct mddev mddev, struct bio *bio);	726	void md_account_bio(struct mddev mddev, struct bio *bio);
725		727
726	extern bool __must_check md_flush_request(struct mddev mddev, struct bio bio);	728	extern bool __must_check md_flush_request(struct mddev mddev, struct bio bio);

drivers/md/raid0.c+28−10

@@ -356,7 +356,21 @@ static sector_t raid0_size(struct mddev *mddev, sector_t sectors, int raid_disks · 355 unmodified lines
356	return array_sectors;	356	return array_sectors;
357	}	357	}
358		358
359	static void raid0_free(struct mddev mddev, void priv);	359	static void free_conf(struct mddev mddev, struct r0conf conf)
		360	{
		361	kfree(conf->strip_zone);
		362	kfree(conf->devlist);
		363	kfree(conf);
		364	mddev->private = NULL;
		365	}
		366
		367	static void raid0_free(struct mddev mddev, void priv)
		368	{
		369	struct r0conf *conf = priv;
		370
		371	free_conf(mddev, conf);
		372	acct_bioset_exit(mddev);
		373	}
360		374
361	static int raid0_run(struct mddev *mddev)	375	static int raid0_run(struct mddev *mddev)
362	{	376	{
@@ -370,11 +384,16 @@ static int raid0_run(struct mddev *mddev) · 7 unmodified lines
370	if (md_check_no_bitmap(mddev))	384	if (md_check_no_bitmap(mddev))
371	return -EINVAL;	385	return -EINVAL;
372		386
		387	if (acct_bioset_init(mddev)) {
		388	pr_err("md/raid0:%s: alloc acct bioset failed.\n", mdname(mddev));
		389	return -ENOMEM;
		390	}
		391
373	/* if private is not null, we are here after takeover */	392	/* if private is not null, we are here after takeover */
374	if (mddev->private == NULL) {	393	if (mddev->private == NULL) {
375	ret = create_strip_zones(mddev, &conf);	394	ret = create_strip_zones(mddev, &conf);
376	if (ret < 0)	395	if (ret < 0)
377	return ret;	396	goto exit_acct_set;
378	mddev->private = conf;	397	mddev->private = conf;
379	}	398	}
380	conf = mddev->private;	399	conf = mddev->private;
@@ -413,17 +432,16 @@ static int raid0_run(struct mddev *mddev) · 32 unmodified lines
413	dump_zones(mddev);	432	dump_zones(mddev);
414		433
415	ret = md_integrity_register(mddev);	434	ret = md_integrity_register(mddev);
		435	if (ret)
		436	goto free;
416		437
417	return ret;	438	return ret;
418	}
419		439
420	static void raid0_free(struct mddev mddev, void priv)	440	free:
421	{	441	free_conf(mddev, conf);
422	struct r0conf *conf = priv;	442	exit_acct_set:
423		443	acct_bioset_exit(mddev);
424	kfree(conf->strip_zone);	444	return ret;
425	kfree(conf->devlist);
426	kfree(conf);
427	}	445	}
428		446
429	static void raid0_handle_discard(struct mddev mddev, struct bio bio)	447	static void raid0_handle_discard(struct mddev mddev, struct bio bio)

drivers/md/raid1-10.c+0−6

 #define BIO_SPECIAL(bio) ((unsigned long)bio <= 2)
-/* When there are this many requests queue to be written by
- * the raid thread, we become 'congested' to provide back-pressure
- * for writeback.
- */
-static int max_queued_requests = 1024;
 /* for managing resync I/O pages */
 struct resync_pages {
 	void		*raid_bio;

drivers/md/raid1.c+56−27

@@ -929,8 +929,10 @@ static void lower_barrier(struct r1conf *conf, sector_t sector_nr) · 928 unmodified lines
929	wake_up(&conf->wait_barrier);	929	wake_up(&conf->wait_barrier);
930	}	930	}
931		931
932	static void _wait_barrier(struct r1conf *conf, int idx)	932	static bool _wait_barrier(struct r1conf *conf, int idx, bool nowait)
933	{	933	{
		934	bool ret = true;
		935
934	/*	936	/*
935	* We need to increase conf->nr_pending[idx] very early here,	937	* We need to increase conf->nr_pending[idx] very early here,
936	* then raise_barrier() can be blocked when it waits for	938	* then raise_barrier() can be blocked when it waits for
@@ -961,7 +963,7 @@ static void _wait_barrier(struct r1conf *conf, int idx) · 24 unmodified lines
961	*/	963	*/
962	if (!READ_ONCE(conf->array_frozen) &&	964	if (!READ_ONCE(conf->array_frozen) &&
963	!atomic_read(&conf->barrier[idx]))	965	!atomic_read(&conf->barrier[idx]))
964	return;	966	return ret;
965		967
966	/*	968	/*
967	* After holding conf->resync_lock, conf->nr_pending[idx]	969	* After holding conf->resync_lock, conf->nr_pending[idx]
@@ -979,18 +981,27 @@ static void _wait_barrier(struct r1conf *conf, int idx) · 11 unmodified lines
979	*/	981	*/
980	wake_up(&conf->wait_barrier);	982	wake_up(&conf->wait_barrier);
981	/* Wait for the barrier in same barrier unit bucket to drop. */	983	/* Wait for the barrier in same barrier unit bucket to drop. */
982	wait_event_lock_irq(conf->wait_barrier,	984
983	!conf->array_frozen &&	985	/* Return false when nowait flag is set */
984	!atomic_read(&conf->barrier[idx]),	986	if (nowait) {
985	conf->resync_lock);	987	ret = false;
986	atomic_inc(&conf->nr_pending[idx]);	988	} else {
		989	wait_event_lock_irq(conf->wait_barrier,
		990	!conf->array_frozen &&
		991	!atomic_read(&conf->barrier[idx]),
		992	conf->resync_lock);
		993	atomic_inc(&conf->nr_pending[idx]);
		994	}
		995
987	atomic_dec(&conf->nr_waiting[idx]);	996	atomic_dec(&conf->nr_waiting[idx]);
988	spin_unlock_irq(&conf->resync_lock);	997	spin_unlock_irq(&conf->resync_lock);
		998	return ret;
989	}	999	}
990		1000
991	static void wait_read_barrier(struct r1conf *conf, sector_t sector_nr)	1001	static bool wait_read_barrier(struct r1conf *conf, sector_t sector_nr, bool nowait)
992	{	1002	{
993	int idx = sector_to_idx(sector_nr);	1003	int idx = sector_to_idx(sector_nr);
		1004	bool ret = true;
994		1005
995	/*	1006	/*
996	* Very similar to _wait_barrier(). The difference is, for read	1007	* Very similar to _wait_barrier(). The difference is, for read
@@ -1002,7 +1013,7 @@ static void wait_read_barrier(struct r1conf *conf, sector_t sector_nr) · 5 unmodified lines
1002	atomic_inc(&conf->nr_pending[idx]);	1013	atomic_inc(&conf->nr_pending[idx]);
1003		1014
1004	if (!READ_ONCE(conf->array_frozen))	1015	if (!READ_ONCE(conf->array_frozen))
1005	return;	1016	return ret;
1006		1017
1007	spin_lock_irq(&conf->resync_lock);	1018	spin_lock_irq(&conf->resync_lock);
1008	atomic_inc(&conf->nr_waiting[idx]);	1019	atomic_inc(&conf->nr_waiting[idx]);
@@ -1013,19 +1024,28 @@ static void wait_read_barrier(struct r1conf *conf, sector_t sector_nr) · 4 unmodified lines
1013	*/	1024	*/
1014	wake_up(&conf->wait_barrier);	1025	wake_up(&conf->wait_barrier);
1015	/* Wait for array to be unfrozen */	1026	/* Wait for array to be unfrozen */
1016	wait_event_lock_irq(conf->wait_barrier,	1027
1017	!conf->array_frozen,	1028	/* Return false when nowait flag is set */
1018	conf->resync_lock);	1029	if (nowait) {
1019	atomic_inc(&conf->nr_pending[idx]);	1030	/* Return false when nowait flag is set */
		1031	ret = false;
		1032	} else {
		1033	wait_event_lock_irq(conf->wait_barrier,
		1034	!conf->array_frozen,
		1035	conf->resync_lock);
		1036	atomic_inc(&conf->nr_pending[idx]);
		1037	}
		1038
1020	atomic_dec(&conf->nr_waiting[idx]);	1039	atomic_dec(&conf->nr_waiting[idx]);
1021	spin_unlock_irq(&conf->resync_lock);	1040	spin_unlock_irq(&conf->resync_lock);
		1041	return ret;
1022	}	1042	}
1023		1043
1024	static void wait_barrier(struct r1conf *conf, sector_t sector_nr)	1044	static bool wait_barrier(struct r1conf *conf, sector_t sector_nr, bool nowait)
1025	{	1045	{
1026	int idx = sector_to_idx(sector_nr);	1046	int idx = sector_to_idx(sector_nr);
1027		1047
1028	_wait_barrier(conf, idx);	1048	return _wait_barrier(conf, idx, nowait);
1029	}	1049	}
1030		1050
1031	static void _allow_barrier(struct r1conf *conf, int idx)	1051	static void _allow_barrier(struct r1conf *conf, int idx)
@@ -1236,7 +1256,11 @@ static void raid1_read_request(struct mddev mddev, struct bio bio, · 204 unmodified lines
1236	* Still need barrier for READ in case that whole	1256	* Still need barrier for READ in case that whole
1237	* array is frozen.	1257	* array is frozen.
1238	*/	1258	*/
1239	wait_read_barrier(conf, bio->bi_iter.bi_sector);	1259	if (!wait_read_barrier(conf, bio->bi_iter.bi_sector,
		1260	bio->bi_opf & REQ_NOWAIT)) {
		1261	bio_wouldblock_error(bio);
		1262	return;
		1263	}
1240		1264
1241	if (!r1_bio)	1265	if (!r1_bio)
1242	r1_bio = alloc_r1bio(mddev, bio);	1266	r1_bio = alloc_r1bio(mddev, bio);
@@ -1336,6 +1360,10 @@ static void raid1_write_request(struct mddev mddev, struct bio bio, · 93 unmodified lines
1336	bio->bi_iter.bi_sector, bio_end_sector(bio))) {	1360	bio->bi_iter.bi_sector, bio_end_sector(bio))) {
1337		1361
1338	DEFINE_WAIT(w);	1362	DEFINE_WAIT(w);
		1363	if (bio->bi_opf & REQ_NOWAIT) {
		1364	bio_wouldblock_error(bio);
		1365	return;
		1366	}
1339	for (;;) {	1367	for (;;) {
1340	prepare_to_wait(&conf->wait_barrier,	1368	prepare_to_wait(&conf->wait_barrier,
1341	&w, TASK_IDLE);	1369	&w, TASK_IDLE);
@@ -1353,17 +1381,15 @@ static void raid1_write_request(struct mddev mddev, struct bio bio, · 11 unmodified lines
1353	* thread has put up a bar for new requests.	1381	* thread has put up a bar for new requests.
1354	* Continue immediately if no resync is active currently.	1382	* Continue immediately if no resync is active currently.
1355	*/	1383	*/
1356	wait_barrier(conf, bio->bi_iter.bi_sector);	1384	if (!wait_barrier(conf, bio->bi_iter.bi_sector,
		1385	bio->bi_opf & REQ_NOWAIT)) {
		1386	bio_wouldblock_error(bio);
		1387	return;
		1388	}
1357		1389
1358	r1_bio = alloc_r1bio(mddev, bio);	1390	r1_bio = alloc_r1bio(mddev, bio);
1359	r1_bio->sectors = max_write_sectors;	1391	r1_bio->sectors = max_write_sectors;
1360		1392
1361	if (conf->pending_count >= max_queued_requests) {
1362	md_wakeup_thread(mddev->thread);
1363	raid1_log(mddev, "wait queued");
1364	wait_event(conf->wait_barrier,
1365	conf->pending_count < max_queued_requests);
1366	}
1367	/* first select target devices under rcu_lock and	1393	/* first select target devices under rcu_lock and
1368	* inc refcount on their rdev. Record them by setting	1394	* inc refcount on their rdev. Record them by setting
1369	* bios[x] to bio	1395	* bios[x] to bio
@@ -1458,9 +1484,14 @@ static void raid1_write_request(struct mddev mddev, struct bio bio, · 88 unmodified lines
1458	rdev_dec_pending(conf->mirrors[j].rdev, mddev);	1484	rdev_dec_pending(conf->mirrors[j].rdev, mddev);
1459	r1_bio->state = 0;	1485	r1_bio->state = 0;
1460	allow_barrier(conf, bio->bi_iter.bi_sector);	1486	allow_barrier(conf, bio->bi_iter.bi_sector);
		1487
		1488	if (bio->bi_opf & REQ_NOWAIT) {
		1489	bio_wouldblock_error(bio);
		1490	return;
		1491	}
1461	raid1_log(mddev, "wait rdev %d blocked", blocked_rdev->raid_disk);	1492	raid1_log(mddev, "wait rdev %d blocked", blocked_rdev->raid_disk);
1462	md_wait_for_blocked_rdev(blocked_rdev, mddev);	1493	md_wait_for_blocked_rdev(blocked_rdev, mddev);
1463	wait_barrier(conf, bio->bi_iter.bi_sector);	1494	wait_barrier(conf, bio->bi_iter.bi_sector, false);
1464	goto retry_write;	1495	goto retry_write;
1465	}	1496	}
1466		1497
@@ -1688,7 +1719,7 @@ static void close_sync(struct r1conf *conf) · 221 unmodified lines
1688	int idx;	1719	int idx;
1689		1720
1690	for (idx = 0; idx < BARRIER_BUCKETS_NR; idx++) {	1721	for (idx = 0; idx < BARRIER_BUCKETS_NR; idx++) {
1691	_wait_barrier(conf, idx);	1722	_wait_barrier(conf, idx, false);
1692	_allow_barrier(conf, idx);	1723	_allow_barrier(conf, idx);
1693	}	1724	}
1694		1725
@@ -3410,5 +3441,3 @@ MODULE_DESCRIPTION("RAID1 (mirroring) personality for MD"); · 1715 unmodified lines
3410	MODULE_ALIAS("md-personality-3"); /* RAID1 */	3441	MODULE_ALIAS("md-personality-3"); /* RAID1 */
3411	MODULE_ALIAS("md-raid1");	3442	MODULE_ALIAS("md-raid1");
3412	MODULE_ALIAS("md-level-1");	3443	MODULE_ALIAS("md-level-1");
3413
3414	module_param(max_queued_requests, int, S_IRUGO\|S_IWUSR);

drivers/md/raid10.c+67−40

 	wake_up(&conf->wait_barrier);
 }
-static void wait_barrier(struct r10conf *conf)
+static bool wait_barrier(struct r10conf *conf, bool nowait)
 {
+	bool ret = true;
 	spin_lock_irq(&conf->resync_lock);
 	if (conf->barrier) {
 		struct bio_list *bio_list = current->bio_list;
 		 * that queue to get the nr_pending
 		 * count down.
 		 */
-		raid10_log(conf->mddev, "wait barrier");
+		/* Return false when nowait flag is set */
-		wait_event_lock_irq(conf->wait_barrier,
+		if (nowait) {
-				    !conf->barrier ||
+			ret = false;
-				    (atomic_read(&conf->nr_pending) &&
+		} else {
-				     bio_list &&
+			raid10_log(conf->mddev, "wait barrier");
-				     (!bio_list_empty(&bio_list[0]) ||
+			wait_event_lock_irq(conf->wait_barrier,
-				      !bio_list_empty(&bio_list[1]))) ||
+					    !conf->barrier ||
-				     /* move on if recovery thread is
+					    (atomic_read(&conf->nr_pending) &&
-				      * blocked by us
+					     bio_list &&
-				      */
+					     (!bio_list_empty(&bio_list[0]) ||
-				     (conf->mddev->thread->tsk == current &&
+					      !bio_list_empty(&bio_list[1]))) ||
-				      test_bit(MD_RECOVERY_RUNNING,
+					     /* move on if recovery thread is
-					       &conf->mddev->recovery) &&
+					      * blocked by us
-				      conf->nr_queued > 0),
+					      */
-				    conf->resync_lock);
+					     (conf->mddev->thread->tsk == current &&
+					      test_bit(MD_RECOVERY_RUNNING,
+						       &conf->mddev->recovery) &&
+					      conf->nr_queued > 0),
+					    conf->resync_lock);
+		}
 		conf->nr_waiting--;
 		if (!conf->nr_waiting)
 			wake_up(&conf->wait_barrier);
 	}
-	atomic_inc(&conf->nr_pending);
+	/* Only increment nr_pending when we wait */
+	if (ret)
+		atomic_inc(&conf->nr_pending);
 	spin_unlock_irq(&conf->resync_lock);
+	return ret;
 }
 static void allow_barrier(struct r10conf *conf)
  * currently.
  * 2. If IO spans the reshape position.  Need to wait for reshape to pass.
  */
-static void regular_request_wait(struct mddev *mddev, struct r10conf *conf,
+static bool regular_request_wait(struct mddev *mddev, struct r10conf *conf,
 				 struct bio *bio, sector_t sectors)
 {
-	wait_barrier(conf);
+	/* Bail out if REQ_NOWAIT is set for the bio */
+	if (!wait_barrier(conf, bio->bi_opf & REQ_NOWAIT)) {
+		bio_wouldblock_error(bio);
+		return false;
+	}
 	while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
 	    bio->bi_iter.bi_sector < conf->reshape_progress &&
 	    bio->bi_iter.bi_sector + sectors > conf->reshape_progress) {
-		raid10_log(conf->mddev, "wait reshape");
 		allow_barrier(conf);
+		if (bio->bi_opf & REQ_NOWAIT) {
+			bio_wouldblock_error(bio);
+			return false;
+		}
+		raid10_log(conf->mddev, "wait reshape");
 		wait_event(conf->wait_barrier,
 			   conf->reshape_progress <= bio->bi_iter.bi_sector ||
 			   conf->reshape_progress >= bio->bi_iter.bi_sector +
 			   sectors);
-		wait_barrier(conf);
+		wait_barrier(conf, false);
 	}
+	return true;
 }
 static void raid10_read_request(struct mddev *mddev, struct bio *bio,
 		rcu_read_unlock();
 	}
-	regular_request_wait(mddev, conf, bio, r10_bio->sectors);
+	if (!regular_request_wait(mddev, conf, bio, r10_bio->sectors))
+		return;
 	rdev = read_balance(conf, r10_bio, &max_sectors);
 	if (!rdev) {
 		if (err_rdev) {
 		bio_chain(split, bio);
 		allow_barrier(conf);
 		submit_bio_noacct(bio);
-		wait_barrier(conf);
+		wait_barrier(conf, false);
 		bio = split;
 		r10_bio->master_bio = bio;
 		r10_bio->sectors = max_sectors;
 		raid10_log(conf->mddev, "%s wait rdev %d blocked",
 				__func__, blocked_rdev->raid_disk);
 		md_wait_for_blocked_rdev(blocked_rdev, mddev);
-		wait_barrier(conf);
+		wait_barrier(conf, false);
 		goto retry_wait;
 	}
 }
 					    bio->bi_iter.bi_sector,
 					    bio_end_sector(bio)))) {
 		DEFINE_WAIT(w);
+		/* Bail out if REQ_NOWAIT is set for the bio */
+		if (bio->bi_opf & REQ_NOWAIT) {
+			bio_wouldblock_error(bio);
+			return;
+		}
 		for (;;) {
 			prepare_to_wait(&conf->wait_barrier,
 					&w, TASK_IDLE);
 	}
 	sectors = r10_bio->sectors;
-	regular_request_wait(mddev, conf, bio, sectors);
+	if (!regular_request_wait(mddev, conf, bio, sectors))
+		return;
 	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
 	    (mddev->reshape_backwards
 	     ? (bio->bi_iter.bi_sector < conf->reshape_safe &&
 		set_mask_bits(&mddev->sb_flags, 0,
 			      BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
 		md_wakeup_thread(mddev->thread);
+		if (bio->bi_opf & REQ_NOWAIT) {
+			allow_barrier(conf);
+			bio_wouldblock_error(bio);
+			return;
+		}
 		raid10_log(conf->mddev, "wait reshape metadata");
 		wait_event(mddev->sb_wait,
 			   !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags));
 		conf->reshape_safe = mddev->reshape_position;
 	}
-	if (conf->pending_count >= max_queued_requests) {
-		md_wakeup_thread(mddev->thread);
-		raid10_log(mddev, "wait queued");
-		wait_event(conf->wait_barrier,
-			   conf->pending_count < max_queued_requests);
-	}
 	/* first select target devices under rcu_lock and
 	 * inc refcount on their rdev.  Record them by setting
 	 * bios[x] to bio
 		bio_chain(split, bio);
 		allow_barrier(conf);
 		submit_bio_noacct(bio);
-		wait_barrier(conf);
+		wait_barrier(conf, false);
 		bio = split;
 		r10_bio->master_bio = bio;
 	}
 	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
 		return -EAGAIN;
-	wait_barrier(conf);
+	if (WARN_ON_ONCE(bio->bi_opf & REQ_NOWAIT)) {
+		bio_wouldblock_error(bio);
+		return 0;
+	}
+	wait_barrier(conf, false);
 	/*
 	 * Check reshape again to avoid reshape happens after checking
 		allow_barrier(conf);
 		/* Resend the fist split part */
 		submit_bio_noacct(split);
-		wait_barrier(conf);
+		wait_barrier(conf, false);
 	}
 	div_u64_rem(bio_end, stripe_size, &remainder);
 	if (remainder) {
 		/* Resend the second split part */
 		submit_bio_noacct(bio);
 		bio = split;
-		wait_barrier(conf);
+		wait_barrier(conf, false);
 	}
 	bio_start = bio->bi_iter.bi_sector;
 		end_disk_offset += geo->stride;
 		atomic_inc(&first_r10bio->remaining);
 		raid_end_discard_bio(r10_bio);
-		wait_barrier(conf);
+		wait_barrier(conf, false);
 		goto retry_discard;
 	}
 static void close_sync(struct r10conf *conf)
 {
-	wait_barrier(conf);
+	wait_barrier(conf, false);
 	allow_barrier(conf);
 	mempool_exit(&conf->r10buf_pool);
 	if (need_flush ||
 	    time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
 		/* Need to update reshape_position in metadata */
-		wait_barrier(conf);
+		wait_barrier(conf, false);
 		mddev->reshape_position = conf->reshape_progress;
 		if (mddev->reshape_backwards)
 			mddev->curr_resync_completed = raid10_size(mddev, 0, 0)
 MODULE_ALIAS("md-personality-9"); /* RAID10 */
 MODULE_ALIAS("md-raid10");
 MODULE_ALIAS("md-level-10");
-module_param(max_queued_requests, int, S_IRUGO|S_IWUSR);

drivers/md/raid5.c+51−16

 	struct r5conf *conf = sh->raid_conf;
 	int level = conf->level;
 	struct raid5_percpu *percpu;
-	unsigned long cpu;
-	cpu = get_cpu();
+	local_lock(&conf->percpu->lock);
-	percpu = per_cpu_ptr(conf->percpu, cpu);
+	percpu = this_cpu_ptr(conf->percpu);
 	if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {
 		ops_run_biofill(sh);
 		overlap_clear++;
 			BUG();
 	}
-	if (overlap_clear && !sh->batch_head)
+	if (overlap_clear && !sh->batch_head) {
 		for (i = disks; i--; ) {
 			struct r5dev *dev = &sh->dev[i];
 			if (test_and_clear_bit(R5_Overlap, &dev->flags))
 				wake_up(&sh->raid_conf->wait_for_overlap);
 		}
-	put_cpu();
+	}
+	local_unlock(&conf->percpu->lock);
 }
 static void free_stripe(struct kmem_cache *sc, struct stripe_head *sh)
 	struct stripe_head *sh;
 	int stripe_sectors;
+	/* We need to handle this when io_uring supports discard/trim */
+	if (WARN_ON_ONCE(bi->bi_opf & REQ_NOWAIT))
+		return;
 	if (mddev->reshape_position != MaxSector)
 		/* Skip discard while reshape is happening */
 		return;
 	last_sector = bio_end_sector(bi);
 	bi->bi_next = NULL;
+	/* Bail out if conflicts with reshape and REQ_NOWAIT is set */
+	if ((bi->bi_opf & REQ_NOWAIT) &&
+	    (conf->reshape_progress != MaxSector) &&
+	    (mddev->reshape_backwards
+	    ? (logical_sector > conf->reshape_progress && logical_sector <= conf->reshape_safe)
+	    : (logical_sector >= conf->reshape_safe && logical_sector < conf->reshape_progress))) {
+		bio_wouldblock_error(bi);
+		if (rw == WRITE)
+			md_write_end(mddev);
+		return true;
+	}
 	md_account_bio(mddev, &bi);
 	prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE);
 	for (; logical_sector < last_sector; logical_sector += RAID5_STRIPE_SECTORS(conf)) {
 		return -ENOMEM;
 	}
+	local_lock_init(&percpu->lock);
 	return 0;
 }
 	struct md_rdev *rdev;
 	struct md_rdev *journal_dev = NULL;
 	sector_t reshape_offset = 0;
-	int i;
+	int i, ret = 0;
 	long long min_offset_diff = 0;
 	int first = 1;
-	if (mddev_init_writes_pending(mddev) < 0)
+	if (acct_bioset_init(mddev)) {
+		pr_err("md/raid456:%s: alloc acct bioset failed.\n", mdname(mddev));
 		return -ENOMEM;
+	}
+	if (mddev_init_writes_pending(mddev) < 0) {
+		ret = -ENOMEM;
+		goto exit_acct_set;
+	}
 	if (mddev->recovery_cp != MaxSector)
 		pr_notice("md/raid:%s: not clean -- starting background reconstruction\n",
 	    (mddev->bitmap_info.offset || mddev->bitmap_info.file)) {
 		pr_notice("md/raid:%s: array cannot have both journal and bitmap\n",
 			  mdname(mddev));
-		return -EINVAL;
+		ret = -EINVAL;
+		goto exit_acct_set;
 	}
 	if (mddev->reshape_position != MaxSector) {
 		if (journal_dev) {
 			pr_warn("md/raid:%s: don't support reshape with journal - aborting.\n",
 				mdname(mddev));
-			return -EINVAL;
+			ret = -EINVAL;
+			goto exit_acct_set;
 		}
 		if (mddev->new_level != mddev->level) {
 			pr_warn("md/raid:%s: unsupported reshape required - aborting.\n",
 				mdname(mddev));
-			return -EINVAL;
+			ret = -EINVAL;
+			goto exit_acct_set;
 		}
 		old_disks = mddev->raid_disks - mddev->delta_disks;
 		/* reshape_position must be on a new-stripe boundary, and one
 		if (sector_div(here_new, chunk_sectors * new_data_disks)) {
 			pr_warn("md/raid:%s: reshape_position not on a stripe boundary\n",
 				mdname(mddev));
-			return -EINVAL;
+			ret = -EINVAL;
+			goto exit_acct_set;
 		}
 		reshape_offset = here_new * chunk_sectors;
 		/* here_new is the stripe we will write to */
 			else if (mddev->ro == 0) {
 				pr_warn("md/raid:%s: in-place reshape must be started in read-only mode - aborting\n",
 					mdname(mddev));
-				return -EINVAL;
+				ret = -EINVAL;
+				goto exit_acct_set;
 			}
 		} else if (mddev->reshape_backwards
 		    ? (here_new * chunk_sectors + min_offset_diff <=
 			/* Reading from the same stripe as writing to - bad */
 			pr_warn("md/raid:%s: reshape_position too early for auto-recovery - aborting.\n",
 				mdname(mddev));
-			return -EINVAL;
+			ret = -EINVAL;
+			goto exit_acct_set;
 		}
 		pr_debug("md/raid:%s: reshape will continue\n", mdname(mddev));
 		/* OK, we should be able to continue; */
 	else
 		conf = mddev->private;
-	if (IS_ERR(conf))
+	if (IS_ERR(conf)) {
-		return PTR_ERR(conf);
+		ret = PTR_ERR(conf);
+		goto exit_acct_set;
+	}
 	if (test_bit(MD_HAS_JOURNAL, &mddev->flags)) {
 		if (!journal_dev) {
 	free_conf(conf);
 	mddev->private = NULL;
 	pr_warn("md/raid:%s: failed to run raid set.\n", mdname(mddev));
-	return -EIO;
+	ret = -EIO;
+exit_acct_set:
+	acct_bioset_exit(mddev);
+	return ret;
 }
 static void raid5_free(struct mddev *mddev, void *priv)
 	struct r5conf *conf = priv;
 	free_conf(conf);
+	acct_bioset_exit(mddev);
 	mddev->to_remove = &raid5_attrs_group;
 }

drivers/md/raid5.h+3−1

 #include <linux/raid/xor.h>
 #include <linux/dmaengine.h>
+#include <linux/local_lock.h>
 /*
  *
 					     * lists and performing address
 					     * conversions
 					     */
-		int scribble_obj_size;
+		int             scribble_obj_size;
+		local_lock_t    lock;
 	} __percpu *percpu;
 	int scribble_disks;
 	int scribble_sectors;

drivers/nvme/host/core.c+1−6

 blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req)
 {
 	struct nvme_command *cmd = nvme_req(req)->cmd;
-	struct nvme_ctrl *ctrl = nvme_req(req)->ctrl;
 	blk_status_t ret = BLK_STS_OK;
 	if (!(req->rq_flags & RQF_DONTPREP))
 		return BLK_STS_IOERR;
 	}
-	if (!(ctrl->quirks & NVME_QUIRK_SKIP_CID_GEN))
-		nvme_req(req)->genctr++;
 	cmd->common.command_id = nvme_cid(req);
 	trace_nvme_setup_cmd(req, cmd);
 	return ret;
 		return -EINVAL;
 	}
 	subsys->awupf = le16_to_cpu(id->awupf);
-#ifdef CONFIG_NVME_MULTIPATH
+	nvme_mpath_default_iopolicy(subsys);
-	subsys->iopolicy = NVME_IOPOLICY_NUMA;
-#endif
 	subsys->dev.class = nvme_subsys_class;
 	subsys->dev.release = nvme_release_subsystem;

drivers/nvme/host/fabrics.c+21−1

@@ -1069,6 +1069,26 @@ out_unlock: · 1068 unmodified lines
1069	return ret ? ret : count;	1069	return ret ? ret : count;
1070	}	1070	}
1071		1071
		1072	static void __nvmf_concat_opt_tokens(struct seq_file *seq_file)
		1073	{
		1074	const struct match_token *tok;
		1075	int idx;
		1076
		1077	/*
		1078	* Add dummy entries for instance and cntlid to
		1079	* signal an invalid/non-existing controller
		1080	*/
		1081	seq_puts(seq_file, "instance=-1,cntlid=-1");
		1082	for (idx = 0; idx < ARRAY_SIZE(opt_tokens); idx++) {
		1083	tok = &opt_tokens[idx];
		1084	if (tok->token == NVMF_OPT_ERR)
		1085	continue;
		1086	seq_puts(seq_file, ",");
		1087	seq_puts(seq_file, tok->pattern);
		1088	}
		1089	seq_puts(seq_file, "\n");
		1090	}
		1091
1072	static int nvmf_dev_show(struct seq_file seq_file, void private)	1092	static int nvmf_dev_show(struct seq_file seq_file, void private)
1073	{	1093	{
1074	struct nvme_ctrl *ctrl;	1094	struct nvme_ctrl *ctrl;
@@ -1077,7 +1097,7 @@ static int nvmf_dev_show(struct seq_file seq_file, void private) · 2 unmodified lines
1077	mutex_lock(&nvmf_dev_mutex);	1097	mutex_lock(&nvmf_dev_mutex);
1078	ctrl = seq_file->private;	1098	ctrl = seq_file->private;
1079	if (!ctrl) {	1099	if (!ctrl) {
1080	ret = -EINVAL;	1100	__nvmf_concat_opt_tokens(seq_file);
1081	goto out_unlock;	1101	goto out_unlock;
1082	}	1102	}
1083		1103

drivers/nvme/host/multipath.c+36−5

@@ -13,6 +13,42 @@ module_param(multipath, bool, 0444); · 12 unmodified lines
13	MODULE_PARM_DESC(multipath,	13	MODULE_PARM_DESC(multipath,
14	"turn on native support for multiple controllers per subsystem");	14	"turn on native support for multiple controllers per subsystem");
15		15
		16	static const char *nvme_iopolicy_names[] = {
		17	[NVME_IOPOLICY_NUMA] = "numa",
		18	[NVME_IOPOLICY_RR] = "round-robin",
		19	};
		20
		21	static int iopolicy = NVME_IOPOLICY_NUMA;
		22
		23	static int nvme_set_iopolicy(const char val, const struct kernel_param kp)
		24	{
		25	if (!val)
		26	return -EINVAL;
		27	if (!strncmp(val, "numa", 4))
		28	iopolicy = NVME_IOPOLICY_NUMA;
		29	else if (!strncmp(val, "round-robin", 11))
		30	iopolicy = NVME_IOPOLICY_RR;
		31	else
		32	return -EINVAL;
		33
		34	return 0;
		35	}
		36
		37	static int nvme_get_iopolicy(char buf, const struct kernel_param kp)
		38	{
		39	return sprintf(buf, "%s\n", nvme_iopolicy_names[iopolicy]);
		40	}
		41
		42	module_param_call(iopolicy, nvme_set_iopolicy, nvme_get_iopolicy,
		43	&iopolicy, 0644);
		44	MODULE_PARM_DESC(iopolicy,
		45	"Default multipath I/O policy; 'numa' (default) or 'round-robin'");
		46
		47	void nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys)
		48	{
		49	subsys->iopolicy = iopolicy;
		50	}
		51
16	void nvme_mpath_unfreeze(struct nvme_subsystem *subsys)	52	void nvme_mpath_unfreeze(struct nvme_subsystem *subsys)
17	{	53	{
18	struct nvme_ns_head *h;	54	struct nvme_ns_head *h;
@@ -706,11 +742,6 @@ void nvme_mpath_stop(struct nvme_ctrl *ctrl) · 687 unmodified lines
706	struct device_attribute subsys_attr_##_name = \	742	struct device_attribute subsys_attr_##_name = \
707	__ATTR(_name, _mode, _show, _store)	743	__ATTR(_name, _mode, _show, _store)
708		744
709	static const char *nvme_iopolicy_names[] = {
710	[NVME_IOPOLICY_NUMA] = "numa",
711	[NVME_IOPOLICY_RR] = "round-robin",
712	};
713
714	static ssize_t nvme_subsys_iopolicy_show(struct device *dev,	745	static ssize_t nvme_subsys_iopolicy_show(struct device *dev,
715	struct device_attribute attr, char buf)	746	struct device_attribute attr, char buf)
716	{	747	{

drivers/nvme/host/nvme.h+8−0

 		union nvme_result result)
 {
 	struct nvme_request *rq = nvme_req(req);
+	struct nvme_ctrl *ctrl = rq->ctrl;
+	if (!(ctrl->quirks & NVME_QUIRK_SKIP_CID_GEN))
+		rq->genctr++;
 	rq->status = le16_to_cpu(status) >> 1;
 	rq->result = result;
 void nvme_mpath_unfreeze(struct nvme_subsystem *subsys);
 void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys);
 void nvme_mpath_start_freeze(struct nvme_subsystem *subsys);
+void nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys);
 bool nvme_mpath_set_disk_name(struct nvme_ns *ns, char *disk_name, int *flags);
 void nvme_failover_req(struct request *req);
 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl);
 static inline void nvme_mpath_start_freeze(struct nvme_subsystem *subsys)
 {
 }
+static inline void nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys)
+{
+}
 #endif /* CONFIG_NVME_MULTIPATH */
 int nvme_revalidate_zones(struct nvme_ns *ns);

Merge tag 'for-5.17/drivers-2022-01-11' of git://git.kernel.dk/linux-block

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code