io context sharing: preliminary support

Detach task state from ioc, instead keep track of how many processes are accessing the ioc.

Jens Axboe committed 18 years ago

Commit d38ecf91 parent fd0928d5 files · +49 −23Browse files

Unified · Split

5 files

block/ll_rw_blk.c +30 −16
fs/ioprio.c +0 −1
include/linux/blkdev.h +1 −1
include/linux/iocontext.h +18 −4
kernel/fork.c +0 −1

@@ -3854,12 +3854,13 @@ int __init blk_dev_init(void) · 3853 unmodified lines
3854	3854	}
3855	3855
3856	3856	/*
3857		− * IO Context helper functions
	3857	+ * IO Context helper functions. put_io_context() returns 1 if there are no
	3858	+ * more users of this io context, 0 otherwise.
3858	3859	*/
3859		−void put_io_context(struct io_context *ioc)
	3860	+int put_io_context(struct io_context *ioc)
3860	3861	{
3861	3862	if (ioc == NULL)
3862		− return;
	3863	+ return 1;
3863	3864
3864	3865	BUG_ON(atomic_read(&ioc->refcount) == 0);
3865	3866
@@ -3878,7 +3879,9 @@ void put_io_context(struct io_context *ioc) · 12 unmodified lines
3878	3879	rcu_read_unlock();
3879	3880
3880	3881	kmem_cache_free(iocontext_cachep, ioc);
	3882	+ return 1;
3881	3883	}
	3884	+ return 0;
3882	3885	}
3883	3886	EXPORT_SYMBOL(put_io_context);
3884	3887
@@ -3893,15 +3896,17 @@ void exit_io_context(void) · 8 unmodified lines
3893	3896	current->io_context = NULL;
3894	3897	task_unlock(current);
3895	3898
3896		− ioc->task = NULL;
3897		− if (ioc->aic && ioc->aic->exit)
3898		− ioc->aic->exit(ioc->aic);
3899		− if (ioc->cic_root.rb_node != NULL) {
3900		− cic = rb_entry(rb_first(&ioc->cic_root), struct cfq_io_context, rb_node);
3901		− cic->exit(ioc);
3902		− }
	3899	+ if (atomic_dec_and_test(&ioc->nr_tasks)) {
	3900	+ if (ioc->aic && ioc->aic->exit)
	3901	+ ioc->aic->exit(ioc->aic);
	3902	+ if (ioc->cic_root.rb_node != NULL) {
	3903	+ cic = rb_entry(rb_first(&ioc->cic_root),
	3904	+ struct cfq_io_context, rb_node);
	3905	+ cic->exit(ioc);
	3906	+ }
3903	3907
3904		− put_io_context(ioc);
	3908	+ put_io_context(ioc);
	3909	+ }
3905	3910	}
3906	3911
3907	3912	struct io_context *alloc_io_context(gfp_t gfp_flags, int node)
@@ -3911,7 +3916,8 @@ struct io_context *alloc_io_context(gfp_t gfp_flags, int node) · 3 unmodified lines
3911	3916	ret = kmem_cache_alloc_node(iocontext_cachep, gfp_flags, node);
3912	3917	if (ret) {
3913	3918	atomic_set(&ret->refcount, 1);
3914		− ret->task = current;
	3919	+ atomic_set(&ret->nr_tasks, 1);
	3920	+ spin_lock_init(&ret->lock);
3915	3921	ret->ioprio_changed = 0;
3916	3922	ret->ioprio = 0;
3917	3923	ret->last_waited = jiffies; /* doesn't matter... */
@@ -3959,10 +3965,18 @@ static struct io_context *current_io_context(gfp_t gfp_flags, int node) · 41 unmodified lines
3959	3965	*/
3960	3966	struct io_context *get_io_context(gfp_t gfp_flags, int node)
3961	3967	{
3962		− struct io_context *ret;
3963		− ret = current_io_context(gfp_flags, node);
3964		− if (likely(ret))
3965		− atomic_inc(&ret->refcount);
	3968	+ struct io_context *ret = NULL;
	3969	+
	3970	+ /*
	3971	+ * Check for unlikely race with exiting task. ioc ref count is
	3972	+ * zero when ioc is being detached.
	3973	+ */
	3974	+ do {
	3975	+ ret = current_io_context(gfp_flags, node);
	3976	+ if (unlikely(!ret))
	3977	+ break;
	3978	+ } while (!atomic_inc_not_zero(&ret->refcount));
	3979	+
3966	3980	return ret;
3967	3981	}
3968	3982	EXPORT_SYMBOL(get_io_context);

fs/ioprio.c+0−1

@@ -54,7 +54,6 @@ static int set_task_ioprio(struct task_struct *task, int ioprio) · 53 unmodified lines
54	54	break;
55	55	}
56	56	task->io_context = ioc;
57		− ioc->task = task;
58	57	} while (1);
59	58
60	59	if (!err) {

include/linux/blkdev.h+1−1

@@ -34,7 +34,7 @@ struct sg_io_hdr; · 33 unmodified lines
34	34	#define BLKDEV_MIN_RQ 4
35	35	#define BLKDEV_MAX_RQ 128 /* Default maximum */
36	36
37		−void put_io_context(struct io_context *ioc);
	37	+int put_io_context(struct io_context *ioc);
38	38	void exit_io_context(void);
39	39	struct io_context *get_io_context(gfp_t gfp_flags, int node);
40	40	struct io_context *alloc_io_context(gfp_t gfp_flags, int node);

include/linux/iocontext.h+18−4

@@ -54,13 +54,15 @@ struct cfq_io_context { · 53 unmodified lines
54	54	};
55	55
56	56	/*
57		− * This is the per-process I/O subsystem state. It is refcounted and
58		− * kmalloc'ed. Currently all fields are modified in process io context
59		− * (apart from the atomic refcount), so require no locking.
	57	+ * I/O subsystem state of the associated processes. It is refcounted
	58	+ * and kmalloc'ed. These could be shared between processes.
60	59	*/
61	60	struct io_context {
62	61	atomic_t refcount;
63		− struct task_struct *task;
	62	+ atomic_t nr_tasks;
	63	+
	64	+ /* all the fields below are protected by this lock */
	65	+ spinlock_t lock;
64	66
65	67	unsigned short ioprio;
66	68	unsigned short ioprio_changed;
@@ -76,4 +78,16 @@ struct io_context { · 9 unmodified lines
76	78	void *ioc_data;
77	79	};
78	80
	81	+static inline struct io_context ioc_task_link(struct io_context ioc)
	82	+{
	83	+ /*
	84	+ * if ref count is zero, don't allow sharing (ioc is going away, it's
	85	+ * a race).
	86	+ */
	87	+ if (ioc && atomic_inc_not_zero(&ioc->refcount))
	88	+ return ioc;
	89	+
	90	+ return NULL;
	91	+}
	92	+
79	93	#endif

kernel/fork.c+0−1

@@ -805,7 +805,6 @@ static int copy_io(struct task_struct *tsk) · 804 unmodified lines
805	805	if (unlikely(!tsk->io_context))
806	806	return -ENOMEM;
807	807
808		− tsk->io_context->task = tsk;
809	808	tsk->io_context->ioprio = ioc->ioprio;
810	809	}
811	810	#endif