Merge tag 'io_uring-6.19-20260122' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux

Pull io_uring fixes from Jens Axboe: - Fix for a potential leak of an iovec, if a specific cleanup path is used and the rw_cache is full at the time of the call - Fix for a regression added in this cycle, where waitid should be using prober release/acquire semantics for updating the wait queue head - Check for the cancelation bit being set for every work item processed by io-wq, not just at the start of the loop. Has no real practical implications other than to shut up syzbot doing crazy things that grossly overload a system, hence slowing down ring exit - A few selftest additions, updating the mini_liburing that selftests use * tag 'io_uring-6.19-20260122' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux: selftests/io_uring: support NO_SQARRAY in miniliburing selftests/io_uring: add io_uring_queue_init_params io_uring/io-wq: check IO_WQ_BIT_EXIT inside work run loop io_uring/waitid: fix KCSAN warning on io_waitid->head io_uring/rw: free potentially allocated iovec on cache put failure

Linus Torvalds committed 9 months ago

Commit 7907f672 parents b33d706 145e0074 files · +59 −23Browse files

Unified · Split

4 files

io_uring/io-wq.c +1 −1
io_uring/rw.c +11 −4
io_uring/waitid.c +3 −3
tools/include/io_uring/mini_liburing.h +44 −15

io_uring/io-wq.c+1−1

@@ -598,9 +598,9 @@ static void io_worker_handle_work(struct io_wq_acct *acct, · 597 unmodified lines
598	598	__releases(&acct->lock)
599	599	{
600	600	struct io_wq *wq = worker->wq;
601		− bool do_kill = test_bit(IO_WQ_BIT_EXIT, &wq->state);
602	601
603	602	do {
	603	+ bool do_kill = test_bit(IO_WQ_BIT_EXIT, &wq->state);
604	604	struct io_wq_work *work;
605	605
606	606	/*

io_uring/rw.c+11−4

@@ -144,19 +144,22 @@ static inline int io_import_rw_buffer(int rw, struct io_kiocb *req, · 143 unmodified lines
144	144	return 0;
145	145	}
146	146
147		−static void io_rw_recycle(struct io_kiocb *req, unsigned int issue_flags)
	147	+static bool io_rw_recycle(struct io_kiocb *req, unsigned int issue_flags)
148	148	{
149	149	struct io_async_rw *rw = req->async_data;
150	150
151	151	if (unlikely(issue_flags & IO_URING_F_UNLOCKED))
152		− return;
	152	+ return false;
153	153
154	154	io_alloc_cache_vec_kasan(&rw->vec);
155	155	if (rw->vec.nr > IO_VEC_CACHE_SOFT_CAP)
156	156	io_vec_free(&rw->vec);
157	157
158		− if (io_alloc_cache_put(&req->ctx->rw_cache, rw))
	158	+ if (io_alloc_cache_put(&req->ctx->rw_cache, rw)) {
159	159	io_req_async_data_clear(req, 0);
	160	+ return true;
	161	+ }
	162	+ return false;
160	163	}
161	164
162	165	static void io_req_rw_cleanup(struct io_kiocb *req, unsigned int issue_flags)
@@ -190,7 +193,11 @@ static void io_req_rw_cleanup(struct io_kiocb *req, unsigned int issue_flags) · 27 unmodified lines
190	193	*/
191	194	if (!(req->flags & (REQ_F_REISSUE \| REQ_F_REFCOUNT))) {
192	195	req->flags &= ~REQ_F_NEED_CLEANUP;
193		− io_rw_recycle(req, issue_flags);
	196	+ if (!io_rw_recycle(req, issue_flags)) {
	197	+ struct io_async_rw *rw = req->async_data;
	198	+
	199	+ io_vec_free(&rw->vec);
	200	+ }
194	201	}
195	202	}
196	203

io_uring/waitid.c+3−3

@@ -114,11 +114,11 @@ static void io_waitid_remove_wq(struct io_kiocb *req) · 113 unmodified lines
114	114	struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
115	115	struct wait_queue_head *head;
116	116
117		− head = READ_ONCE(iw->head);
	117	+ head = smp_load_acquire(&iw->head);
118	118	if (head) {
119	119	struct io_waitid_async *iwa = req->async_data;
120	120
121		− iw->head = NULL;
	121	+ smp_store_release(&iw->head, NULL);
122	122	spin_lock_irq(&head->lock);
123	123	list_del_init(&iwa->wo.child_wait.entry);
124	124	spin_unlock_irq(&head->lock);
@@ -246,7 +246,7 @@ static int io_waitid_wait(struct wait_queue_entry *wait, unsigned mode, · 121 unmodified lines
246	246	return 0;
247	247
248	248	list_del_init(&wait->entry);
249		− iw->head = NULL;
	249	+ smp_store_release(&iw->head, NULL);
250	250
251	251	/* cancel is in progress */
252	252	if (atomic_fetch_inc(&iw->refs) & IO_WAITID_REF_MASK)

tools/include/io_uring/mini_liburing.h+44−15

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	6	#include <stdio.h>
7	7	#include <string.h>
8	8	#include <unistd.h>
	9	+#include <sys/uio.h>
9	10
10	11	struct io_sq_ring {
11	12	unsigned int *head;
@@ -55,6 +56,7 @@ struct io_uring { · 43 unmodified lines
55	56	struct io_uring_sq sq;
56	57	struct io_uring_cq cq;
57	58	int ring_fd;
	59	+ unsigned flags;
58	60	};
59	61
60	62	#if defined(__x86_64) \|\| defined(__i386__)
@@ -72,7 +74,14 @@ static inline int io_uring_mmap(int fd, struct io_uring_params *p, · 11 unmodified lines
72	74	void *ptr;
73	75	int ret;
74	76
75		− sq->ring_sz = p->sq_off.array + p->sq_entries * sizeof(unsigned int);
	77	+ if (p->flags & IORING_SETUP_NO_SQARRAY) {
	78	+ sq->ring_sz = p->cq_off.cqes;
	79	+ sq->ring_sz += p->cq_entries * sizeof(struct io_uring_cqe);
	80	+ } else {
	81	+ sq->ring_sz = p->sq_off.array;
	82	+ sq->ring_sz += p->sq_entries * sizeof(unsigned int);
	83	+ }
	84	+
76	85	ptr = mmap(0, sq->ring_sz, PROT_READ \| PROT_WRITE,
77	86	MAP_SHARED \| MAP_POPULATE, fd, IORING_OFF_SQ_RING);
78	87	if (ptr == MAP_FAILED)
@@ -83,7 +92,8 @@ static inline int io_uring_mmap(int fd, struct io_uring_params *p, · 4 unmodified lines
83	92	sq->kring_entries = ptr + p->sq_off.ring_entries;
84	93	sq->kflags = ptr + p->sq_off.flags;
85	94	sq->kdropped = ptr + p->sq_off.dropped;
86		− sq->array = ptr + p->sq_off.array;
	95	+ if (!(p->flags & IORING_SETUP_NO_SQARRAY))
	96	+ sq->array = ptr + p->sq_off.array;
87	97
88	98	size = p->sq_entries * sizeof(struct io_uring_sqe);
89	99	sq->sqes = mmap(0, size, PROT_READ \| PROT_WRITE,
@@ -126,28 +136,39 @@ static inline int io_uring_enter(int fd, unsigned int to_submit, · 36 unmodified lines
126	136	flags, sig, _NSIG / 8);
127	137	}
128	138
129		−static inline int io_uring_queue_init(unsigned int entries,
130		− struct io_uring *ring,
131		− unsigned int flags)
	139	+static inline int io_uring_queue_init_params(unsigned int entries,
	140	+ struct io_uring *ring,
	141	+ struct io_uring_params *p)
132	142	{
133		− struct io_uring_params p;
134	143	int fd, ret;
135	144
136	145	memset(ring, 0, sizeof(*ring));
137		− memset(&p, 0, sizeof(p));
138		− p.flags = flags;
139	146
140		− fd = io_uring_setup(entries, &p);
	147	+ fd = io_uring_setup(entries, p);
141	148	if (fd < 0)
142	149	return fd;
143		− ret = io_uring_mmap(fd, &p, &ring->sq, &ring->cq);
144		− if (!ret)
	150	+ ret = io_uring_mmap(fd, p, &ring->sq, &ring->cq);
	151	+ if (!ret) {
145	152	ring->ring_fd = fd;
146		− else
	153	+ ring->flags = p->flags;
	154	+ } else {
147	155	close(fd);
	156	+ }
148	157	return ret;
149	158	}
150	159
	160	+static inline int io_uring_queue_init(unsigned int entries,
	161	+ struct io_uring *ring,
	162	+ unsigned int flags)
	163	+{
	164	+ struct io_uring_params p;
	165	+
	166	+ memset(&p, 0, sizeof(p));
	167	+ p.flags = flags;
	168	+
	169	+ return io_uring_queue_init_params(entries, ring, &p);
	170	+}
	171	+
151	172	/* Get a sqe */
152	173	static inline struct io_uring_sqe io_uring_get_sqe(struct io_uring ring)
153	174	{
@@ -199,10 +220,18 @@ static inline int io_uring_submit(struct io_uring *ring) · 45 unmodified lines
199	220
200	221	ktail = *sq->ktail;
201	222	to_submit = sq->sqe_tail - sq->sqe_head;
202		− for (submitted = 0; submitted < to_submit; submitted++) {
203		− read_barrier();
204		− sq->array[ktail++ & mask] = sq->sqe_head++ & mask;
	223	+
	224	+ if (!(ring->flags & IORING_SETUP_NO_SQARRAY)) {
	225	+ for (submitted = 0; submitted < to_submit; submitted++) {
	226	+ read_barrier();
	227	+ sq->array[ktail++ & mask] = sq->sqe_head++ & mask;
	228	+ }
	229	+ } else {
	230	+ ktail += to_submit;
	231	+ sq->sqe_head += to_submit;
	232	+ submitted = to_submit;
205	233	}
	234	+
206	235	if (!submitted)
207	236	return 0;
208	237

Merge tag 'io_uring-6.19-20260122' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code