History

Merge tag 'io_uring-6.19-20260122' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux

Pull io_uring fixes from Jens Axboe: - Fix for a potential leak of an iovec, if a specific cleanup path is used and the rw_cache is full at the time of the call - Fix for a regression added in this cycle, where waitid should be using prober release/acquire semantics for updating the wait queue head - Check for the cancelation bit being set for every work item processed by io-wq, not just at the start of the loop. Has no real practical implications other than to shut up syzbot doing crazy things that grossly overload a system, hence slowing down ring exit - A few selftest additions, updating the mini_liburing that selftests use * tag 'io_uring-6.19-20260122' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux: selftests/io_uring: support NO_SQARRAY in miniliburing selftests/io_uring: add io_uring_queue_init_params io_uring/io-wq: check IO_WQ_BIT_EXIT inside work run loop io_uring/waitid: fix KCSAN warning on io_waitid->head io_uring/rw: free potentially allocated iovec on cache put failure

Linus Torvalds committed 9 months ago

Commit 7907f672 parents b33d706 145e0074 files · +59 −23Browse files

Unified · Split

4 files
  1. io_uring/io-wq.c +1 −1
  2. io_uring/rw.c +11 −4
  3. io_uring/waitid.c +3 −3
  4. tools/include/io_uring/mini_liburing.h +44 −15
io_uring/io-wq.c+1−1
@@ -598,9 +598,9 @@ static void io_worker_handle_work(struct io_wq_acct *acct, · 597 unmodified lines
598 __releases(&acct->lock)598 __releases(&acct->lock)
599{599{
600 struct io_wq *wq = worker->wq;600 struct io_wq *wq = worker->wq;
601 bool do_kill = test_bit(IO_WQ_BIT_EXIT, &wq->state);
602 601
603 do {602 do {
603 bool do_kill = test_bit(IO_WQ_BIT_EXIT, &wq->state);
604 struct io_wq_work *work;604 struct io_wq_work *work;
605 605
606 /*606 /*
io_uring/rw.c+11−4
@@ -144,19 +144,22 @@ static inline int io_import_rw_buffer(int rw, struct io_kiocb *req, · 143 unmodified lines
144 return 0;144 return 0;
145}145}
146 146
147static void io_rw_recycle(struct io_kiocb *req, unsigned int issue_flags)147static bool io_rw_recycle(struct io_kiocb *req, unsigned int issue_flags)
148{148{
149 struct io_async_rw *rw = req->async_data;149 struct io_async_rw *rw = req->async_data;
150 150
151 if (unlikely(issue_flags & IO_URING_F_UNLOCKED))151 if (unlikely(issue_flags & IO_URING_F_UNLOCKED))
152 return;152 return false;
153 153
154 io_alloc_cache_vec_kasan(&rw->vec);154 io_alloc_cache_vec_kasan(&rw->vec);
155 if (rw->vec.nr > IO_VEC_CACHE_SOFT_CAP)155 if (rw->vec.nr > IO_VEC_CACHE_SOFT_CAP)
156 io_vec_free(&rw->vec);156 io_vec_free(&rw->vec);
157 157
158 if (io_alloc_cache_put(&req->ctx->rw_cache, rw))158 if (io_alloc_cache_put(&req->ctx->rw_cache, rw)) {
159 io_req_async_data_clear(req, 0);159 io_req_async_data_clear(req, 0);
160 return true;
161 }
162 return false;
160}163}
161 164
162static void io_req_rw_cleanup(struct io_kiocb *req, unsigned int issue_flags)165static void io_req_rw_cleanup(struct io_kiocb *req, unsigned int issue_flags)
@@ -190,7 +193,11 @@ static void io_req_rw_cleanup(struct io_kiocb *req, unsigned int issue_flags) · 27 unmodified lines
190 */193 */
191 if (!(req->flags & (REQ_F_REISSUE | REQ_F_REFCOUNT))) {194 if (!(req->flags & (REQ_F_REISSUE | REQ_F_REFCOUNT))) {
192 req->flags &= ~REQ_F_NEED_CLEANUP;195 req->flags &= ~REQ_F_NEED_CLEANUP;
193 io_rw_recycle(req, issue_flags);196 if (!io_rw_recycle(req, issue_flags)) {
197 struct io_async_rw *rw = req->async_data;
198
199 io_vec_free(&rw->vec);
200 }
194 }201 }
195}202}
196 203
io_uring/waitid.c+3−3
@@ -114,11 +114,11 @@ static void io_waitid_remove_wq(struct io_kiocb *req) · 113 unmodified lines
114 struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);114 struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid);
115 struct wait_queue_head *head;115 struct wait_queue_head *head;
116 116
117 head = READ_ONCE(iw->head);117 head = smp_load_acquire(&iw->head);
118 if (head) {118 if (head) {
119 struct io_waitid_async *iwa = req->async_data;119 struct io_waitid_async *iwa = req->async_data;
120 120
121 iw->head = NULL;121 smp_store_release(&iw->head, NULL);
122 spin_lock_irq(&head->lock);122 spin_lock_irq(&head->lock);
123 list_del_init(&iwa->wo.child_wait.entry);123 list_del_init(&iwa->wo.child_wait.entry);
124 spin_unlock_irq(&head->lock);124 spin_unlock_irq(&head->lock);
@@ -246,7 +246,7 @@ static int io_waitid_wait(struct wait_queue_entry *wait, unsigned mode, · 121 unmodified lines
246 return 0;246 return 0;
247 247
248 list_del_init(&wait->entry);248 list_del_init(&wait->entry);
249 iw->head = NULL;249 smp_store_release(&iw->head, NULL);
250 250
251 /* cancel is in progress */251 /* cancel is in progress */
252 if (atomic_fetch_inc(&iw->refs) & IO_WAITID_REF_MASK)252 if (atomic_fetch_inc(&iw->refs) & IO_WAITID_REF_MASK)
tools/include/io_uring/mini_liburing.h+44−15
@@ -6,6 +6,7 @@ · 5 unmodified lines
6#include <stdio.h>6#include <stdio.h>
7#include <string.h>7#include <string.h>
8#include <unistd.h>8#include <unistd.h>
9#include <sys/uio.h>
9 10
10struct io_sq_ring {11struct io_sq_ring {
11 unsigned int *head;12 unsigned int *head;
@@ -55,6 +56,7 @@ struct io_uring { · 43 unmodified lines
55 struct io_uring_sq sq;56 struct io_uring_sq sq;
56 struct io_uring_cq cq;57 struct io_uring_cq cq;
57 int ring_fd;58 int ring_fd;
59 unsigned flags;
58};60};
59 61
60#if defined(__x86_64) || defined(__i386__)62#if defined(__x86_64) || defined(__i386__)
@@ -72,7 +74,14 @@ static inline int io_uring_mmap(int fd, struct io_uring_params *p, · 11 unmodified lines
72 void *ptr;74 void *ptr;
73 int ret;75 int ret;
74 76
75 sq->ring_sz = p->sq_off.array + p->sq_entries * sizeof(unsigned int);77 if (p->flags & IORING_SETUP_NO_SQARRAY) {
78 sq->ring_sz = p->cq_off.cqes;
79 sq->ring_sz += p->cq_entries * sizeof(struct io_uring_cqe);
80 } else {
81 sq->ring_sz = p->sq_off.array;
82 sq->ring_sz += p->sq_entries * sizeof(unsigned int);
83 }
84
76 ptr = mmap(0, sq->ring_sz, PROT_READ | PROT_WRITE,85 ptr = mmap(0, sq->ring_sz, PROT_READ | PROT_WRITE,
77 MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQ_RING);86 MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQ_RING);
78 if (ptr == MAP_FAILED)87 if (ptr == MAP_FAILED)
@@ -83,7 +92,8 @@ static inline int io_uring_mmap(int fd, struct io_uring_params *p, · 4 unmodified lines
83 sq->kring_entries = ptr + p->sq_off.ring_entries;92 sq->kring_entries = ptr + p->sq_off.ring_entries;
84 sq->kflags = ptr + p->sq_off.flags;93 sq->kflags = ptr + p->sq_off.flags;
85 sq->kdropped = ptr + p->sq_off.dropped;94 sq->kdropped = ptr + p->sq_off.dropped;
86 sq->array = ptr + p->sq_off.array;95 if (!(p->flags & IORING_SETUP_NO_SQARRAY))
96 sq->array = ptr + p->sq_off.array;
87 97
88 size = p->sq_entries * sizeof(struct io_uring_sqe);98 size = p->sq_entries * sizeof(struct io_uring_sqe);
89 sq->sqes = mmap(0, size, PROT_READ | PROT_WRITE,99 sq->sqes = mmap(0, size, PROT_READ | PROT_WRITE,
@@ -126,28 +136,39 @@ static inline int io_uring_enter(int fd, unsigned int to_submit, · 36 unmodified lines
126 flags, sig, _NSIG / 8);136 flags, sig, _NSIG / 8);
127}137}
128 138
129static inline int io_uring_queue_init(unsigned int entries,139static inline int io_uring_queue_init_params(unsigned int entries,
130 struct io_uring *ring,140 struct io_uring *ring,
131 unsigned int flags)141 struct io_uring_params *p)
132{142{
133 struct io_uring_params p;
134 int fd, ret;143 int fd, ret;
135 144
136 memset(ring, 0, sizeof(*ring));145 memset(ring, 0, sizeof(*ring));
137 memset(&p, 0, sizeof(p));
138 p.flags = flags;
139 146
140 fd = io_uring_setup(entries, &p);147 fd = io_uring_setup(entries, p);
141 if (fd < 0)148 if (fd < 0)
142 return fd;149 return fd;
143 ret = io_uring_mmap(fd, &p, &ring->sq, &ring->cq);150 ret = io_uring_mmap(fd, p, &ring->sq, &ring->cq);
144 if (!ret)151 if (!ret) {
145 ring->ring_fd = fd;152 ring->ring_fd = fd;
146 else153 ring->flags = p->flags;
154 } else {
147 close(fd);155 close(fd);
156 }
148 return ret;157 return ret;
149}158}
150 159
160static inline int io_uring_queue_init(unsigned int entries,
161 struct io_uring *ring,
162 unsigned int flags)
163{
164 struct io_uring_params p;
165
166 memset(&p, 0, sizeof(p));
167 p.flags = flags;
168
169 return io_uring_queue_init_params(entries, ring, &p);
170}
171
151/* Get a sqe */172/* Get a sqe */
152static inline struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring)173static inline struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring)
153{174{
@@ -199,10 +220,18 @@ static inline int io_uring_submit(struct io_uring *ring) · 45 unmodified lines
199 220
200 ktail = *sq->ktail;221 ktail = *sq->ktail;
201 to_submit = sq->sqe_tail - sq->sqe_head;222 to_submit = sq->sqe_tail - sq->sqe_head;
202 for (submitted = 0; submitted < to_submit; submitted++) {223
203 read_barrier();224 if (!(ring->flags & IORING_SETUP_NO_SQARRAY)) {
204 sq->array[ktail++ & mask] = sq->sqe_head++ & mask;225 for (submitted = 0; submitted < to_submit; submitted++) {
226 read_barrier();
227 sq->array[ktail++ & mask] = sq->sqe_head++ & mask;
228 }
229 } else {
230 ktail += to_submit;
231 sq->sqe_head += to_submit;
232 submitted = to_submit;
205 }233 }
234
206 if (!submitted)235 if (!submitted)
207 return 0;236 return 0;
208 237

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t