| @@ -14,10 +14,16 @@ · 13 unmodified lines |
| 14 | 14 | |
| 15 | 15 | struct io_futex { |
| 16 | 16 | struct file *file; |
| 17 | | − u32 __user *uaddr; |
| 17 | + union { |
| 18 | + u32 __user *uaddr; |
| 19 | + struct futex_waitv __user *uwaitv; |
| 20 | + }; |
| 18 | 21 | unsigned long futex_val; |
| 19 | 22 | unsigned long futex_mask; |
| 23 | + unsigned long futexv_owned; |
| 20 | 24 | u32 futex_flags; |
| 25 | + unsigned int futex_nr; |
| 26 | + bool futexv_unqueued; |
| 21 | 27 | }; |
| 22 | 28 | |
| 23 | 29 | struct io_futex_data { |
| @@ -44,6 +50,13 @@ void io_futex_cache_free(struct io_ring_ctx *ctx) · 20 unmodified lines |
| 44 | 50 | io_alloc_cache_free(&ctx->futex_cache, io_futex_cache_entry_free); |
| 45 | 51 | } |
| 46 | 52 | |
| 53 | +static void __io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts) |
| 54 | +{ |
| 55 | + req->async_data = NULL; |
| 56 | + hlist_del_init(&req->hash_node); |
| 57 | + io_req_task_complete(req, ts); |
| 58 | +} |
| 59 | + |
| 47 | 60 | static void io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts) |
| 48 | 61 | { |
| 49 | 62 | struct io_futex_data *ifd = req->async_data; |
| @@ -52,22 +65,56 @@ static void io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts) · 2 unmodified lines |
| 52 | 65 | io_tw_lock(ctx, ts); |
| 53 | 66 | if (!io_alloc_cache_put(&ctx->futex_cache, &ifd->cache)) |
| 54 | 67 | kfree(ifd); |
| 55 | | − req->async_data = NULL; |
| 56 | | − hlist_del_init(&req->hash_node); |
| 57 | | − io_req_task_complete(req, ts); |
| 68 | + __io_futex_complete(req, ts); |
| 58 | 69 | } |
| 59 | 70 | |
| 60 | | −static bool __io_futex_cancel(struct io_ring_ctx *ctx, struct io_kiocb *req) |
| 71 | +static void io_futexv_complete(struct io_kiocb *req, struct io_tw_state *ts) |
| 61 | 72 | { |
| 62 | | − struct io_futex_data *ifd = req->async_data; |
| 73 | + struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); |
| 74 | + struct futex_vector *futexv = req->async_data; |
| 63 | 75 | |
| 64 | | − /* futex wake already done or in progress */ |
| 65 | | − if (!futex_unqueue(&ifd->q)) |
| 76 | + io_tw_lock(req->ctx, ts); |
| 77 | + |
| 78 | + if (!iof->futexv_unqueued) { |
| 79 | + int res; |
| 80 | + |
| 81 | + res = futex_unqueue_multiple(futexv, iof->futex_nr); |
| 82 | + if (res != -1) |
| 83 | + io_req_set_res(req, res, 0); |
| 84 | + } |
| 85 | + |
| 86 | + kfree(req->async_data); |
| 87 | + req->flags &= ~REQ_F_ASYNC_DATA; |
| 88 | + __io_futex_complete(req, ts); |
| 89 | +} |
| 90 | + |
| 91 | +static bool io_futexv_claim(struct io_futex *iof) |
| 92 | +{ |
| 93 | + if (test_bit(0, &iof->futexv_owned) || |
| 94 | + test_and_set_bit_lock(0, &iof->futexv_owned)) |
| 66 | 95 | return false; |
| 96 | + return true; |
| 97 | +} |
| 98 | + |
| 99 | +static bool __io_futex_cancel(struct io_ring_ctx *ctx, struct io_kiocb *req) |
| 100 | +{ |
| 101 | + /* futex wake already done or in progress */ |
| 102 | + if (req->opcode == IORING_OP_FUTEX_WAIT) { |
| 103 | + struct io_futex_data *ifd = req->async_data; |
| 104 | + |
| 105 | + if (!futex_unqueue(&ifd->q)) |
| 106 | + return false; |
| 107 | + req->io_task_work.func = io_futex_complete; |
| 108 | + } else { |
| 109 | + struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); |
| 110 | + |
| 111 | + if (!io_futexv_claim(iof)) |
| 112 | + return false; |
| 113 | + req->io_task_work.func = io_futexv_complete; |
| 114 | + } |
| 67 | 115 | |
| 68 | 116 | hlist_del_init(&req->hash_node); |
| 69 | 117 | io_req_set_res(req, -ECANCELED, 0); |
| 70 | | − req->io_task_work.func = io_futex_complete; |
| 71 | 118 | io_req_task_work_add(req); |
| 72 | 119 | return true; |
| 73 | 120 | } |
| @@ -147,6 +194,55 @@ int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 73 unmodified lines |
| 147 | 194 | return 0; |
| 148 | 195 | } |
| 149 | 196 | |
| 197 | +static void io_futex_wakev_fn(struct wake_q_head *wake_q, struct futex_q *q) |
| 198 | +{ |
| 199 | + struct io_kiocb *req = q->wake_data; |
| 200 | + struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); |
| 201 | + |
| 202 | + if (!io_futexv_claim(iof)) |
| 203 | + return; |
| 204 | + if (unlikely(!__futex_wake_mark(q))) |
| 205 | + return; |
| 206 | + |
| 207 | + io_req_set_res(req, 0, 0); |
| 208 | + req->io_task_work.func = io_futexv_complete; |
| 209 | + io_req_task_work_add(req); |
| 210 | +} |
| 211 | + |
| 212 | +int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) |
| 213 | +{ |
| 214 | + struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); |
| 215 | + struct futex_vector *futexv; |
| 216 | + int ret; |
| 217 | + |
| 218 | + /* No flags or mask supported for waitv */ |
| 219 | + if (unlikely(sqe->fd || sqe->buf_index || sqe->file_index || |
| 220 | + sqe->addr2 || sqe->futex_flags || sqe->addr3)) |
| 221 | + return -EINVAL; |
| 222 | + |
| 223 | + iof->uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr)); |
| 224 | + iof->futex_nr = READ_ONCE(sqe->len); |
| 225 | + if (!iof->futex_nr || iof->futex_nr > FUTEX_WAITV_MAX) |
| 226 | + return -EINVAL; |
| 227 | + |
| 228 | + futexv = kcalloc(iof->futex_nr, sizeof(*futexv), GFP_KERNEL); |
| 229 | + if (!futexv) |
| 230 | + return -ENOMEM; |
| 231 | + |
| 232 | + ret = futex_parse_waitv(futexv, iof->uwaitv, iof->futex_nr, |
| 233 | + io_futex_wakev_fn, req); |
| 234 | + if (ret) { |
| 235 | + kfree(futexv); |
| 236 | + return ret; |
| 237 | + } |
| 238 | + |
| 239 | + iof->futexv_owned = 0; |
| 240 | + iof->futexv_unqueued = 0; |
| 241 | + req->flags |= REQ_F_ASYNC_DATA; |
| 242 | + req->async_data = futexv; |
| 243 | + return 0; |
| 244 | +} |
| 245 | + |
| 150 | 246 | static void io_futex_wake_fn(struct wake_q_head *wake_q, struct futex_q *q) |
| 151 | 247 | { |
| 152 | 248 | struct io_futex_data *ifd = container_of(q, struct io_futex_data, q); |
| @@ -171,6 +267,61 @@ static struct io_futex_data *io_alloc_ifd(struct io_ring_ctx *ctx) · 18 unmodified lines |
| 171 | 267 | return kmalloc(sizeof(struct io_futex_data), GFP_NOWAIT); |
| 172 | 268 | } |
| 173 | 269 | |
| 270 | +int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags) |
| 271 | +{ |
| 272 | + struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); |
| 273 | + struct futex_vector *futexv = req->async_data; |
| 274 | + struct io_ring_ctx *ctx = req->ctx; |
| 275 | + int ret, woken = -1; |
| 276 | + |
| 277 | + io_ring_submit_lock(ctx, issue_flags); |
| 278 | + |
| 279 | + ret = futex_wait_multiple_setup(futexv, iof->futex_nr, &woken); |
| 280 | + |
| 281 | + /* |
| 282 | + * Error case, ret is < 0. Mark the request as failed. |
| 283 | + */ |
| 284 | + if (unlikely(ret < 0)) { |
| 285 | + io_ring_submit_unlock(ctx, issue_flags); |
| 286 | + req_set_fail(req); |
| 287 | + io_req_set_res(req, ret, 0); |
| 288 | + kfree(futexv); |
| 289 | + req->async_data = NULL; |
| 290 | + req->flags &= ~REQ_F_ASYNC_DATA; |
| 291 | + return IOU_OK; |
| 292 | + } |
| 293 | + |
| 294 | + /* |
| 295 | + * 0 return means that we successfully setup the waiters, and that |
| 296 | + * nobody triggered a wakeup while we were doing so. If the wakeup |
| 297 | + * happened post setup, the task_work will be run post this issue and |
| 298 | + * under the submission lock. 1 means We got woken while setting up, |
| 299 | + * let that side do the completion. Note that |
| 300 | + * futex_wait_multiple_setup() will have unqueued all the futexes in |
| 301 | + * this case. Mark us as having done that already, since this is |
| 302 | + * different from normal wakeup. |
| 303 | + */ |
| 304 | + if (!ret) { |
| 305 | + /* |
| 306 | + * If futex_wait_multiple_setup() returns 0 for a |
| 307 | + * successful setup, then the task state will not be |
| 308 | + * runnable. This is fine for the sync syscall, as |
| 309 | + * it'll be blocking unless we already got one of the |
| 310 | + * futexes woken, but it obviously won't work for an |
| 311 | + * async invocation. Mark us runnable again. |
| 312 | + */ |
| 313 | + __set_current_state(TASK_RUNNING); |
| 314 | + hlist_add_head(&req->hash_node, &ctx->futex_list); |
| 315 | + } else { |
| 316 | + iof->futexv_unqueued = 1; |
| 317 | + if (woken != -1) |
| 318 | + io_req_set_res(req, woken, 0); |
| 319 | + } |
| 320 | + |
| 321 | + io_ring_submit_unlock(ctx, issue_flags); |
| 322 | + return IOU_ISSUE_SKIP_COMPLETE; |
| 323 | +} |
| 324 | + |
| 174 | 325 | int io_futex_wait(struct io_kiocb *req, unsigned int issue_flags) |
| 175 | 326 | { |
| 176 | 327 | struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); |