| @@ -14,10 +14,16 @@ · 13 unmodified lines |
| 14 | | 14 | |
| 15 | struct io_futex { | 15 | struct io_futex { |
| 16 | struct file *file; | 16 | struct file *file; |
| 17 | u32 __user *uaddr; | 17 | union { |
| | 18 | u32 __user *uaddr; |
| | 19 | struct futex_waitv __user *uwaitv; |
| | 20 | }; |
| 18 | unsigned long futex_val; | 21 | unsigned long futex_val; |
| 19 | unsigned long futex_mask; | 22 | unsigned long futex_mask; |
| | 23 | unsigned long futexv_owned; |
| 20 | u32 futex_flags; | 24 | u32 futex_flags; |
| | 25 | unsigned int futex_nr; |
| | 26 | bool futexv_unqueued; |
| 21 | }; | 27 | }; |
| 22 | | 28 | |
| 23 | struct io_futex_data { | 29 | struct io_futex_data { |
| @@ -44,6 +50,13 @@ void io_futex_cache_free(struct io_ring_ctx *ctx) · 20 unmodified lines |
| 44 | io_alloc_cache_free(&ctx->futex_cache, io_futex_cache_entry_free); | 50 | io_alloc_cache_free(&ctx->futex_cache, io_futex_cache_entry_free); |
| 45 | } | 51 | } |
| 46 | | 52 | |
| | 53 | static void __io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts) |
| | 54 | { |
| | 55 | req->async_data = NULL; |
| | 56 | hlist_del_init(&req->hash_node); |
| | 57 | io_req_task_complete(req, ts); |
| | 58 | } |
| | 59 | |
| 47 | static void io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts) | 60 | static void io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts) |
| 48 | { | 61 | { |
| 49 | struct io_futex_data *ifd = req->async_data; | 62 | struct io_futex_data *ifd = req->async_data; |
| @@ -52,22 +65,56 @@ static void io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts) · 2 unmodified lines |
| 52 | io_tw_lock(ctx, ts); | 65 | io_tw_lock(ctx, ts); |
| 53 | if (!io_alloc_cache_put(&ctx->futex_cache, &ifd->cache)) | 66 | if (!io_alloc_cache_put(&ctx->futex_cache, &ifd->cache)) |
| 54 | kfree(ifd); | 67 | kfree(ifd); |
| 55 | req->async_data = NULL; | 68 | __io_futex_complete(req, ts); |
| 56 | hlist_del_init(&req->hash_node); | | |
| 57 | io_req_task_complete(req, ts); | | |
| 58 | } | 69 | } |
| 59 | | 70 | |
| 60 | static bool __io_futex_cancel(struct io_ring_ctx *ctx, struct io_kiocb *req) | 71 | static void io_futexv_complete(struct io_kiocb *req, struct io_tw_state *ts) |
| 61 | { | 72 | { |
| 62 | struct io_futex_data *ifd = req->async_data; | 73 | struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); |
| | 74 | struct futex_vector *futexv = req->async_data; |
| 63 | | 75 | |
| 64 | /* futex wake already done or in progress */ | 76 | io_tw_lock(req->ctx, ts); |
| 65 | if (!futex_unqueue(&ifd->q)) | 77 | |
| | 78 | if (!iof->futexv_unqueued) { |
| | 79 | int res; |
| | 80 | |
| | 81 | res = futex_unqueue_multiple(futexv, iof->futex_nr); |
| | 82 | if (res != -1) |
| | 83 | io_req_set_res(req, res, 0); |
| | 84 | } |
| | 85 | |
| | 86 | kfree(req->async_data); |
| | 87 | req->flags &= ~REQ_F_ASYNC_DATA; |
| | 88 | __io_futex_complete(req, ts); |
| | 89 | } |
| | 90 | |
| | 91 | static bool io_futexv_claim(struct io_futex *iof) |
| | 92 | { |
| | 93 | if (test_bit(0, &iof->futexv_owned) || |
| | 94 | test_and_set_bit_lock(0, &iof->futexv_owned)) |
| 66 | return false; | 95 | return false; |
| | 96 | return true; |
| | 97 | } |
| | 98 | |
| | 99 | static bool __io_futex_cancel(struct io_ring_ctx *ctx, struct io_kiocb *req) |
| | 100 | { |
| | 101 | /* futex wake already done or in progress */ |
| | 102 | if (req->opcode == IORING_OP_FUTEX_WAIT) { |
| | 103 | struct io_futex_data *ifd = req->async_data; |
| | 104 | |
| | 105 | if (!futex_unqueue(&ifd->q)) |
| | 106 | return false; |
| | 107 | req->io_task_work.func = io_futex_complete; |
| | 108 | } else { |
| | 109 | struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); |
| | 110 | |
| | 111 | if (!io_futexv_claim(iof)) |
| | 112 | return false; |
| | 113 | req->io_task_work.func = io_futexv_complete; |
| | 114 | } |
| 67 | | 115 | |
| 68 | hlist_del_init(&req->hash_node); | 116 | hlist_del_init(&req->hash_node); |
| 69 | io_req_set_res(req, -ECANCELED, 0); | 117 | io_req_set_res(req, -ECANCELED, 0); |
| 70 | req->io_task_work.func = io_futex_complete; | | |
| 71 | io_req_task_work_add(req); | 118 | io_req_task_work_add(req); |
| 72 | return true; | 119 | return true; |
| 73 | } | 120 | } |
| @@ -147,6 +194,55 @@ int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) · 73 unmodified lines |
| 147 | return 0; | 194 | return 0; |
| 148 | } | 195 | } |
| 149 | | 196 | |
| | 197 | static void io_futex_wakev_fn(struct wake_q_head *wake_q, struct futex_q *q) |
| | 198 | { |
| | 199 | struct io_kiocb *req = q->wake_data; |
| | 200 | struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); |
| | 201 | |
| | 202 | if (!io_futexv_claim(iof)) |
| | 203 | return; |
| | 204 | if (unlikely(!__futex_wake_mark(q))) |
| | 205 | return; |
| | 206 | |
| | 207 | io_req_set_res(req, 0, 0); |
| | 208 | req->io_task_work.func = io_futexv_complete; |
| | 209 | io_req_task_work_add(req); |
| | 210 | } |
| | 211 | |
| | 212 | int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) |
| | 213 | { |
| | 214 | struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); |
| | 215 | struct futex_vector *futexv; |
| | 216 | int ret; |
| | 217 | |
| | 218 | /* No flags or mask supported for waitv */ |
| | 219 | if (unlikely(sqe->fd || sqe->buf_index || sqe->file_index || |
| | 220 | sqe->addr2 || sqe->futex_flags || sqe->addr3)) |
| | 221 | return -EINVAL; |
| | 222 | |
| | 223 | iof->uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr)); |
| | 224 | iof->futex_nr = READ_ONCE(sqe->len); |
| | 225 | if (!iof->futex_nr || iof->futex_nr > FUTEX_WAITV_MAX) |
| | 226 | return -EINVAL; |
| | 227 | |
| | 228 | futexv = kcalloc(iof->futex_nr, sizeof(*futexv), GFP_KERNEL); |
| | 229 | if (!futexv) |
| | 230 | return -ENOMEM; |
| | 231 | |
| | 232 | ret = futex_parse_waitv(futexv, iof->uwaitv, iof->futex_nr, |
| | 233 | io_futex_wakev_fn, req); |
| | 234 | if (ret) { |
| | 235 | kfree(futexv); |
| | 236 | return ret; |
| | 237 | } |
| | 238 | |
| | 239 | iof->futexv_owned = 0; |
| | 240 | iof->futexv_unqueued = 0; |
| | 241 | req->flags |= REQ_F_ASYNC_DATA; |
| | 242 | req->async_data = futexv; |
| | 243 | return 0; |
| | 244 | } |
| | 245 | |
| 150 | static void io_futex_wake_fn(struct wake_q_head *wake_q, struct futex_q *q) | 246 | static void io_futex_wake_fn(struct wake_q_head *wake_q, struct futex_q *q) |
| 151 | { | 247 | { |
| 152 | struct io_futex_data *ifd = container_of(q, struct io_futex_data, q); | 248 | struct io_futex_data *ifd = container_of(q, struct io_futex_data, q); |
| @@ -171,6 +267,61 @@ static struct io_futex_data *io_alloc_ifd(struct io_ring_ctx *ctx) · 18 unmodified lines |
| 171 | return kmalloc(sizeof(struct io_futex_data), GFP_NOWAIT); | 267 | return kmalloc(sizeof(struct io_futex_data), GFP_NOWAIT); |
| 172 | } | 268 | } |
| 173 | | 269 | |
| | 270 | int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags) |
| | 271 | { |
| | 272 | struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); |
| | 273 | struct futex_vector *futexv = req->async_data; |
| | 274 | struct io_ring_ctx *ctx = req->ctx; |
| | 275 | int ret, woken = -1; |
| | 276 | |
| | 277 | io_ring_submit_lock(ctx, issue_flags); |
| | 278 | |
| | 279 | ret = futex_wait_multiple_setup(futexv, iof->futex_nr, &woken); |
| | 280 | |
| | 281 | /* |
| | 282 | * Error case, ret is < 0. Mark the request as failed. |
| | 283 | */ |
| | 284 | if (unlikely(ret < 0)) { |
| | 285 | io_ring_submit_unlock(ctx, issue_flags); |
| | 286 | req_set_fail(req); |
| | 287 | io_req_set_res(req, ret, 0); |
| | 288 | kfree(futexv); |
| | 289 | req->async_data = NULL; |
| | 290 | req->flags &= ~REQ_F_ASYNC_DATA; |
| | 291 | return IOU_OK; |
| | 292 | } |
| | 293 | |
| | 294 | /* |
| | 295 | * 0 return means that we successfully setup the waiters, and that |
| | 296 | * nobody triggered a wakeup while we were doing so. If the wakeup |
| | 297 | * happened post setup, the task_work will be run post this issue and |
| | 298 | * under the submission lock. 1 means We got woken while setting up, |
| | 299 | * let that side do the completion. Note that |
| | 300 | * futex_wait_multiple_setup() will have unqueued all the futexes in |
| | 301 | * this case. Mark us as having done that already, since this is |
| | 302 | * different from normal wakeup. |
| | 303 | */ |
| | 304 | if (!ret) { |
| | 305 | /* |
| | 306 | * If futex_wait_multiple_setup() returns 0 for a |
| | 307 | * successful setup, then the task state will not be |
| | 308 | * runnable. This is fine for the sync syscall, as |
| | 309 | * it'll be blocking unless we already got one of the |
| | 310 | * futexes woken, but it obviously won't work for an |
| | 311 | * async invocation. Mark us runnable again. |
| | 312 | */ |
| | 313 | __set_current_state(TASK_RUNNING); |
| | 314 | hlist_add_head(&req->hash_node, &ctx->futex_list); |
| | 315 | } else { |
| | 316 | iof->futexv_unqueued = 1; |
| | 317 | if (woken != -1) |
| | 318 | io_req_set_res(req, woken, 0); |
| | 319 | } |
| | 320 | |
| | 321 | io_ring_submit_unlock(ctx, issue_flags); |
| | 322 | return IOU_ISSUE_SKIP_COMPLETE; |
| | 323 | } |
| | 324 | |
| 174 | int io_futex_wait(struct io_kiocb *req, unsigned int issue_flags) | 325 | int io_futex_wait(struct io_kiocb *req, unsigned int issue_flags) |
| 175 | { | 326 | { |
| 176 | struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); | 327 | struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); |