Al Viro root-caused a race where the IOCB_CMD_POLL handling of fget/fput() could cause us to access the file pointer after it had already been freed: "In more details - normally IOCB_CMD_POLL handling looks so: 1) io_submit(2) allocates aio_kiocb instance and passes it to aio_poll() 2) aio_poll() resolves the descriptor to struct file by req->file = fget(iocb->aio_fildes) 3) aio_poll() sets ->woken to false and raises ->ki_refcnt of that aio_kiocb to 2 (bumps by 1, that is). 4) aio_poll() calls vfs_poll(). After sanity checks (basically, "poll_wait() had been called and only once") it locks the queue. That's what the extra reference to iocb had been for - we know we can safely access it. 5) With queue locked, we check if ->woken has already been set to true (by aio_poll_wake()) and, if it had been, we unlock the queue, drop a reference to aio_kiocb and bugger off - at that point it's a responsibility to aio_poll_wake() and the stuff called/scheduled by it. That code will drop the reference to file in req->file, along with the other reference to our aio_kiocb. 6) otherwise, we see whether we need to wait. If we do, we unlock the queue, drop one reference to aio_kiocb and go away - eventual wakeup (or cancel) will deal with the reference to file and with the other reference to aio_kiocb 7) otherwise we remove ourselves from waitqueue (still under the queue lock), so that wakeup won't get us. No async activity will be happening, so we can safely drop req->file and iocb ourselves. If wakeup happens while we are in vfs_poll(), we are fine - aio_kiocb won't get freed under us, so we can do all the checks and locking safely. And we don't touch ->file if we detect that case. However, vfs_poll() most certainly *does* touch the file it had been given. So wakeup coming while we are still in ->poll() might end up doing fput() on that file. That case is not too rare, and usually we are saved by the still present reference from descriptor table - that fput() is not the final one. But if another thread closes that descriptor right after our fget() and wakeup does happen before ->poll() returns, we are in trouble - final fput() done while we are in the middle of a method: Al also wrote a patch to take an extra reference to the file descriptor to fix this, but I instead suggested we just streamline the whole file pointer handling by submit_io() so that the generic aio submission code simply keeps the file pointer around until the aio has completed.
Unified · Split
| @@ -167,9 +167,13 @@ struct kioctx { · 166 unmodified lines | |||
| 167 | unsigned id; | 167 | unsigned id; |
| 168 | }; | 168 | }; |
| 169 | 169 | ||
| 170 | /* | ||
| 171 | * First field must be the file pointer in all the | ||
| 172 | * iocb unions! See also 'struct kiocb' in <linux/fs.h> | ||
| 173 | */ | ||
| 170 | struct fsync_iocb { | 174 | struct fsync_iocb { |
| 171 | struct work_struct work; | ||
| 172 | struct file *file; | 175 | struct file *file; |
| 176 | struct work_struct work; | ||
| 173 | bool datasync; | 177 | bool datasync; |
| 174 | }; | 178 | }; |
| 175 | 179 | ||
| @@ -183,8 +187,15 @@ struct poll_iocb { · 7 unmodified lines | |||
| 183 | struct work_struct work; | 187 | struct work_struct work; |
| 184 | }; | 188 | }; |
| 185 | 189 | ||
| 190 | /* | ||
| 191 | * NOTE! Each of the iocb union members has the file pointer | ||
| 192 | * as the first entry in their struct definition. So you can | ||
| 193 | * access the file pointer through any of the sub-structs, | ||
| 194 | * or directly as just 'ki_filp' in this struct. | ||
| 195 | */ | ||
| 186 | struct aio_kiocb { | 196 | struct aio_kiocb { |
| 187 | union { | 197 | union { |
| 198 | struct file *ki_filp; | ||
| 188 | struct kiocb rw; | 199 | struct kiocb rw; |
| 189 | struct fsync_iocb fsync; | 200 | struct fsync_iocb fsync; |
| 190 | struct poll_iocb poll; | 201 | struct poll_iocb poll; |
| @@ -1060,6 +1071,8 @@ static inline void iocb_put(struct aio_kiocb *iocb) · 869 unmodified lines | |||
| 1060 | { | 1071 | { |
| 1061 | if (refcount_read(&iocb->ki_refcnt) == 0 || | 1072 | if (refcount_read(&iocb->ki_refcnt) == 0 || |
| 1062 | refcount_dec_and_test(&iocb->ki_refcnt)) { | 1073 | refcount_dec_and_test(&iocb->ki_refcnt)) { |
| 1074 | if (iocb->ki_filp) | ||
| 1075 | fput(iocb->ki_filp); | ||
| 1063 | percpu_ref_put(&iocb->ki_ctx->reqs); | 1076 | percpu_ref_put(&iocb->ki_ctx->reqs); |
| 1064 | kmem_cache_free(kiocb_cachep, iocb); | 1077 | kmem_cache_free(kiocb_cachep, iocb); |
| 1065 | } | 1078 | } |
| @@ -1424,7 +1437,6 @@ static void aio_complete_rw(struct kiocb *kiocb, long res, long res2) · 358 unmodified lines | |||
| 1424 | file_end_write(kiocb->ki_filp); | 1437 | file_end_write(kiocb->ki_filp); |
| 1425 | } | 1438 | } |
| 1426 | 1439 | ||
| 1427 | fput(kiocb->ki_filp); | ||
| 1428 | aio_complete(iocb, res, res2); | 1440 | aio_complete(iocb, res, res2); |
| 1429 | } | 1441 | } |
| 1430 | 1442 | ||
| @@ -1432,9 +1444,6 @@ static int aio_prep_rw(struct kiocb *req, const struct iocb *iocb) · 1 unmodified line | |||
| 1432 | { | 1444 | { |
| 1433 | int ret; | 1445 | int ret; |
| 1434 | 1446 | ||
| 1435 | req->ki_filp = fget(iocb->aio_fildes); | ||
| 1436 | if (unlikely(!req->ki_filp)) | ||
| 1437 | return -EBADF; | ||
| 1438 | req->ki_complete = aio_complete_rw; | 1447 | req->ki_complete = aio_complete_rw; |
| 1439 | req->private = NULL; | 1448 | req->private = NULL; |
| 1440 | req->ki_pos = iocb->aio_offset; | 1449 | req->ki_pos = iocb->aio_offset; |
| @@ -1451,7 +1460,7 @@ static int aio_prep_rw(struct kiocb *req, const struct iocb *iocb) · 10 unmodified lines | |||
| 1451 | ret = ioprio_check_cap(iocb->aio_reqprio); | 1460 | ret = ioprio_check_cap(iocb->aio_reqprio); |
| 1452 | if (ret) { | 1461 | if (ret) { |
| 1453 | pr_debug("aio ioprio check cap error: %d\n", ret); | 1462 | pr_debug("aio ioprio check cap error: %d\n", ret); |
| 1454 | goto out_fput; | 1463 | return ret; |
| 1455 | } | 1464 | } |
| 1456 | 1465 | ||
| 1457 | req->ki_ioprio = iocb->aio_reqprio; | 1466 | req->ki_ioprio = iocb->aio_reqprio; |
| @@ -1460,14 +1469,10 @@ static int aio_prep_rw(struct kiocb *req, const struct iocb *iocb) · 2 unmodified lines | |||
| 1460 | 1469 | ||
| 1461 | ret = kiocb_set_rw_flags(req, iocb->aio_rw_flags); | 1470 | ret = kiocb_set_rw_flags(req, iocb->aio_rw_flags); |
| 1462 | if (unlikely(ret)) | 1471 | if (unlikely(ret)) |
| 1463 | goto out_fput; | 1472 | return ret; |
| 1464 | 1473 | ||
| 1465 | req->ki_flags &= ~IOCB_HIPRI; /* no one is going to poll for this I/O */ | 1474 | req->ki_flags &= ~IOCB_HIPRI; /* no one is going to poll for this I/O */ |
| 1466 | return 0; | 1475 | return 0; |
| 1467 | |||
| 1468 | out_fput: | ||
| 1469 | fput(req->ki_filp); | ||
| 1470 | return ret; | ||
| 1471 | } | 1476 | } |
| 1472 | 1477 | ||
| 1473 | static int aio_setup_rw(int rw, const struct iocb *iocb, struct iovec **iovec, | 1478 | static int aio_setup_rw(int rw, const struct iocb *iocb, struct iovec **iovec, |
| @@ -1521,24 +1526,19 @@ static ssize_t aio_read(struct kiocb *req, const struct iocb *iocb, · 47 unmodified lines | |||
| 1521 | if (ret) | 1526 | if (ret) |
| 1522 | return ret; | 1527 | return ret; |
| 1523 | file = req->ki_filp; | 1528 | file = req->ki_filp; |
| 1524 | |||
| 1525 | ret = -EBADF; | ||
| 1526 | if (unlikely(!(file->f_mode & FMODE_READ))) | 1529 | if (unlikely(!(file->f_mode & FMODE_READ))) |
| 1527 | goto out_fput; | 1530 | return -EBADF; |
| 1528 | ret = -EINVAL; | 1531 | ret = -EINVAL; |
| 1529 | if (unlikely(!file->f_op->read_iter)) | 1532 | if (unlikely(!file->f_op->read_iter)) |
| 1530 | goto out_fput; | 1533 | return -EINVAL; |
| 1531 | 1534 | ||
| 1532 | ret = aio_setup_rw(READ, iocb, &iovec, vectored, compat, &iter); | 1535 | ret = aio_setup_rw(READ, iocb, &iovec, vectored, compat, &iter); |
| 1533 | if (ret) | 1536 | if (ret) |
| 1534 | goto out_fput; | 1537 | return ret; |
| 1535 | ret = rw_verify_area(READ, file, &req->ki_pos, iov_iter_count(&iter)); | 1538 | ret = rw_verify_area(READ, file, &req->ki_pos, iov_iter_count(&iter)); |
| 1536 | if (!ret) | 1539 | if (!ret) |
| 1537 | aio_rw_done(req, call_read_iter(file, req, &iter)); | 1540 | aio_rw_done(req, call_read_iter(file, req, &iter)); |
| 1538 | kfree(iovec); | 1541 | kfree(iovec); |
| 1539 | out_fput: | ||
| 1540 | if (unlikely(ret)) | ||
| 1541 | fput(file); | ||
| 1542 | return ret; | 1542 | return ret; |
| 1543 | } | 1543 | } |
| 1544 | 1544 | ||
| @@ -1555,16 +1555,14 @@ static ssize_t aio_write(struct kiocb *req, const struct iocb *iocb, · 10 unmodified lines | |||
| 1555 | return ret; | 1555 | return ret; |
| 1556 | file = req->ki_filp; | 1556 | file = req->ki_filp; |
| 1557 | 1557 | ||
| 1558 | ret = -EBADF; | ||
| 1559 | if (unlikely(!(file->f_mode & FMODE_WRITE))) | 1558 | if (unlikely(!(file->f_mode & FMODE_WRITE))) |
| 1560 | goto out_fput; | 1559 | return -EBADF; |
| 1561 | ret = -EINVAL; | ||
| 1562 | if (unlikely(!file->f_op->write_iter)) | 1560 | if (unlikely(!file->f_op->write_iter)) |
| 1563 | goto out_fput; | 1561 | return -EINVAL; |
| 1564 | 1562 | ||
| 1565 | ret = aio_setup_rw(WRITE, iocb, &iovec, vectored, compat, &iter); | 1563 | ret = aio_setup_rw(WRITE, iocb, &iovec, vectored, compat, &iter); |
| 1566 | if (ret) | 1564 | if (ret) |
| 1567 | goto out_fput; | 1565 | return ret; |
| 1568 | ret = rw_verify_area(WRITE, file, &req->ki_pos, iov_iter_count(&iter)); | 1566 | ret = rw_verify_area(WRITE, file, &req->ki_pos, iov_iter_count(&iter)); |
| 1569 | if (!ret) { | 1567 | if (!ret) { |
| 1570 | /* | 1568 | /* |
| @@ -1582,9 +1580,6 @@ static ssize_t aio_write(struct kiocb *req, const struct iocb *iocb, · 11 unmodified lines | |||
| 1582 | aio_rw_done(req, call_write_iter(file, req, &iter)); | 1580 | aio_rw_done(req, call_write_iter(file, req, &iter)); |
| 1583 | } | 1581 | } |
| 1584 | kfree(iovec); | 1582 | kfree(iovec); |
| 1585 | out_fput: | ||
| 1586 | if (unlikely(ret)) | ||
| 1587 | fput(file); | ||
| 1588 | return ret; | 1583 | return ret; |
| 1589 | } | 1584 | } |
| 1590 | 1585 | ||
| @@ -1594,7 +1589,6 @@ static void aio_fsync_work(struct work_struct *work) · 3 unmodified lines | |||
| 1594 | int ret; | 1589 | int ret; |
| 1595 | 1590 | ||
| 1596 | ret = vfs_fsync(req->file, req->datasync); | 1591 | ret = vfs_fsync(req->file, req->datasync); |
| 1597 | fput(req->file); | ||
| 1598 | aio_complete(container_of(req, struct aio_kiocb, fsync), ret, 0); | 1592 | aio_complete(container_of(req, struct aio_kiocb, fsync), ret, 0); |
| 1599 | } | 1593 | } |
| 1600 | 1594 | ||
| @@ -1605,13 +1599,8 @@ static int aio_fsync(struct fsync_iocb *req, const struct iocb *iocb, · 4 unmodified lines | |||
| 1605 | iocb->aio_rw_flags)) | 1599 | iocb->aio_rw_flags)) |
| 1606 | return -EINVAL; | 1600 | return -EINVAL; |
| 1607 | 1601 | ||
| 1608 | req->file = fget(iocb->aio_fildes); | 1602 | if (unlikely(!req->file->f_op->fsync)) |
| 1609 | if (unlikely(!req->file)) | ||
| 1610 | return -EBADF; | ||
| 1611 | if (unlikely(!req->file->f_op->fsync)) { | ||
| 1612 | fput(req->file); | ||
| 1613 | return -EINVAL; | 1603 | return -EINVAL; |
| 1614 | } | ||
| 1615 | 1604 | ||
| 1616 | req->datasync = datasync; | 1605 | req->datasync = datasync; |
| 1617 | INIT_WORK(&req->work, aio_fsync_work); | 1606 | INIT_WORK(&req->work, aio_fsync_work); |
| @@ -1621,10 +1610,7 @@ static int aio_fsync(struct fsync_iocb *req, const struct iocb *iocb, · 3 unmodified lines | |||
| 1621 | 1610 | ||
| 1622 | static inline void aio_poll_complete(struct aio_kiocb *iocb, __poll_t mask) | 1611 | static inline void aio_poll_complete(struct aio_kiocb *iocb, __poll_t mask) |
| 1623 | { | 1612 | { |
| 1624 | struct file *file = iocb->poll.file; | ||
| 1625 | |||
| 1626 | aio_complete(iocb, mangle_poll(mask), 0); | 1613 | aio_complete(iocb, mangle_poll(mask), 0); |
| 1627 | fput(file); | ||
| 1628 | } | 1614 | } |
| 1629 | 1615 | ||
| 1630 | static void aio_poll_complete_work(struct work_struct *work) | 1616 | static void aio_poll_complete_work(struct work_struct *work) |
| @@ -1743,9 +1729,6 @@ static ssize_t aio_poll(struct aio_kiocb *aiocb, const struct iocb *iocb) · 112 unmodified lines | |||
| 1743 | 1729 | ||
| 1744 | INIT_WORK(&req->work, aio_poll_complete_work); | 1730 | INIT_WORK(&req->work, aio_poll_complete_work); |
| 1745 | req->events = demangle_poll(iocb->aio_buf) | EPOLLERR | EPOLLHUP; | 1731 | req->events = demangle_poll(iocb->aio_buf) | EPOLLERR | EPOLLHUP; |
| 1746 | req->file = fget(iocb->aio_fildes); | ||
| 1747 | if (unlikely(!req->file)) | ||
| 1748 | return -EBADF; | ||
| 1749 | 1732 | ||
| 1750 | req->head = NULL; | 1733 | req->head = NULL; |
| 1751 | req->woken = false; | 1734 | req->woken = false; |
| @@ -1788,10 +1771,8 @@ static ssize_t aio_poll(struct aio_kiocb *aiocb, const struct iocb *iocb) · 36 unmodified lines | |||
| 1788 | spin_unlock_irq(&ctx->ctx_lock); | 1771 | spin_unlock_irq(&ctx->ctx_lock); |
| 1789 | 1772 | ||
| 1790 | out: | 1773 | out: |
| 1791 | if (unlikely(apt.error)) { | 1774 | if (unlikely(apt.error)) |
| 1792 | fput(req->file); | ||
| 1793 | return apt.error; | 1775 | return apt.error; |
| 1794 | } | ||
| 1795 | 1776 | ||
| 1796 | if (mask) | 1777 | if (mask) |
| 1797 | aio_poll_complete(aiocb, mask); | 1778 | aio_poll_complete(aiocb, mask); |
| @@ -1829,6 +1810,11 @@ static int __io_submit_one(struct kioctx *ctx, const struct iocb *iocb, · 31 unmodified lines | |||
| 1829 | if (unlikely(!req)) | 1810 | if (unlikely(!req)) |
| 1830 | goto out_put_reqs_available; | 1811 | goto out_put_reqs_available; |
| 1831 | 1812 | ||
| 1813 | req->ki_filp = fget(iocb->aio_fildes); | ||
| 1814 | ret = -EBADF; | ||
| 1815 | if (unlikely(!req->ki_filp)) | ||
| 1816 | goto out_put_req; | ||
| 1817 | |||
| 1832 | if (iocb->aio_flags & IOCB_FLAG_RESFD) { | 1818 | if (iocb->aio_flags & IOCB_FLAG_RESFD) { |
| 1833 | /* | 1819 | /* |
| 1834 | * If the IOCB_FLAG_RESFD flag of aio_flags is set, get an | 1820 | * If the IOCB_FLAG_RESFD flag of aio_flags is set, get an |
| @@ -304,13 +304,19 @@ enum rw_hint { · 303 unmodified lines | |||
| 304 | 304 | ||
| 305 | struct kiocb { | 305 | struct kiocb { |
| 306 | struct file *ki_filp; | 306 | struct file *ki_filp; |
| 307 | |||
| 308 | /* The 'ki_filp' pointer is shared in a union for aio */ | ||
| 309 | randomized_struct_fields_start | ||
| 310 | |||
| 307 | loff_t ki_pos; | 311 | loff_t ki_pos; |
| 308 | void (*ki_complete)(struct kiocb *iocb, long ret, long ret2); | 312 | void (*ki_complete)(struct kiocb *iocb, long ret, long ret2); |
| 309 | void *private; | 313 | void *private; |
| 310 | int ki_flags; | 314 | int ki_flags; |
| 311 | u16 ki_hint; | 315 | u16 ki_hint; |
| 312 | u16 ki_ioprio; /* See linux/ioprio.h */ | 316 | u16 ki_ioprio; /* See linux/ioprio.h */ |
| 313 | } __randomize_layout; | 317 | |
| 318 | randomized_struct_fields_end | ||
| 319 | }; | ||
| 314 | 320 | ||
| 315 | static inline bool is_sync_kiocb(struct kiocb *kiocb) | 321 | static inline bool is_sync_kiocb(struct kiocb *kiocb) |
| 316 | { | 322 | { |