History

aio: simplify - and fix - fget/fput for io_submit()

Al Viro root-caused a race where the IOCB_CMD_POLL handling of fget/fput() could cause us to access the file pointer after it had already been freed: "In more details - normally IOCB_CMD_POLL handling looks so: 1) io_submit(2) allocates aio_kiocb instance and passes it to aio_poll() 2) aio_poll() resolves the descriptor to struct file by req->file = fget(iocb->aio_fildes) 3) aio_poll() sets ->woken to false and raises ->ki_refcnt of that aio_kiocb to 2 (bumps by 1, that is). 4) aio_poll() calls vfs_poll(). After sanity checks (basically, "poll_wait() had been called and only once") it locks the queue. That's what the extra reference to iocb had been for - we know we can safely access it. 5) With queue locked, we check if ->woken has already been set to true (by aio_poll_wake()) and, if it had been, we unlock the queue, drop a reference to aio_kiocb and bugger off - at that point it's a responsibility to aio_poll_wake() and the stuff called/scheduled by it. That code will drop the reference to file in req->file, along with the other reference to our aio_kiocb. 6) otherwise, we see whether we need to wait. If we do, we unlock the queue, drop one reference to aio_kiocb and go away - eventual wakeup (or cancel) will deal with the reference to file and with the other reference to aio_kiocb 7) otherwise we remove ourselves from waitqueue (still under the queue lock), so that wakeup won't get us. No async activity will be happening, so we can safely drop req->file and iocb ourselves. If wakeup happens while we are in vfs_poll(), we are fine - aio_kiocb won't get freed under us, so we can do all the checks and locking safely. And we don't touch ->file if we detect that case. However, vfs_poll() most certainly *does* touch the file it had been given. So wakeup coming while we are still in ->poll() might end up doing fput() on that file. That case is not too rare, and usually we are saved by the still present reference from descriptor table - that fput() is not the final one. But if another thread closes that descriptor right after our fget() and wakeup does happen before ->poll() returns, we are in trouble - final fput() done while we are in the middle of a method: Al also wrote a patch to take an extra reference to the file descriptor to fix this, but I instead suggested we just streamline the whole file pointer handling by submit_io() so that the generic aio submission code simply keeps the file pointer around until the aio has completed.

Linus Torvalds committed 7 years ago

Commit 84c4e1f1 parent 00c42372 files · +36 −44Browse files

Unified · Split

2 files
  1. fs/aio.c +29 −43
  2. include/linux/fs.h +7 −1
fs/aio.c+29−43
@@ -167,9 +167,13 @@ struct kioctx { · 166 unmodified lines
167 unsigned id;167 unsigned id;
168};168};
169 169
170/*
171 * First field must be the file pointer in all the
172 * iocb unions! See also 'struct kiocb' in <linux/fs.h>
173 */
170struct fsync_iocb {174struct fsync_iocb {
171 struct work_struct work;
172 struct file *file;175 struct file *file;
176 struct work_struct work;
173 bool datasync;177 bool datasync;
174};178};
175 179
@@ -183,8 +187,15 @@ struct poll_iocb { · 7 unmodified lines
183 struct work_struct work;187 struct work_struct work;
184};188};
185 189
190/*
191 * NOTE! Each of the iocb union members has the file pointer
192 * as the first entry in their struct definition. So you can
193 * access the file pointer through any of the sub-structs,
194 * or directly as just 'ki_filp' in this struct.
195 */
186struct aio_kiocb {196struct aio_kiocb {
187 union {197 union {
198 struct file *ki_filp;
188 struct kiocb rw;199 struct kiocb rw;
189 struct fsync_iocb fsync;200 struct fsync_iocb fsync;
190 struct poll_iocb poll;201 struct poll_iocb poll;
@@ -1060,6 +1071,8 @@ static inline void iocb_put(struct aio_kiocb *iocb) · 869 unmodified lines
1060{1071{
1061 if (refcount_read(&iocb->ki_refcnt) == 0 ||1072 if (refcount_read(&iocb->ki_refcnt) == 0 ||
1062 refcount_dec_and_test(&iocb->ki_refcnt)) {1073 refcount_dec_and_test(&iocb->ki_refcnt)) {
1074 if (iocb->ki_filp)
1075 fput(iocb->ki_filp);
1063 percpu_ref_put(&iocb->ki_ctx->reqs);1076 percpu_ref_put(&iocb->ki_ctx->reqs);
1064 kmem_cache_free(kiocb_cachep, iocb);1077 kmem_cache_free(kiocb_cachep, iocb);
1065 }1078 }
@@ -1424,7 +1437,6 @@ static void aio_complete_rw(struct kiocb *kiocb, long res, long res2) · 358 unmodified lines
1424 file_end_write(kiocb->ki_filp);1437 file_end_write(kiocb->ki_filp);
1425 }1438 }
1426 1439
1427 fput(kiocb->ki_filp);
1428 aio_complete(iocb, res, res2);1440 aio_complete(iocb, res, res2);
1429}1441}
1430 1442
@@ -1432,9 +1444,6 @@ static int aio_prep_rw(struct kiocb *req, const struct iocb *iocb) · 1 unmodified line
1432{1444{
1433 int ret;1445 int ret;
1434 1446
1435 req->ki_filp = fget(iocb->aio_fildes);
1436 if (unlikely(!req->ki_filp))
1437 return -EBADF;
1438 req->ki_complete = aio_complete_rw;1447 req->ki_complete = aio_complete_rw;
1439 req->private = NULL;1448 req->private = NULL;
1440 req->ki_pos = iocb->aio_offset;1449 req->ki_pos = iocb->aio_offset;
@@ -1451,7 +1460,7 @@ static int aio_prep_rw(struct kiocb *req, const struct iocb *iocb) · 10 unmodified lines
1451 ret = ioprio_check_cap(iocb->aio_reqprio);1460 ret = ioprio_check_cap(iocb->aio_reqprio);
1452 if (ret) {1461 if (ret) {
1453 pr_debug("aio ioprio check cap error: %d\n", ret);1462 pr_debug("aio ioprio check cap error: %d\n", ret);
1454 goto out_fput;1463 return ret;
1455 }1464 }
1456 1465
1457 req->ki_ioprio = iocb->aio_reqprio;1466 req->ki_ioprio = iocb->aio_reqprio;
@@ -1460,14 +1469,10 @@ static int aio_prep_rw(struct kiocb *req, const struct iocb *iocb) · 2 unmodified lines
1460 1469
1461 ret = kiocb_set_rw_flags(req, iocb->aio_rw_flags);1470 ret = kiocb_set_rw_flags(req, iocb->aio_rw_flags);
1462 if (unlikely(ret))1471 if (unlikely(ret))
1463 goto out_fput;1472 return ret;
1464 1473
1465 req->ki_flags &= ~IOCB_HIPRI; /* no one is going to poll for this I/O */1474 req->ki_flags &= ~IOCB_HIPRI; /* no one is going to poll for this I/O */
1466 return 0;1475 return 0;
1467
1468out_fput:
1469 fput(req->ki_filp);
1470 return ret;
1471}1476}
1472 1477
1473static int aio_setup_rw(int rw, const struct iocb *iocb, struct iovec **iovec,1478static int aio_setup_rw(int rw, const struct iocb *iocb, struct iovec **iovec,
@@ -1521,24 +1526,19 @@ static ssize_t aio_read(struct kiocb *req, const struct iocb *iocb, · 47 unmodified lines
1521 if (ret)1526 if (ret)
1522 return ret;1527 return ret;
1523 file = req->ki_filp;1528 file = req->ki_filp;
1524
1525 ret = -EBADF;
1526 if (unlikely(!(file->f_mode & FMODE_READ)))1529 if (unlikely(!(file->f_mode & FMODE_READ)))
1527 goto out_fput;1530 return -EBADF;
1528 ret = -EINVAL;1531 ret = -EINVAL;
1529 if (unlikely(!file->f_op->read_iter))1532 if (unlikely(!file->f_op->read_iter))
1530 goto out_fput;1533 return -EINVAL;
1531 1534
1532 ret = aio_setup_rw(READ, iocb, &iovec, vectored, compat, &iter);1535 ret = aio_setup_rw(READ, iocb, &iovec, vectored, compat, &iter);
1533 if (ret)1536 if (ret)
1534 goto out_fput;1537 return ret;
1535 ret = rw_verify_area(READ, file, &req->ki_pos, iov_iter_count(&iter));1538 ret = rw_verify_area(READ, file, &req->ki_pos, iov_iter_count(&iter));
1536 if (!ret)1539 if (!ret)
1537 aio_rw_done(req, call_read_iter(file, req, &iter));1540 aio_rw_done(req, call_read_iter(file, req, &iter));
1538 kfree(iovec);1541 kfree(iovec);
1539out_fput:
1540 if (unlikely(ret))
1541 fput(file);
1542 return ret;1542 return ret;
1543}1543}
1544 1544
@@ -1555,16 +1555,14 @@ static ssize_t aio_write(struct kiocb *req, const struct iocb *iocb, · 10 unmodified lines
1555 return ret;1555 return ret;
1556 file = req->ki_filp;1556 file = req->ki_filp;
1557 1557
1558 ret = -EBADF;
1559 if (unlikely(!(file->f_mode & FMODE_WRITE)))1558 if (unlikely(!(file->f_mode & FMODE_WRITE)))
1560 goto out_fput;1559 return -EBADF;
1561 ret = -EINVAL;
1562 if (unlikely(!file->f_op->write_iter))1560 if (unlikely(!file->f_op->write_iter))
1563 goto out_fput;1561 return -EINVAL;
1564 1562
1565 ret = aio_setup_rw(WRITE, iocb, &iovec, vectored, compat, &iter);1563 ret = aio_setup_rw(WRITE, iocb, &iovec, vectored, compat, &iter);
1566 if (ret)1564 if (ret)
1567 goto out_fput;1565 return ret;
1568 ret = rw_verify_area(WRITE, file, &req->ki_pos, iov_iter_count(&iter));1566 ret = rw_verify_area(WRITE, file, &req->ki_pos, iov_iter_count(&iter));
1569 if (!ret) {1567 if (!ret) {
1570 /*1568 /*
@@ -1582,9 +1580,6 @@ static ssize_t aio_write(struct kiocb *req, const struct iocb *iocb, · 11 unmodified lines
1582 aio_rw_done(req, call_write_iter(file, req, &iter));1580 aio_rw_done(req, call_write_iter(file, req, &iter));
1583 }1581 }
1584 kfree(iovec);1582 kfree(iovec);
1585out_fput:
1586 if (unlikely(ret))
1587 fput(file);
1588 return ret;1583 return ret;
1589}1584}
1590 1585
@@ -1594,7 +1589,6 @@ static void aio_fsync_work(struct work_struct *work) · 3 unmodified lines
1594 int ret;1589 int ret;
1595 1590
1596 ret = vfs_fsync(req->file, req->datasync);1591 ret = vfs_fsync(req->file, req->datasync);
1597 fput(req->file);
1598 aio_complete(container_of(req, struct aio_kiocb, fsync), ret, 0);1592 aio_complete(container_of(req, struct aio_kiocb, fsync), ret, 0);
1599}1593}
1600 1594
@@ -1605,13 +1599,8 @@ static int aio_fsync(struct fsync_iocb *req, const struct iocb *iocb, · 4 unmodified lines
1605 iocb->aio_rw_flags))1599 iocb->aio_rw_flags))
1606 return -EINVAL;1600 return -EINVAL;
1607 1601
1608 req->file = fget(iocb->aio_fildes);1602 if (unlikely(!req->file->f_op->fsync))
1609 if (unlikely(!req->file))
1610 return -EBADF;
1611 if (unlikely(!req->file->f_op->fsync)) {
1612 fput(req->file);
1613 return -EINVAL;1603 return -EINVAL;
1614 }
1615 1604
1616 req->datasync = datasync;1605 req->datasync = datasync;
1617 INIT_WORK(&req->work, aio_fsync_work);1606 INIT_WORK(&req->work, aio_fsync_work);
@@ -1621,10 +1610,7 @@ static int aio_fsync(struct fsync_iocb *req, const struct iocb *iocb, · 3 unmodified lines
1621 1610
1622static inline void aio_poll_complete(struct aio_kiocb *iocb, __poll_t mask)1611static inline void aio_poll_complete(struct aio_kiocb *iocb, __poll_t mask)
1623{1612{
1624 struct file *file = iocb->poll.file;
1625
1626 aio_complete(iocb, mangle_poll(mask), 0);1613 aio_complete(iocb, mangle_poll(mask), 0);
1627 fput(file);
1628}1614}
1629 1615
1630static void aio_poll_complete_work(struct work_struct *work)1616static void aio_poll_complete_work(struct work_struct *work)
@@ -1743,9 +1729,6 @@ static ssize_t aio_poll(struct aio_kiocb *aiocb, const struct iocb *iocb) · 112 unmodified lines
1743 1729
1744 INIT_WORK(&req->work, aio_poll_complete_work);1730 INIT_WORK(&req->work, aio_poll_complete_work);
1745 req->events = demangle_poll(iocb->aio_buf) | EPOLLERR | EPOLLHUP;1731 req->events = demangle_poll(iocb->aio_buf) | EPOLLERR | EPOLLHUP;
1746 req->file = fget(iocb->aio_fildes);
1747 if (unlikely(!req->file))
1748 return -EBADF;
1749 1732
1750 req->head = NULL;1733 req->head = NULL;
1751 req->woken = false;1734 req->woken = false;
@@ -1788,10 +1771,8 @@ static ssize_t aio_poll(struct aio_kiocb *aiocb, const struct iocb *iocb) · 36 unmodified lines
1788 spin_unlock_irq(&ctx->ctx_lock);1771 spin_unlock_irq(&ctx->ctx_lock);
1789 1772
1790out:1773out:
1791 if (unlikely(apt.error)) {1774 if (unlikely(apt.error))
1792 fput(req->file);
1793 return apt.error;1775 return apt.error;
1794 }
1795 1776
1796 if (mask)1777 if (mask)
1797 aio_poll_complete(aiocb, mask);1778 aio_poll_complete(aiocb, mask);
@@ -1829,6 +1810,11 @@ static int __io_submit_one(struct kioctx *ctx, const struct iocb *iocb, · 31 unmodified lines
1829 if (unlikely(!req))1810 if (unlikely(!req))
1830 goto out_put_reqs_available;1811 goto out_put_reqs_available;
1831 1812
1813 req->ki_filp = fget(iocb->aio_fildes);
1814 ret = -EBADF;
1815 if (unlikely(!req->ki_filp))
1816 goto out_put_req;
1817
1832 if (iocb->aio_flags & IOCB_FLAG_RESFD) {1818 if (iocb->aio_flags & IOCB_FLAG_RESFD) {
1833 /*1819 /*
1834 * If the IOCB_FLAG_RESFD flag of aio_flags is set, get an1820 * If the IOCB_FLAG_RESFD flag of aio_flags is set, get an
include/linux/fs.h+7−1
@@ -304,13 +304,19 @@ enum rw_hint { · 303 unmodified lines
304 304
305struct kiocb {305struct kiocb {
306 struct file *ki_filp;306 struct file *ki_filp;
307
308 /* The 'ki_filp' pointer is shared in a union for aio */
309 randomized_struct_fields_start
310
307 loff_t ki_pos;311 loff_t ki_pos;
308 void (*ki_complete)(struct kiocb *iocb, long ret, long ret2);312 void (*ki_complete)(struct kiocb *iocb, long ret, long ret2);
309 void *private;313 void *private;
310 int ki_flags;314 int ki_flags;
311 u16 ki_hint;315 u16 ki_hint;
312 u16 ki_ioprio; /* See linux/ioprio.h */316 u16 ki_ioprio; /* See linux/ioprio.h */
313} __randomize_layout;317
318 randomized_struct_fields_end
319};
314 320
315static inline bool is_sync_kiocb(struct kiocb *kiocb)321static inline bool is_sync_kiocb(struct kiocb *kiocb)
316{322{

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t