History

aio: simplify - and fix - fget/fput for io_submit()

Al Viro root-caused a race where the IOCB_CMD_POLL handling of fget/fput() could cause us to access the file pointer after it had already been freed: "In more details - normally IOCB_CMD_POLL handling looks so: 1) io_submit(2) allocates aio_kiocb instance and passes it to aio_poll() 2) aio_poll() resolves the descriptor to struct file by req->file = fget(iocb->aio_fildes) 3) aio_poll() sets ->woken to false and raises ->ki_refcnt of that aio_kiocb to 2 (bumps by 1, that is). 4) aio_poll() calls vfs_poll(). After sanity checks (basically, "poll_wait() had been called and only once") it locks the queue. That's what the extra reference to iocb had been for - we know we can safely access it. 5) With queue locked, we check if ->woken has already been set to true (by aio_poll_wake()) and, if it had been, we unlock the queue, drop a reference to aio_kiocb and bugger off - at that point it's a responsibility to aio_poll_wake() and the stuff called/scheduled by it. That code will drop the reference to file in req->file, along with the other reference to our aio_kiocb. 6) otherwise, we see whether we need to wait. If we do, we unlock the queue, drop one reference to aio_kiocb and go away - eventual wakeup (or cancel) will deal with the reference to file and with the other reference to aio_kiocb 7) otherwise we remove ourselves from waitqueue (still under the queue lock), so that wakeup won't get us. No async activity will be happening, so we can safely drop req->file and iocb ourselves. If wakeup happens while we are in vfs_poll(), we are fine - aio_kiocb won't get freed under us, so we can do all the checks and locking safely. And we don't touch ->file if we detect that case. However, vfs_poll() most certainly *does* touch the file it had been given. So wakeup coming while we are still in ->poll() might end up doing fput() on that file. That case is not too rare, and usually we are saved by the still present reference from descriptor table - that fput() is not the final one. But if another thread closes that descriptor right after our fget() and wakeup does happen before ->poll() returns, we are in trouble - final fput() done while we are in the middle of a method: Al also wrote a patch to take an extra reference to the file descriptor to fix this, but I instead suggested we just streamline the whole file pointer handling by submit_io() so that the generic aio submission code simply keeps the file pointer around until the aio has completed.

Linus Torvalds committed 7 years ago

Commit 84c4e1f1 parent 00c42372 files · +36 −44Browse files

Unified · Split

2 files
  1. fs/aio.c +29 −43
  2. include/linux/fs.h +7 −1
fs/aio.c+29−43
@@ -167,9 +167,13 @@ struct kioctx { · 166 unmodified lines
167167 unsigned id;
168168 };
169169
170+/*
171+ * First field must be the file pointer in all the
172+ * iocb unions! See also 'struct kiocb' in <linux/fs.h>
173+ */
170174 struct fsync_iocb {
171− struct work_struct work;
172175 struct file *file;
176+ struct work_struct work;
173177 bool datasync;
174178 };
175179
@@ -183,8 +187,15 @@ struct poll_iocb { · 7 unmodified lines
183187 struct work_struct work;
184188 };
185189
190+/*
191+ * NOTE! Each of the iocb union members has the file pointer
192+ * as the first entry in their struct definition. So you can
193+ * access the file pointer through any of the sub-structs,
194+ * or directly as just 'ki_filp' in this struct.
195+ */
186196 struct aio_kiocb {
187197 union {
198+ struct file *ki_filp;
188199 struct kiocb rw;
189200 struct fsync_iocb fsync;
190201 struct poll_iocb poll;
@@ -1060,6 +1071,8 @@ static inline void iocb_put(struct aio_kiocb *iocb) · 869 unmodified lines
10601071 {
10611072 if (refcount_read(&iocb->ki_refcnt) == 0 ||
10621073 refcount_dec_and_test(&iocb->ki_refcnt)) {
1074+ if (iocb->ki_filp)
1075+ fput(iocb->ki_filp);
10631076 percpu_ref_put(&iocb->ki_ctx->reqs);
10641077 kmem_cache_free(kiocb_cachep, iocb);
10651078 }
@@ -1424,7 +1437,6 @@ static void aio_complete_rw(struct kiocb *kiocb, long res, long res2) · 358 unmodified lines
14241437 file_end_write(kiocb->ki_filp);
14251438 }
14261439
1427− fput(kiocb->ki_filp);
14281440 aio_complete(iocb, res, res2);
14291441 }
14301442
@@ -1432,9 +1444,6 @@ static int aio_prep_rw(struct kiocb *req, const struct iocb *iocb) · 1 unmodified line
14321444 {
14331445 int ret;
14341446
1435− req->ki_filp = fget(iocb->aio_fildes);
1436− if (unlikely(!req->ki_filp))
1437− return -EBADF;
14381447 req->ki_complete = aio_complete_rw;
14391448 req->private = NULL;
14401449 req->ki_pos = iocb->aio_offset;
@@ -1451,7 +1460,7 @@ static int aio_prep_rw(struct kiocb *req, const struct iocb *iocb) · 10 unmodified lines
14511460 ret = ioprio_check_cap(iocb->aio_reqprio);
14521461 if (ret) {
14531462 pr_debug("aio ioprio check cap error: %d\n", ret);
1454− goto out_fput;
1463+ return ret;
14551464 }
14561465
14571466 req->ki_ioprio = iocb->aio_reqprio;
@@ -1460,14 +1469,10 @@ static int aio_prep_rw(struct kiocb *req, const struct iocb *iocb) · 2 unmodified lines
14601469
14611470 ret = kiocb_set_rw_flags(req, iocb->aio_rw_flags);
14621471 if (unlikely(ret))
1463− goto out_fput;
1472+ return ret;
14641473
14651474 req->ki_flags &= ~IOCB_HIPRI; /* no one is going to poll for this I/O */
14661475 return 0;
1467−
1468−out_fput:
1469− fput(req->ki_filp);
1470− return ret;
14711476 }
14721477
14731478 static int aio_setup_rw(int rw, const struct iocb *iocb, struct iovec **iovec,
@@ -1521,24 +1526,19 @@ static ssize_t aio_read(struct kiocb *req, const struct iocb *iocb, · 47 unmodified lines
15211526 if (ret)
15221527 return ret;
15231528 file = req->ki_filp;
1524−
1525− ret = -EBADF;
15261529 if (unlikely(!(file->f_mode & FMODE_READ)))
1527− goto out_fput;
1530+ return -EBADF;
15281531 ret = -EINVAL;
15291532 if (unlikely(!file->f_op->read_iter))
1530− goto out_fput;
1533+ return -EINVAL;
15311534
15321535 ret = aio_setup_rw(READ, iocb, &iovec, vectored, compat, &iter);
15331536 if (ret)
1534− goto out_fput;
1537+ return ret;
15351538 ret = rw_verify_area(READ, file, &req->ki_pos, iov_iter_count(&iter));
15361539 if (!ret)
15371540 aio_rw_done(req, call_read_iter(file, req, &iter));
15381541 kfree(iovec);
1539−out_fput:
1540− if (unlikely(ret))
1541− fput(file);
15421542 return ret;
15431543 }
15441544
@@ -1555,16 +1555,14 @@ static ssize_t aio_write(struct kiocb *req, const struct iocb *iocb, · 10 unmodified lines
15551555 return ret;
15561556 file = req->ki_filp;
15571557
1558− ret = -EBADF;
15591558 if (unlikely(!(file->f_mode & FMODE_WRITE)))
1560− goto out_fput;
1561− ret = -EINVAL;
1559+ return -EBADF;
15621560 if (unlikely(!file->f_op->write_iter))
1563− goto out_fput;
1561+ return -EINVAL;
15641562
15651563 ret = aio_setup_rw(WRITE, iocb, &iovec, vectored, compat, &iter);
15661564 if (ret)
1567− goto out_fput;
1565+ return ret;
15681566 ret = rw_verify_area(WRITE, file, &req->ki_pos, iov_iter_count(&iter));
15691567 if (!ret) {
15701568 /*
@@ -1582,9 +1580,6 @@ static ssize_t aio_write(struct kiocb *req, const struct iocb *iocb, · 11 unmodified lines
15821580 aio_rw_done(req, call_write_iter(file, req, &iter));
15831581 }
15841582 kfree(iovec);
1585−out_fput:
1586− if (unlikely(ret))
1587− fput(file);
15881583 return ret;
15891584 }
15901585
@@ -1594,7 +1589,6 @@ static void aio_fsync_work(struct work_struct *work) · 3 unmodified lines
15941589 int ret;
15951590
15961591 ret = vfs_fsync(req->file, req->datasync);
1597− fput(req->file);
15981592 aio_complete(container_of(req, struct aio_kiocb, fsync), ret, 0);
15991593 }
16001594
@@ -1605,13 +1599,8 @@ static int aio_fsync(struct fsync_iocb *req, const struct iocb *iocb, · 4 unmodified lines
16051599 iocb->aio_rw_flags))
16061600 return -EINVAL;
16071601
1608− req->file = fget(iocb->aio_fildes);
1609− if (unlikely(!req->file))
1610− return -EBADF;
1611− if (unlikely(!req->file->f_op->fsync)) {
1612− fput(req->file);
1602+ if (unlikely(!req->file->f_op->fsync))
16131603 return -EINVAL;
1614− }
16151604
16161605 req->datasync = datasync;
16171606 INIT_WORK(&req->work, aio_fsync_work);
@@ -1621,10 +1610,7 @@ static int aio_fsync(struct fsync_iocb *req, const struct iocb *iocb, · 3 unmodified lines
16211610
16221611 static inline void aio_poll_complete(struct aio_kiocb *iocb, __poll_t mask)
16231612 {
1624− struct file *file = iocb->poll.file;
1625−
16261613 aio_complete(iocb, mangle_poll(mask), 0);
1627− fput(file);
16281614 }
16291615
16301616 static void aio_poll_complete_work(struct work_struct *work)
@@ -1743,9 +1729,6 @@ static ssize_t aio_poll(struct aio_kiocb *aiocb, const struct iocb *iocb) · 112 unmodified lines
17431729
17441730 INIT_WORK(&req->work, aio_poll_complete_work);
17451731 req->events = demangle_poll(iocb->aio_buf) | EPOLLERR | EPOLLHUP;
1746− req->file = fget(iocb->aio_fildes);
1747− if (unlikely(!req->file))
1748− return -EBADF;
17491732
17501733 req->head = NULL;
17511734 req->woken = false;
@@ -1788,10 +1771,8 @@ static ssize_t aio_poll(struct aio_kiocb *aiocb, const struct iocb *iocb) · 36 unmodified lines
17881771 spin_unlock_irq(&ctx->ctx_lock);
17891772
17901773 out:
1791− if (unlikely(apt.error)) {
1792− fput(req->file);
1774+ if (unlikely(apt.error))
17931775 return apt.error;
1794− }
17951776
17961777 if (mask)
17971778 aio_poll_complete(aiocb, mask);
@@ -1829,6 +1810,11 @@ static int __io_submit_one(struct kioctx *ctx, const struct iocb *iocb, · 31 unmodified lines
18291810 if (unlikely(!req))
18301811 goto out_put_reqs_available;
18311812
1813+ req->ki_filp = fget(iocb->aio_fildes);
1814+ ret = -EBADF;
1815+ if (unlikely(!req->ki_filp))
1816+ goto out_put_req;
1817+
18321818 if (iocb->aio_flags & IOCB_FLAG_RESFD) {
18331819 /*
18341820 * If the IOCB_FLAG_RESFD flag of aio_flags is set, get an
include/linux/fs.h+7−1
@@ -304,13 +304,19 @@ enum rw_hint { · 303 unmodified lines
304304
305305 struct kiocb {
306306 struct file *ki_filp;
307+
308+ /* The 'ki_filp' pointer is shared in a union for aio */
309+ randomized_struct_fields_start
310+
307311 loff_t ki_pos;
308312 void (*ki_complete)(struct kiocb *iocb, long ret, long ret2);
309313 void *private;
310314 int ki_flags;
311315 u16 ki_hint;
312316 u16 ki_ioprio; /* See linux/ioprio.h */
313−} __randomize_layout;
317+
318+ randomized_struct_fields_end
319+};
314320
315321 static inline bool is_sync_kiocb(struct kiocb *kiocb)
316322 {

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t