History

netfs: Speed up buffered reading

Improve the efficiency of buffered reads in a number of ways: (1) Overhaul the algorithm in general so that it's a lot more compact and split the read submission code between buffered and unbuffered versions. The unbuffered version can be vastly simplified. (2) Read-result collection is handed off to a work queue rather than being done in the I/O thread. Multiple subrequests can be processes simultaneously. (3) When a subrequest is collected, any folios it fully spans are collected and "spare" data on either side is donated to either the previous or the next subrequest in the sequence. Notes: (*) Readahead expansion is massively slows down fio, presumably because it causes a load of extra allocations, both folio and xarray, up front before RPC requests can be transmitted. (*) RDMA with cifs does appear to work, both with SIW and RXE. (*) PG_private_2-based reading and copy-to-cache is split out into its own file and altered to use folio_queue. Note that the copy to the cache now creates a new write transaction against the cache and adds the folios to be copied into it. This allows it to use part of the writeback I/O code.

David Howells committed 2 years ago

Commit ee4cdf71 parent 2e45b9228 files · +2058 −470Browse files

Unified · Split

28 files
  1. fs/9p/vfs_addr.c +8 −3
  2. fs/afs/file.c +15 −6
  3. fs/afs/fsclient.c +7 −2
  4. fs/afs/yfsclient.c +7 −2
  5. fs/ceph/addr.c +46 −30
  6. fs/netfs/Makefile +3 −1
  7. fs/netfs/buffered_read.c +471 −295
  8. fs/netfs/direct_read.c +141 −6
  9. fs/netfs/internal.h +28 −7
  10. fs/netfs/iterator.c +50 −0
  11. fs/netfs/main.c +3 −1
  12. fs/netfs/objects.c +5 −3
  13. fs/netfs/read_collect.c +544 −0
  14. fs/netfs/read_pgpriv2.c +264 −0
  15. fs/netfs/read_retry.c +256 −0
  16. fs/netfs/stats.c +4 −2
  17. fs/netfs/write_collect.c +8 −1
  18. fs/netfs/write_issue.c +7 −10
  19. fs/nfs/fscache.c +6 −13
  20. fs/nfs/fscache.h +3 −4
  21. fs/smb/client/cifsglob.h +0 −1
  22. fs/smb/client/cifssmb.c +2 −4
  23. fs/smb/client/file.c +28 −54
  24. fs/smb/client/smb2ops.c +2 −1
  25. fs/smb/client/smb2pdu.c +16 −11
  26. include/linux/folio_queue.h +18 −0
  27. include/linux/netfs.h +18 −8
  28. include/trace/events/netfs.h +98 −5
fs/9p/vfs_addr.c+8−3
@@ -68,17 +68,22 @@ static void v9fs_issue_read(struct netfs_io_subrequest *subreq) · 67 unmodified lines
6868 {
6969 struct netfs_io_request *rreq = subreq->rreq;
7070 struct p9_fid *fid = rreq->netfs_priv;
71+ unsigned long long pos = subreq->start + subreq->transferred;
7172 int total, err;
7273
73− total = p9_client_read(fid, subreq->start + subreq->transferred,
74− &subreq->io_iter, &err);
74+ total = p9_client_read(fid, pos, &subreq->io_iter, &err);
7575
7676 /* if we just extended the file size, any portion not in
7777 * cache won't be on server and is zeroes */
7878 if (subreq->rreq->origin != NETFS_DIO_READ)
7979 __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags);
80+ if (pos + total >= i_size_read(rreq->inode))
81+ __set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags);
8082
81− netfs_subreq_terminated(subreq, err ?: total, false);
83+ if (!err)
84+ subreq->transferred += total;
85+
86+ netfs_read_subreq_terminated(subreq, err, false);
8287 }
8388
8489 /**
fs/afs/file.c+15−6
@@ -16,6 +16,7 @@ · 15 unmodified lines
1616 #include <linux/mm.h>
1717 #include <linux/swap.h>
1818 #include <linux/netfs.h>
19+#include <trace/events/netfs.h>
1920 #include "internal.h"
2021
2122 static int afs_file_mmap(struct file *file, struct vm_area_struct *vma);
@@ -242,9 +243,10 @@ static void afs_fetch_data_notify(struct afs_operation *op) · 220 unmodified lines
242243
243244 req->error = error;
244245 if (subreq) {
245− if (subreq->rreq->origin != NETFS_DIO_READ)
246− __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags);
247− netfs_subreq_terminated(subreq, error ?: req->actual_len, false);
246+ subreq->rreq->i_size = req->file_size;
247+ if (req->pos + req->actual_len >= req->file_size)
248+ __set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags);
249+ netfs_read_subreq_terminated(subreq, error, false);
248250 req->subreq = NULL;
249251 } else if (req->done) {
250252 req->done(req);
@@ -262,6 +264,12 @@ static void afs_fetch_data_success(struct afs_operation *op) · 11 unmodified lines
262264 afs_fetch_data_notify(op);
263265 }
264266
267+static void afs_fetch_data_aborted(struct afs_operation *op)
268+{
269+ afs_check_for_remote_deletion(op);
270+ afs_fetch_data_notify(op);
271+}
272+
265273 static void afs_fetch_data_put(struct afs_operation *op)
266274 {
267275 op->fetch.req->error = afs_op_error(op);
@@ -272,7 +280,7 @@ static const struct afs_operation_ops afs_fetch_data_operation = { · 4 unmodified lines
272280 .issue_afs_rpc = afs_fs_fetch_data,
273281 .issue_yfs_rpc = yfs_fs_fetch_data,
274282 .success = afs_fetch_data_success,
275− .aborted = afs_check_for_remote_deletion,
283+ .aborted = afs_fetch_data_aborted,
276284 .failed = afs_fetch_data_notify,
277285 .put = afs_fetch_data_put,
278286 };
@@ -294,7 +302,7 @@ int afs_fetch_data(struct afs_vnode *vnode, struct afs_read *req) · 15 unmodified lines
294302 op = afs_alloc_operation(req->key, vnode->volume);
295303 if (IS_ERR(op)) {
296304 if (req->subreq)
297− netfs_subreq_terminated(req->subreq, PTR_ERR(op), false);
305+ netfs_read_subreq_terminated(req->subreq, PTR_ERR(op), false);
298306 return PTR_ERR(op);
299307 }
300308
@@ -313,7 +321,7 @@ static void afs_read_worker(struct work_struct *work) · 12 unmodified lines
313321
314322 fsreq = afs_alloc_read(GFP_NOFS);
315323 if (!fsreq)
316− return netfs_subreq_terminated(subreq, -ENOMEM, false);
324+ return netfs_read_subreq_terminated(subreq, -ENOMEM, false);
317325
318326 fsreq->subreq = subreq;
319327 fsreq->pos = subreq->start + subreq->transferred;
@@ -322,6 +330,7 @@ static void afs_read_worker(struct work_struct *work) · 2 unmodified lines
322330 fsreq->vnode = vnode;
323331 fsreq->iter = &subreq->io_iter;
324332
333+ trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
325334 afs_fetch_data(fsreq->vnode, fsreq);
326335 afs_put_read(fsreq);
327336 }
fs/afs/fsclient.c+7−2
@@ -304,6 +304,7 @@ static int afs_deliver_fs_fetch_data(struct afs_call *call) · 303 unmodified lines
304304 struct afs_vnode_param *vp = &op->file[0];
305305 struct afs_read *req = op->fetch.req;
306306 const __be32 *bp;
307+ size_t count_before;
307308 int ret;
308309
309310 _enter("{%u,%zu,%zu/%llu}",
@@ -345,10 +346,14 @@ static int afs_deliver_fs_fetch_data(struct afs_call *call) · 35 unmodified lines
345346
346347 /* extract the returned data */
347348 case 2:
348− _debug("extract data %zu/%llu",
349− iov_iter_count(call->iter), req->actual_len);
349+ count_before = call->iov_len;
350+ _debug("extract data %zu/%llu", count_before, req->actual_len);
350351
351352 ret = afs_extract_data(call, true);
353+ if (req->subreq) {
354+ req->subreq->transferred += count_before - call->iov_len;
355+ netfs_read_subreq_progress(req->subreq, false);
356+ }
352357 if (ret < 0)
353358 return ret;
354359
fs/afs/yfsclient.c+7−2
@@ -355,6 +355,7 @@ static int yfs_deliver_fs_fetch_data64(struct afs_call *call) · 354 unmodified lines
355355 struct afs_vnode_param *vp = &op->file[0];
356356 struct afs_read *req = op->fetch.req;
357357 const __be32 *bp;
358+ size_t count_before;
358359 int ret;
359360
360361 _enter("{%u,%zu, %zu/%llu}",
@@ -391,10 +392,14 @@ static int yfs_deliver_fs_fetch_data64(struct afs_call *call) · 30 unmodified lines
391392
392393 /* extract the returned data */
393394 case 2:
394− _debug("extract data %zu/%llu",
395− iov_iter_count(call->iter), req->actual_len);
395+ count_before = call->iov_len;
396+ _debug("extract data %zu/%llu", count_before, req->actual_len);
396397
397398 ret = afs_extract_data(call, true);
399+ if (req->subreq) {
400+ req->subreq->transferred += count_before - call->iov_len;
401+ netfs_read_subreq_progress(req->subreq, false);
402+ }
398403 if (ret < 0)
399404 return ret;
400405
fs/ceph/addr.c+46−30
@@ -13,6 +13,7 @@ · 12 unmodified lines
1313 #include <linux/iversion.h>
1414 #include <linux/ktime.h>
1515 #include <linux/netfs.h>
16+#include <trace/events/netfs.h>
1617
1718 #include "super.h"
1819 #include "mds_client.h"
@@ -205,21 +206,6 @@ static void ceph_netfs_expand_readahead(struct netfs_io_request *rreq) · 186 unmodified lines
205206 }
206207 }
207208
208−static bool ceph_netfs_clamp_length(struct netfs_io_subrequest *subreq)
209−{
210− struct inode *inode = subreq->rreq->inode;
211− struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
212− struct ceph_inode_info *ci = ceph_inode(inode);
213− u64 objno, objoff;
214− u32 xlen;
215−
216− /* Truncate the extent at the end of the current block */
217− ceph_calc_file_object_mapping(&ci->i_layout, subreq->start, subreq->len,
218− &objno, &objoff, &xlen);
219− subreq->len = min(xlen, fsc->mount_options->rsize);
220− return true;
221−}
222−
223209 static void finish_netfs_read(struct ceph_osd_request *req)
224210 {
225211 struct inode *inode = req->r_inode;
@@ -264,7 +250,12 @@ static void finish_netfs_read(struct ceph_osd_request *req) · 38 unmodified lines
264250 calc_pages_for(osd_data->alignment,
265251 osd_data->length), false);
266252 }
267− netfs_subreq_terminated(subreq, err, false);
253+ if (err > 0) {
254+ subreq->transferred = err;
255+ err = 0;
256+ }
257+ trace_netfs_sreq(subreq, netfs_sreq_trace_io_progress);
258+ netfs_read_subreq_terminated(subreq, err, false);
268259 iput(req->r_inode);
269260 ceph_dec_osd_stopping_blocker(fsc->mdsc);
270261 }
@@ -278,7 +269,6 @@ static bool ceph_netfs_issue_op_inline(struct netfs_io_subrequest *subreq) · 7 unmodified lines
278269 struct ceph_mds_request *req;
279270 struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb);
280271 struct ceph_inode_info *ci = ceph_inode(inode);
281− struct iov_iter iter;
282272 ssize_t err = 0;
283273 size_t len;
284274 int mode;
@@ -301,6 +291,7 @@ static bool ceph_netfs_issue_op_inline(struct netfs_io_subrequest *subreq) · 16 unmodified lines
301291 req->r_args.getattr.mask = cpu_to_le32(CEPH_STAT_CAP_INLINE_DATA);
302292 req->r_num_caps = 2;
303293
294+ trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
304295 err = ceph_mdsc_do_request(mdsc, NULL, req);
305296 if (err < 0)
306297 goto out;
@@ -314,17 +305,36 @@ static bool ceph_netfs_issue_op_inline(struct netfs_io_subrequest *subreq) · 7 unmodified lines
314305 }
315306
316307 len = min_t(size_t, iinfo->inline_len - subreq->start, subreq->len);
317− iov_iter_xarray(&iter, ITER_DEST, &rreq->mapping->i_pages, subreq->start, len);
318− err = copy_to_iter(iinfo->inline_data + subreq->start, len, &iter);
319− if (err == 0)
308+ err = copy_to_iter(iinfo->inline_data + subreq->start, len, &subreq->io_iter);
309+ if (err == 0) {
320310 err = -EFAULT;
311+ } else {
312+ subreq->transferred += err;
313+ err = 0;
314+ }
321315
322316 ceph_mdsc_put_request(req);
323317 out:
324− netfs_subreq_terminated(subreq, err, false);
318+ netfs_read_subreq_terminated(subreq, err, false);
325319 return true;
326320 }
327321
322+static int ceph_netfs_prepare_read(struct netfs_io_subrequest *subreq)
323+{
324+ struct netfs_io_request *rreq = subreq->rreq;
325+ struct inode *inode = rreq->inode;
326+ struct ceph_inode_info *ci = ceph_inode(inode);
327+ struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
328+ u64 objno, objoff;
329+ u32 xlen;
330+
331+ /* Truncate the extent at the end of the current block */
332+ ceph_calc_file_object_mapping(&ci->i_layout, subreq->start, subreq->len,
333+ &objno, &objoff, &xlen);
334+ rreq->io_streams[0].sreq_max_len = umin(xlen, fsc->mount_options->rsize);
335+ return 0;
336+}
337+
328338 static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq)
329339 {
330340 struct netfs_io_request *rreq = subreq->rreq;
@@ -334,9 +344,8 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) · 3 unmodified lines
334344 struct ceph_client *cl = fsc->client;
335345 struct ceph_osd_request *req = NULL;
336346 struct ceph_vino vino = ceph_vino(inode);
337− struct iov_iter iter;
338− int err = 0;
339− u64 len = subreq->len;
347+ int err;
348+ u64 len;
340349 bool sparse = IS_ENCRYPTED(inode) || ceph_test_mount_opt(fsc, SPARSEREAD);
341350 u64 off = subreq->start;
342351 int extent_cnt;
@@ -349,6 +358,12 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) · 6 unmodified lines
349358 if (ceph_has_inline_data(ci) && ceph_netfs_issue_op_inline(subreq))
350359 return;
351360
361+ // TODO: This rounding here is slightly dodgy. It *should* work, for
362+ // now, as the cache only deals in blocks that are a multiple of
363+ // PAGE_SIZE and fscrypt blocks are at most PAGE_SIZE. What needs to
364+ // happen is for the fscrypt driving to be moved into netfslib and the
365+ // data in the cache also to be stored encrypted.
366+ len = subreq->len;
352367 ceph_fscrypt_adjust_off_and_len(inode, &off, &len);
353368
354369 req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, vino,
@@ -371,8 +386,6 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) · 16 unmodified lines
371386 doutc(cl, "%llx.%llx pos=%llu orig_len=%zu len=%llu\n",
372387 ceph_vinop(inode), subreq->start, subreq->len, len);
373388
374− iov_iter_xarray(&iter, ITER_DEST, &rreq->mapping->i_pages, subreq->start, len);
375−
376389 /*
377390 * FIXME: For now, use CEPH_OSD_DATA_TYPE_PAGES instead of _ITER for
378391 * encrypted inodes. We'd need infrastructure that handles an iov_iter
@@ -384,7 +397,7 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) · 5 unmodified lines
384397 struct page **pages;
385398 size_t page_off;
386399
387− err = iov_iter_get_pages_alloc2(&iter, &pages, len, &page_off);
400+ err = iov_iter_get_pages_alloc2(&subreq->io_iter, &pages, len, &page_off);
388401 if (err < 0) {
389402 doutc(cl, "%llx.%llx failed to allocate pages, %d\n",
390403 ceph_vinop(inode), err);
@@ -399,7 +412,7 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) · 8 unmodified lines
399412 osd_req_op_extent_osd_data_pages(req, 0, pages, len, 0, false,
400413 false);
401414 } else {
402− osd_req_op_extent_osd_iter(req, 0, &iter);
415+ osd_req_op_extent_osd_iter(req, 0, &subreq->io_iter);
403416 }
404417 if (!ceph_inc_osd_stopping_blocker(fsc->mdsc)) {
405418 err = -EIO;
@@ -410,17 +423,19 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) · 4 unmodified lines
410423 req->r_inode = inode;
411424 ihold(inode);
412425
426+ trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
413427 ceph_osdc_start_request(req->r_osdc, req);
414428 out:
415429 ceph_osdc_put_request(req);
416430 if (err)
417− netfs_subreq_terminated(subreq, err, false);
431+ netfs_read_subreq_terminated(subreq, err, false);
418432 doutc(cl, "%llx.%llx result %d\n", ceph_vinop(inode), err);
419433 }
420434
421435 static int ceph_init_request(struct netfs_io_request *rreq, struct file *file)
422436 {
423437 struct inode *inode = rreq->inode;
438+ struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
424439 struct ceph_client *cl = ceph_inode_to_client(inode);
425440 int got = 0, want = CEPH_CAP_FILE_CACHE;
426441 struct ceph_netfs_request_data *priv;
@@ -472,6 +487,7 @@ static int ceph_init_request(struct netfs_io_request *rreq, struct file *file) · 45 unmodified lines
472487
473488 priv->caps = got;
474489 rreq->netfs_priv = priv;
490+ rreq->io_streams[0].sreq_max_len = fsc->mount_options->rsize;
475491
476492 out:
477493 if (ret < 0)
@@ -496,9 +512,9 @@ static void ceph_netfs_free_request(struct netfs_io_request *rreq) · 18 unmodified lines
496512 const struct netfs_request_ops ceph_netfs_ops = {
497513 .init_request = ceph_init_request,
498514 .free_request = ceph_netfs_free_request,
515+ .prepare_read = ceph_netfs_prepare_read,
499516 .issue_read = ceph_netfs_issue_read,
500517 .expand_readahead = ceph_netfs_expand_readahead,
501− .clamp_length = ceph_netfs_clamp_length,
502518 .check_write_begin = ceph_netfs_check_write_begin,
503519 };
504520
fs/netfs/Makefile+3−1
@@ -5,12 +5,14 @@ netfs-y := \ · 4 unmodified lines
55 buffered_write.o \
66 direct_read.o \
77 direct_write.o \
8− io.o \
98 iterator.o \
109 locking.o \
1110 main.o \
1211 misc.o \
1312 objects.o \
13+ read_collect.o \
14+ read_pgpriv2.o \
15+ read_retry.o \
1416 write_collect.o \
1517 write_issue.o
1618
fs/netfs/buffered_read.c+471−295
@@ -9,266 +9,388 @@ · 8 unmodified lines
99 #include <linux/task_io_accounting_ops.h>
1010 #include "internal.h"
1111
12−/*
13− * [DEPRECATED] Unlock the folios in a read operation for when the filesystem
14− * is using PG_private_2 and direct writing to the cache from here rather than
15− * marking the page for writeback.
16− *
17− * Note that we don't touch folio->private in this code.
18− */
19−static void netfs_rreq_unlock_folios_pgpriv2(struct netfs_io_request *rreq,
20− size_t *account)
12+static void netfs_cache_expand_readahead(struct netfs_io_request *rreq,
13+ unsigned long long *_start,
14+ unsigned long long *_len,
15+ unsigned long long i_size)
2116 {
22− struct netfs_io_subrequest *subreq;
23− struct folio *folio;
24− pgoff_t start_page = rreq->start / PAGE_SIZE;
25− pgoff_t last_page = ((rreq->start + rreq->len) / PAGE_SIZE) - 1;
26− bool subreq_failed = false;
17+ struct netfs_cache_resources *cres = &rreq->cache_resources;
2718
28− XA_STATE(xas, &rreq->mapping->i_pages, start_page);
19+ if (cres->ops && cres->ops->expand_readahead)
20+ cres->ops->expand_readahead(cres, _start, _len, i_size);
21+}
2922
30− /* Walk through the pagecache and the I/O request lists simultaneously.
31− * We may have a mixture of cached and uncached sections and we only
32− * really want to write out the uncached sections. This is slightly
33− * complicated by the possibility that we might have huge pages with a
34− * mixture inside.
23+static void netfs_rreq_expand(struct netfs_io_request *rreq,
24+ struct readahead_control *ractl)
25+{
26+ /* Give the cache a chance to change the request parameters. The
27+ * resultant request must contain the original region.
3528 */
36− subreq = list_first_entry(&rreq->subrequests,
37− struct netfs_io_subrequest, rreq_link);
38− subreq_failed = (subreq->error < 0);
29+ netfs_cache_expand_readahead(rreq, &rreq->start, &rreq->len, rreq->i_size);
3930
40− trace_netfs_rreq(rreq, netfs_rreq_trace_unlock_pgpriv2);
31+ /* Give the netfs a chance to change the request parameters. The
32+ * resultant request must contain the original region.
33+ */
34+ if (rreq->netfs_ops->expand_readahead)
35+ rreq->netfs_ops->expand_readahead(rreq);
4136
42− rcu_read_lock();
43− xas_for_each(&xas, folio, last_page) {
44− loff_t pg_end;
45− bool pg_failed = false;
46− bool folio_started = false;
37+ /* Expand the request if the cache wants it to start earlier. Note
38+ * that the expansion may get further extended if the VM wishes to
39+ * insert THPs and the preferred start and/or end wind up in the middle
40+ * of THPs.
41+ *
42+ * If this is the case, however, the THP size should be an integer
43+ * multiple of the cache granule size, so we get a whole number of
44+ * granules to deal with.
45+ */
46+ if (rreq->start != readahead_pos(ractl) ||
47+ rreq->len != readahead_length(ractl)) {
48+ readahead_expand(ractl, rreq->start, rreq->len);
49+ rreq->start = readahead_pos(ractl);
50+ rreq->len = readahead_length(ractl);
4751
48− if (xas_retry(&xas, folio))
49− continue;
52+ trace_netfs_read(rreq, readahead_pos(ractl), readahead_length(ractl),
53+ netfs_read_trace_expanded);
54+ }
55+}
5056
51− pg_end = folio_pos(folio) + folio_size(folio) - 1;
57+/*
58+ * Begin an operation, and fetch the stored zero point value from the cookie if
59+ * available.
60+ */
61+static int netfs_begin_cache_read(struct netfs_io_request *rreq, struct netfs_inode *ctx)
62+{
63+ return fscache_begin_read_operation(&rreq->cache_resources, netfs_i_cookie(ctx));
64+}
5265
53− for (;;) {
54− loff_t sreq_end;
66+/*
67+ * Decant the list of folios to read into a rolling buffer.
68+ */
69+static size_t netfs_load_buffer_from_ra(struct netfs_io_request *rreq,
70+ struct folio_queue *folioq)
71+{
72+ unsigned int order, nr;
73+ size_t size = 0;
74+
75+ nr = __readahead_batch(rreq->ractl, (struct page **)folioq->vec.folios,
76+ ARRAY_SIZE(folioq->vec.folios));
77+ folioq->vec.nr = nr;
78+ for (int i = 0; i < nr; i++) {
79+ struct folio *folio = folioq_folio(folioq, i);
80+
81+ trace_netfs_folio(folio, netfs_folio_trace_read);
82+ order = folio_order(folio);
83+ folioq->orders[i] = order;
84+ size += PAGE_SIZE << order;
85+ }
5586
56− if (!subreq) {
57− pg_failed = true;
58− break;
59− }
87+ for (int i = nr; i < folioq_nr_slots(folioq); i++)
88+ folioq_clear(folioq, i);
6089
61− if (!folio_started &&
62− test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags) &&
63− fscache_operation_valid(&rreq->cache_resources)) {
64− trace_netfs_folio(folio, netfs_folio_trace_copy_to_cache);
65− folio_start_private_2(folio);
66− folio_started = true;
67− }
90+ return size;
91+}
6892
69− pg_failed |= subreq_failed;
70− sreq_end = subreq->start + subreq->len - 1;
71− if (pg_end < sreq_end)
72− break;
93+/*
94+ * netfs_prepare_read_iterator - Prepare the subreq iterator for I/O
95+ * @subreq: The subrequest to be set up
96+ *
97+ * Prepare the I/O iterator representing the read buffer on a subrequest for
98+ * the filesystem to use for I/O (it can be passed directly to a socket). This
99+ * is intended to be called from the ->issue_read() method once the filesystem
100+ * has trimmed the request to the size it wants.
101+ *
102+ * Returns the limited size if successful and -ENOMEM if insufficient memory
103+ * available.
104+ *
105+ * [!] NOTE: This must be run in the same thread as ->issue_read() was called
106+ * in as we access the readahead_control struct.
107+ */
108+static ssize_t netfs_prepare_read_iterator(struct netfs_io_subrequest *subreq)
109+{
110+ struct netfs_io_request *rreq = subreq->rreq;
111+ size_t rsize = subreq->len;
112+
113+ if (subreq->source == NETFS_DOWNLOAD_FROM_SERVER)
114+ rsize = umin(rsize, rreq->io_streams[0].sreq_max_len);
115+
116+ if (rreq->ractl) {
117+ /* If we don't have sufficient folios in the rolling buffer,
118+ * extract a folioq's worth from the readahead region at a time
119+ * into the buffer. Note that this acquires a ref on each page
120+ * that we will need to release later - but we don't want to do
121+ * that until after we've started the I/O.
122+ */
123+ while (rreq->submitted < subreq->start + rsize) {
124+ struct folio_queue *tail = rreq->buffer_tail, *new;
125+ size_t added;
126+
127+ new = kmalloc(sizeof(*new), GFP_NOFS);
128+ if (!new)
129+ return -ENOMEM;
130+ netfs_stat(&netfs_n_folioq);
131+ folioq_init(new);
132+ new->prev = tail;
133+ tail->next = new;
134+ rreq->buffer_tail = new;
135+ added = netfs_load_buffer_from_ra(rreq, new);
136+ rreq->iter.count += added;
137+ rreq->submitted += added;
138+ }
139+ }
73140
74− *account += subreq->transferred;
75− if (!list_is_last(&subreq->rreq_link, &rreq->subrequests)) {
76− subreq = list_next_entry(subreq, rreq_link);
77− subreq_failed = (subreq->error < 0);
78− } else {
79− subreq = NULL;
80− subreq_failed = false;
81− }
141+ subreq->len = rsize;
142+ if (unlikely(rreq->io_streams[0].sreq_max_segs)) {
143+ size_t limit = netfs_limit_iter(&rreq->iter, 0, rsize,
144+ rreq->io_streams[0].sreq_max_segs);
82145
83− if (pg_end == sreq_end)
84− break;
146+ if (limit < rsize) {
147+ subreq->len = limit;
148+ trace_netfs_sreq(subreq, netfs_sreq_trace_limited);
85149 }
150+ }
86151
87− if (!pg_failed) {
88− flush_dcache_folio(folio);
89− folio_mark_uptodate(folio);
90− }
152+ subreq->io_iter = rreq->iter;
91153
92− if (!test_bit(NETFS_RREQ_DONT_UNLOCK_FOLIOS, &rreq->flags)) {
93− if (folio->index == rreq->no_unlock_folio &&
94− test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags))
95− _debug("no unlock");
96− else
97− folio_unlock(folio);
154+ if (iov_iter_is_folioq(&subreq->io_iter)) {
155+ if (subreq->io_iter.folioq_slot >= folioq_nr_slots(subreq->io_iter.folioq)) {
156+ subreq->io_iter.folioq = subreq->io_iter.folioq->next;
157+ subreq->io_iter.folioq_slot = 0;
98158 }
159+ subreq->curr_folioq = (struct folio_queue *)subreq->io_iter.folioq;
160+ subreq->curr_folioq_slot = subreq->io_iter.folioq_slot;
161+ subreq->curr_folio_order = subreq->curr_folioq->orders[subreq->curr_folioq_slot];
99162 }
100− rcu_read_unlock();
163+
164+ iov_iter_truncate(&subreq->io_iter, subreq->len);
165+ iov_iter_advance(&rreq->iter, subreq->len);
166+ return subreq->len;
101167 }
102168
103−/*
104− * Unlock the folios in a read operation. We need to set PG_writeback on any
105− * folios we're going to write back before we unlock them.
106− *
107− * Note that if the deprecated NETFS_RREQ_USE_PGPRIV2 is set then we use
108− * PG_private_2 and do a direct write to the cache from here instead.
109− */
110−void netfs_rreq_unlock_folios(struct netfs_io_request *rreq)
169+static enum netfs_io_source netfs_cache_prepare_read(struct netfs_io_request *rreq,
170+ struct netfs_io_subrequest *subreq,
171+ loff_t i_size)
111172 {
112− struct netfs_io_subrequest *subreq;
113− struct netfs_folio *finfo;
114− struct folio *folio;
115− pgoff_t start_page = rreq->start / PAGE_SIZE;
116− pgoff_t last_page = ((rreq->start + rreq->len) / PAGE_SIZE) - 1;
117− size_t account = 0;
118− bool subreq_failed = false;
173+ struct netfs_cache_resources *cres = &rreq->cache_resources;
119174
120− XA_STATE(xas, &rreq->mapping->i_pages, start_page);
175+ if (!cres->ops)
176+ return NETFS_DOWNLOAD_FROM_SERVER;
177+ return cres->ops->prepare_read(subreq, i_size);
178+}
121179
122− if (test_bit(NETFS_RREQ_FAILED, &rreq->flags)) {
123− __clear_bit(NETFS_RREQ_COPY_TO_CACHE, &rreq->flags);
124− list_for_each_entry(subreq, &rreq->subrequests, rreq_link) {
125− __clear_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags);
126− }
127− }
180+static void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error,
181+ bool was_async)
182+{
183+ struct netfs_io_subrequest *subreq = priv;
128184
129− /* Handle deprecated PG_private_2 case. */
130− if (test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)) {
131− netfs_rreq_unlock_folios_pgpriv2(rreq, &account);
132− goto out;
185+ if (transferred_or_error < 0) {
186+ netfs_read_subreq_terminated(subreq, transferred_or_error, was_async);
187+ return;
133188 }
134189
135− /* Walk through the pagecache and the I/O request lists simultaneously.
136− * We may have a mixture of cached and uncached sections and we only
137− * really want to write out the uncached sections. This is slightly
138− * complicated by the possibility that we might have huge pages with a
139− * mixture inside.
140− */
141− subreq = list_first_entry(&rreq->subrequests,
142− struct netfs_io_subrequest, rreq_link);
143− subreq_failed = (subreq->error < 0);
144−
145− trace_netfs_rreq(rreq, netfs_rreq_trace_unlock);
190+ if (transferred_or_error > 0)
191+ subreq->transferred += transferred_or_error;
192+ netfs_read_subreq_terminated(subreq, 0, was_async);
193+}
146194
147− rcu_read_lock();
148− xas_for_each(&xas, folio, last_page) {
149− loff_t pg_end;
150− bool pg_failed = false;
151− bool wback_to_cache = false;
195+/*
196+ * Issue a read against the cache.
197+ * - Eats the caller's ref on subreq.
198+ */
199+static void netfs_read_cache_to_pagecache(struct netfs_io_request *rreq,
200+ struct netfs_io_subrequest *subreq)
201+{
202+ struct netfs_cache_resources *cres = &rreq->cache_resources;
152203
153− if (xas_retry(&xas, folio))
154− continue;
204+ netfs_stat(&netfs_n_rh_read);
205+ cres->ops->read(cres, subreq->start, &subreq->io_iter, NETFS_READ_HOLE_IGNORE,
206+ netfs_cache_read_terminated, subreq);
207+}
155208
156− pg_end = folio_pos(folio) + folio_size(folio) - 1;
209+/*
210+ * Perform a read to the pagecache from a series of sources of different types,
211+ * slicing up the region to be read according to available cache blocks and
212+ * network rsize.
213+ */
214+static void netfs_read_to_pagecache(struct netfs_io_request *rreq)
215+{
216+ struct netfs_inode *ictx = netfs_inode(rreq->inode);
217+ unsigned long long start = rreq->start;
218+ ssize_t size = rreq->len;
219+ int ret = 0;
220+
221+ atomic_inc(&rreq->nr_outstanding);
222+
223+ do {
224+ struct netfs_io_subrequest *subreq;
225+ enum netfs_io_source source = NETFS_DOWNLOAD_FROM_SERVER;
226+ ssize_t slice;
227+
228+ subreq = netfs_alloc_subrequest(rreq);
229+ if (!subreq) {
230+ ret = -ENOMEM;
231+ break;
232+ }
157233
158− for (;;) {
159− loff_t sreq_end;
234+ subreq->start = start;
235+ subreq->len = size;
236+
237+ atomic_inc(&rreq->nr_outstanding);
238+ spin_lock_bh(&rreq->lock);
239+ list_add_tail(&subreq->rreq_link, &rreq->subrequests);
240+ subreq->prev_donated = rreq->prev_donated;
241+ rreq->prev_donated = 0;
242+ trace_netfs_sreq(subreq, netfs_sreq_trace_added);
243+ spin_unlock_bh(&rreq->lock);
244+
245+ source = netfs_cache_prepare_read(rreq, subreq, rreq->i_size);
246+ subreq->source = source;
247+ if (source == NETFS_DOWNLOAD_FROM_SERVER) {
248+ unsigned long long zp = umin(ictx->zero_point, rreq->i_size);
249+ size_t len = subreq->len;
250+
251+ if (subreq->start >= zp) {
252+ subreq->source = source = NETFS_FILL_WITH_ZEROES;
253+ goto fill_with_zeroes;
254+ }
160255
161− if (!subreq) {
162− pg_failed = true;
256+ if (len > zp - subreq->start)
257+ len = zp - subreq->start;
258+ if (len == 0) {
259+ pr_err("ZERO-LEN READ: R=%08x[%x] l=%zx/%zx s=%llx z=%llx i=%llx",
260+ rreq->debug_id, subreq->debug_index,
261+ subreq->len, size,
262+ subreq->start, ictx->zero_point, rreq->i_size);
163263 break;
164264 }
265+ subreq->len = len;
266+
267+ netfs_stat(&netfs_n_rh_download);
268+ if (rreq->netfs_ops->prepare_read) {
269+ ret = rreq->netfs_ops->prepare_read(subreq);
270+ if (ret < 0) {
271+ atomic_dec(&rreq->nr_outstanding);
272+ netfs_put_subrequest(subreq, false,
273+ netfs_sreq_trace_put_cancel);
274+ break;
275+ }
276+ trace_netfs_sreq(subreq, netfs_sreq_trace_prepare);
277+ }
165278
166− wback_to_cache |= test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags);
167− pg_failed |= subreq_failed;
168− sreq_end = subreq->start + subreq->len - 1;
169− if (pg_end < sreq_end)
279+ slice = netfs_prepare_read_iterator(subreq);
280+ if (slice < 0) {
281+ atomic_dec(&rreq->nr_outstanding);
282+ netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_cancel);
283+ ret = slice;
170284 break;
171−
172− account += subreq->transferred;
173− if (!list_is_last(&subreq->rreq_link, &rreq->subrequests)) {
174− subreq = list_next_entry(subreq, rreq_link);
175− subreq_failed = (subreq->error < 0);
176− } else {
177− subreq = NULL;
178− subreq_failed = false;
179285 }
180286
181− if (pg_end == sreq_end)
182− break;
287+ rreq->netfs_ops->issue_read(subreq);
288+ goto done;
183289 }
184290
185− if (!pg_failed) {
186− flush_dcache_folio(folio);
187− finfo = netfs_folio_info(folio);
188− if (finfo) {
189− trace_netfs_folio(folio, netfs_folio_trace_filled_gaps);
190− if (finfo->netfs_group)
191− folio_change_private(folio, finfo->netfs_group);
192− else
193− folio_detach_private(folio);
194− kfree(finfo);
195− }
196− folio_mark_uptodate(folio);
197− if (wback_to_cache && !WARN_ON_ONCE(folio_get_private(folio) != NULL)) {
198− trace_netfs_folio(folio, netfs_folio_trace_copy_to_cache);
199− folio_attach_private(folio, NETFS_FOLIO_COPY_TO_CACHE);
200− filemap_dirty_folio(folio->mapping, folio);
201− }
291+ fill_with_zeroes:
292+ if (source == NETFS_FILL_WITH_ZEROES) {
293+ subreq->source = NETFS_FILL_WITH_ZEROES;
294+ trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
295+ netfs_stat(&netfs_n_rh_zero);
296+ slice = netfs_prepare_read_iterator(subreq);
297+ __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags);
298+ netfs_read_subreq_terminated(subreq, 0, false);
299+ goto done;
202300 }
203301
204− if (!test_bit(NETFS_RREQ_DONT_UNLOCK_FOLIOS, &rreq->flags)) {
205− if (folio->index == rreq->no_unlock_folio &&
206− test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags))
207− _debug("no unlock");
208− else
209− folio_unlock(folio);
302+ if (source == NETFS_READ_FROM_CACHE) {
303+ trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
304+ slice = netfs_prepare_read_iterator(subreq);
305+ netfs_read_cache_to_pagecache(rreq, subreq);
306+ goto done;
210307 }
211− }
212− rcu_read_unlock();
213308
214−out:
215− task_io_account_read(account);
216− if (rreq->netfs_ops->done)
217− rreq->netfs_ops->done(rreq);
218−}
309+ pr_err("Unexpected read source %u\n", source);
310+ WARN_ON_ONCE(1);
311+ break;
219312
220−static void netfs_cache_expand_readahead(struct netfs_io_request *rreq,
221− unsigned long long *_start,
222− unsigned long long *_len,
223− unsigned long long i_size)
224−{
225− struct netfs_cache_resources *cres = &rreq->cache_resources;
313+ done:
314+ size -= slice;
315+ start += slice;
316+ cond_resched();
317+ } while (size > 0);
226318
227− if (cres->ops && cres->ops->expand_readahead)
228− cres->ops->expand_readahead(cres, _start, _len, i_size);
319+ if (atomic_dec_and_test(&rreq->nr_outstanding))
320+ netfs_rreq_terminated(rreq, false);
321+
322+ /* Defer error return as we may need to wait for outstanding I/O. */
323+ cmpxchg(&rreq->error, 0, ret);
229324 }
230325
231−static void netfs_rreq_expand(struct netfs_io_request *rreq,
232− struct readahead_control *ractl)
326+/*
327+ * Wait for the read operation to complete, successfully or otherwise.
328+ */
329+static int netfs_wait_for_read(struct netfs_io_request *rreq)
233330 {
234− /* Give the cache a chance to change the request parameters. The
235− * resultant request must contain the original region.
236− */
237− netfs_cache_expand_readahead(rreq, &rreq->start, &rreq->len, rreq->i_size);
331+ int ret;
238332
239− /* Give the netfs a chance to change the request parameters. The
240− * resultant request must contain the original region.
241− */
242− if (rreq->netfs_ops->expand_readahead)
243− rreq->netfs_ops->expand_readahead(rreq);
333+ trace_netfs_rreq(rreq, netfs_rreq_trace_wait_ip);
334+ wait_on_bit(&rreq->flags, NETFS_RREQ_IN_PROGRESS, TASK_UNINTERRUPTIBLE);
335+ ret = rreq->error;
336+ if (ret == 0 && rreq->submitted < rreq->len) {
337+ trace_netfs_failure(rreq, NULL, ret, netfs_fail_short_read);
338+ ret = -EIO;
339+ }
244340
245− /* Expand the request if the cache wants it to start earlier. Note
246− * that the expansion may get further extended if the VM wishes to
247− * insert THPs and the preferred start and/or end wind up in the middle
248− * of THPs.
249− *
250− * If this is the case, however, the THP size should be an integer
251− * multiple of the cache granule size, so we get a whole number of
252− * granules to deal with.
253− */
254− if (rreq->start != readahead_pos(ractl) ||
255− rreq->len != readahead_length(ractl)) {
256− readahead_expand(ractl, rreq->start, rreq->len);
257− rreq->start = readahead_pos(ractl);
258− rreq->len = readahead_length(ractl);
341+ return ret;
342+}
259343
260− trace_netfs_read(rreq, readahead_pos(ractl), readahead_length(ractl),
261− netfs_read_trace_expanded);
262− }
344+/*
345+ * Set up the initial folioq of buffer folios in the rolling buffer and set the
346+ * iterator to refer to it.
347+ */
348+static int netfs_prime_buffer(struct netfs_io_request *rreq)
349+{
350+ struct folio_queue *folioq;
351+ size_t added;
352+
353+ folioq = kmalloc(sizeof(*folioq), GFP_KERNEL);
354+ if (!folioq)
355+ return -ENOMEM;
356+ netfs_stat(&netfs_n_folioq);
357+ folioq_init(folioq);
358+ rreq->buffer = folioq;
359+ rreq->buffer_tail = folioq;
360+ rreq->submitted = rreq->start;
361+ iov_iter_folio_queue(&rreq->iter, ITER_DEST, folioq, 0, 0, 0);
362+
363+ added = netfs_load_buffer_from_ra(rreq, folioq);
364+ rreq->iter.count += added;
365+ rreq->submitted += added;
366+ return 0;
263367 }
264368
265369 /*
266− * Begin an operation, and fetch the stored zero point value from the cookie if
267− * available.
370+ * Drop the ref on each folio that we inherited from the VM readahead code. We
371+ * still have the folio locks to pin the page until we complete the I/O.
372+ *
373+ * Note that we can't just release the batch in each queue struct as we use the
374+ * occupancy count in other places.
268375 */
269−static int netfs_begin_cache_read(struct netfs_io_request *rreq, struct netfs_inode *ctx)
376+static void netfs_put_ra_refs(struct folio_queue *folioq)
270377 {
271− return fscache_begin_read_operation(&rreq->cache_resources, netfs_i_cookie(ctx));
378+ struct folio_batch fbatch;
379+
380+ folio_batch_init(&fbatch);
381+ while (folioq) {
382+ for (unsigned int slot = 0; slot < folioq_count(folioq); slot++) {
383+ struct folio *folio = folioq_folio(folioq, slot);
384+ if (!folio)
385+ continue;
386+ trace_netfs_folio(folio, netfs_folio_trace_read_put);
387+ if (!folio_batch_add(&fbatch, folio))
388+ folio_batch_release(&fbatch);
389+ }
390+ folioq = folioq->next;
391+ }
392+
393+ folio_batch_release(&fbatch);
272394 }
273395
274396 /**
@@ -289,22 +411,17 @@ static int netfs_begin_cache_read(struct netfs_io_request *rreq, struct netfs_in · 14 unmodified lines
289411 void netfs_readahead(struct readahead_control *ractl)
290412 {
291413 struct netfs_io_request *rreq;
292− struct netfs_inode *ctx = netfs_inode(ractl->mapping->host);
414+ struct netfs_inode *ictx = netfs_inode(ractl->mapping->host);
415+ unsigned long long start = readahead_pos(ractl);
416+ size_t size = readahead_length(ractl);
293417 int ret;
294418
295− _enter("%lx,%x", readahead_index(ractl), readahead_count(ractl));
296−
297− if (readahead_count(ractl) == 0)
298− return;
299−
300− rreq = netfs_alloc_request(ractl->mapping, ractl->file,
301− readahead_pos(ractl),
302− readahead_length(ractl),
419+ rreq = netfs_alloc_request(ractl->mapping, ractl->file, start, size,
303420 NETFS_READAHEAD);
304421 if (IS_ERR(rreq))
305422 return;
306423
307− ret = netfs_begin_cache_read(rreq, ctx);
424+ ret = netfs_begin_cache_read(rreq, ictx);
308425 if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS)
309426 goto cleanup_free;
310427
@@ -314,18 +431,15 @@ void netfs_readahead(struct readahead_control *ractl) · 3 unmodified lines
314431
315432 netfs_rreq_expand(rreq, ractl);
316433
317− /* Set up the output buffer */
318− iov_iter_xarray(&rreq->iter, ITER_DEST, &ractl->mapping->i_pages,
319− rreq->start, rreq->len);
434+ rreq->ractl = ractl;
435+ if (netfs_prime_buffer(rreq) < 0)
436+ goto cleanup_free;
437+ netfs_read_to_pagecache(rreq);
320438
321− /* Drop the refs on the folios here rather than in the cache or
322− * filesystem. The locks will be dropped in netfs_rreq_unlock().
323− */
324− while (readahead_folio(ractl))
325− ;
439+ /* Release the folio refs whilst we're waiting for the I/O. */
440+ netfs_put_ra_refs(rreq->buffer);
326441
327− netfs_begin_read(rreq, false);
328− netfs_put_request(rreq, false, netfs_rreq_trace_put_return);
442+ netfs_put_request(rreq, true, netfs_rreq_trace_put_return);
329443 return;
330444
331445 cleanup_free:
@@ -334,6 +448,117 @@ cleanup_free: · 2 unmodified lines
334448 }
335449 EXPORT_SYMBOL(netfs_readahead);
336450
451+/*
452+ * Create a rolling buffer with a single occupying folio.
453+ */
454+static int netfs_create_singular_buffer(struct netfs_io_request *rreq, struct folio *folio)
455+{
456+ struct folio_queue *folioq;
457+
458+ folioq = kmalloc(sizeof(*folioq), GFP_KERNEL);
459+ if (!folioq)
460+ return -ENOMEM;
461+
462+ netfs_stat(&netfs_n_folioq);
463+ folioq_init(folioq);
464+ folioq_append(folioq, folio);
465+ BUG_ON(folioq_folio(folioq, 0) != folio);
466+ BUG_ON(folioq_folio_order(folioq, 0) != folio_order(folio));
467+ rreq->buffer = folioq;
468+ rreq->buffer_tail = folioq;
469+ rreq->submitted = rreq->start + rreq->len;
470+ iov_iter_folio_queue(&rreq->iter, ITER_DEST, folioq, 0, 0, rreq->len);
471+ rreq->ractl = (struct readahead_control *)1UL;
472+ return 0;
473+}
474+
475+/*
476+ * Read into gaps in a folio partially filled by a streaming write.
477+ */
478+static int netfs_read_gaps(struct file *file, struct folio *folio)
479+{
480+ struct netfs_io_request *rreq;
481+ struct address_space *mapping = folio->mapping;
482+ struct netfs_folio *finfo = netfs_folio_info(folio);
483+ struct netfs_inode *ctx = netfs_inode(mapping->host);
484+ struct folio *sink = NULL;
485+ struct bio_vec *bvec;
486+ unsigned int from = finfo->dirty_offset;
487+ unsigned int to = from + finfo->dirty_len;
488+ unsigned int off = 0, i = 0;
489+ size_t flen = folio_size(folio);
490+ size_t nr_bvec = flen / PAGE_SIZE + 2;
491+ size_t part;
492+ int ret;
493+
494+ _enter("%lx", folio->index);
495+
496+ rreq = netfs_alloc_request(mapping, file, folio_pos(folio), flen, NETFS_READ_GAPS);
497+ if (IS_ERR(rreq)) {
498+ ret = PTR_ERR(rreq);
499+ goto alloc_error;
500+ }
501+
502+ ret = netfs_begin_cache_read(rreq, ctx);
503+ if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS)
504+ goto discard;
505+
506+ netfs_stat(&netfs_n_rh_read_folio);
507+ trace_netfs_read(rreq, rreq->start, rreq->len, netfs_read_trace_read_gaps);
508+
509+ /* Fiddle the buffer so that a gap at the beginning and/or a gap at the
510+ * end get copied to, but the middle is discarded.
511+ */
512+ ret = -ENOMEM;
513+ bvec = kmalloc_array(nr_bvec, sizeof(*bvec), GFP_KERNEL);
514+ if (!bvec)
515+ goto discard;
516+
517+ sink = folio_alloc(GFP_KERNEL, 0);
518+ if (!sink) {
519+ kfree(bvec);
520+ goto discard;
521+ }
522+
523+ trace_netfs_folio(folio, netfs_folio_trace_read_gaps);
524+
525+ rreq->direct_bv = bvec;
526+ rreq->direct_bv_count = nr_bvec;
527+ if (from > 0) {
528+ bvec_set_folio(&bvec[i++], folio, from, 0);
529+ off = from;
530+ }
531+ while (off < to) {
532+ part = min_t(size_t, to - off, PAGE_SIZE);
533+ bvec_set_folio(&bvec[i++], sink, part, 0);
534+ off += part;
535+ }
536+ if (to < flen)
537+ bvec_set_folio(&bvec[i++], folio, flen - to, to);
538+ iov_iter_bvec(&rreq->iter, ITER_DEST, bvec, i, rreq->len);
539+ rreq->submitted = rreq->start + flen;
540+
541+ netfs_read_to_pagecache(rreq);
542+
543+ if (sink)
544+ folio_put(sink);
545+
546+ ret = netfs_wait_for_read(rreq);
547+ if (ret == 0) {
548+ flush_dcache_folio(folio);
549+ folio_mark_uptodate(folio);
550+ }
551+ folio_unlock(folio);
552+ netfs_put_request(rreq, false, netfs_rreq_trace_put_return);
553+ return ret < 0 ? ret : 0;
554+
555+discard:
556+ netfs_put_request(rreq, false, netfs_rreq_trace_put_discard);
557+alloc_error:
558+ folio_unlock(folio);
559+ return ret;
560+}
561+
337562 /**
338563 * netfs_read_folio - Helper to manage a read_folio request
339564 * @file: The file to read from
@@ -353,9 +578,13 @@ int netfs_read_folio(struct file *file, struct folio *folio) · 13 unmodified lines
353578 struct address_space *mapping = folio->mapping;
354579 struct netfs_io_request *rreq;
355580 struct netfs_inode *ctx = netfs_inode(mapping->host);
356− struct folio *sink = NULL;
357581 int ret;
358582
583+ if (folio_test_dirty(folio)) {
584+ trace_netfs_folio(folio, netfs_folio_trace_read_gaps);
585+ return netfs_read_gaps(file, folio);
586+ }
587+
359588 _enter("%lx", folio->index);
360589
361590 rreq = netfs_alloc_request(mapping, file,
@@ -374,54 +603,12 @@ int netfs_read_folio(struct file *file, struct folio *folio) · 12 unmodified lines
374603 trace_netfs_read(rreq, rreq->start, rreq->len, netfs_read_trace_readpage);
375604
376605 /* Set up the output buffer */
377− if (folio_test_dirty(folio)) {
378− /* Handle someone trying to read from an unflushed streaming
379− * write. We fiddle the buffer so that a gap at the beginning
380− * and/or a gap at the end get copied to, but the middle is
381− * discarded.
382− */
383− struct netfs_folio *finfo = netfs_folio_info(folio);
384− struct bio_vec *bvec;
385− unsigned int from = finfo->dirty_offset;
386− unsigned int to = from + finfo->dirty_len;
387− unsigned int off = 0, i = 0;
388− size_t flen = folio_size(folio);
389− size_t nr_bvec = flen / PAGE_SIZE + 2;
390− size_t part;
391−
392− ret = -ENOMEM;
393− bvec = kmalloc_array(nr_bvec, sizeof(*bvec), GFP_KERNEL);
394− if (!bvec)
395− goto discard;
396−
397− sink = folio_alloc(GFP_KERNEL, 0);
398− if (!sink)
399− goto discard;
400−
401− trace_netfs_folio(folio, netfs_folio_trace_read_gaps);
402−
403− rreq->direct_bv = bvec;
404− rreq->direct_bv_count = nr_bvec;
405− if (from > 0) {
406− bvec_set_folio(&bvec[i++], folio, from, 0);
407− off = from;
408− }
409− while (off < to) {
410− part = min_t(size_t, to - off, PAGE_SIZE);
411− bvec_set_folio(&bvec[i++], sink, part, 0);
412− off += part;
413− }
414− if (to < flen)
415− bvec_set_folio(&bvec[i++], folio, flen - to, to);
416− iov_iter_bvec(&rreq->iter, ITER_DEST, bvec, i, rreq->len);
417− } else {
418− iov_iter_xarray(&rreq->iter, ITER_DEST, &mapping->i_pages,
419− rreq->start, rreq->len);
420− }
606+ ret = netfs_create_singular_buffer(rreq, folio);
607+ if (ret < 0)
608+ goto discard;
421609
422− ret = netfs_begin_read(rreq, true);
423− if (sink)
424− folio_put(sink);
610+ netfs_read_to_pagecache(rreq);
611+ ret = netfs_wait_for_read(rreq);
425612 netfs_put_request(rreq, false, netfs_rreq_trace_put_return);
426613 return ret < 0 ? ret : 0;
427614
@@ -494,13 +681,10 @@ zero_out: · 66 unmodified lines
494681 *
495682 * Pre-read data for a write-begin request by drawing data from the cache if
496683 * possible, or the netfs if not. Space beyond the EOF is zero-filled.
497− * Multiple I/O requests from different sources will get munged together. If
498− * necessary, the readahead window can be expanded in either direction to a
499− * more convenient alighment for RPC efficiency or to make storage in the cache
500− * feasible.
684+ * Multiple I/O requests from different sources will get munged together.
501685 *
502686 * The calling netfs must provide a table of operations, only one of which,
503− * issue_op, is mandatory.
687+ * issue_read, is mandatory.
504688 *
505689 * The check_write_begin() operation can be provided to check for and flush
506690 * conflicting writes once the folio is grabbed and locked. It is passed a
@@ -528,8 +712,6 @@ int netfs_write_begin(struct netfs_inode *ctx, · 21 unmodified lines
528712 pgoff_t index = pos >> PAGE_SHIFT;
529713 int ret;
530714
531− DEFINE_READAHEAD(ractl, file, NULL, mapping, index);
532−
533715 retry:
534716 folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN,
535717 mapping_gfp_mask(mapping));
@@ -577,22 +759,13 @@ retry: · 41 unmodified lines
577759 netfs_stat(&netfs_n_rh_write_begin);
578760 trace_netfs_read(rreq, pos, len, netfs_read_trace_write_begin);
579761
580− /* Expand the request to meet caching requirements and download
581− * preferences.
582− */
583− ractl._nr_pages = folio_nr_pages(folio);
584− netfs_rreq_expand(rreq, &ractl);
585−
586762 /* Set up the output buffer */
587− iov_iter_xarray(&rreq->iter, ITER_DEST, &mapping->i_pages,
588− rreq->start, rreq->len);
589−
590− /* We hold the folio locks, so we can drop the references */
591− folio_get(folio);
592− while (readahead_folio(&ractl))
593− ;
763+ ret = netfs_create_singular_buffer(rreq, folio);
764+ if (ret < 0)
765+ goto error_put;
594766
595− ret = netfs_begin_read(rreq, true);
767+ netfs_read_to_pagecache(rreq);
768+ ret = netfs_wait_for_read(rreq);
596769 if (ret < 0)
597770 goto error;
598771 netfs_put_request(rreq, false, netfs_rreq_trace_put_return);
@@ -652,10 +825,13 @@ int netfs_prefetch_for_write(struct file *file, struct folio *folio, · 53 unmodified lines
652825 trace_netfs_read(rreq, start, flen, netfs_read_trace_prefetch_for_write);
653826
654827 /* Set up the output buffer */
655− iov_iter_xarray(&rreq->iter, ITER_DEST, &mapping->i_pages,
656− rreq->start, rreq->len);
828+ ret = netfs_create_singular_buffer(rreq, folio);
829+ if (ret < 0)
830+ goto error_put;
657831
658− ret = netfs_begin_read(rreq, true);
832+ folioq_mark2(rreq->buffer, 0);
833+ netfs_read_to_pagecache(rreq);
834+ ret = netfs_wait_for_read(rreq);
659835 netfs_put_request(rreq, false, netfs_rreq_trace_put_return);
660836 return ret;
661837
fs/netfs/direct_read.c+141−6
@@ -16,6 +16,143 @@ · 15 unmodified lines
1616 #include <linux/netfs.h>
1717 #include "internal.h"
1818
19+static void netfs_prepare_dio_read_iterator(struct netfs_io_subrequest *subreq)
20+{
21+ struct netfs_io_request *rreq = subreq->rreq;
22+ size_t rsize;
23+
24+ rsize = umin(subreq->len, rreq->io_streams[0].sreq_max_len);
25+ subreq->len = rsize;
26+
27+ if (unlikely(rreq->io_streams[0].sreq_max_segs)) {
28+ size_t limit = netfs_limit_iter(&rreq->iter, 0, rsize,
29+ rreq->io_streams[0].sreq_max_segs);
30+
31+ if (limit < rsize) {
32+ subreq->len = limit;
33+ trace_netfs_sreq(subreq, netfs_sreq_trace_limited);
34+ }
35+ }
36+
37+ trace_netfs_sreq(subreq, netfs_sreq_trace_prepare);
38+
39+ subreq->io_iter = rreq->iter;
40+ iov_iter_truncate(&subreq->io_iter, subreq->len);
41+ iov_iter_advance(&rreq->iter, subreq->len);
42+}
43+
44+/*
45+ * Perform a read to a buffer from the server, slicing up the region to be read
46+ * according to the network rsize.
47+ */
48+static int netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq)
49+{
50+ unsigned long long start = rreq->start;
51+ ssize_t size = rreq->len;
52+ int ret = 0;
53+
54+ atomic_set(&rreq->nr_outstanding, 1);
55+
56+ do {
57+ struct netfs_io_subrequest *subreq;
58+ ssize_t slice;
59+
60+ subreq = netfs_alloc_subrequest(rreq);
61+ if (!subreq) {
62+ ret = -ENOMEM;
63+ break;
64+ }
65+
66+ subreq->source = NETFS_DOWNLOAD_FROM_SERVER;
67+ subreq->start = start;
68+ subreq->len = size;
69+
70+ atomic_inc(&rreq->nr_outstanding);
71+ spin_lock_bh(&rreq->lock);
72+ list_add_tail(&subreq->rreq_link, &rreq->subrequests);
73+ subreq->prev_donated = rreq->prev_donated;
74+ rreq->prev_donated = 0;
75+ trace_netfs_sreq(subreq, netfs_sreq_trace_added);
76+ spin_unlock_bh(&rreq->lock);
77+
78+ netfs_stat(&netfs_n_rh_download);
79+ if (rreq->netfs_ops->prepare_read) {
80+ ret = rreq->netfs_ops->prepare_read(subreq);
81+ if (ret < 0) {
82+ atomic_dec(&rreq->nr_outstanding);
83+ netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_cancel);
84+ break;
85+ }
86+ }
87+
88+ netfs_prepare_dio_read_iterator(subreq);
89+ slice = subreq->len;
90+ rreq->netfs_ops->issue_read(subreq);
91+
92+ size -= slice;
93+ start += slice;
94+ rreq->submitted += slice;
95+
96+ if (test_bit(NETFS_RREQ_BLOCKED, &rreq->flags) &&
97+ test_bit(NETFS_RREQ_NONBLOCK, &rreq->flags))
98+ break;
99+ cond_resched();
100+ } while (size > 0);
101+
102+ if (atomic_dec_and_test(&rreq->nr_outstanding))
103+ netfs_rreq_terminated(rreq, false);
104+ return ret;
105+}
106+
107+/*
108+ * Perform a read to an application buffer, bypassing the pagecache and the
109+ * local disk cache.
110+ */
111+static int netfs_unbuffered_read(struct netfs_io_request *rreq, bool sync)
112+{
113+ int ret;
114+
115+ _enter("R=%x %llx-%llx",
116+ rreq->debug_id, rreq->start, rreq->start + rreq->len - 1);
117+
118+ if (rreq->len == 0) {
119+ pr_err("Zero-sized read [R=%x]\n", rreq->debug_id);
120+ return -EIO;
121+ }
122+
123+ // TODO: Use bounce buffer if requested
124+
125+ inode_dio_begin(rreq->inode);
126+
127+ ret = netfs_dispatch_unbuffered_reads(rreq);
128+
129+ if (!rreq->submitted) {
130+ netfs_put_request(rreq, false, netfs_rreq_trace_put_no_submit);
131+ inode_dio_end(rreq->inode);
132+ ret = 0;
133+ goto out;
134+ }
135+
136+ if (sync) {
137+ trace_netfs_rreq(rreq, netfs_rreq_trace_wait_ip);
138+ wait_on_bit(&rreq->flags, NETFS_RREQ_IN_PROGRESS,
139+ TASK_UNINTERRUPTIBLE);
140+
141+ ret = rreq->error;
142+ if (ret == 0 && rreq->submitted < rreq->len &&
143+ rreq->origin != NETFS_DIO_READ) {
144+ trace_netfs_failure(rreq, NULL, ret, netfs_fail_short_read);
145+ ret = -EIO;
146+ }
147+ } else {
148+ ret = -EIOCBQUEUED;
149+ }
150+
151+out:
152+ _leave(" = %d", ret);
153+ return ret;
154+}
155+
19156 /**
20157 * netfs_unbuffered_read_iter_locked - Perform an unbuffered or direct I/O read
21158 * @iocb: The I/O control descriptor describing the read
@@ -31,7 +168,7 @@ ssize_t netfs_unbuffered_read_iter_locked(struct kiocb *iocb, struct iov_iter *i · 9 unmodified lines
31168 struct netfs_io_request *rreq;
32169 ssize_t ret;
33170 size_t orig_count = iov_iter_count(iter);
34− bool async = !is_sync_kiocb(iocb);
171+ bool sync = is_sync_kiocb(iocb);
35172
36173 _enter("");
37174
@@ -78,13 +215,13 @@ ssize_t netfs_unbuffered_read_iter_locked(struct kiocb *iocb, struct iov_iter *i · 40 unmodified lines
78215
79216 // TODO: Set up bounce buffer if needed
80217
81− if (async)
218+ if (!sync)
82219 rreq->iocb = iocb;
83220
84− ret = netfs_begin_read(rreq, is_sync_kiocb(iocb));
221+ ret = netfs_unbuffered_read(rreq, sync);
85222 if (ret < 0)
86223 goto out; /* May be -EIOCBQUEUED */
87− if (!async) {
224+ if (sync) {
88225 // TODO: Copy from bounce buffer
89226 iocb->ki_pos += rreq->transferred;
90227 ret = rreq->transferred;
@@ -94,8 +231,6 @@ out: · 3 unmodified lines
94231 netfs_put_request(rreq, false, netfs_rreq_trace_put_return);
95232 if (ret > 0)
96233 orig_count -= ret;
97− if (ret != -EIOCBQUEUED)
98− iov_iter_revert(iter, orig_count - iov_iter_count(iter));
99234 return ret;
100235 }
101236 EXPORT_SYMBOL(netfs_unbuffered_read_iter_locked);
fs/netfs/internal.h+28−7
@@ -23,16 +23,9 @@ · 22 unmodified lines
2323 /*
2424 * buffered_read.c
2525 */
26−void netfs_rreq_unlock_folios(struct netfs_io_request *rreq);
2726 int netfs_prefetch_for_write(struct file *file, struct folio *folio,
2827 size_t offset, size_t len);
2928
30−/*
31− * io.c
32− */
33−void netfs_rreq_work(struct work_struct *work);
34−int netfs_begin_read(struct netfs_io_request *rreq, bool sync);
35−
3629 /*
3730 * main.c
3831 */
@@ -90,6 +83,28 @@ static inline void netfs_see_request(struct netfs_io_request *rreq, · 51 unmodified lines
9083 trace_netfs_rreq_ref(rreq->debug_id, refcount_read(&rreq->ref), what);
9184 }
9285
86+/*
87+ * read_collect.c
88+ */
89+void netfs_read_termination_worker(struct work_struct *work);
90+void netfs_rreq_terminated(struct netfs_io_request *rreq, bool was_async);
91+
92+/*
93+ * read_pgpriv2.c
94+ */
95+void netfs_pgpriv2_mark_copy_to_cache(struct netfs_io_subrequest *subreq,
96+ struct netfs_io_request *rreq,
97+ struct folio_queue *folioq,
98+ int slot);
99+void netfs_pgpriv2_write_to_the_cache(struct netfs_io_request *rreq);
100+bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *wreq);
101+
102+/*
103+ * read_retry.c
104+ */
105+void netfs_retry_reads(struct netfs_io_request *rreq);
106+void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq);
107+
93108 /*
94109 * stats.c
95110 */
@@ -117,6 +132,7 @@ extern atomic_t netfs_n_wh_buffered_write; · 21 unmodified lines
117132 extern atomic_t netfs_n_wh_writethrough;
118133 extern atomic_t netfs_n_wh_dio_write;
119134 extern atomic_t netfs_n_wh_writepages;
135+extern atomic_t netfs_n_wh_copy_to_cache;
120136 extern atomic_t netfs_n_wh_wstream_conflict;
121137 extern atomic_t netfs_n_wh_upload;
122138 extern atomic_t netfs_n_wh_upload_done;
@@ -162,6 +178,11 @@ struct netfs_io_request *netfs_create_write_req(struct address_space *mapping, · 39 unmodified lines
162178 void netfs_reissue_write(struct netfs_io_stream *stream,
163179 struct netfs_io_subrequest *subreq,
164180 struct iov_iter *source);
181+void netfs_issue_write(struct netfs_io_request *wreq,
182+ struct netfs_io_stream *stream);
183+int netfs_advance_write(struct netfs_io_request *wreq,
184+ struct netfs_io_stream *stream,
185+ loff_t start, size_t len, bool to_eof);
165186 struct netfs_io_request *netfs_begin_writethrough(struct kiocb *iocb, size_t len);
166187 int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc,
167188 struct folio *folio, size_t copied, bool to_page_end,
fs/netfs/iterator.c+50−0
@@ -188,9 +188,59 @@ static size_t netfs_limit_xarray(const struct iov_iter *iter, size_t start_offse · 187 unmodified lines
188188 return min(span, max_size);
189189 }
190190
191+/*
192+ * Select the span of a folio queue iterator we're going to use. Limit it by
193+ * both maximum size and maximum number of segments. Returns the size of the
194+ * span in bytes.
195+ */
196+static size_t netfs_limit_folioq(const struct iov_iter *iter, size_t start_offset,
197+ size_t max_size, size_t max_segs)
198+{
199+ const struct folio_queue *folioq = iter->folioq;
200+ unsigned int nsegs = 0;
201+ unsigned int slot = iter->folioq_slot;
202+ size_t span = 0, n = iter->count;
203+
204+ if (WARN_ON(!iov_iter_is_folioq(iter)) ||
205+ WARN_ON(start_offset > n) ||
206+ n == 0)
207+ return 0;
208+ max_size = umin(max_size, n - start_offset);
209+
210+ if (slot >= folioq_nr_slots(folioq)) {
211+ folioq = folioq->next;
212+ slot = 0;
213+ }
214+
215+ start_offset += iter->iov_offset;
216+ do {
217+ size_t flen = folioq_folio_size(folioq, slot);
218+
219+ if (start_offset < flen) {
220+ span += flen - start_offset;
221+ nsegs++;
222+ start_offset = 0;
223+ } else {
224+ start_offset -= flen;
225+ }
226+ if (span >= max_size || nsegs >= max_segs)
227+ break;
228+
229+ slot++;
230+ if (slot >= folioq_nr_slots(folioq)) {
231+ folioq = folioq->next;
232+ slot = 0;
233+ }
234+ } while (folioq);
235+
236+ return umin(span, max_size);
237+}
238+
191239 size_t netfs_limit_iter(const struct iov_iter *iter, size_t start_offset,
192240 size_t max_size, size_t max_segs)
193241 {
242+ if (iov_iter_is_folioq(iter))
243+ return netfs_limit_folioq(iter, start_offset, max_size, max_segs);
194244 if (iov_iter_is_bvec(iter))
195245 return netfs_limit_bvec(iter, start_offset, max_size, max_segs);
196246 if (iov_iter_is_xarray(iter))
fs/netfs/main.c+3−1
@@ -36,12 +36,14 @@ DEFINE_SPINLOCK(netfs_proc_lock); · 35 unmodified lines
3636 static const char *netfs_origins[nr__netfs_io_origin] = {
3737 [NETFS_READAHEAD] = "RA",
3838 [NETFS_READPAGE] = "RP",
39+ [NETFS_READ_GAPS] = "RG",
3940 [NETFS_READ_FOR_WRITE] = "RW",
4041 [NETFS_DIO_READ] = "DR",
4142 [NETFS_WRITEBACK] = "WB",
4243 [NETFS_WRITETHROUGH] = "WT",
4344 [NETFS_UNBUFFERED_WRITE] = "UW",
4445 [NETFS_DIO_WRITE] = "DW",
46+ [NETFS_PGPRIV2_COPY_TO_CACHE] = "2C",
4547 };
4648
4749 /*
@@ -61,7 +63,7 @@ static int netfs_requests_seq_show(struct seq_file *m, void *v) · 13 unmodified lines
6163
6264 rreq = list_entry(v, struct netfs_io_request, proc_link);
6365 seq_printf(m,
64− "%08x %s %3d %2lx %4d %3d @%04llx %llx/%llx",
66+ "%08x %s %3d %2lx %4ld %3d @%04llx %llx/%llx",
6567 rreq->debug_id,
6668 netfs_origins[rreq->origin],
6769 refcount_read(&rreq->ref),
fs/netfs/objects.c+5−3
@@ -36,7 +36,6 @@ struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, · 35 unmodified lines
3636 memset(rreq, 0, kmem_cache_size(cache));
3737 rreq->start = start;
3838 rreq->len = len;
39− rreq->upper_len = len;
4039 rreq->origin = origin;
4140 rreq->netfs_ops = ctx->ops;
4241 rreq->mapping = mapping;
@@ -44,6 +43,8 @@ struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, · 1 unmodified line
4443 rreq->i_size = i_size_read(inode);
4544 rreq->debug_id = atomic_inc_return(&debug_ids);
4645 rreq->wsize = INT_MAX;
46+ rreq->io_streams[0].sreq_max_len = ULONG_MAX;
47+ rreq->io_streams[0].sreq_max_segs = 0;
4748 spin_lock_init(&rreq->lock);
4849 INIT_LIST_HEAD(&rreq->io_streams[0].subrequests);
4950 INIT_LIST_HEAD(&rreq->io_streams[1].subrequests);
@@ -52,9 +53,10 @@ struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, · 2 unmodified lines
5253
5354 if (origin == NETFS_READAHEAD ||
5455 origin == NETFS_READPAGE ||
56+ origin == NETFS_READ_GAPS ||
5557 origin == NETFS_READ_FOR_WRITE ||
5658 origin == NETFS_DIO_READ)
57− INIT_WORK(&rreq->work, netfs_rreq_work);
59+ INIT_WORK(&rreq->work, netfs_read_termination_worker);
5860 else
5961 INIT_WORK(&rreq->work, netfs_write_collection_worker);
6062
@@ -163,7 +165,7 @@ void netfs_put_request(struct netfs_io_request *rreq, bool was_async, · 102 unmodified lines
163165 if (was_async) {
164166 rreq->work.func = netfs_free_request;
165167 if (!queue_work(system_unbound_wq, &rreq->work))
166− BUG();
168+ WARN_ON(1);
167169 } else {
168170 netfs_free_request(&rreq->work);
169171 }
fs/netfs/read_collect.c+544−0
@@ -0,0 +1,544 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/* Network filesystem read subrequest result collection, assessment and
3+ * retrying.
4+ *
5+ * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
6+ * Written by David Howells (dhowells@redhat.com)
7+ */
8+
9+#include <linux/export.h>
10+#include <linux/fs.h>
11+#include <linux/mm.h>
12+#include <linux/pagemap.h>
13+#include <linux/slab.h>
14+#include <linux/task_io_accounting_ops.h>
15+#include "internal.h"
16+
17+/*
18+ * Clear the unread part of an I/O request.
19+ */
20+static void netfs_clear_unread(struct netfs_io_subrequest *subreq)
21+{
22+ netfs_reset_iter(subreq);
23+ WARN_ON_ONCE(subreq->len - subreq->transferred != iov_iter_count(&subreq->io_iter));
24+ iov_iter_zero(iov_iter_count(&subreq->io_iter), &subreq->io_iter);
25+ if (subreq->start + subreq->transferred >= subreq->rreq->i_size)
26+ __set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags);
27+}
28+
29+/*
30+ * Flush, mark and unlock a folio that's now completely read. If we want to
31+ * cache the folio, we set the group to NETFS_FOLIO_COPY_TO_CACHE, mark it
32+ * dirty and let writeback handle it.
33+ */
34+static void netfs_unlock_read_folio(struct netfs_io_subrequest *subreq,
35+ struct netfs_io_request *rreq,
36+ struct folio_queue *folioq,
37+ int slot)
38+{
39+ struct netfs_folio *finfo;
40+ struct folio *folio = folioq_folio(folioq, slot);
41+
42+ flush_dcache_folio(folio);
43+ folio_mark_uptodate(folio);
44+
45+ if (!test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)) {
46+ finfo = netfs_folio_info(folio);
47+ if (finfo) {
48+ trace_netfs_folio(folio, netfs_folio_trace_filled_gaps);
49+ if (finfo->netfs_group)
50+ folio_change_private(folio, finfo->netfs_group);
51+ else
52+ folio_detach_private(folio);
53+ kfree(finfo);
54+ }
55+
56+ if (test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags)) {
57+ if (!WARN_ON_ONCE(folio_get_private(folio) != NULL)) {
58+ trace_netfs_folio(folio, netfs_folio_trace_copy_to_cache);
59+ folio_attach_private(folio, NETFS_FOLIO_COPY_TO_CACHE);
60+ folio_mark_dirty(folio);
61+ }
62+ } else {
63+ trace_netfs_folio(folio, netfs_folio_trace_read_done);
64+ }
65+ } else {
66+ // TODO: Use of PG_private_2 is deprecated.
67+ if (test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags))
68+ netfs_pgpriv2_mark_copy_to_cache(subreq, rreq, folioq, slot);
69+ }
70+
71+ if (!test_bit(NETFS_RREQ_DONT_UNLOCK_FOLIOS, &rreq->flags)) {
72+ if (folio->index == rreq->no_unlock_folio &&
73+ test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags)) {
74+ _debug("no unlock");
75+ } else {
76+ trace_netfs_folio(folio, netfs_folio_trace_read_unlock);
77+ folio_unlock(folio);
78+ }
79+ }
80+}
81+
82+/*
83+ * Unlock any folios that are now completely read. Returns true if the
84+ * subrequest is removed from the list.
85+ */
86+static bool netfs_consume_read_data(struct netfs_io_subrequest *subreq, bool was_async)
87+{
88+ struct netfs_io_subrequest *prev, *next;
89+ struct netfs_io_request *rreq = subreq->rreq;
90+ struct folio_queue *folioq = subreq->curr_folioq;
91+ size_t avail, prev_donated, next_donated, fsize, part, excess;
92+ loff_t fpos, start;
93+ loff_t fend;
94+ int slot = subreq->curr_folioq_slot;
95+
96+ if (WARN(subreq->transferred > subreq->len,
97+ "Subreq overread: R%x[%x] %zu > %zu",
98+ rreq->debug_id, subreq->debug_index,
99+ subreq->transferred, subreq->len))
100+ subreq->transferred = subreq->len;
101+
102+next_folio:
103+ fsize = PAGE_SIZE << subreq->curr_folio_order;
104+ fpos = round_down(subreq->start + subreq->consumed, fsize);
105+ fend = fpos + fsize;
106+
107+ if (WARN_ON_ONCE(!folioq) ||
108+ WARN_ON_ONCE(!folioq_folio(folioq, slot)) ||
109+ WARN_ON_ONCE(folioq_folio(folioq, slot)->index != fpos / PAGE_SIZE)) {
110+ pr_err("R=%08x[%x] s=%llx-%llx ctl=%zx/%zx/%zx sl=%u\n",
111+ rreq->debug_id, subreq->debug_index,
112+ subreq->start, subreq->start + subreq->transferred - 1,
113+ subreq->consumed, subreq->transferred, subreq->len,
114+ slot);
115+ if (folioq) {
116+ struct folio *folio = folioq_folio(folioq, slot);
117+
118+ pr_err("folioq: orders=%02x%02x%02x%02x\n",
119+ folioq->orders[0], folioq->orders[1],
120+ folioq->orders[2], folioq->orders[3]);
121+ if (folio)
122+ pr_err("folio: %llx-%llx ix=%llx o=%u qo=%u\n",
123+ fpos, fend - 1, folio_pos(folio), folio_order(folio),
124+ folioq_folio_order(folioq, slot));
125+ }
126+ }
127+
128+donation_changed:
129+ /* Try to consume the current folio if we've hit or passed the end of
130+ * it. There's a possibility that this subreq doesn't start at the
131+ * beginning of the folio, in which case we need to donate to/from the
132+ * preceding subreq.
133+ *
134+ * We also need to include any potential donation back from the
135+ * following subreq.
136+ */
137+ prev_donated = READ_ONCE(subreq->prev_donated);
138+ next_donated = READ_ONCE(subreq->next_donated);
139+ if (prev_donated || next_donated) {
140+ spin_lock_bh(&rreq->lock);
141+ prev_donated = subreq->prev_donated;
142+ next_donated = subreq->next_donated;
143+ subreq->start -= prev_donated;
144+ subreq->len += prev_donated;
145+ subreq->transferred += prev_donated;
146+ prev_donated = subreq->prev_donated = 0;
147+ if (subreq->transferred == subreq->len) {
148+ subreq->len += next_donated;
149+ subreq->transferred += next_donated;
150+ next_donated = subreq->next_donated = 0;
151+ }
152+ trace_netfs_sreq(subreq, netfs_sreq_trace_add_donations);
153+ spin_unlock_bh(&rreq->lock);
154+ }
155+
156+ avail = subreq->transferred;
157+ if (avail == subreq->len)
158+ avail += next_donated;
159+ start = subreq->start;
160+ if (subreq->consumed == 0) {
161+ start -= prev_donated;
162+ avail += prev_donated;
163+ } else {
164+ start += subreq->consumed;
165+ avail -= subreq->consumed;
166+ }
167+ part = umin(avail, fsize);
168+
169+ trace_netfs_progress(subreq, start, avail, part);
170+
171+ if (start + avail >= fend) {
172+ if (fpos == start) {
173+ /* Flush, unlock and mark for caching any folio we've just read. */
174+ subreq->consumed = fend - subreq->start;
175+ netfs_unlock_read_folio(subreq, rreq, folioq, slot);
176+ folioq_mark2(folioq, slot);
177+ if (subreq->consumed >= subreq->len)
178+ goto remove_subreq;
179+ } else if (fpos < start) {
180+ excess = fend - subreq->start;
181+
182+ spin_lock_bh(&rreq->lock);
183+ /* If we complete first on a folio split with the
184+ * preceding subreq, donate to that subreq - otherwise
185+ * we get the responsibility.
186+ */
187+ if (subreq->prev_donated != prev_donated) {
188+ spin_unlock_bh(&rreq->lock);
189+ goto donation_changed;
190+ }
191+
192+ if (list_is_first(&subreq->rreq_link, &rreq->subrequests)) {
193+ spin_unlock_bh(&rreq->lock);
194+ pr_err("Can't donate prior to front\n");
195+ goto bad;
196+ }
197+
198+ prev = list_prev_entry(subreq, rreq_link);
199+ WRITE_ONCE(prev->next_donated, prev->next_donated + excess);
200+ subreq->start += excess;
201+ subreq->len -= excess;
202+ subreq->transferred -= excess;
203+ trace_netfs_donate(rreq, subreq, prev, excess,
204+ netfs_trace_donate_tail_to_prev);
205+ trace_netfs_sreq(subreq, netfs_sreq_trace_donate_to_prev);
206+
207+ if (subreq->consumed >= subreq->len)
208+ goto remove_subreq_locked;
209+ spin_unlock_bh(&rreq->lock);
210+ } else {
211+ pr_err("fpos > start\n");
212+ goto bad;
213+ }
214+
215+ /* Advance the rolling buffer to the next folio. */
216+ slot++;
217+ if (slot >= folioq_nr_slots(folioq)) {
218+ slot = 0;
219+ folioq = folioq->next;
220+ subreq->curr_folioq = folioq;
221+ }
222+ subreq->curr_folioq_slot = slot;
223+ if (folioq && folioq_folio(folioq, slot))
224+ subreq->curr_folio_order = folioq->orders[slot];
225+ if (!was_async)
226+ cond_resched();
227+ goto next_folio;
228+ }
229+
230+ /* Deal with partial progress. */
231+ if (subreq->transferred < subreq->len)
232+ return false;
233+
234+ /* Donate the remaining downloaded data to one of the neighbouring
235+ * subrequests. Note that we may race with them doing the same thing.
236+ */
237+ spin_lock_bh(&rreq->lock);
238+
239+ if (subreq->prev_donated != prev_donated ||
240+ subreq->next_donated != next_donated) {
241+ spin_unlock_bh(&rreq->lock);
242+ cond_resched();
243+ goto donation_changed;
244+ }
245+
246+ /* Deal with the trickiest case: that this subreq is in the middle of a
247+ * folio, not touching either edge, but finishes first. In such a
248+ * case, we donate to the previous subreq, if there is one, so that the
249+ * donation is only handled when that completes - and remove this
250+ * subreq from the list.
251+ *
252+ * If the previous subreq finished first, we will have acquired their
253+ * donation and should be able to unlock folios and/or donate nextwards.
254+ */
255+ if (!subreq->consumed &&
256+ !prev_donated &&
257+ !list_is_first(&subreq->rreq_link, &rreq->subrequests)) {
258+ prev = list_prev_entry(subreq, rreq_link);
259+ WRITE_ONCE(prev->next_donated, prev->next_donated + subreq->len);
260+ subreq->start += subreq->len;
261+ subreq->len = 0;
262+ subreq->transferred = 0;
263+ trace_netfs_donate(rreq, subreq, prev, subreq->len,
264+ netfs_trace_donate_to_prev);
265+ trace_netfs_sreq(subreq, netfs_sreq_trace_donate_to_prev);
266+ goto remove_subreq_locked;
267+ }
268+
269+ /* If we can't donate down the chain, donate up the chain instead. */
270+ excess = subreq->len - subreq->consumed + next_donated;
271+
272+ if (!subreq->consumed)
273+ excess += prev_donated;
274+
275+ if (list_is_last(&subreq->rreq_link, &rreq->subrequests)) {
276+ rreq->prev_donated = excess;
277+ trace_netfs_donate(rreq, subreq, NULL, excess,
278+ netfs_trace_donate_to_deferred_next);
279+ } else {
280+ next = list_next_entry(subreq, rreq_link);
281+ WRITE_ONCE(next->prev_donated, excess);
282+ trace_netfs_donate(rreq, subreq, next, excess,
283+ netfs_trace_donate_to_next);
284+ }
285+ trace_netfs_sreq(subreq, netfs_sreq_trace_donate_to_next);
286+ subreq->len = subreq->consumed;
287+ subreq->transferred = subreq->consumed;
288+ goto remove_subreq_locked;
289+
290+remove_subreq:
291+ spin_lock_bh(&rreq->lock);
292+remove_subreq_locked:
293+ subreq->consumed = subreq->len;
294+ list_del(&subreq->rreq_link);
295+ spin_unlock_bh(&rreq->lock);
296+ netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_consumed);
297+ return true;
298+
299+bad:
300+ /* Errr... prev and next both donated to us, but insufficient to finish
301+ * the folio.
302+ */
303+ printk("R=%08x[%x] s=%llx-%llx %zx/%zx/%zx\n",
304+ rreq->debug_id, subreq->debug_index,
305+ subreq->start, subreq->start + subreq->transferred - 1,
306+ subreq->consumed, subreq->transferred, subreq->len);
307+ printk("folio: %llx-%llx\n", fpos, fend - 1);
308+ printk("donated: prev=%zx next=%zx\n", prev_donated, next_donated);
309+ printk("s=%llx av=%zx part=%zx\n", start, avail, part);
310+ BUG();
311+}
312+
313+/*
314+ * Do page flushing and suchlike after DIO.
315+ */
316+static void netfs_rreq_assess_dio(struct netfs_io_request *rreq)
317+{
318+ struct netfs_io_subrequest *subreq;
319+ unsigned int i;
320+
321+ /* Collect unbuffered reads and direct reads, adding up the transfer
322+ * sizes until we find the first short or failed subrequest.
323+ */
324+ list_for_each_entry(subreq, &rreq->subrequests, rreq_link) {
325+ rreq->transferred += subreq->transferred;
326+
327+ if (subreq->transferred < subreq->len ||
328+ test_bit(NETFS_SREQ_FAILED, &subreq->flags)) {
329+ rreq->error = subreq->error;
330+ break;
331+ }
332+ }
333+
334+ if (rreq->origin == NETFS_DIO_READ) {
335+ for (i = 0; i < rreq->direct_bv_count; i++) {
336+ flush_dcache_page(rreq->direct_bv[i].bv_page);
337+ // TODO: cifs marks pages in the destination buffer
338+ // dirty under some circumstances after a read. Do we
339+ // need to do that too?
340+ set_page_dirty(rreq->direct_bv[i].bv_page);
341+ }
342+ }
343+
344+ if (rreq->iocb) {
345+ rreq->iocb->ki_pos += rreq->transferred;
346+ if (rreq->iocb->ki_complete)
347+ rreq->iocb->ki_complete(
348+ rreq->iocb, rreq->error ? rreq->error : rreq->transferred);
349+ }
350+ if (rreq->netfs_ops->done)
351+ rreq->netfs_ops->done(rreq);
352+ if (rreq->origin == NETFS_DIO_READ)
353+ inode_dio_end(rreq->inode);
354+}
355+
356+/*
357+ * Assess the state of a read request and decide what to do next.
358+ *
359+ * Note that we're in normal kernel thread context at this point, possibly
360+ * running on a workqueue.
361+ */
362+static void netfs_rreq_assess(struct netfs_io_request *rreq)
363+{
364+ trace_netfs_rreq(rreq, netfs_rreq_trace_assess);
365+
366+ //netfs_rreq_is_still_valid(rreq);
367+
368+ if (test_and_clear_bit(NETFS_RREQ_NEED_RETRY, &rreq->flags)) {
369+ netfs_retry_reads(rreq);
370+ return;
371+ }
372+
373+ if (rreq->origin == NETFS_DIO_READ ||
374+ rreq->origin == NETFS_READ_GAPS)
375+ netfs_rreq_assess_dio(rreq);
376+ task_io_account_read(rreq->transferred);
377+
378+ trace_netfs_rreq(rreq, netfs_rreq_trace_wake_ip);
379+ clear_bit_unlock(NETFS_RREQ_IN_PROGRESS, &rreq->flags);
380+ wake_up_bit(&rreq->flags, NETFS_RREQ_IN_PROGRESS);
381+
382+ trace_netfs_rreq(rreq, netfs_rreq_trace_done);
383+ netfs_clear_subrequests(rreq, false);
384+ netfs_unlock_abandoned_read_pages(rreq);
385+ if (unlikely(test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)))
386+ netfs_pgpriv2_write_to_the_cache(rreq);
387+}
388+
389+void netfs_read_termination_worker(struct work_struct *work)
390+{
391+ struct netfs_io_request *rreq =
392+ container_of(work, struct netfs_io_request, work);
393+ netfs_see_request(rreq, netfs_rreq_trace_see_work);
394+ netfs_rreq_assess(rreq);
395+ netfs_put_request(rreq, false, netfs_rreq_trace_put_work_complete);
396+}
397+
398+/*
399+ * Handle the completion of all outstanding I/O operations on a read request.
400+ * We inherit a ref from the caller.
401+ */
402+void netfs_rreq_terminated(struct netfs_io_request *rreq, bool was_async)
403+{
404+ if (!was_async)
405+ return netfs_rreq_assess(rreq);
406+ if (!work_pending(&rreq->work)) {
407+ netfs_get_request(rreq, netfs_rreq_trace_get_work);
408+ if (!queue_work(system_unbound_wq, &rreq->work))
409+ netfs_put_request(rreq, was_async, netfs_rreq_trace_put_work_nq);
410+ }
411+}
412+
413+/**
414+ * netfs_read_subreq_progress - Note progress of a read operation.
415+ * @subreq: The read request that has terminated.
416+ * @was_async: True if we're in an asynchronous context.
417+ *
418+ * This tells the read side of netfs lib that a contributory I/O operation has
419+ * made some progress and that it may be possible to unlock some folios.
420+ *
421+ * Before calling, the filesystem should update subreq->transferred to track
422+ * the amount of data copied into the output buffer.
423+ *
424+ * If @was_async is true, the caller might be running in softirq or interrupt
425+ * context and we can't sleep.
426+ */
427+void netfs_read_subreq_progress(struct netfs_io_subrequest *subreq,
428+ bool was_async)
429+{
430+ struct netfs_io_request *rreq = subreq->rreq;
431+
432+ trace_netfs_sreq(subreq, netfs_sreq_trace_progress);
433+
434+ if (subreq->transferred > subreq->consumed &&
435+ (rreq->origin == NETFS_READAHEAD ||
436+ rreq->origin == NETFS_READPAGE ||
437+ rreq->origin == NETFS_READ_FOR_WRITE)) {
438+ netfs_consume_read_data(subreq, was_async);
439+ __clear_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags);
440+ }
441+}
442+EXPORT_SYMBOL(netfs_read_subreq_progress);
443+
444+/**
445+ * netfs_read_subreq_terminated - Note the termination of an I/O operation.
446+ * @subreq: The I/O request that has terminated.
447+ * @error: Error code indicating type of completion.
448+ * @was_async: The termination was asynchronous
449+ *
450+ * This tells the read helper that a contributory I/O operation has terminated,
451+ * one way or another, and that it should integrate the results.
452+ *
453+ * The caller indicates the outcome of the operation through @error, supplying
454+ * 0 to indicate a successful or retryable transfer (if NETFS_SREQ_NEED_RETRY
455+ * is set) or a negative error code. The helper will look after reissuing I/O
456+ * operations as appropriate and writing downloaded data to the cache.
457+ *
458+ * Before calling, the filesystem should update subreq->transferred to track
459+ * the amount of data copied into the output buffer.
460+ *
461+ * If @was_async is true, the caller might be running in softirq or interrupt
462+ * context and we can't sleep.
463+ */
464+void netfs_read_subreq_terminated(struct netfs_io_subrequest *subreq,
465+ int error, bool was_async)
466+{
467+ struct netfs_io_request *rreq = subreq->rreq;
468+
469+ switch (subreq->source) {
470+ case NETFS_READ_FROM_CACHE:
471+ netfs_stat(&netfs_n_rh_read_done);
472+ break;
473+ case NETFS_DOWNLOAD_FROM_SERVER:
474+ netfs_stat(&netfs_n_rh_download_done);
475+ break;
476+ default:
477+ break;
478+ }
479+
480+ if (rreq->origin != NETFS_DIO_READ) {
481+ /* Collect buffered reads.
482+ *
483+ * If the read completed validly short, then we can clear the
484+ * tail before going on to unlock the folios.
485+ */
486+ if (error == 0 && subreq->transferred < subreq->len &&
487+ (test_bit(NETFS_SREQ_HIT_EOF, &subreq->flags) ||
488+ test_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags))) {
489+ netfs_clear_unread(subreq);
490+ subreq->transferred = subreq->len;
491+ trace_netfs_sreq(subreq, netfs_sreq_trace_clear);
492+ }
493+ if (subreq->transferred > subreq->consumed &&
494+ (rreq->origin == NETFS_READAHEAD ||
495+ rreq->origin == NETFS_READPAGE ||
496+ rreq->origin == NETFS_READ_FOR_WRITE)) {
497+ netfs_consume_read_data(subreq, was_async);
498+ __clear_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags);
499+ }
500+ rreq->transferred += subreq->transferred;
501+ }
502+
503+ /* Deal with retry requests, short reads and errors. If we retry
504+ * but don't make progress, we abandon the attempt.
505+ */
506+ if (!error && subreq->transferred < subreq->len) {
507+ if (test_bit(NETFS_SREQ_HIT_EOF, &subreq->flags)) {
508+ trace_netfs_sreq(subreq, netfs_sreq_trace_hit_eof);
509+ } else {
510+ trace_netfs_sreq(subreq, netfs_sreq_trace_short);
511+ if (subreq->transferred > subreq->consumed) {
512+ __set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
513+ __clear_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags);
514+ set_bit(NETFS_RREQ_NEED_RETRY, &rreq->flags);
515+ } else if (!__test_and_set_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags)) {
516+ __set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
517+ set_bit(NETFS_RREQ_NEED_RETRY, &rreq->flags);
518+ } else {
519+ __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
520+ error = -ENODATA;
521+ }
522+ }
523+ }
524+
525+ subreq->error = error;
526+ trace_netfs_sreq(subreq, netfs_sreq_trace_terminated);
527+
528+ if (unlikely(error < 0)) {
529+ trace_netfs_failure(rreq, subreq, error, netfs_fail_read);
530+ if (subreq->source == NETFS_READ_FROM_CACHE) {
531+ netfs_stat(&netfs_n_rh_read_failed);
532+ } else {
533+ netfs_stat(&netfs_n_rh_download_failed);
534+ set_bit(NETFS_RREQ_FAILED, &rreq->flags);
535+ rreq->error = subreq->error;
536+ }
537+ }
538+
539+ if (atomic_dec_and_test(&rreq->nr_outstanding))
540+ netfs_rreq_terminated(rreq, was_async);
541+
542+ netfs_put_subrequest(subreq, was_async, netfs_sreq_trace_put_terminated);
543+}
544+EXPORT_SYMBOL(netfs_read_subreq_terminated);
fs/netfs/read_pgpriv2.c+264−0
@@ -0,0 +1,264 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/* Read with PG_private_2 [DEPRECATED].
3+ *
4+ * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
5+ * Written by David Howells (dhowells@redhat.com)
6+ */
7+
8+#include <linux/export.h>
9+#include <linux/fs.h>
10+#include <linux/mm.h>
11+#include <linux/pagemap.h>
12+#include <linux/slab.h>
13+#include <linux/task_io_accounting_ops.h>
14+#include "internal.h"
15+
16+/*
17+ * [DEPRECATED] Mark page as requiring copy-to-cache using PG_private_2. The
18+ * third mark in the folio queue is used to indicate that this folio needs
19+ * writing.
20+ */
21+void netfs_pgpriv2_mark_copy_to_cache(struct netfs_io_subrequest *subreq,
22+ struct netfs_io_request *rreq,
23+ struct folio_queue *folioq,
24+ int slot)
25+{
26+ struct folio *folio = folioq_folio(folioq, slot);
27+
28+ trace_netfs_folio(folio, netfs_folio_trace_copy_to_cache);
29+ folio_start_private_2(folio);
30+ folioq_mark3(folioq, slot);
31+}
32+
33+/*
34+ * [DEPRECATED] Cancel PG_private_2 on all marked folios in the event of an
35+ * unrecoverable error.
36+ */
37+static void netfs_pgpriv2_cancel(struct folio_queue *folioq)
38+{
39+ struct folio *folio;
40+ int slot;
41+
42+ while (folioq) {
43+ if (!folioq->marks3) {
44+ folioq = folioq->next;
45+ continue;
46+ }
47+
48+ slot = __ffs(folioq->marks3);
49+ folio = folioq_folio(folioq, slot);
50+
51+ trace_netfs_folio(folio, netfs_folio_trace_cancel_copy);
52+ folio_end_private_2(folio);
53+ folioq_unmark3(folioq, slot);
54+ }
55+}
56+
57+/*
58+ * [DEPRECATED] Copy a folio to the cache with PG_private_2 set.
59+ */
60+static int netfs_pgpriv2_copy_folio(struct netfs_io_request *wreq, struct folio *folio)
61+{
62+ struct netfs_io_stream *cache = &wreq->io_streams[1];
63+ size_t fsize = folio_size(folio), flen = fsize;
64+ loff_t fpos = folio_pos(folio), i_size;
65+ bool to_eof = false;
66+
67+ _enter("");
68+
69+ /* netfs_perform_write() may shift i_size around the page or from out
70+ * of the page to beyond it, but cannot move i_size into or through the
71+ * page since we have it locked.
72+ */
73+ i_size = i_size_read(wreq->inode);
74+
75+ if (fpos >= i_size) {
76+ /* mmap beyond eof. */
77+ _debug("beyond eof");
78+ folio_end_private_2(folio);
79+ return 0;
80+ }
81+
82+ if (fpos + fsize > wreq->i_size)
83+ wreq->i_size = i_size;
84+
85+ if (flen > i_size - fpos) {
86+ flen = i_size - fpos;
87+ to_eof = true;
88+ } else if (flen == i_size - fpos) {
89+ to_eof = true;
90+ }
91+
92+ _debug("folio %zx %zx", flen, fsize);
93+
94+ trace_netfs_folio(folio, netfs_folio_trace_store_copy);
95+
96+ /* Attach the folio to the rolling buffer. */
97+ if (netfs_buffer_append_folio(wreq, folio, false) < 0)
98+ return -ENOMEM;
99+
100+ cache->submit_max_len = fsize;
101+ cache->submit_off = 0;
102+ cache->submit_len = flen;
103+
104+ /* Attach the folio to one or more subrequests. For a big folio, we
105+ * could end up with thousands of subrequests if the wsize is small -
106+ * but we might need to wait during the creation of subrequests for
107+ * network resources (eg. SMB credits).
108+ */
109+ do {
110+ ssize_t part;
111+
112+ wreq->io_iter.iov_offset = cache->submit_off;
113+
114+ atomic64_set(&wreq->issued_to, fpos + cache->submit_off);
115+ part = netfs_advance_write(wreq, cache, fpos + cache->submit_off,
116+ cache->submit_len, to_eof);
117+ cache->submit_off += part;
118+ cache->submit_max_len -= part;
119+ if (part > cache->submit_len)
120+ cache->submit_len = 0;
121+ else
122+ cache->submit_len -= part;
123+ } while (cache->submit_len > 0);
124+
125+ wreq->io_iter.iov_offset = 0;
126+ iov_iter_advance(&wreq->io_iter, fsize);
127+ atomic64_set(&wreq->issued_to, fpos + fsize);
128+
129+ if (flen < fsize)
130+ netfs_issue_write(wreq, cache);
131+
132+ _leave(" = 0");
133+ return 0;
134+}
135+
136+/*
137+ * [DEPRECATED] Go through the buffer and write any folios that are marked with
138+ * the third mark to the cache.
139+ */
140+void netfs_pgpriv2_write_to_the_cache(struct netfs_io_request *rreq)
141+{
142+ struct netfs_io_request *wreq;
143+ struct folio_queue *folioq;
144+ struct folio *folio;
145+ int error = 0;
146+ int slot = 0;
147+
148+ _enter("");
149+
150+ if (!fscache_resources_valid(&rreq->cache_resources))
151+ goto couldnt_start;
152+
153+ /* Need the first folio to be able to set up the op. */
154+ for (folioq = rreq->buffer; folioq; folioq = folioq->next) {
155+ if (folioq->marks3) {
156+ slot = __ffs(folioq->marks3);
157+ break;
158+ }
159+ }
160+ if (!folioq)
161+ return;
162+ folio = folioq_folio(folioq, slot);
163+
164+ wreq = netfs_create_write_req(rreq->mapping, NULL, folio_pos(folio),
165+ NETFS_PGPRIV2_COPY_TO_CACHE);
166+ if (IS_ERR(wreq)) {
167+ kleave(" [create %ld]", PTR_ERR(wreq));
168+ goto couldnt_start;
169+ }
170+
171+ trace_netfs_write(wreq, netfs_write_trace_copy_to_cache);
172+ netfs_stat(&netfs_n_wh_copy_to_cache);
173+
174+ for (;;) {
175+ error = netfs_pgpriv2_copy_folio(wreq, folio);
176+ if (error < 0)
177+ break;
178+
179+ folioq_unmark3(folioq, slot);
180+ if (!folioq->marks3) {
181+ folioq = folioq->next;
182+ if (!folioq)
183+ break;
184+ }
185+
186+ slot = __ffs(folioq->marks3);
187+ folio = folioq_folio(folioq, slot);
188+ }
189+
190+ netfs_issue_write(wreq, &wreq->io_streams[1]);
191+ smp_wmb(); /* Write lists before ALL_QUEUED. */
192+ set_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags);
193+
194+ netfs_put_request(wreq, false, netfs_rreq_trace_put_return);
195+ _leave(" = %d", error);
196+couldnt_start:
197+ netfs_pgpriv2_cancel(rreq->buffer);
198+}
199+
200+/*
201+ * [DEPRECATED] Remove the PG_private_2 mark from any folios we've finished
202+ * copying.
203+ */
204+bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *wreq)
205+{
206+ struct folio_queue *folioq = wreq->buffer;
207+ unsigned long long collected_to = wreq->collected_to;
208+ unsigned int slot = wreq->buffer_head_slot;
209+ bool made_progress = false;
210+
211+ if (slot >= folioq_nr_slots(folioq)) {
212+ folioq = netfs_delete_buffer_head(wreq);
213+ slot = 0;
214+ }
215+
216+ for (;;) {
217+ struct folio *folio;
218+ unsigned long long fpos, fend;
219+ size_t fsize, flen;
220+
221+ folio = folioq_folio(folioq, slot);
222+ if (WARN_ONCE(!folio_test_private_2(folio),
223+ "R=%08x: folio %lx is not marked private_2\n",
224+ wreq->debug_id, folio->index))
225+ trace_netfs_folio(folio, netfs_folio_trace_not_under_wback);
226+
227+ fpos = folio_pos(folio);
228+ fsize = folio_size(folio);
229+ flen = fsize;
230+
231+ fend = min_t(unsigned long long, fpos + flen, wreq->i_size);
232+
233+ trace_netfs_collect_folio(wreq, folio, fend, collected_to);
234+
235+ /* Unlock any folio we've transferred all of. */
236+ if (collected_to < fend)
237+ break;
238+
239+ trace_netfs_folio(folio, netfs_folio_trace_end_copy);
240+ folio_end_private_2(folio);
241+ wreq->cleaned_to = fpos + fsize;
242+ made_progress = true;
243+
244+ /* Clean up the head folioq. If we clear an entire folioq, then
245+ * we can get rid of it provided it's not also the tail folioq
246+ * being filled by the issuer.
247+ */
248+ folioq_clear(folioq, slot);
249+ slot++;
250+ if (slot >= folioq_nr_slots(folioq)) {
251+ if (READ_ONCE(wreq->buffer_tail) == folioq)
252+ break;
253+ folioq = netfs_delete_buffer_head(wreq);
254+ slot = 0;
255+ }
256+
257+ if (fpos + fsize >= collected_to)
258+ break;
259+ }
260+
261+ wreq->buffer = folioq;
262+ wreq->buffer_head_slot = slot;
263+ return made_progress;
264+}
fs/netfs/read_retry.c+256−0
@@ -0,0 +1,256 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/* Network filesystem read subrequest retrying.
3+ *
4+ * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
5+ * Written by David Howells (dhowells@redhat.com)
6+ */
7+
8+#include <linux/fs.h>
9+#include <linux/slab.h>
10+#include "internal.h"
11+
12+static void netfs_reissue_read(struct netfs_io_request *rreq,
13+ struct netfs_io_subrequest *subreq)
14+{
15+ struct iov_iter *io_iter = &subreq->io_iter;
16+
17+ if (iov_iter_is_folioq(io_iter)) {
18+ subreq->curr_folioq = (struct folio_queue *)io_iter->folioq;
19+ subreq->curr_folioq_slot = io_iter->folioq_slot;
20+ subreq->curr_folio_order = subreq->curr_folioq->orders[subreq->curr_folioq_slot];
21+ }
22+
23+ atomic_inc(&rreq->nr_outstanding);
24+ __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
25+ netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit);
26+ subreq->rreq->netfs_ops->issue_read(subreq);
27+}
28+
29+/*
30+ * Go through the list of failed/short reads, retrying all retryable ones. We
31+ * need to switch failed cache reads to network downloads.
32+ */
33+static void netfs_retry_read_subrequests(struct netfs_io_request *rreq)
34+{
35+ struct netfs_io_subrequest *subreq;
36+ struct netfs_io_stream *stream0 = &rreq->io_streams[0];
37+ LIST_HEAD(sublist);
38+ LIST_HEAD(queue);
39+
40+ _enter("R=%x", rreq->debug_id);
41+
42+ if (list_empty(&rreq->subrequests))
43+ return;
44+
45+ if (rreq->netfs_ops->retry_request)
46+ rreq->netfs_ops->retry_request(rreq, NULL);
47+
48+ /* If there's no renegotiation to do, just resend each retryable subreq
49+ * up to the first permanently failed one.
50+ */
51+ if (!rreq->netfs_ops->prepare_read &&
52+ !test_bit(NETFS_RREQ_COPY_TO_CACHE, &rreq->flags)) {
53+ struct netfs_io_subrequest *subreq;
54+
55+ list_for_each_entry(subreq, &rreq->subrequests, rreq_link) {
56+ if (test_bit(NETFS_SREQ_FAILED, &subreq->flags))
57+ break;
58+ if (__test_and_clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) {
59+ netfs_reset_iter(subreq);
60+ netfs_reissue_read(rreq, subreq);
61+ }
62+ }
63+ return;
64+ }
65+
66+ /* Okay, we need to renegotiate all the download requests and flip any
67+ * failed cache reads over to being download requests and negotiate
68+ * those also. All fully successful subreqs have been removed from the
69+ * list and any spare data from those has been donated.
70+ *
71+ * What we do is decant the list and rebuild it one subreq at a time so
72+ * that we don't end up with donations jumping over a gap we're busy
73+ * populating with smaller subrequests. In the event that the subreq
74+ * we just launched finishes before we insert the next subreq, it'll
75+ * fill in rreq->prev_donated instead.
76+
77+ * Note: Alternatively, we could split the tail subrequest right before
78+ * we reissue it and fix up the donations under lock.
79+ */
80+ list_splice_init(&rreq->subrequests, &queue);
81+
82+ do {
83+ struct netfs_io_subrequest *from;
84+ struct iov_iter source;
85+ unsigned long long start, len;
86+ size_t part, deferred_next_donated = 0;
87+ bool boundary = false;
88+
89+ /* Go through the subreqs and find the next span of contiguous
90+ * buffer that we then rejig (cifs, for example, needs the
91+ * rsize renegotiating) and reissue.
92+ */
93+ from = list_first_entry(&queue, struct netfs_io_subrequest, rreq_link);
94+ list_move_tail(&from->rreq_link, &sublist);
95+ start = from->start + from->transferred;
96+ len = from->len - from->transferred;
97+
98+ _debug("from R=%08x[%x] s=%llx ctl=%zx/%zx/%zx",
99+ rreq->debug_id, from->debug_index,
100+ from->start, from->consumed, from->transferred, from->len);
101+
102+ if (test_bit(NETFS_SREQ_FAILED, &from->flags) ||
103+ !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags))
104+ goto abandon;
105+
106+ deferred_next_donated = from->next_donated;
107+ while ((subreq = list_first_entry_or_null(
108+ &queue, struct netfs_io_subrequest, rreq_link))) {
109+ if (subreq->start != start + len ||
110+ subreq->transferred > 0 ||
111+ !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags))
112+ break;
113+ list_move_tail(&subreq->rreq_link, &sublist);
114+ len += subreq->len;
115+ deferred_next_donated = subreq->next_donated;
116+ if (test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags))
117+ break;
118+ }
119+
120+ _debug(" - range: %llx-%llx %llx", start, start + len - 1, len);
121+
122+ /* Determine the set of buffers we're going to use. Each
123+ * subreq gets a subset of a single overall contiguous buffer.
124+ */
125+ netfs_reset_iter(from);
126+ source = from->io_iter;
127+ source.count = len;
128+
129+ /* Work through the sublist. */
130+ while ((subreq = list_first_entry_or_null(
131+ &sublist, struct netfs_io_subrequest, rreq_link))) {
132+ list_del(&subreq->rreq_link);
133+
134+ subreq->source = NETFS_DOWNLOAD_FROM_SERVER;
135+ subreq->start = start - subreq->transferred;
136+ subreq->len = len + subreq->transferred;
137+ stream0->sreq_max_len = subreq->len;
138+
139+ __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
140+ __set_bit(NETFS_SREQ_RETRYING, &subreq->flags);
141+
142+ spin_lock_bh(&rreq->lock);
143+ list_add_tail(&subreq->rreq_link, &rreq->subrequests);
144+ subreq->prev_donated += rreq->prev_donated;
145+ rreq->prev_donated = 0;
146+ trace_netfs_sreq(subreq, netfs_sreq_trace_retry);
147+ spin_unlock_bh(&rreq->lock);
148+
149+ BUG_ON(!len);
150+
151+ /* Renegotiate max_len (rsize) */
152+ if (rreq->netfs_ops->prepare_read(subreq) < 0) {
153+ trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed);
154+ __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
155+ }
156+
157+ part = umin(len, stream0->sreq_max_len);
158+ if (unlikely(rreq->io_streams[0].sreq_max_segs))
159+ part = netfs_limit_iter(&source, 0, part, stream0->sreq_max_segs);
160+ subreq->len = subreq->transferred + part;
161+ subreq->io_iter = source;
162+ iov_iter_truncate(&subreq->io_iter, part);
163+ iov_iter_advance(&source, part);
164+ len -= part;
165+ start += part;
166+ if (!len) {
167+ if (boundary)
168+ __set_bit(NETFS_SREQ_BOUNDARY, &subreq->flags);
169+ subreq->next_donated = deferred_next_donated;
170+ } else {
171+ __clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags);
172+ subreq->next_donated = 0;
173+ }
174+
175+ netfs_reissue_read(rreq, subreq);
176+ if (!len)
177+ break;
178+
179+ /* If we ran out of subrequests, allocate another. */
180+ if (list_empty(&sublist)) {
181+ subreq = netfs_alloc_subrequest(rreq);
182+ if (!subreq)
183+ goto abandon;
184+ subreq->source = NETFS_DOWNLOAD_FROM_SERVER;
185+ subreq->start = start;
186+
187+ /* We get two refs, but need just one. */
188+ netfs_put_subrequest(subreq, false, netfs_sreq_trace_new);
189+ trace_netfs_sreq(subreq, netfs_sreq_trace_split);
190+ list_add_tail(&subreq->rreq_link, &sublist);
191+ }
192+ }
193+
194+ /* If we managed to use fewer subreqs, we can discard the
195+ * excess.
196+ */
197+ while ((subreq = list_first_entry_or_null(
198+ &sublist, struct netfs_io_subrequest, rreq_link))) {
199+ trace_netfs_sreq(subreq, netfs_sreq_trace_discard);
200+ list_del(&subreq->rreq_link);
201+ netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_done);
202+ }
203+
204+ } while (!list_empty(&queue));
205+
206+ return;
207+
208+ /* If we hit ENOMEM, fail all remaining subrequests */
209+abandon:
210+ list_splice_init(&sublist, &queue);
211+ list_for_each_entry(subreq, &queue, rreq_link) {
212+ if (!subreq->error)
213+ subreq->error = -ENOMEM;
214+ __clear_bit(NETFS_SREQ_FAILED, &subreq->flags);
215+ __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
216+ __clear_bit(NETFS_SREQ_RETRYING, &subreq->flags);
217+ }
218+ spin_lock_bh(&rreq->lock);
219+ list_splice_tail_init(&queue, &rreq->subrequests);
220+ spin_unlock_bh(&rreq->lock);
221+}
222+
223+/*
224+ * Retry reads.
225+ */
226+void netfs_retry_reads(struct netfs_io_request *rreq)
227+{
228+ trace_netfs_rreq(rreq, netfs_rreq_trace_resubmit);
229+
230+ atomic_inc(&rreq->nr_outstanding);
231+
232+ netfs_retry_read_subrequests(rreq);
233+
234+ if (atomic_dec_and_test(&rreq->nr_outstanding))
235+ netfs_rreq_terminated(rreq, false);
236+}
237+
238+/*
239+ * Unlock any the pages that haven't been unlocked yet due to abandoned
240+ * subrequests.
241+ */
242+void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq)
243+{
244+ struct folio_queue *p;
245+
246+ for (p = rreq->buffer; p; p = p->next) {
247+ for (int slot = 0; slot < folioq_count(p); slot++) {
248+ struct folio *folio = folioq_folio(p, slot);
249+
250+ if (folio && !folioq_is_marked2(p, slot)) {
251+ trace_netfs_folio(folio, netfs_folio_trace_abandon);
252+ folio_unlock(folio);
253+ }
254+ }
255+ }
256+}
fs/netfs/stats.c+4−2
@@ -32,6 +32,7 @@ atomic_t netfs_n_wh_buffered_write; · 31 unmodified lines
3232 atomic_t netfs_n_wh_writethrough;
3333 atomic_t netfs_n_wh_dio_write;
3434 atomic_t netfs_n_wh_writepages;
35+atomic_t netfs_n_wh_copy_to_cache;
3536 atomic_t netfs_n_wh_wstream_conflict;
3637 atomic_t netfs_n_wh_upload;
3738 atomic_t netfs_n_wh_upload_done;
@@ -51,11 +52,12 @@ int netfs_stats_show(struct seq_file *m, void *v) · 13 unmodified lines
5152 atomic_read(&netfs_n_rh_read_folio),
5253 atomic_read(&netfs_n_rh_write_begin),
5354 atomic_read(&netfs_n_rh_write_zskip));
54− seq_printf(m, "Writes : BW=%u WT=%u DW=%u WP=%u\n",
55+ seq_printf(m, "Writes : BW=%u WT=%u DW=%u WP=%u 2C=%u\n",
5556 atomic_read(&netfs_n_wh_buffered_write),
5657 atomic_read(&netfs_n_wh_writethrough),
5758 atomic_read(&netfs_n_wh_dio_write),
58− atomic_read(&netfs_n_wh_writepages));
59+ atomic_read(&netfs_n_wh_writepages),
60+ atomic_read(&netfs_n_wh_copy_to_cache));
5961 seq_printf(m, "ZeroOps: ZR=%u sh=%u sk=%u\n",
6062 atomic_read(&netfs_n_rh_zero),
6163 atomic_read(&netfs_n_rh_short_read),
fs/netfs/write_collect.c+8−1
@@ -87,6 +87,12 @@ static void netfs_writeback_unlock_folios(struct netfs_io_request *wreq, · 86 unmodified lines
8787 unsigned long long collected_to = wreq->collected_to;
8888 unsigned int slot = wreq->buffer_head_slot;
8989
90+ if (wreq->origin == NETFS_PGPRIV2_COPY_TO_CACHE) {
91+ if (netfs_pgpriv2_unlock_copied_folios(wreq))
92+ *notes |= MADE_PROGRESS;
93+ return;
94+ }
95+
9096 if (slot >= folioq_nr_slots(folioq)) {
9197 folioq = netfs_delete_buffer_head(wreq);
9298 slot = 0;
@@ -383,7 +389,8 @@ reassess_streams: · 290 unmodified lines
383389 smp_rmb();
384390 collected_to = ULLONG_MAX;
385391 if (wreq->origin == NETFS_WRITEBACK ||
386− wreq->origin == NETFS_WRITETHROUGH)
392+ wreq->origin == NETFS_WRITETHROUGH ||
393+ wreq->origin == NETFS_PGPRIV2_COPY_TO_CACHE)
387394 notes = BUFFERED;
388395 else
389396 notes = 0;
fs/netfs/write_issue.c+7−10
@@ -95,7 +95,8 @@ struct netfs_io_request *netfs_create_write_req(struct address_space *mapping, · 94 unmodified lines
9595 struct netfs_io_request *wreq;
9696 struct netfs_inode *ictx;
9797 bool is_buffered = (origin == NETFS_WRITEBACK ||
98− origin == NETFS_WRITETHROUGH);
98+ origin == NETFS_WRITETHROUGH ||
99+ origin == NETFS_PGPRIV2_COPY_TO_CACHE);
99100
100101 wreq = netfs_alloc_request(mapping, file, start, 0, origin);
101102 if (IS_ERR(wreq))
@@ -161,10 +162,6 @@ static void netfs_prepare_write(struct netfs_io_request *wreq, · 59 unmodified lines
161162
162163 _enter("R=%x[%x]", wreq->debug_id, subreq->debug_index);
163164
164− trace_netfs_sreq_ref(wreq->debug_id, subreq->debug_index,
165− refcount_read(&subreq->ref),
166− netfs_sreq_trace_new);
167−
168165 trace_netfs_sreq(subreq, netfs_sreq_trace_prepare);
169166
170167 stream->sreq_max_len = UINT_MAX;
@@ -241,8 +238,8 @@ void netfs_reissue_write(struct netfs_io_stream *stream, · 70 unmodified lines
241238 netfs_do_issue_write(stream, subreq);
242239 }
243240
244−static void netfs_issue_write(struct netfs_io_request *wreq,
245− struct netfs_io_stream *stream)
241+void netfs_issue_write(struct netfs_io_request *wreq,
242+ struct netfs_io_stream *stream)
246243 {
247244 struct netfs_io_subrequest *subreq = stream->construct;
248245
@@ -259,9 +256,9 @@ static void netfs_issue_write(struct netfs_io_request *wreq, · 10 unmodified lines
259256 * we can avoid overrunning the credits obtained (cifs) and try to parallelise
260257 * content-crypto preparation with network writes.
261258 */
262−static int netfs_advance_write(struct netfs_io_request *wreq,
263− struct netfs_io_stream *stream,
264− loff_t start, size_t len, bool to_eof)
259+int netfs_advance_write(struct netfs_io_request *wreq,
260+ struct netfs_io_stream *stream,
261+ loff_t start, size_t len, bool to_eof)
265262 {
266263 struct netfs_io_subrequest *subreq = stream->construct;
267264 size_t part;
fs/nfs/fscache.c+6−13
@@ -267,6 +267,7 @@ static int nfs_netfs_init_request(struct netfs_io_request *rreq, struct file *fi · 266 unmodified lines
267267 rreq->debug_id = atomic_inc_return(&nfs_netfs_debug_id);
268268 /* [DEPRECATED] Use PG_private_2 to mark folio being written to the cache. */
269269 __set_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags);
270+ rreq->io_streams[0].sreq_max_len = NFS_SB(rreq->inode->i_sb)->rsize;
270271
271272 return 0;
272273 }
@@ -288,14 +289,6 @@ static struct nfs_netfs_io_data *nfs_netfs_alloc(struct netfs_io_subrequest *sre · 15 unmodified lines
288289 return netfs;
289290 }
290291
291−static bool nfs_netfs_clamp_length(struct netfs_io_subrequest *sreq)
292−{
293− size_t rsize = NFS_SB(sreq->rreq->inode->i_sb)->rsize;
294−
295− sreq->len = min(sreq->len, rsize);
296− return true;
297−}
298−
299292 static void nfs_netfs_issue_read(struct netfs_io_subrequest *sreq)
300293 {
301294 struct nfs_netfs_io_data *netfs;
@@ -304,17 +297,18 @@ static void nfs_netfs_issue_read(struct netfs_io_subrequest *sreq) · 2 unmodified lines
304297 struct nfs_open_context *ctx = sreq->rreq->netfs_priv;
305298 struct page *page;
306299 unsigned long idx;
300+ pgoff_t start, last;
307301 int err;
308− pgoff_t start = (sreq->start + sreq->transferred) >> PAGE_SHIFT;
309− pgoff_t last = ((sreq->start + sreq->len -
310− sreq->transferred - 1) >> PAGE_SHIFT);
302+
303+ start = (sreq->start + sreq->transferred) >> PAGE_SHIFT;
304+ last = ((sreq->start + sreq->len - sreq->transferred - 1) >> PAGE_SHIFT);
311305
312306 nfs_pageio_init_read(&pgio, inode, false,
313307 &nfs_async_read_completion_ops);
314308
315309 netfs = nfs_netfs_alloc(sreq);
316310 if (!netfs)
317− return netfs_subreq_terminated(sreq, -ENOMEM, false);
311+ return netfs_read_subreq_terminated(sreq, -ENOMEM, false);
318312
319313 pgio.pg_netfs = netfs; /* used in completion */
320314
@@ -380,5 +374,4 @@ const struct netfs_request_ops nfs_netfs_ops = { · 59 unmodified lines
380374 .init_request = nfs_netfs_init_request,
381375 .free_request = nfs_netfs_free_request,
382376 .issue_read = nfs_netfs_issue_read,
383− .clamp_length = nfs_netfs_clamp_length
384377 };
fs/nfs/fscache.h+3−4
@@ -60,8 +60,6 @@ static inline void nfs_netfs_get(struct nfs_netfs_io_data *netfs) · 59 unmodified lines
6060
6161 static inline void nfs_netfs_put(struct nfs_netfs_io_data *netfs)
6262 {
63− ssize_t final_len;
64−
6563 /* Only the last RPC completion should call netfs_subreq_terminated() */
6664 if (!refcount_dec_and_test(&netfs->refcount))
6765 return;
@@ -74,8 +72,9 @@ static inline void nfs_netfs_put(struct nfs_netfs_io_data *netfs) · 6 unmodified lines
7472 * Correct the final length here to be no larger than the netfs subrequest
7573 * length, and thus avoid netfs's "Subreq overread" warning message.
7674 */
77− final_len = min_t(s64, netfs->sreq->len, atomic64_read(&netfs->transferred));
78− netfs_subreq_terminated(netfs->sreq, netfs->error ?: final_len, false);
75+ netfs->sreq->transferred = min_t(s64, netfs->sreq->len,
76+ atomic64_read(&netfs->transferred));
77+ netfs_read_subreq_terminated(netfs->sreq, netfs->error, false);
7978 kfree(netfs);
8079 }
8180 static inline void nfs_netfs_inode_init(struct nfs_inode *nfsi)
fs/smb/client/cifsglob.h+0−1
@@ -1485,7 +1485,6 @@ struct cifs_io_subrequest { · 1484 unmodified lines
14851485 struct cifs_io_request *req;
14861486 };
14871487 ssize_t got_bytes;
1488− size_t actual_len;
14891488 unsigned int xid;
14901489 int result;
14911490 bool have_xid;
fs/smb/client/cifssmb.c+2−4
@@ -1309,10 +1309,8 @@ cifs_readv_callback(struct mid_q_entry *mid) · 1308 unmodified lines
13091309 if (rdata->result == 0 || rdata->result == -EAGAIN)
13101310 iov_iter_advance(&rdata->subreq.io_iter, rdata->got_bytes);
13111311 rdata->credits.value = 0;
1312− netfs_subreq_terminated(&rdata->subreq,
1313− (rdata->result == 0 || rdata->result == -EAGAIN) ?
1314− rdata->got_bytes : rdata->result,
1315− false);
1312+ rdata->subreq.transferred += rdata->got_bytes;
1313+ netfs_read_subreq_terminated(&rdata->subreq, rdata->result, false);
13161314 release_mid(mid);
13171315 add_credits(server, &credits, 0);
13181316 }
fs/smb/client/file.c+28−54
@@ -112,7 +112,6 @@ static void cifs_issue_write(struct netfs_io_subrequest *subreq) · 111 unmodified lines
112112 goto fail;
113113 }
114114
115− wdata->actual_len = wdata->subreq.len;
116115 rc = adjust_credits(wdata->server, wdata, cifs_trace_rw_credits_issue_write_adjust);
117116 if (rc)
118117 goto fail;
@@ -141,25 +140,22 @@ static void cifs_netfs_invalidate_cache(struct netfs_io_request *wreq) · 22 unmodified lines
141140 }
142141
143142 /*
144− * Split the read up according to how many credits we can get for each piece.
145− * It's okay to sleep here if we need to wait for more credit to become
146− * available.
147− *
148− * We also choose the server and allocate an operation ID to be cleaned up
149− * later.
143+ * Negotiate the size of a read operation on behalf of the netfs library.
150144 */
151−static bool cifs_clamp_length(struct netfs_io_subrequest *subreq)
145+static int cifs_prepare_read(struct netfs_io_subrequest *subreq)
152146 {
153147 struct netfs_io_request *rreq = subreq->rreq;
154− struct netfs_io_stream *stream = &rreq->io_streams[subreq->stream_nr];
155148 struct cifs_io_subrequest *rdata = container_of(subreq, struct cifs_io_subrequest, subreq);
156149 struct cifs_io_request *req = container_of(subreq->rreq, struct cifs_io_request, rreq);
157150 struct TCP_Server_Info *server = req->server;
158151 struct cifs_sb_info *cifs_sb = CIFS_SB(rreq->inode->i_sb);
159− int rc;
152+ size_t size;
153+ int rc = 0;
160154
161− rdata->xid = get_xid();
162− rdata->have_xid = true;
155+ if (!rdata->have_xid) {
156+ rdata->xid = get_xid();
157+ rdata->have_xid = true;
158+ }
163159 rdata->server = server;
164160
165161 if (cifs_sb->ctx->rsize == 0)
@@ -167,13 +163,12 @@ static bool cifs_clamp_length(struct netfs_io_subrequest *subreq) · 1 unmodified line
167163 server->ops->negotiate_rsize(tlink_tcon(req->cfile->tlink),
168164 cifs_sb->ctx);
169165
170−
171166 rc = server->ops->wait_mtu_credits(server, cifs_sb->ctx->rsize,
172− &stream->sreq_max_len, &rdata->credits);
173− if (rc) {
174− subreq->error = rc;
175− return false;
176− }
167+ &size, &rdata->credits);
168+ if (rc)
169+ return rc;
170+
171+ rreq->io_streams[0].sreq_max_len = size;
177172
178173 rdata->credits.in_flight_check = 1;
179174 rdata->credits.rreq_debug_id = rreq->debug_id;
@@ -185,14 +180,11 @@ static bool cifs_clamp_length(struct netfs_io_subrequest *subreq) · 5 unmodified lines
185180 server->credits, server->in_flight, 0,
186181 cifs_trace_rw_credits_read_submit);
187182
188− subreq->len = umin(subreq->len, stream->sreq_max_len);
189− rdata->actual_len = subreq->len;
190−
191183 #ifdef CONFIG_CIFS_SMB_DIRECT
192184 if (server->smbd_conn)
193− stream->sreq_max_segs = server->smbd_conn->max_frmr_depth;
185+ rreq->io_streams[0].sreq_max_segs = server->smbd_conn->max_frmr_depth;
194186 #endif
195− return true;
187+ return 0;
196188 }
197189
198190 /*
@@ -201,59 +193,41 @@ static bool cifs_clamp_length(struct netfs_io_subrequest *subreq) · 2 unmodified lines
201193 * to only read a portion of that, but as long as we read something, the netfs
202194 * helper will call us again so that we can issue another read.
203195 */
204−static void cifs_req_issue_read(struct netfs_io_subrequest *subreq)
196+static void cifs_issue_read(struct netfs_io_subrequest *subreq)
205197 {
206198 struct netfs_io_request *rreq = subreq->rreq;
207199 struct cifs_io_subrequest *rdata = container_of(subreq, struct cifs_io_subrequest, subreq);
208200 struct cifs_io_request *req = container_of(subreq->rreq, struct cifs_io_request, rreq);
209201 struct TCP_Server_Info *server = req->server;
210− struct cifs_sb_info *cifs_sb = CIFS_SB(rreq->inode->i_sb);
211202 int rc = 0;
212203
213204 cifs_dbg(FYI, "%s: op=%08x[%x] mapping=%p len=%zu/%zu\n",
214205 __func__, rreq->debug_id, subreq->debug_index, rreq->mapping,
215206 subreq->transferred, subreq->len);
216207
217− if (test_bit(NETFS_SREQ_RETRYING, &subreq->flags)) {
218− /*
219− * As we're issuing a retry, we need to negotiate some new
220− * credits otherwise the server may reject the op with
221− * INVALID_PARAMETER. Note, however, we may get back less
222− * credit than we need to complete the op, in which case, we
223− * shorten the op and rely on additional rounds of retry.
224− */
225− size_t rsize = umin(subreq->len - subreq->transferred,
226− cifs_sb->ctx->rsize);
227−
228− rc = server->ops->wait_mtu_credits(server, rsize, &rdata->actual_len,
229− &rdata->credits);
230− if (rc)
231− goto out;
232−
233− rdata->credits.in_flight_check = 1;
234−
235− trace_smb3_rw_credits(rdata->rreq->debug_id,
236− rdata->subreq.debug_index,
237− rdata->credits.value,
238− server->credits, server->in_flight, 0,
239− cifs_trace_rw_credits_read_resubmit);
240− }
208+ rc = adjust_credits(server, rdata, cifs_trace_rw_credits_issue_read_adjust);
209+ if (rc)
210+ goto failed;
241211
242212 if (req->cfile->invalidHandle) {
243213 do {
244214 rc = cifs_reopen_file(req->cfile, true);
245215 } while (rc == -EAGAIN);
246216 if (rc)
247− goto out;
217+ goto failed;
248218 }
249219
250220 if (subreq->rreq->origin != NETFS_DIO_READ)
251221 __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags);
252222
223+ trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
253224 rc = rdata->server->ops->async_readv(rdata);
254−out:
255225 if (rc)
256− netfs_subreq_terminated(subreq, rc, false);
226+ goto failed;
227+ return;
228+
229+failed:
230+ netfs_read_subreq_terminated(subreq, rc, false);
257231 }
258232
259233 /*
@@ -364,8 +338,8 @@ const struct netfs_request_ops cifs_req_ops = { · 104 unmodified lines
364338 .init_request = cifs_init_request,
365339 .free_request = cifs_free_request,
366340 .free_subrequest = cifs_free_subrequest,
367− .clamp_length = cifs_clamp_length,
368− .issue_read = cifs_req_issue_read,
341+ .prepare_read = cifs_prepare_read,
342+ .issue_read = cifs_issue_read,
369343 .done = cifs_rreq_done,
370344 .begin_writeback = cifs_begin_writeback,
371345 .prepare_write = cifs_prepare_write,
fs/smb/client/smb2ops.c+2−1
@@ -301,7 +301,8 @@ smb2_adjust_credits(struct TCP_Server_Info *server, · 300 unmodified lines
301301 unsigned int /*enum smb3_rw_credits_trace*/ trace)
302302 {
303303 struct cifs_credits *credits = &subreq->credits;
304− int new_val = DIV_ROUND_UP(subreq->actual_len, SMB2_MAX_BUFFER_SIZE);
304+ int new_val = DIV_ROUND_UP(subreq->subreq.len - subreq->subreq.transferred,
305+ SMB2_MAX_BUFFER_SIZE);
305306 int scredits, in_flight;
306307
307308 if (!credits->value || credits->value == new_val)
fs/smb/client/smb2pdu.c+16−11
@@ -4498,9 +4498,7 @@ static void smb2_readv_worker(struct work_struct *work) · 4497 unmodified lines
44984498 struct cifs_io_subrequest *rdata =
44994499 container_of(work, struct cifs_io_subrequest, subreq.work);
45004500
4501− netfs_subreq_terminated(&rdata->subreq,
4502− (rdata->result == 0 || rdata->result == -EAGAIN) ?
4503− rdata->got_bytes : rdata->result, true);
4501+ netfs_read_subreq_terminated(&rdata->subreq, rdata->result, false);
45044502 }
45054503
45064504 static void
@@ -4532,7 +4530,7 @@ smb2_readv_callback(struct mid_q_entry *mid) · 25 unmodified lines
45324530
45334531 cifs_dbg(FYI, "%s: mid=%llu state=%d result=%d bytes=%zu/%zu\n",
45344532 __func__, mid->mid, mid->mid_state, rdata->result,
4535− rdata->actual_len, rdata->subreq.len - rdata->subreq.transferred);
4533+ rdata->got_bytes, rdata->subreq.len - rdata->subreq.transferred);
45364534
45374535 switch (mid->mid_state) {
45384536 case MID_RESPONSE_RECEIVED:
@@ -4554,6 +4552,7 @@ smb2_readv_callback(struct mid_q_entry *mid) · 15 unmodified lines
45544552 break;
45554553 case MID_REQUEST_SUBMITTED:
45564554 case MID_RETRY_NEEDED:
4555+ __set_bit(NETFS_SREQ_NEED_RETRY, &rdata->subreq.flags);
45574556 rdata->result = -EAGAIN;
45584557 if (server->sign && rdata->got_bytes)
45594558 /* reset bytes number since we can not check a sign */
@@ -4588,7 +4587,7 @@ smb2_readv_callback(struct mid_q_entry *mid) · 28 unmodified lines
45884587 rdata->req->cfile->fid.persistent_fid,
45894588 tcon->tid, tcon->ses->Suid,
45904589 rdata->subreq.start + rdata->subreq.transferred,
4591− rdata->actual_len,
4590+ rdata->subreq.len - rdata->subreq.transferred,
45924591 rdata->result);
45934592 } else
45944593 trace_smb3_read_done(rdata->rreq->debug_id,
@@ -4603,9 +4602,9 @@ smb2_readv_callback(struct mid_q_entry *mid) · 8 unmodified lines
46034602 __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags);
46044603 rdata->result = 0;
46054604 } else {
4606− if (rdata->got_bytes < rdata->actual_len &&
4607− rdata->subreq.start + rdata->subreq.transferred + rdata->got_bytes ==
4608− ictx->remote_i_size) {
4605+ size_t trans = rdata->subreq.transferred + rdata->got_bytes;
4606+ if (trans < rdata->subreq.len &&
4607+ rdata->subreq.start + trans == ictx->remote_i_size) {
46094608 __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags);
46104609 rdata->result = 0;
46114610 }
@@ -4614,6 +4613,10 @@ smb2_readv_callback(struct mid_q_entry *mid) · 2 unmodified lines
46144613 server->credits, server->in_flight,
46154614 0, cifs_trace_rw_credits_read_response_clear);
46164615 rdata->credits.value = 0;
4616+ rdata->subreq.transferred += rdata->got_bytes;
4617+ if (rdata->subreq.start + rdata->subreq.transferred >= rdata->subreq.rreq->i_size)
4618+ __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags);
4619+ trace_netfs_sreq(&rdata->subreq, netfs_sreq_trace_io_progress);
46174620 INIT_WORK(&rdata->subreq.work, smb2_readv_worker);
46184621 queue_work(cifsiod_wq, &rdata->subreq.work);
46194622 release_mid(mid);
@@ -4648,7 +4651,7 @@ smb2_async_readv(struct cifs_io_subrequest *rdata) · 28 unmodified lines
46484651 io_parms.tcon = tlink_tcon(rdata->req->cfile->tlink);
46494652 io_parms.server = server = rdata->server;
46504653 io_parms.offset = subreq->start + subreq->transferred;
4651− io_parms.length = rdata->actual_len;
4654+ io_parms.length = subreq->len - subreq->transferred;
46524655 io_parms.persistent_fid = rdata->req->cfile->fid.persistent_fid;
46534656 io_parms.volatile_fid = rdata->req->cfile->fid.volatile_fid;
46544657 io_parms.pid = rdata->req->pid;
@@ -4669,7 +4672,7 @@ smb2_async_readv(struct cifs_io_subrequest *rdata) · 14 unmodified lines
46694672 shdr = (struct smb2_hdr *)buf;
46704673
46714674 if (rdata->credits.value > 0) {
4672− shdr->CreditCharge = cpu_to_le16(DIV_ROUND_UP(rdata->actual_len,
4675+ shdr->CreditCharge = cpu_to_le16(DIV_ROUND_UP(io_parms.length,
46734676 SMB2_MAX_BUFFER_SIZE));
46744677 credit_request = le16_to_cpu(shdr->CreditCharge) + 8;
46754678 if (server->credits >= server->max_credits)
@@ -4697,7 +4700,8 @@ smb2_async_readv(struct cifs_io_subrequest *rdata) · 21 unmodified lines
46974700 rdata->xid, io_parms.persistent_fid,
46984701 io_parms.tcon->tid,
46994702 io_parms.tcon->ses->Suid,
4700− io_parms.offset, rdata->actual_len, rc);
4703+ io_parms.offset,
4704+ subreq->len - subreq->transferred, rc);
47014705 }
47024706
47034707 async_readv_out:
@@ -4880,6 +4884,7 @@ smb2_writev_callback(struct mid_q_entry *mid) · 176 unmodified lines
48804884 server->credits, server->in_flight,
48814885 0, cifs_trace_rw_credits_write_response_clear);
48824886 wdata->credits.value = 0;
4887+ trace_netfs_sreq(&wdata->subreq, netfs_sreq_trace_io_progress);
48834888 cifs_write_subrequest_terminated(wdata, result ?: written, true);
48844889 release_mid(mid);
48854890 trace_smb3_rw_credits(rreq_debug_id, subreq_debug_index, 0,
include/linux/folio_queue.h+18−0
@@ -27,6 +27,7 @@ struct folio_queue { · 26 unmodified lines
2727 struct folio_queue *prev; /* Previous queue segment of NULL */
2828 unsigned long marks; /* 1-bit mark per folio */
2929 unsigned long marks2; /* Second 1-bit mark per folio */
30+ unsigned long marks3; /* Third 1-bit mark per folio */
3031 #if PAGEVEC_SIZE > BITS_PER_LONG
3132 #error marks is not big enough
3233 #endif
@@ -39,6 +40,7 @@ static inline void folioq_init(struct folio_queue *folioq) · 6 unmodified lines
3940 folioq->prev = NULL;
4041 folioq->marks = 0;
4142 folioq->marks2 = 0;
43+ folioq->marks3 = 0;
4244 }
4345
4446 static inline unsigned int folioq_nr_slots(const struct folio_queue *folioq)
@@ -87,6 +89,21 @@ static inline void folioq_unmark2(struct folio_queue *folioq, unsigned int slot) · 42 unmodified lines
8789 clear_bit(slot, &folioq->marks2);
8890 }
8991
92+static inline bool folioq_is_marked3(const struct folio_queue *folioq, unsigned int slot)
93+{
94+ return test_bit(slot, &folioq->marks3);
95+}
96+
97+static inline void folioq_mark3(struct folio_queue *folioq, unsigned int slot)
98+{
99+ set_bit(slot, &folioq->marks3);
100+}
101+
102+static inline void folioq_unmark3(struct folio_queue *folioq, unsigned int slot)
103+{
104+ clear_bit(slot, &folioq->marks3);
105+}
106+
90107 static inline unsigned int __folio_order(struct folio *folio)
91108 {
92109 if (!folio_test_large(folio))
@@ -133,6 +150,7 @@ static inline void folioq_clear(struct folio_queue *folioq, unsigned int slot) · 40 unmodified lines
133150 folioq->vec.folios[slot] = NULL;
134151 folioq_unmark(folioq, slot);
135152 folioq_unmark2(folioq, slot);
153+ folioq_unmark3(folioq, slot);
136154 }
137155
138156 #endif /* _LINUX_FOLIO_QUEUE_H */
include/linux/netfs.h+18−8
@@ -178,36 +178,43 @@ struct netfs_io_subrequest { · 177 unmodified lines
178178 unsigned long long start; /* Where to start the I/O */
179179 size_t len; /* Size of the I/O */
180180 size_t transferred; /* Amount of data transferred */
181+ size_t consumed; /* Amount of read data consumed */
182+ size_t prev_donated; /* Amount of data donated from previous subreq */
183+ size_t next_donated; /* Amount of data donated from next subreq */
181184 refcount_t ref;
182185 short error; /* 0 or error that occurred */
183186 unsigned short debug_index; /* Index in list (for debugging output) */
184187 unsigned int nr_segs; /* Number of segs in io_iter */
185188 enum netfs_io_source source; /* Where to read from/write to */
186189 unsigned char stream_nr; /* I/O stream this belongs to */
190+ unsigned char curr_folioq_slot; /* Folio currently being read */
191+ unsigned char curr_folio_order; /* Order of folio */
192+ struct folio_queue *curr_folioq; /* Queue segment in which current folio resides */
187193 unsigned long flags;
188194 #define NETFS_SREQ_COPY_TO_CACHE 0 /* Set if should copy the data to the cache */
189195 #define NETFS_SREQ_CLEAR_TAIL 1 /* Set if the rest of the read should be cleared */
190−#define NETFS_SREQ_SHORT_IO 2 /* Set if the I/O was short */
191196 #define NETFS_SREQ_SEEK_DATA_READ 3 /* Set if ->read() should SEEK_DATA first */
192197 #define NETFS_SREQ_NO_PROGRESS 4 /* Set if we didn't manage to read any data */
193198 #define NETFS_SREQ_ONDEMAND 5 /* Set if it's from on-demand read mode */
194199 #define NETFS_SREQ_BOUNDARY 6 /* Set if ends on hard boundary (eg. ceph object) */
200+#define NETFS_SREQ_HIT_EOF 7 /* Set if short due to EOF */
195201 #define NETFS_SREQ_IN_PROGRESS 8 /* Unlocked when the subrequest completes */
196202 #define NETFS_SREQ_NEED_RETRY 9 /* Set if the filesystem requests a retry */
197203 #define NETFS_SREQ_RETRYING 10 /* Set if we're retrying */
198204 #define NETFS_SREQ_FAILED 11 /* Set if the subreq failed unretryably */
199−#define NETFS_SREQ_HIT_EOF 12 /* Set if we hit the EOF */
200205 };
201206
202207 enum netfs_io_origin {
203208 NETFS_READAHEAD, /* This read was triggered by readahead */
204209 NETFS_READPAGE, /* This read is a synchronous read */
210+ NETFS_READ_GAPS, /* This read is a synchronous read to fill gaps */
205211 NETFS_READ_FOR_WRITE, /* This read is to prepare a write */
206212 NETFS_DIO_READ, /* This is a direct I/O read */
207213 NETFS_WRITEBACK, /* This write was triggered by writepages */
208214 NETFS_WRITETHROUGH, /* This write was made by netfs_perform_write() */
209215 NETFS_UNBUFFERED_WRITE, /* This is an unbuffered write */
210216 NETFS_DIO_WRITE, /* This is a direct I/O write */
217+ NETFS_PGPRIV2_COPY_TO_CACHE, /* [DEPRECATED] This is writing read data to the cache */
211218 nr__netfs_io_origin
212219 } __mode(byte);
213220
@@ -224,6 +231,7 @@ struct netfs_io_request { · 10 unmodified lines
224231 struct address_space *mapping; /* The mapping being accessed */
225232 struct kiocb *iocb; /* AIO completion vector */
226233 struct netfs_cache_resources cache_resources;
234+ struct readahead_control *ractl; /* Readahead descriptor */
227235 struct list_head proc_link; /* Link in netfs_iorequests */
228236 struct list_head subrequests; /* Contributory I/O operations */
229237 struct netfs_io_stream io_streams[2]; /* Streams of parallel I/O operations */
@@ -244,12 +252,10 @@ struct netfs_io_request { · 14 unmodified lines
244252 unsigned int nr_group_rel; /* Number of refs to release on ->group */
245253 spinlock_t lock; /* Lock for queuing subreqs */
246254 atomic_t nr_outstanding; /* Number of ops in progress */
247− atomic_t nr_copy_ops; /* Number of copy-to-cache ops in progress */
248− size_t upper_len; /* Length can be extended to here */
249255 unsigned long long submitted; /* Amount submitted for I/O so far */
250256 unsigned long long len; /* Length of the request */
251257 size_t transferred; /* Amount to be indicated as transferred */
252− short error; /* 0 or error that occurred */
258+ long error; /* 0 or error that occurred */
253259 enum netfs_io_origin origin; /* Origin of the request */
254260 bool direct_bv_unpin; /* T if direct_bv[] must be unpinned */
255261 u8 buffer_head_slot; /* First slot in ->buffer */
@@ -260,9 +266,9 @@ struct netfs_io_request { · 4 unmodified lines
260266 unsigned long long collected_to; /* Point we've collected to */
261267 unsigned long long cleaned_to; /* Position we've cleaned folios to */
262268 pgoff_t no_unlock_folio; /* Don't unlock this folio after read */
269+ size_t prev_donated; /* Fallback for subreq->prev_donated */
263270 refcount_t ref;
264271 unsigned long flags;
265−#define NETFS_RREQ_INCOMPLETE_IO 0 /* Some ioreqs terminated short or with error */
266272 #define NETFS_RREQ_COPY_TO_CACHE 1 /* Need to write to the cache */
267273 #define NETFS_RREQ_NO_UNLOCK_FOLIO 2 /* Don't unlock no_unlock_folio on completion */
268274 #define NETFS_RREQ_DONT_UNLOCK_FOLIOS 3 /* Don't unlock the folios on completion */
@@ -274,6 +280,7 @@ struct netfs_io_request { · 5 unmodified lines
274280 #define NETFS_RREQ_PAUSE 11 /* Pause subrequest generation */
275281 #define NETFS_RREQ_USE_IO_ITER 12 /* Use ->io_iter rather than ->i_pages */
276282 #define NETFS_RREQ_ALL_QUEUED 13 /* All subreqs are now queued */
283+#define NETFS_RREQ_NEED_RETRY 14 /* Need to try retrying */
277284 #define NETFS_RREQ_USE_PGPRIV2 31 /* [DEPRECATED] Use PG_private_2 to mark
278285 * write to cache on read */
279286 const struct netfs_request_ops *netfs_ops;
@@ -292,7 +299,7 @@ struct netfs_request_ops { · 12 unmodified lines
292299
293300 /* Read request handling */
294301 void (*expand_readahead)(struct netfs_io_request *rreq);
295− bool (*clamp_length)(struct netfs_io_subrequest *subreq);
302+ int (*prepare_read)(struct netfs_io_subrequest *subreq);
296303 void (*issue_read)(struct netfs_io_subrequest *subreq);
297304 bool (*is_still_valid)(struct netfs_io_request *rreq);
298305 int (*check_write_begin)(struct file *file, loff_t pos, unsigned len,
@@ -422,7 +429,10 @@ bool netfs_release_folio(struct folio *folio, gfp_t gfp); · 123 unmodified lines
422429 vm_fault_t netfs_page_mkwrite(struct vm_fault *vmf, struct netfs_group *netfs_group);
423430
424431 /* (Sub)request management API. */
425−void netfs_subreq_terminated(struct netfs_io_subrequest *, ssize_t, bool);
432+void netfs_read_subreq_progress(struct netfs_io_subrequest *subreq,
433+ bool was_async);
434+void netfs_read_subreq_terminated(struct netfs_io_subrequest *subreq,
435+ int error, bool was_async);
426436 void netfs_get_subrequest(struct netfs_io_subrequest *subreq,
427437 enum netfs_sreq_ref_trace what);
428438 void netfs_put_subrequest(struct netfs_io_subrequest *subreq,
include/trace/events/netfs.h+98−5
@@ -20,6 +20,7 @@ · 19 unmodified lines
2020 EM(netfs_read_trace_expanded, "EXPANDED ") \
2121 EM(netfs_read_trace_readahead, "READAHEAD") \
2222 EM(netfs_read_trace_readpage, "READPAGE ") \
23+ EM(netfs_read_trace_read_gaps, "READ-GAPS") \
2324 EM(netfs_read_trace_prefetch_for_write, "PREFETCHW") \
2425 E_(netfs_read_trace_write_begin, "WRITEBEGN")
2526
@@ -33,12 +34,14 @@ · 7 unmodified lines
3334 #define netfs_rreq_origins \
3435 EM(NETFS_READAHEAD, "RA") \
3536 EM(NETFS_READPAGE, "RP") \
37+ EM(NETFS_READ_GAPS, "RG") \
3638 EM(NETFS_READ_FOR_WRITE, "RW") \
3739 EM(NETFS_DIO_READ, "DR") \
3840 EM(NETFS_WRITEBACK, "WB") \
3941 EM(NETFS_WRITETHROUGH, "WT") \
4042 EM(NETFS_UNBUFFERED_WRITE, "UW") \
41− E_(NETFS_DIO_WRITE, "DW")
43+ EM(NETFS_DIO_WRITE, "DW") \
44+ E_(NETFS_PGPRIV2_COPY_TO_CACHE, "2C")
4245
4346 #define netfs_rreq_traces \
4447 EM(netfs_rreq_trace_assess, "ASSESS ") \
@@ -69,15 +72,25 @@ · 24 unmodified lines
6972 E_(NETFS_INVALID_WRITE, "INVL")
7073
7174 #define netfs_sreq_traces \
75+ EM(netfs_sreq_trace_add_donations, "+DON ") \
76+ EM(netfs_sreq_trace_added, "ADD ") \
77+ EM(netfs_sreq_trace_clear, "CLEAR") \
7278 EM(netfs_sreq_trace_discard, "DSCRD") \
79+ EM(netfs_sreq_trace_donate_to_prev, "DON-P") \
80+ EM(netfs_sreq_trace_donate_to_next, "DON-N") \
7381 EM(netfs_sreq_trace_download_instead, "RDOWN") \
7482 EM(netfs_sreq_trace_fail, "FAIL ") \
7583 EM(netfs_sreq_trace_free, "FREE ") \
84+ EM(netfs_sreq_trace_hit_eof, "EOF ") \
85+ EM(netfs_sreq_trace_io_progress, "IO ") \
7686 EM(netfs_sreq_trace_limited, "LIMIT") \
7787 EM(netfs_sreq_trace_prepare, "PREP ") \
7888 EM(netfs_sreq_trace_prep_failed, "PRPFL") \
79− EM(netfs_sreq_trace_resubmit_short, "SHORT") \
89+ EM(netfs_sreq_trace_progress, "PRGRS") \
90+ EM(netfs_sreq_trace_reprep_failed, "REPFL") \
8091 EM(netfs_sreq_trace_retry, "RETRY") \
92+ EM(netfs_sreq_trace_short, "SHORT") \
93+ EM(netfs_sreq_trace_split, "SPLIT") \
8194 EM(netfs_sreq_trace_submit, "SUBMT") \
8295 EM(netfs_sreq_trace_terminated, "TERM ") \
8396 EM(netfs_sreq_trace_write, "WRITE") \
@@ -118,7 +131,7 @@ · 34 unmodified lines
118131 EM(netfs_sreq_trace_new, "NEW ") \
119132 EM(netfs_sreq_trace_put_cancel, "PUT CANCEL ") \
120133 EM(netfs_sreq_trace_put_clear, "PUT CLEAR ") \
121− EM(netfs_sreq_trace_put_discard, "PUT DISCARD") \
134+ EM(netfs_sreq_trace_put_consumed, "PUT CONSUME") \
122135 EM(netfs_sreq_trace_put_done, "PUT DONE ") \
123136 EM(netfs_sreq_trace_put_failed, "PUT FAILED ") \
124137 EM(netfs_sreq_trace_put_merged, "PUT MERGED ") \
@@ -138,6 +151,7 @@ · 13 unmodified lines
138151 EM(netfs_flush_content, "flush") \
139152 EM(netfs_streaming_filled_page, "mod-streamw-f") \
140153 EM(netfs_streaming_cont_filled_page, "mod-streamw-f+") \
154+ EM(netfs_folio_trace_abandon, "abandon") \
141155 EM(netfs_folio_trace_cancel_copy, "cancel-copy") \
142156 EM(netfs_folio_trace_clear, "clear") \
143157 EM(netfs_folio_trace_clear_cc, "clear-cc") \
@@ -154,7 +168,11 @@ · 10 unmodified lines
154168 EM(netfs_folio_trace_mkwrite_plus, "mkwrite+") \
155169 EM(netfs_folio_trace_not_under_wback, "!wback") \
156170 EM(netfs_folio_trace_put, "put") \
171+ EM(netfs_folio_trace_read, "read") \
172+ EM(netfs_folio_trace_read_done, "read-done") \
157173 EM(netfs_folio_trace_read_gaps, "read-gaps") \
174+ EM(netfs_folio_trace_read_put, "read-put") \
175+ EM(netfs_folio_trace_read_unlock, "read-unlock") \
158176 EM(netfs_folio_trace_redirtied, "redirtied") \
159177 EM(netfs_folio_trace_store, "store") \
160178 EM(netfs_folio_trace_store_copy, "store-copy") \
@@ -167,6 +185,12 @@ · 6 unmodified lines
167185 EM(netfs_contig_trace_jump, "-->JUMP-->") \
168186 E_(netfs_contig_trace_unlock, "Unlock")
169187
188+#define netfs_donate_traces \
189+ EM(netfs_trace_donate_tail_to_prev, "tail-to-prev") \
190+ EM(netfs_trace_donate_to_prev, "to-prev") \
191+ EM(netfs_trace_donate_to_next, "to-next") \
192+ E_(netfs_trace_donate_to_deferred_next, "defer-next")
193+
170194 #ifndef __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY
171195 #define __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY
172196
@@ -184,6 +208,7 @@ enum netfs_rreq_ref_trace { netfs_rreq_ref_traces } __mode(byte); · 11 unmodified lines
184208 enum netfs_sreq_ref_trace { netfs_sreq_ref_traces } __mode(byte);
185209 enum netfs_folio_trace { netfs_folio_traces } __mode(byte);
186210 enum netfs_collect_contig_trace { netfs_collect_contig_traces } __mode(byte);
211+enum netfs_donate_trace { netfs_donate_traces } __mode(byte);
187212
188213 #endif
189214
@@ -206,6 +231,7 @@ netfs_rreq_ref_traces; · 16 unmodified lines
206231 netfs_sreq_ref_traces;
207232 netfs_folio_traces;
208233 netfs_collect_contig_traces;
234+netfs_donate_traces;
209235
210236 /*
211237 * Now redefine the EM() and E_() macros to map the enums to the strings that
@@ -226,6 +252,7 @@ TRACE_EVENT(netfs_read, · 14 unmodified lines
226252 TP_STRUCT__entry(
227253 __field(unsigned int, rreq )
228254 __field(unsigned int, cookie )
255+ __field(loff_t, i_size )
229256 __field(loff_t, start )
230257 __field(size_t, len )
231258 __field(enum netfs_read_trace, what )
@@ -235,18 +262,19 @@ TRACE_EVENT(netfs_read, · 3 unmodified lines
235262 TP_fast_assign(
236263 __entry->rreq = rreq->debug_id;
237264 __entry->cookie = rreq->cache_resources.debug_id;
265+ __entry->i_size = rreq->i_size;
238266 __entry->start = start;
239267 __entry->len = len;
240268 __entry->what = what;
241269 __entry->netfs_inode = rreq->inode->i_ino;
242270 ),
243271
244− TP_printk("R=%08x %s c=%08x ni=%x s=%llx %zx",
272+ TP_printk("R=%08x %s c=%08x ni=%x s=%llx l=%zx sz=%llx",
245273 __entry->rreq,
246274 __print_symbolic(__entry->what, netfs_read_traces),
247275 __entry->cookie,
248276 __entry->netfs_inode,
249− __entry->start, __entry->len)
277+ __entry->start, __entry->len, __entry->i_size)
250278 );
251279
252280 TRACE_EVENT(netfs_rreq,
@@ -651,6 +679,71 @@ TRACE_EVENT(netfs_collect_stream, · 398 unmodified lines
651679 __entry->collected_to, __entry->front)
652680 );
653681
682+TRACE_EVENT(netfs_progress,
683+ TP_PROTO(const struct netfs_io_subrequest *subreq,
684+ unsigned long long start, size_t avail, size_t part),
685+
686+ TP_ARGS(subreq, start, avail, part),
687+
688+ TP_STRUCT__entry(
689+ __field(unsigned int, rreq)
690+ __field(unsigned int, subreq)
691+ __field(unsigned int, consumed)
692+ __field(unsigned int, transferred)
693+ __field(unsigned long long, f_start)
694+ __field(unsigned int, f_avail)
695+ __field(unsigned int, f_part)
696+ __field(unsigned char, slot)
697+ ),
698+
699+ TP_fast_assign(
700+ __entry->rreq = subreq->rreq->debug_id;
701+ __entry->subreq = subreq->debug_index;
702+ __entry->consumed = subreq->consumed;
703+ __entry->transferred = subreq->transferred;
704+ __entry->f_start = start;
705+ __entry->f_avail = avail;
706+ __entry->f_part = part;
707+ __entry->slot = subreq->curr_folioq_slot;
708+ ),
709+
710+ TP_printk("R=%08x[%02x] s=%llx ct=%x/%x pa=%x/%x sl=%x",
711+ __entry->rreq, __entry->subreq, __entry->f_start,
712+ __entry->consumed, __entry->transferred,
713+ __entry->f_part, __entry->f_avail, __entry->slot)
714+ );
715+
716+TRACE_EVENT(netfs_donate,
717+ TP_PROTO(const struct netfs_io_request *rreq,
718+ const struct netfs_io_subrequest *from,
719+ const struct netfs_io_subrequest *to,
720+ size_t amount,
721+ enum netfs_donate_trace trace),
722+
723+ TP_ARGS(rreq, from, to, amount, trace),
724+
725+ TP_STRUCT__entry(
726+ __field(unsigned int, rreq)
727+ __field(unsigned int, from)
728+ __field(unsigned int, to)
729+ __field(unsigned int, amount)
730+ __field(enum netfs_donate_trace, trace)
731+ ),
732+
733+ TP_fast_assign(
734+ __entry->rreq = rreq->debug_id;
735+ __entry->from = from->debug_index;
736+ __entry->to = to ? to->debug_index : -1;
737+ __entry->amount = amount;
738+ __entry->trace = trace;
739+ ),
740+
741+ TP_printk("R=%08x[%02x] -> [%02x] %s am=%x",
742+ __entry->rreq, __entry->from, __entry->to,
743+ __print_symbolic(__entry->trace, netfs_donate_traces),
744+ __entry->amount)
745+ );
746+
654747 #undef EM
655748 #undef E_
656749 #endif /* _TRACE_NETFS_H */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t