History

netfs: Speed up buffered reading

Improve the efficiency of buffered reads in a number of ways: (1) Overhaul the algorithm in general so that it's a lot more compact and split the read submission code between buffered and unbuffered versions. The unbuffered version can be vastly simplified. (2) Read-result collection is handed off to a work queue rather than being done in the I/O thread. Multiple subrequests can be processes simultaneously. (3) When a subrequest is collected, any folios it fully spans are collected and "spare" data on either side is donated to either the previous or the next subrequest in the sequence. Notes: (*) Readahead expansion is massively slows down fio, presumably because it causes a load of extra allocations, both folio and xarray, up front before RPC requests can be transmitted. (*) RDMA with cifs does appear to work, both with SIW and RXE. (*) PG_private_2-based reading and copy-to-cache is split out into its own file and altered to use folio_queue. Note that the copy to the cache now creates a new write transaction against the cache and adds the folios to be copied into it. This allows it to use part of the writeback I/O code.

David Howells committed 2 years ago

Commit ee4cdf71 parent 2e45b9228 files · +2058 −470Browse files

Unified · Split

28 files
  1. fs/9p/vfs_addr.c +8 −3
  2. fs/afs/file.c +15 −6
  3. fs/afs/fsclient.c +7 −2
  4. fs/afs/yfsclient.c +7 −2
  5. fs/ceph/addr.c +46 −30
  6. fs/netfs/Makefile +3 −1
  7. fs/netfs/buffered_read.c +471 −295
  8. fs/netfs/direct_read.c +141 −6
  9. fs/netfs/internal.h +28 −7
  10. fs/netfs/iterator.c +50 −0
  11. fs/netfs/main.c +3 −1
  12. fs/netfs/objects.c +5 −3
  13. fs/netfs/read_collect.c +544 −0
  14. fs/netfs/read_pgpriv2.c +264 −0
  15. fs/netfs/read_retry.c +256 −0
  16. fs/netfs/stats.c +4 −2
  17. fs/netfs/write_collect.c +8 −1
  18. fs/netfs/write_issue.c +7 −10
  19. fs/nfs/fscache.c +6 −13
  20. fs/nfs/fscache.h +3 −4
  21. fs/smb/client/cifsglob.h +0 −1
  22. fs/smb/client/cifssmb.c +2 −4
  23. fs/smb/client/file.c +28 −54
  24. fs/smb/client/smb2ops.c +2 −1
  25. fs/smb/client/smb2pdu.c +16 −11
  26. include/linux/folio_queue.h +18 −0
  27. include/linux/netfs.h +18 −8
  28. include/trace/events/netfs.h +98 −5
fs/9p/vfs_addr.c+8−3
@@ -68,17 +68,22 @@ static void v9fs_issue_read(struct netfs_io_subrequest *subreq) · 67 unmodified lines
68{68{
69 struct netfs_io_request *rreq = subreq->rreq;69 struct netfs_io_request *rreq = subreq->rreq;
70 struct p9_fid *fid = rreq->netfs_priv;70 struct p9_fid *fid = rreq->netfs_priv;
71 unsigned long long pos = subreq->start + subreq->transferred;
71 int total, err;72 int total, err;
72 73
73 total = p9_client_read(fid, subreq->start + subreq->transferred,74 total = p9_client_read(fid, pos, &subreq->io_iter, &err);
74 &subreq->io_iter, &err);
75 75
76 /* if we just extended the file size, any portion not in76 /* if we just extended the file size, any portion not in
77 * cache won't be on server and is zeroes */77 * cache won't be on server and is zeroes */
78 if (subreq->rreq->origin != NETFS_DIO_READ)78 if (subreq->rreq->origin != NETFS_DIO_READ)
79 __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags);79 __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags);
80 if (pos + total >= i_size_read(rreq->inode))
81 __set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags);
80 82
81 netfs_subreq_terminated(subreq, err ?: total, false);83 if (!err)
84 subreq->transferred += total;
85
86 netfs_read_subreq_terminated(subreq, err, false);
82}87}
83 88
84/**89/**
fs/afs/file.c+15−6
@@ -16,6 +16,7 @@ · 15 unmodified lines
16#include <linux/mm.h>16#include <linux/mm.h>
17#include <linux/swap.h>17#include <linux/swap.h>
18#include <linux/netfs.h>18#include <linux/netfs.h>
19#include <trace/events/netfs.h>
19#include "internal.h"20#include "internal.h"
20 21
21static int afs_file_mmap(struct file *file, struct vm_area_struct *vma);22static int afs_file_mmap(struct file *file, struct vm_area_struct *vma);
@@ -242,9 +243,10 @@ static void afs_fetch_data_notify(struct afs_operation *op) · 220 unmodified lines
242 243
243 req->error = error;244 req->error = error;
244 if (subreq) {245 if (subreq) {
245 if (subreq->rreq->origin != NETFS_DIO_READ)246 subreq->rreq->i_size = req->file_size;
246 __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags);247 if (req->pos + req->actual_len >= req->file_size)
247 netfs_subreq_terminated(subreq, error ?: req->actual_len, false);248 __set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags);
249 netfs_read_subreq_terminated(subreq, error, false);
248 req->subreq = NULL;250 req->subreq = NULL;
249 } else if (req->done) {251 } else if (req->done) {
250 req->done(req);252 req->done(req);
@@ -262,6 +264,12 @@ static void afs_fetch_data_success(struct afs_operation *op) · 11 unmodified lines
262 afs_fetch_data_notify(op);264 afs_fetch_data_notify(op);
263}265}
264 266
267static void afs_fetch_data_aborted(struct afs_operation *op)
268{
269 afs_check_for_remote_deletion(op);
270 afs_fetch_data_notify(op);
271}
272
265static void afs_fetch_data_put(struct afs_operation *op)273static void afs_fetch_data_put(struct afs_operation *op)
266{274{
267 op->fetch.req->error = afs_op_error(op);275 op->fetch.req->error = afs_op_error(op);
@@ -272,7 +280,7 @@ static const struct afs_operation_ops afs_fetch_data_operation = { · 4 unmodified lines
272 .issue_afs_rpc = afs_fs_fetch_data,280 .issue_afs_rpc = afs_fs_fetch_data,
273 .issue_yfs_rpc = yfs_fs_fetch_data,281 .issue_yfs_rpc = yfs_fs_fetch_data,
274 .success = afs_fetch_data_success,282 .success = afs_fetch_data_success,
275 .aborted = afs_check_for_remote_deletion,283 .aborted = afs_fetch_data_aborted,
276 .failed = afs_fetch_data_notify,284 .failed = afs_fetch_data_notify,
277 .put = afs_fetch_data_put,285 .put = afs_fetch_data_put,
278};286};
@@ -294,7 +302,7 @@ int afs_fetch_data(struct afs_vnode *vnode, struct afs_read *req) · 15 unmodified lines
294 op = afs_alloc_operation(req->key, vnode->volume);302 op = afs_alloc_operation(req->key, vnode->volume);
295 if (IS_ERR(op)) {303 if (IS_ERR(op)) {
296 if (req->subreq)304 if (req->subreq)
297 netfs_subreq_terminated(req->subreq, PTR_ERR(op), false);305 netfs_read_subreq_terminated(req->subreq, PTR_ERR(op), false);
298 return PTR_ERR(op);306 return PTR_ERR(op);
299 }307 }
300 308
@@ -313,7 +321,7 @@ static void afs_read_worker(struct work_struct *work) · 12 unmodified lines
313 321
314 fsreq = afs_alloc_read(GFP_NOFS);322 fsreq = afs_alloc_read(GFP_NOFS);
315 if (!fsreq)323 if (!fsreq)
316 return netfs_subreq_terminated(subreq, -ENOMEM, false);324 return netfs_read_subreq_terminated(subreq, -ENOMEM, false);
317 325
318 fsreq->subreq = subreq;326 fsreq->subreq = subreq;
319 fsreq->pos = subreq->start + subreq->transferred;327 fsreq->pos = subreq->start + subreq->transferred;
@@ -322,6 +330,7 @@ static void afs_read_worker(struct work_struct *work) · 2 unmodified lines
322 fsreq->vnode = vnode;330 fsreq->vnode = vnode;
323 fsreq->iter = &subreq->io_iter;331 fsreq->iter = &subreq->io_iter;
324 332
333 trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
325 afs_fetch_data(fsreq->vnode, fsreq);334 afs_fetch_data(fsreq->vnode, fsreq);
326 afs_put_read(fsreq);335 afs_put_read(fsreq);
327}336}
fs/afs/fsclient.c+7−2
@@ -304,6 +304,7 @@ static int afs_deliver_fs_fetch_data(struct afs_call *call) · 303 unmodified lines
304 struct afs_vnode_param *vp = &op->file[0];304 struct afs_vnode_param *vp = &op->file[0];
305 struct afs_read *req = op->fetch.req;305 struct afs_read *req = op->fetch.req;
306 const __be32 *bp;306 const __be32 *bp;
307 size_t count_before;
307 int ret;308 int ret;
308 309
309 _enter("{%u,%zu,%zu/%llu}",310 _enter("{%u,%zu,%zu/%llu}",
@@ -345,10 +346,14 @@ static int afs_deliver_fs_fetch_data(struct afs_call *call) · 35 unmodified lines
345 346
346 /* extract the returned data */347 /* extract the returned data */
347 case 2:348 case 2:
348 _debug("extract data %zu/%llu",349 count_before = call->iov_len;
349 iov_iter_count(call->iter), req->actual_len);350 _debug("extract data %zu/%llu", count_before, req->actual_len);
350 351
351 ret = afs_extract_data(call, true);352 ret = afs_extract_data(call, true);
353 if (req->subreq) {
354 req->subreq->transferred += count_before - call->iov_len;
355 netfs_read_subreq_progress(req->subreq, false);
356 }
352 if (ret < 0)357 if (ret < 0)
353 return ret;358 return ret;
354 359
fs/afs/yfsclient.c+7−2
@@ -355,6 +355,7 @@ static int yfs_deliver_fs_fetch_data64(struct afs_call *call) · 354 unmodified lines
355 struct afs_vnode_param *vp = &op->file[0];355 struct afs_vnode_param *vp = &op->file[0];
356 struct afs_read *req = op->fetch.req;356 struct afs_read *req = op->fetch.req;
357 const __be32 *bp;357 const __be32 *bp;
358 size_t count_before;
358 int ret;359 int ret;
359 360
360 _enter("{%u,%zu, %zu/%llu}",361 _enter("{%u,%zu, %zu/%llu}",
@@ -391,10 +392,14 @@ static int yfs_deliver_fs_fetch_data64(struct afs_call *call) · 30 unmodified lines
391 392
392 /* extract the returned data */393 /* extract the returned data */
393 case 2:394 case 2:
394 _debug("extract data %zu/%llu",395 count_before = call->iov_len;
395 iov_iter_count(call->iter), req->actual_len);396 _debug("extract data %zu/%llu", count_before, req->actual_len);
396 397
397 ret = afs_extract_data(call, true);398 ret = afs_extract_data(call, true);
399 if (req->subreq) {
400 req->subreq->transferred += count_before - call->iov_len;
401 netfs_read_subreq_progress(req->subreq, false);
402 }
398 if (ret < 0)403 if (ret < 0)
399 return ret;404 return ret;
400 405
fs/ceph/addr.c+46−30
@@ -13,6 +13,7 @@ · 12 unmodified lines
13#include <linux/iversion.h>13#include <linux/iversion.h>
14#include <linux/ktime.h>14#include <linux/ktime.h>
15#include <linux/netfs.h>15#include <linux/netfs.h>
16#include <trace/events/netfs.h>
16 17
17#include "super.h"18#include "super.h"
18#include "mds_client.h"19#include "mds_client.h"
@@ -205,21 +206,6 @@ static void ceph_netfs_expand_readahead(struct netfs_io_request *rreq) · 186 unmodified lines
205 }206 }
206}207}
207 208
208static bool ceph_netfs_clamp_length(struct netfs_io_subrequest *subreq)
209{
210 struct inode *inode = subreq->rreq->inode;
211 struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
212 struct ceph_inode_info *ci = ceph_inode(inode);
213 u64 objno, objoff;
214 u32 xlen;
215
216 /* Truncate the extent at the end of the current block */
217 ceph_calc_file_object_mapping(&ci->i_layout, subreq->start, subreq->len,
218 &objno, &objoff, &xlen);
219 subreq->len = min(xlen, fsc->mount_options->rsize);
220 return true;
221}
222
223static void finish_netfs_read(struct ceph_osd_request *req)209static void finish_netfs_read(struct ceph_osd_request *req)
224{210{
225 struct inode *inode = req->r_inode;211 struct inode *inode = req->r_inode;
@@ -264,7 +250,12 @@ static void finish_netfs_read(struct ceph_osd_request *req) · 38 unmodified lines
264 calc_pages_for(osd_data->alignment,250 calc_pages_for(osd_data->alignment,
265 osd_data->length), false);251 osd_data->length), false);
266 }252 }
267 netfs_subreq_terminated(subreq, err, false);253 if (err > 0) {
254 subreq->transferred = err;
255 err = 0;
256 }
257 trace_netfs_sreq(subreq, netfs_sreq_trace_io_progress);
258 netfs_read_subreq_terminated(subreq, err, false);
268 iput(req->r_inode);259 iput(req->r_inode);
269 ceph_dec_osd_stopping_blocker(fsc->mdsc);260 ceph_dec_osd_stopping_blocker(fsc->mdsc);
270}261}
@@ -278,7 +269,6 @@ static bool ceph_netfs_issue_op_inline(struct netfs_io_subrequest *subreq) · 7 unmodified lines
278 struct ceph_mds_request *req;269 struct ceph_mds_request *req;
279 struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb);270 struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb);
280 struct ceph_inode_info *ci = ceph_inode(inode);271 struct ceph_inode_info *ci = ceph_inode(inode);
281 struct iov_iter iter;
282 ssize_t err = 0;272 ssize_t err = 0;
283 size_t len;273 size_t len;
284 int mode;274 int mode;
@@ -301,6 +291,7 @@ static bool ceph_netfs_issue_op_inline(struct netfs_io_subrequest *subreq) · 16 unmodified lines
301 req->r_args.getattr.mask = cpu_to_le32(CEPH_STAT_CAP_INLINE_DATA);291 req->r_args.getattr.mask = cpu_to_le32(CEPH_STAT_CAP_INLINE_DATA);
302 req->r_num_caps = 2;292 req->r_num_caps = 2;
303 293
294 trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
304 err = ceph_mdsc_do_request(mdsc, NULL, req);295 err = ceph_mdsc_do_request(mdsc, NULL, req);
305 if (err < 0)296 if (err < 0)
306 goto out;297 goto out;
@@ -314,17 +305,36 @@ static bool ceph_netfs_issue_op_inline(struct netfs_io_subrequest *subreq) · 7 unmodified lines
314 }305 }
315 306
316 len = min_t(size_t, iinfo->inline_len - subreq->start, subreq->len);307 len = min_t(size_t, iinfo->inline_len - subreq->start, subreq->len);
317 iov_iter_xarray(&iter, ITER_DEST, &rreq->mapping->i_pages, subreq->start, len);308 err = copy_to_iter(iinfo->inline_data + subreq->start, len, &subreq->io_iter);
318 err = copy_to_iter(iinfo->inline_data + subreq->start, len, &iter);309 if (err == 0) {
319 if (err == 0)
320 err = -EFAULT;310 err = -EFAULT;
311 } else {
312 subreq->transferred += err;
313 err = 0;
314 }
321 315
322 ceph_mdsc_put_request(req);316 ceph_mdsc_put_request(req);
323out:317out:
324 netfs_subreq_terminated(subreq, err, false);318 netfs_read_subreq_terminated(subreq, err, false);
325 return true;319 return true;
326}320}
327 321
322static int ceph_netfs_prepare_read(struct netfs_io_subrequest *subreq)
323{
324 struct netfs_io_request *rreq = subreq->rreq;
325 struct inode *inode = rreq->inode;
326 struct ceph_inode_info *ci = ceph_inode(inode);
327 struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
328 u64 objno, objoff;
329 u32 xlen;
330
331 /* Truncate the extent at the end of the current block */
332 ceph_calc_file_object_mapping(&ci->i_layout, subreq->start, subreq->len,
333 &objno, &objoff, &xlen);
334 rreq->io_streams[0].sreq_max_len = umin(xlen, fsc->mount_options->rsize);
335 return 0;
336}
337
328static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq)338static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq)
329{339{
330 struct netfs_io_request *rreq = subreq->rreq;340 struct netfs_io_request *rreq = subreq->rreq;
@@ -334,9 +344,8 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) · 3 unmodified lines
334 struct ceph_client *cl = fsc->client;344 struct ceph_client *cl = fsc->client;
335 struct ceph_osd_request *req = NULL;345 struct ceph_osd_request *req = NULL;
336 struct ceph_vino vino = ceph_vino(inode);346 struct ceph_vino vino = ceph_vino(inode);
337 struct iov_iter iter;347 int err;
338 int err = 0;348 u64 len;
339 u64 len = subreq->len;
340 bool sparse = IS_ENCRYPTED(inode) || ceph_test_mount_opt(fsc, SPARSEREAD);349 bool sparse = IS_ENCRYPTED(inode) || ceph_test_mount_opt(fsc, SPARSEREAD);
341 u64 off = subreq->start;350 u64 off = subreq->start;
342 int extent_cnt;351 int extent_cnt;
@@ -349,6 +358,12 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) · 6 unmodified lines
349 if (ceph_has_inline_data(ci) && ceph_netfs_issue_op_inline(subreq))358 if (ceph_has_inline_data(ci) && ceph_netfs_issue_op_inline(subreq))
350 return;359 return;
351 360
361 // TODO: This rounding here is slightly dodgy. It *should* work, for
362 // now, as the cache only deals in blocks that are a multiple of
363 // PAGE_SIZE and fscrypt blocks are at most PAGE_SIZE. What needs to
364 // happen is for the fscrypt driving to be moved into netfslib and the
365 // data in the cache also to be stored encrypted.
366 len = subreq->len;
352 ceph_fscrypt_adjust_off_and_len(inode, &off, &len);367 ceph_fscrypt_adjust_off_and_len(inode, &off, &len);
353 368
354 req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, vino,369 req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, vino,
@@ -371,8 +386,6 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) · 16 unmodified lines
371 doutc(cl, "%llx.%llx pos=%llu orig_len=%zu len=%llu\n",386 doutc(cl, "%llx.%llx pos=%llu orig_len=%zu len=%llu\n",
372 ceph_vinop(inode), subreq->start, subreq->len, len);387 ceph_vinop(inode), subreq->start, subreq->len, len);
373 388
374 iov_iter_xarray(&iter, ITER_DEST, &rreq->mapping->i_pages, subreq->start, len);
375
376 /*389 /*
377 * FIXME: For now, use CEPH_OSD_DATA_TYPE_PAGES instead of _ITER for390 * FIXME: For now, use CEPH_OSD_DATA_TYPE_PAGES instead of _ITER for
378 * encrypted inodes. We'd need infrastructure that handles an iov_iter391 * encrypted inodes. We'd need infrastructure that handles an iov_iter
@@ -384,7 +397,7 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) · 5 unmodified lines
384 struct page **pages;397 struct page **pages;
385 size_t page_off;398 size_t page_off;
386 399
387 err = iov_iter_get_pages_alloc2(&iter, &pages, len, &page_off);400 err = iov_iter_get_pages_alloc2(&subreq->io_iter, &pages, len, &page_off);
388 if (err < 0) {401 if (err < 0) {
389 doutc(cl, "%llx.%llx failed to allocate pages, %d\n",402 doutc(cl, "%llx.%llx failed to allocate pages, %d\n",
390 ceph_vinop(inode), err);403 ceph_vinop(inode), err);
@@ -399,7 +412,7 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) · 8 unmodified lines
399 osd_req_op_extent_osd_data_pages(req, 0, pages, len, 0, false,412 osd_req_op_extent_osd_data_pages(req, 0, pages, len, 0, false,
400 false);413 false);
401 } else {414 } else {
402 osd_req_op_extent_osd_iter(req, 0, &iter);415 osd_req_op_extent_osd_iter(req, 0, &subreq->io_iter);
403 }416 }
404 if (!ceph_inc_osd_stopping_blocker(fsc->mdsc)) {417 if (!ceph_inc_osd_stopping_blocker(fsc->mdsc)) {
405 err = -EIO;418 err = -EIO;
@@ -410,17 +423,19 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) · 4 unmodified lines
410 req->r_inode = inode;423 req->r_inode = inode;
411 ihold(inode);424 ihold(inode);
412 425
426 trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
413 ceph_osdc_start_request(req->r_osdc, req);427 ceph_osdc_start_request(req->r_osdc, req);
414out:428out:
415 ceph_osdc_put_request(req);429 ceph_osdc_put_request(req);
416 if (err)430 if (err)
417 netfs_subreq_terminated(subreq, err, false);431 netfs_read_subreq_terminated(subreq, err, false);
418 doutc(cl, "%llx.%llx result %d\n", ceph_vinop(inode), err);432 doutc(cl, "%llx.%llx result %d\n", ceph_vinop(inode), err);
419}433}
420 434
421static int ceph_init_request(struct netfs_io_request *rreq, struct file *file)435static int ceph_init_request(struct netfs_io_request *rreq, struct file *file)
422{436{
423 struct inode *inode = rreq->inode;437 struct inode *inode = rreq->inode;
438 struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
424 struct ceph_client *cl = ceph_inode_to_client(inode);439 struct ceph_client *cl = ceph_inode_to_client(inode);
425 int got = 0, want = CEPH_CAP_FILE_CACHE;440 int got = 0, want = CEPH_CAP_FILE_CACHE;
426 struct ceph_netfs_request_data *priv;441 struct ceph_netfs_request_data *priv;
@@ -472,6 +487,7 @@ static int ceph_init_request(struct netfs_io_request *rreq, struct file *file) · 45 unmodified lines
472 487
473 priv->caps = got;488 priv->caps = got;
474 rreq->netfs_priv = priv;489 rreq->netfs_priv = priv;
490 rreq->io_streams[0].sreq_max_len = fsc->mount_options->rsize;
475 491
476out:492out:
477 if (ret < 0)493 if (ret < 0)
@@ -496,9 +512,9 @@ static void ceph_netfs_free_request(struct netfs_io_request *rreq) · 18 unmodified lines
496const struct netfs_request_ops ceph_netfs_ops = {512const struct netfs_request_ops ceph_netfs_ops = {
497 .init_request = ceph_init_request,513 .init_request = ceph_init_request,
498 .free_request = ceph_netfs_free_request,514 .free_request = ceph_netfs_free_request,
515 .prepare_read = ceph_netfs_prepare_read,
499 .issue_read = ceph_netfs_issue_read,516 .issue_read = ceph_netfs_issue_read,
500 .expand_readahead = ceph_netfs_expand_readahead,517 .expand_readahead = ceph_netfs_expand_readahead,
501 .clamp_length = ceph_netfs_clamp_length,
502 .check_write_begin = ceph_netfs_check_write_begin,518 .check_write_begin = ceph_netfs_check_write_begin,
503};519};
504 520
fs/netfs/Makefile+3−1
@@ -5,12 +5,14 @@ netfs-y := \ · 4 unmodified lines
5 buffered_write.o \5 buffered_write.o \
6 direct_read.o \6 direct_read.o \
7 direct_write.o \7 direct_write.o \
8 io.o \
9 iterator.o \8 iterator.o \
10 locking.o \9 locking.o \
11 main.o \10 main.o \
12 misc.o \11 misc.o \
13 objects.o \12 objects.o \
13 read_collect.o \
14 read_pgpriv2.o \
15 read_retry.o \
14 write_collect.o \16 write_collect.o \
15 write_issue.o17 write_issue.o
16 18
fs/netfs/buffered_read.c+471−295
@@ -9,266 +9,388 @@ · 8 unmodified lines
9#include <linux/task_io_accounting_ops.h>9#include <linux/task_io_accounting_ops.h>
10#include "internal.h"10#include "internal.h"
11 11
12/*12static void netfs_cache_expand_readahead(struct netfs_io_request *rreq,
13 * [DEPRECATED] Unlock the folios in a read operation for when the filesystem13 unsigned long long *_start,
14 * is using PG_private_2 and direct writing to the cache from here rather than14 unsigned long long *_len,
15 * marking the page for writeback.15 unsigned long long i_size)
16 *
17 * Note that we don't touch folio->private in this code.
18 */
19static void netfs_rreq_unlock_folios_pgpriv2(struct netfs_io_request *rreq,
20 size_t *account)
21{16{
22 struct netfs_io_subrequest *subreq;17 struct netfs_cache_resources *cres = &rreq->cache_resources;
23 struct folio *folio;
24 pgoff_t start_page = rreq->start / PAGE_SIZE;
25 pgoff_t last_page = ((rreq->start + rreq->len) / PAGE_SIZE) - 1;
26 bool subreq_failed = false;
27 18
28 XA_STATE(xas, &rreq->mapping->i_pages, start_page);19 if (cres->ops && cres->ops->expand_readahead)
20 cres->ops->expand_readahead(cres, _start, _len, i_size);
21}
29 22
30 /* Walk through the pagecache and the I/O request lists simultaneously.23static void netfs_rreq_expand(struct netfs_io_request *rreq,
31 * We may have a mixture of cached and uncached sections and we only24 struct readahead_control *ractl)
32 * really want to write out the uncached sections. This is slightly25{
33 * complicated by the possibility that we might have huge pages with a26 /* Give the cache a chance to change the request parameters. The
34 * mixture inside.27 * resultant request must contain the original region.
35 */28 */
36 subreq = list_first_entry(&rreq->subrequests,29 netfs_cache_expand_readahead(rreq, &rreq->start, &rreq->len, rreq->i_size);
37 struct netfs_io_subrequest, rreq_link);
38 subreq_failed = (subreq->error < 0);
39 30
40 trace_netfs_rreq(rreq, netfs_rreq_trace_unlock_pgpriv2);31 /* Give the netfs a chance to change the request parameters. The
32 * resultant request must contain the original region.
33 */
34 if (rreq->netfs_ops->expand_readahead)
35 rreq->netfs_ops->expand_readahead(rreq);
41 36
42 rcu_read_lock();37 /* Expand the request if the cache wants it to start earlier. Note
43 xas_for_each(&xas, folio, last_page) {38 * that the expansion may get further extended if the VM wishes to
44 loff_t pg_end;39 * insert THPs and the preferred start and/or end wind up in the middle
45 bool pg_failed = false;40 * of THPs.
46 bool folio_started = false;41 *
42 * If this is the case, however, the THP size should be an integer
43 * multiple of the cache granule size, so we get a whole number of
44 * granules to deal with.
45 */
46 if (rreq->start != readahead_pos(ractl) ||
47 rreq->len != readahead_length(ractl)) {
48 readahead_expand(ractl, rreq->start, rreq->len);
49 rreq->start = readahead_pos(ractl);
50 rreq->len = readahead_length(ractl);
47 51
48 if (xas_retry(&xas, folio))52 trace_netfs_read(rreq, readahead_pos(ractl), readahead_length(ractl),
49 continue;53 netfs_read_trace_expanded);
54 }
55}
50 56
51 pg_end = folio_pos(folio) + folio_size(folio) - 1;57/*
58 * Begin an operation, and fetch the stored zero point value from the cookie if
59 * available.
60 */
61static int netfs_begin_cache_read(struct netfs_io_request *rreq, struct netfs_inode *ctx)
62{
63 return fscache_begin_read_operation(&rreq->cache_resources, netfs_i_cookie(ctx));
64}
52 65
53 for (;;) {66/*
54 loff_t sreq_end;67 * Decant the list of folios to read into a rolling buffer.
68 */
69static size_t netfs_load_buffer_from_ra(struct netfs_io_request *rreq,
70 struct folio_queue *folioq)
71{
72 unsigned int order, nr;
73 size_t size = 0;
74
75 nr = __readahead_batch(rreq->ractl, (struct page **)folioq->vec.folios,
76 ARRAY_SIZE(folioq->vec.folios));
77 folioq->vec.nr = nr;
78 for (int i = 0; i < nr; i++) {
79 struct folio *folio = folioq_folio(folioq, i);
80
81 trace_netfs_folio(folio, netfs_folio_trace_read);
82 order = folio_order(folio);
83 folioq->orders[i] = order;
84 size += PAGE_SIZE << order;
85 }
55 86
56 if (!subreq) {87 for (int i = nr; i < folioq_nr_slots(folioq); i++)
57 pg_failed = true;88 folioq_clear(folioq, i);
58 break;
59 }
60 89
61 if (!folio_started &&90 return size;
62 test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags) &&91}
63 fscache_operation_valid(&rreq->cache_resources)) {
64 trace_netfs_folio(folio, netfs_folio_trace_copy_to_cache);
65 folio_start_private_2(folio);
66 folio_started = true;
67 }
68 92
69 pg_failed |= subreq_failed;93/*
70 sreq_end = subreq->start + subreq->len - 1;94 * netfs_prepare_read_iterator - Prepare the subreq iterator for I/O
71 if (pg_end < sreq_end)95 * @subreq: The subrequest to be set up
72 break;96 *
97 * Prepare the I/O iterator representing the read buffer on a subrequest for
98 * the filesystem to use for I/O (it can be passed directly to a socket). This
99 * is intended to be called from the ->issue_read() method once the filesystem
100 * has trimmed the request to the size it wants.
101 *
102 * Returns the limited size if successful and -ENOMEM if insufficient memory
103 * available.
104 *
105 * [!] NOTE: This must be run in the same thread as ->issue_read() was called
106 * in as we access the readahead_control struct.
107 */
108static ssize_t netfs_prepare_read_iterator(struct netfs_io_subrequest *subreq)
109{
110 struct netfs_io_request *rreq = subreq->rreq;
111 size_t rsize = subreq->len;
112
113 if (subreq->source == NETFS_DOWNLOAD_FROM_SERVER)
114 rsize = umin(rsize, rreq->io_streams[0].sreq_max_len);
115
116 if (rreq->ractl) {
117 /* If we don't have sufficient folios in the rolling buffer,
118 * extract a folioq's worth from the readahead region at a time
119 * into the buffer. Note that this acquires a ref on each page
120 * that we will need to release later - but we don't want to do
121 * that until after we've started the I/O.
122 */
123 while (rreq->submitted < subreq->start + rsize) {
124 struct folio_queue *tail = rreq->buffer_tail, *new;
125 size_t added;
126
127 new = kmalloc(sizeof(*new), GFP_NOFS);
128 if (!new)
129 return -ENOMEM;
130 netfs_stat(&netfs_n_folioq);
131 folioq_init(new);
132 new->prev = tail;
133 tail->next = new;
134 rreq->buffer_tail = new;
135 added = netfs_load_buffer_from_ra(rreq, new);
136 rreq->iter.count += added;
137 rreq->submitted += added;
138 }
139 }
73 140
74 *account += subreq->transferred;141 subreq->len = rsize;
75 if (!list_is_last(&subreq->rreq_link, &rreq->subrequests)) {142 if (unlikely(rreq->io_streams[0].sreq_max_segs)) {
76 subreq = list_next_entry(subreq, rreq_link);143 size_t limit = netfs_limit_iter(&rreq->iter, 0, rsize,
77 subreq_failed = (subreq->error < 0);144 rreq->io_streams[0].sreq_max_segs);
78 } else {
79 subreq = NULL;
80 subreq_failed = false;
81 }
82 145
83 if (pg_end == sreq_end)146 if (limit < rsize) {
84 break;147 subreq->len = limit;
148 trace_netfs_sreq(subreq, netfs_sreq_trace_limited);
85 }149 }
150 }
86 151
87 if (!pg_failed) {152 subreq->io_iter = rreq->iter;
88 flush_dcache_folio(folio);
89 folio_mark_uptodate(folio);
90 }
91 153
92 if (!test_bit(NETFS_RREQ_DONT_UNLOCK_FOLIOS, &rreq->flags)) {154 if (iov_iter_is_folioq(&subreq->io_iter)) {
93 if (folio->index == rreq->no_unlock_folio &&155 if (subreq->io_iter.folioq_slot >= folioq_nr_slots(subreq->io_iter.folioq)) {
94 test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags))156 subreq->io_iter.folioq = subreq->io_iter.folioq->next;
95 _debug("no unlock");157 subreq->io_iter.folioq_slot = 0;
96 else
97 folio_unlock(folio);
98 }158 }
159 subreq->curr_folioq = (struct folio_queue *)subreq->io_iter.folioq;
160 subreq->curr_folioq_slot = subreq->io_iter.folioq_slot;
161 subreq->curr_folio_order = subreq->curr_folioq->orders[subreq->curr_folioq_slot];
99 }162 }
100 rcu_read_unlock();163
164 iov_iter_truncate(&subreq->io_iter, subreq->len);
165 iov_iter_advance(&rreq->iter, subreq->len);
166 return subreq->len;
101}167}
102 168
103/*169static enum netfs_io_source netfs_cache_prepare_read(struct netfs_io_request *rreq,
104 * Unlock the folios in a read operation. We need to set PG_writeback on any170 struct netfs_io_subrequest *subreq,
105 * folios we're going to write back before we unlock them.171 loff_t i_size)
106 *
107 * Note that if the deprecated NETFS_RREQ_USE_PGPRIV2 is set then we use
108 * PG_private_2 and do a direct write to the cache from here instead.
109 */
110void netfs_rreq_unlock_folios(struct netfs_io_request *rreq)
111{172{
112 struct netfs_io_subrequest *subreq;173 struct netfs_cache_resources *cres = &rreq->cache_resources;
113 struct netfs_folio *finfo;
114 struct folio *folio;
115 pgoff_t start_page = rreq->start / PAGE_SIZE;
116 pgoff_t last_page = ((rreq->start + rreq->len) / PAGE_SIZE) - 1;
117 size_t account = 0;
118 bool subreq_failed = false;
119 174
120 XA_STATE(xas, &rreq->mapping->i_pages, start_page);175 if (!cres->ops)
176 return NETFS_DOWNLOAD_FROM_SERVER;
177 return cres->ops->prepare_read(subreq, i_size);
178}
121 179
122 if (test_bit(NETFS_RREQ_FAILED, &rreq->flags)) {180static void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error,
123 __clear_bit(NETFS_RREQ_COPY_TO_CACHE, &rreq->flags);181 bool was_async)
124 list_for_each_entry(subreq, &rreq->subrequests, rreq_link) {182{
125 __clear_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags);183 struct netfs_io_subrequest *subreq = priv;
126 }
127 }
128 184
129 /* Handle deprecated PG_private_2 case. */185 if (transferred_or_error < 0) {
130 if (test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)) {186 netfs_read_subreq_terminated(subreq, transferred_or_error, was_async);
131 netfs_rreq_unlock_folios_pgpriv2(rreq, &account);187 return;
132 goto out;
133 }188 }
134 189
135 /* Walk through the pagecache and the I/O request lists simultaneously.190 if (transferred_or_error > 0)
136 * We may have a mixture of cached and uncached sections and we only191 subreq->transferred += transferred_or_error;
137 * really want to write out the uncached sections. This is slightly192 netfs_read_subreq_terminated(subreq, 0, was_async);
138 * complicated by the possibility that we might have huge pages with a193}
139 * mixture inside.
140 */
141 subreq = list_first_entry(&rreq->subrequests,
142 struct netfs_io_subrequest, rreq_link);
143 subreq_failed = (subreq->error < 0);
144
145 trace_netfs_rreq(rreq, netfs_rreq_trace_unlock);
146 194
147 rcu_read_lock();195/*
148 xas_for_each(&xas, folio, last_page) {196 * Issue a read against the cache.
149 loff_t pg_end;197 * - Eats the caller's ref on subreq.
150 bool pg_failed = false;198 */
151 bool wback_to_cache = false;199static void netfs_read_cache_to_pagecache(struct netfs_io_request *rreq,
200 struct netfs_io_subrequest *subreq)
201{
202 struct netfs_cache_resources *cres = &rreq->cache_resources;
152 203
153 if (xas_retry(&xas, folio))204 netfs_stat(&netfs_n_rh_read);
154 continue;205 cres->ops->read(cres, subreq->start, &subreq->io_iter, NETFS_READ_HOLE_IGNORE,
206 netfs_cache_read_terminated, subreq);
207}
155 208
156 pg_end = folio_pos(folio) + folio_size(folio) - 1;209/*
210 * Perform a read to the pagecache from a series of sources of different types,
211 * slicing up the region to be read according to available cache blocks and
212 * network rsize.
213 */
214static void netfs_read_to_pagecache(struct netfs_io_request *rreq)
215{
216 struct netfs_inode *ictx = netfs_inode(rreq->inode);
217 unsigned long long start = rreq->start;
218 ssize_t size = rreq->len;
219 int ret = 0;
220
221 atomic_inc(&rreq->nr_outstanding);
222
223 do {
224 struct netfs_io_subrequest *subreq;
225 enum netfs_io_source source = NETFS_DOWNLOAD_FROM_SERVER;
226 ssize_t slice;
227
228 subreq = netfs_alloc_subrequest(rreq);
229 if (!subreq) {
230 ret = -ENOMEM;
231 break;
232 }
157 233
158 for (;;) {234 subreq->start = start;
159 loff_t sreq_end;235 subreq->len = size;
236
237 atomic_inc(&rreq->nr_outstanding);
238 spin_lock_bh(&rreq->lock);
239 list_add_tail(&subreq->rreq_link, &rreq->subrequests);
240 subreq->prev_donated = rreq->prev_donated;
241 rreq->prev_donated = 0;
242 trace_netfs_sreq(subreq, netfs_sreq_trace_added);
243 spin_unlock_bh(&rreq->lock);
244
245 source = netfs_cache_prepare_read(rreq, subreq, rreq->i_size);
246 subreq->source = source;
247 if (source == NETFS_DOWNLOAD_FROM_SERVER) {
248 unsigned long long zp = umin(ictx->zero_point, rreq->i_size);
249 size_t len = subreq->len;
250
251 if (subreq->start >= zp) {
252 subreq->source = source = NETFS_FILL_WITH_ZEROES;
253 goto fill_with_zeroes;
254 }
160 255
161 if (!subreq) {256 if (len > zp - subreq->start)
162 pg_failed = true;257 len = zp - subreq->start;
258 if (len == 0) {
259 pr_err("ZERO-LEN READ: R=%08x[%x] l=%zx/%zx s=%llx z=%llx i=%llx",
260 rreq->debug_id, subreq->debug_index,
261 subreq->len, size,
262 subreq->start, ictx->zero_point, rreq->i_size);
163 break;263 break;
164 }264 }
265 subreq->len = len;
266
267 netfs_stat(&netfs_n_rh_download);
268 if (rreq->netfs_ops->prepare_read) {
269 ret = rreq->netfs_ops->prepare_read(subreq);
270 if (ret < 0) {
271 atomic_dec(&rreq->nr_outstanding);
272 netfs_put_subrequest(subreq, false,
273 netfs_sreq_trace_put_cancel);
274 break;
275 }
276 trace_netfs_sreq(subreq, netfs_sreq_trace_prepare);
277 }
165 278
166 wback_to_cache |= test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags);279 slice = netfs_prepare_read_iterator(subreq);
167 pg_failed |= subreq_failed;280 if (slice < 0) {
168 sreq_end = subreq->start + subreq->len - 1;281 atomic_dec(&rreq->nr_outstanding);
169 if (pg_end < sreq_end)282 netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_cancel);
283 ret = slice;
170 break;284 break;
171
172 account += subreq->transferred;
173 if (!list_is_last(&subreq->rreq_link, &rreq->subrequests)) {
174 subreq = list_next_entry(subreq, rreq_link);
175 subreq_failed = (subreq->error < 0);
176 } else {
177 subreq = NULL;
178 subreq_failed = false;
179 }285 }
180 286
181 if (pg_end == sreq_end)287 rreq->netfs_ops->issue_read(subreq);
182 break;288 goto done;
183 }289 }
184 290
185 if (!pg_failed) {291 fill_with_zeroes:
186 flush_dcache_folio(folio);292 if (source == NETFS_FILL_WITH_ZEROES) {
187 finfo = netfs_folio_info(folio);293 subreq->source = NETFS_FILL_WITH_ZEROES;
188 if (finfo) {294 trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
189 trace_netfs_folio(folio, netfs_folio_trace_filled_gaps);295 netfs_stat(&netfs_n_rh_zero);
190 if (finfo->netfs_group)296 slice = netfs_prepare_read_iterator(subreq);
191 folio_change_private(folio, finfo->netfs_group);297 __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags);
192 else298 netfs_read_subreq_terminated(subreq, 0, false);
193 folio_detach_private(folio);299 goto done;
194 kfree(finfo);
195 }
196 folio_mark_uptodate(folio);
197 if (wback_to_cache && !WARN_ON_ONCE(folio_get_private(folio) != NULL)) {
198 trace_netfs_folio(folio, netfs_folio_trace_copy_to_cache);
199 folio_attach_private(folio, NETFS_FOLIO_COPY_TO_CACHE);
200 filemap_dirty_folio(folio->mapping, folio);
201 }
202 }300 }
203 301
204 if (!test_bit(NETFS_RREQ_DONT_UNLOCK_FOLIOS, &rreq->flags)) {302 if (source == NETFS_READ_FROM_CACHE) {
205 if (folio->index == rreq->no_unlock_folio &&303 trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
206 test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags))304 slice = netfs_prepare_read_iterator(subreq);
207 _debug("no unlock");305 netfs_read_cache_to_pagecache(rreq, subreq);
208 else306 goto done;
209 folio_unlock(folio);
210 }307 }
211 }
212 rcu_read_unlock();
213 308
214out:309 pr_err("Unexpected read source %u\n", source);
215 task_io_account_read(account);310 WARN_ON_ONCE(1);
216 if (rreq->netfs_ops->done)311 break;
217 rreq->netfs_ops->done(rreq);
218}
219 312
220static void netfs_cache_expand_readahead(struct netfs_io_request *rreq,313 done:
221 unsigned long long *_start,314 size -= slice;
222 unsigned long long *_len,315 start += slice;
223 unsigned long long i_size)316 cond_resched();
224{317 } while (size > 0);
225 struct netfs_cache_resources *cres = &rreq->cache_resources;
226 318
227 if (cres->ops && cres->ops->expand_readahead)319 if (atomic_dec_and_test(&rreq->nr_outstanding))
228 cres->ops->expand_readahead(cres, _start, _len, i_size);320 netfs_rreq_terminated(rreq, false);
321
322 /* Defer error return as we may need to wait for outstanding I/O. */
323 cmpxchg(&rreq->error, 0, ret);
229}324}
230 325
231static void netfs_rreq_expand(struct netfs_io_request *rreq,326/*
232 struct readahead_control *ractl)327 * Wait for the read operation to complete, successfully or otherwise.
328 */
329static int netfs_wait_for_read(struct netfs_io_request *rreq)
233{330{
234 /* Give the cache a chance to change the request parameters. The331 int ret;
235 * resultant request must contain the original region.
236 */
237 netfs_cache_expand_readahead(rreq, &rreq->start, &rreq->len, rreq->i_size);
238 332
239 /* Give the netfs a chance to change the request parameters. The333 trace_netfs_rreq(rreq, netfs_rreq_trace_wait_ip);
240 * resultant request must contain the original region.334 wait_on_bit(&rreq->flags, NETFS_RREQ_IN_PROGRESS, TASK_UNINTERRUPTIBLE);
241 */335 ret = rreq->error;
242 if (rreq->netfs_ops->expand_readahead)336 if (ret == 0 && rreq->submitted < rreq->len) {
243 rreq->netfs_ops->expand_readahead(rreq);337 trace_netfs_failure(rreq, NULL, ret, netfs_fail_short_read);
338 ret = -EIO;
339 }
244 340
245 /* Expand the request if the cache wants it to start earlier. Note341 return ret;
246 * that the expansion may get further extended if the VM wishes to342}
247 * insert THPs and the preferred start and/or end wind up in the middle
248 * of THPs.
249 *
250 * If this is the case, however, the THP size should be an integer
251 * multiple of the cache granule size, so we get a whole number of
252 * granules to deal with.
253 */
254 if (rreq->start != readahead_pos(ractl) ||
255 rreq->len != readahead_length(ractl)) {
256 readahead_expand(ractl, rreq->start, rreq->len);
257 rreq->start = readahead_pos(ractl);
258 rreq->len = readahead_length(ractl);
259 343
260 trace_netfs_read(rreq, readahead_pos(ractl), readahead_length(ractl),344/*
261 netfs_read_trace_expanded);345 * Set up the initial folioq of buffer folios in the rolling buffer and set the
262 }346 * iterator to refer to it.
347 */
348static int netfs_prime_buffer(struct netfs_io_request *rreq)
349{
350 struct folio_queue *folioq;
351 size_t added;
352
353 folioq = kmalloc(sizeof(*folioq), GFP_KERNEL);
354 if (!folioq)
355 return -ENOMEM;
356 netfs_stat(&netfs_n_folioq);
357 folioq_init(folioq);
358 rreq->buffer = folioq;
359 rreq->buffer_tail = folioq;
360 rreq->submitted = rreq->start;
361 iov_iter_folio_queue(&rreq->iter, ITER_DEST, folioq, 0, 0, 0);
362
363 added = netfs_load_buffer_from_ra(rreq, folioq);
364 rreq->iter.count += added;
365 rreq->submitted += added;
366 return 0;
263}367}
264 368
265/*369/*
266 * Begin an operation, and fetch the stored zero point value from the cookie if370 * Drop the ref on each folio that we inherited from the VM readahead code. We
267 * available.371 * still have the folio locks to pin the page until we complete the I/O.
372 *
373 * Note that we can't just release the batch in each queue struct as we use the
374 * occupancy count in other places.
268 */375 */
269static int netfs_begin_cache_read(struct netfs_io_request *rreq, struct netfs_inode *ctx)376static void netfs_put_ra_refs(struct folio_queue *folioq)
270{377{
271 return fscache_begin_read_operation(&rreq->cache_resources, netfs_i_cookie(ctx));378 struct folio_batch fbatch;
379
380 folio_batch_init(&fbatch);
381 while (folioq) {
382 for (unsigned int slot = 0; slot < folioq_count(folioq); slot++) {
383 struct folio *folio = folioq_folio(folioq, slot);
384 if (!folio)
385 continue;
386 trace_netfs_folio(folio, netfs_folio_trace_read_put);
387 if (!folio_batch_add(&fbatch, folio))
388 folio_batch_release(&fbatch);
389 }
390 folioq = folioq->next;
391 }
392
393 folio_batch_release(&fbatch);
272}394}
273 395
274/**396/**
@@ -289,22 +411,17 @@ static int netfs_begin_cache_read(struct netfs_io_request *rreq, struct netfs_in · 14 unmodified lines
289void netfs_readahead(struct readahead_control *ractl)411void netfs_readahead(struct readahead_control *ractl)
290{412{
291 struct netfs_io_request *rreq;413 struct netfs_io_request *rreq;
292 struct netfs_inode *ctx = netfs_inode(ractl->mapping->host);414 struct netfs_inode *ictx = netfs_inode(ractl->mapping->host);
415 unsigned long long start = readahead_pos(ractl);
416 size_t size = readahead_length(ractl);
293 int ret;417 int ret;
294 418
295 _enter("%lx,%x", readahead_index(ractl), readahead_count(ractl));419 rreq = netfs_alloc_request(ractl->mapping, ractl->file, start, size,
296
297 if (readahead_count(ractl) == 0)
298 return;
299
300 rreq = netfs_alloc_request(ractl->mapping, ractl->file,
301 readahead_pos(ractl),
302 readahead_length(ractl),
303 NETFS_READAHEAD);420 NETFS_READAHEAD);
304 if (IS_ERR(rreq))421 if (IS_ERR(rreq))
305 return;422 return;
306 423
307 ret = netfs_begin_cache_read(rreq, ctx);424 ret = netfs_begin_cache_read(rreq, ictx);
308 if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS)425 if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS)
309 goto cleanup_free;426 goto cleanup_free;
310 427
@@ -314,18 +431,15 @@ void netfs_readahead(struct readahead_control *ractl) · 3 unmodified lines
314 431
315 netfs_rreq_expand(rreq, ractl);432 netfs_rreq_expand(rreq, ractl);
316 433
317 /* Set up the output buffer */434 rreq->ractl = ractl;
318 iov_iter_xarray(&rreq->iter, ITER_DEST, &ractl->mapping->i_pages,435 if (netfs_prime_buffer(rreq) < 0)
319 rreq->start, rreq->len);436 goto cleanup_free;
437 netfs_read_to_pagecache(rreq);
320 438
321 /* Drop the refs on the folios here rather than in the cache or439 /* Release the folio refs whilst we're waiting for the I/O. */
322 * filesystem. The locks will be dropped in netfs_rreq_unlock().440 netfs_put_ra_refs(rreq->buffer);
323 */
324 while (readahead_folio(ractl))
325 ;
326 441
327 netfs_begin_read(rreq, false);442 netfs_put_request(rreq, true, netfs_rreq_trace_put_return);
328 netfs_put_request(rreq, false, netfs_rreq_trace_put_return);
329 return;443 return;
330 444
331cleanup_free:445cleanup_free:
@@ -334,6 +448,117 @@ cleanup_free: · 2 unmodified lines
334}448}
335EXPORT_SYMBOL(netfs_readahead);449EXPORT_SYMBOL(netfs_readahead);
336 450
451/*
452 * Create a rolling buffer with a single occupying folio.
453 */
454static int netfs_create_singular_buffer(struct netfs_io_request *rreq, struct folio *folio)
455{
456 struct folio_queue *folioq;
457
458 folioq = kmalloc(sizeof(*folioq), GFP_KERNEL);
459 if (!folioq)
460 return -ENOMEM;
461
462 netfs_stat(&netfs_n_folioq);
463 folioq_init(folioq);
464 folioq_append(folioq, folio);
465 BUG_ON(folioq_folio(folioq, 0) != folio);
466 BUG_ON(folioq_folio_order(folioq, 0) != folio_order(folio));
467 rreq->buffer = folioq;
468 rreq->buffer_tail = folioq;
469 rreq->submitted = rreq->start + rreq->len;
470 iov_iter_folio_queue(&rreq->iter, ITER_DEST, folioq, 0, 0, rreq->len);
471 rreq->ractl = (struct readahead_control *)1UL;
472 return 0;
473}
474
475/*
476 * Read into gaps in a folio partially filled by a streaming write.
477 */
478static int netfs_read_gaps(struct file *file, struct folio *folio)
479{
480 struct netfs_io_request *rreq;
481 struct address_space *mapping = folio->mapping;
482 struct netfs_folio *finfo = netfs_folio_info(folio);
483 struct netfs_inode *ctx = netfs_inode(mapping->host);
484 struct folio *sink = NULL;
485 struct bio_vec *bvec;
486 unsigned int from = finfo->dirty_offset;
487 unsigned int to = from + finfo->dirty_len;
488 unsigned int off = 0, i = 0;
489 size_t flen = folio_size(folio);
490 size_t nr_bvec = flen / PAGE_SIZE + 2;
491 size_t part;
492 int ret;
493
494 _enter("%lx", folio->index);
495
496 rreq = netfs_alloc_request(mapping, file, folio_pos(folio), flen, NETFS_READ_GAPS);
497 if (IS_ERR(rreq)) {
498 ret = PTR_ERR(rreq);
499 goto alloc_error;
500 }
501
502 ret = netfs_begin_cache_read(rreq, ctx);
503 if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS)
504 goto discard;
505
506 netfs_stat(&netfs_n_rh_read_folio);
507 trace_netfs_read(rreq, rreq->start, rreq->len, netfs_read_trace_read_gaps);
508
509 /* Fiddle the buffer so that a gap at the beginning and/or a gap at the
510 * end get copied to, but the middle is discarded.
511 */
512 ret = -ENOMEM;
513 bvec = kmalloc_array(nr_bvec, sizeof(*bvec), GFP_KERNEL);
514 if (!bvec)
515 goto discard;
516
517 sink = folio_alloc(GFP_KERNEL, 0);
518 if (!sink) {
519 kfree(bvec);
520 goto discard;
521 }
522
523 trace_netfs_folio(folio, netfs_folio_trace_read_gaps);
524
525 rreq->direct_bv = bvec;
526 rreq->direct_bv_count = nr_bvec;
527 if (from > 0) {
528 bvec_set_folio(&bvec[i++], folio, from, 0);
529 off = from;
530 }
531 while (off < to) {
532 part = min_t(size_t, to - off, PAGE_SIZE);
533 bvec_set_folio(&bvec[i++], sink, part, 0);
534 off += part;
535 }
536 if (to < flen)
537 bvec_set_folio(&bvec[i++], folio, flen - to, to);
538 iov_iter_bvec(&rreq->iter, ITER_DEST, bvec, i, rreq->len);
539 rreq->submitted = rreq->start + flen;
540
541 netfs_read_to_pagecache(rreq);
542
543 if (sink)
544 folio_put(sink);
545
546 ret = netfs_wait_for_read(rreq);
547 if (ret == 0) {
548 flush_dcache_folio(folio);
549 folio_mark_uptodate(folio);
550 }
551 folio_unlock(folio);
552 netfs_put_request(rreq, false, netfs_rreq_trace_put_return);
553 return ret < 0 ? ret : 0;
554
555discard:
556 netfs_put_request(rreq, false, netfs_rreq_trace_put_discard);
557alloc_error:
558 folio_unlock(folio);
559 return ret;
560}
561
337/**562/**
338 * netfs_read_folio - Helper to manage a read_folio request563 * netfs_read_folio - Helper to manage a read_folio request
339 * @file: The file to read from564 * @file: The file to read from
@@ -353,9 +578,13 @@ int netfs_read_folio(struct file *file, struct folio *folio) · 13 unmodified lines
353 struct address_space *mapping = folio->mapping;578 struct address_space *mapping = folio->mapping;
354 struct netfs_io_request *rreq;579 struct netfs_io_request *rreq;
355 struct netfs_inode *ctx = netfs_inode(mapping->host);580 struct netfs_inode *ctx = netfs_inode(mapping->host);
356 struct folio *sink = NULL;
357 int ret;581 int ret;
358 582
583 if (folio_test_dirty(folio)) {
584 trace_netfs_folio(folio, netfs_folio_trace_read_gaps);
585 return netfs_read_gaps(file, folio);
586 }
587
359 _enter("%lx", folio->index);588 _enter("%lx", folio->index);
360 589
361 rreq = netfs_alloc_request(mapping, file,590 rreq = netfs_alloc_request(mapping, file,
@@ -374,54 +603,12 @@ int netfs_read_folio(struct file *file, struct folio *folio) · 12 unmodified lines
374 trace_netfs_read(rreq, rreq->start, rreq->len, netfs_read_trace_readpage);603 trace_netfs_read(rreq, rreq->start, rreq->len, netfs_read_trace_readpage);
375 604
376 /* Set up the output buffer */605 /* Set up the output buffer */
377 if (folio_test_dirty(folio)) {606 ret = netfs_create_singular_buffer(rreq, folio);
378 /* Handle someone trying to read from an unflushed streaming607 if (ret < 0)
379 * write. We fiddle the buffer so that a gap at the beginning608 goto discard;
380 * and/or a gap at the end get copied to, but the middle is
381 * discarded.
382 */
383 struct netfs_folio *finfo = netfs_folio_info(folio);
384 struct bio_vec *bvec;
385 unsigned int from = finfo->dirty_offset;
386 unsigned int to = from + finfo->dirty_len;
387 unsigned int off = 0, i = 0;
388 size_t flen = folio_size(folio);
389 size_t nr_bvec = flen / PAGE_SIZE + 2;
390 size_t part;
391
392 ret = -ENOMEM;
393 bvec = kmalloc_array(nr_bvec, sizeof(*bvec), GFP_KERNEL);
394 if (!bvec)
395 goto discard;
396
397 sink = folio_alloc(GFP_KERNEL, 0);
398 if (!sink)
399 goto discard;
400
401 trace_netfs_folio(folio, netfs_folio_trace_read_gaps);
402
403 rreq->direct_bv = bvec;
404 rreq->direct_bv_count = nr_bvec;
405 if (from > 0) {
406 bvec_set_folio(&bvec[i++], folio, from, 0);
407 off = from;
408 }
409 while (off < to) {
410 part = min_t(size_t, to - off, PAGE_SIZE);
411 bvec_set_folio(&bvec[i++], sink, part, 0);
412 off += part;
413 }
414 if (to < flen)
415 bvec_set_folio(&bvec[i++], folio, flen - to, to);
416 iov_iter_bvec(&rreq->iter, ITER_DEST, bvec, i, rreq->len);
417 } else {
418 iov_iter_xarray(&rreq->iter, ITER_DEST, &mapping->i_pages,
419 rreq->start, rreq->len);
420 }
421 609
422 ret = netfs_begin_read(rreq, true);610 netfs_read_to_pagecache(rreq);
423 if (sink)611 ret = netfs_wait_for_read(rreq);
424 folio_put(sink);
425 netfs_put_request(rreq, false, netfs_rreq_trace_put_return);612 netfs_put_request(rreq, false, netfs_rreq_trace_put_return);
426 return ret < 0 ? ret : 0;613 return ret < 0 ? ret : 0;
427 614
@@ -494,13 +681,10 @@ zero_out: · 66 unmodified lines
494 *681 *
495 * Pre-read data for a write-begin request by drawing data from the cache if682 * Pre-read data for a write-begin request by drawing data from the cache if
496 * possible, or the netfs if not. Space beyond the EOF is zero-filled.683 * possible, or the netfs if not. Space beyond the EOF is zero-filled.
497 * Multiple I/O requests from different sources will get munged together. If684 * Multiple I/O requests from different sources will get munged together.
498 * necessary, the readahead window can be expanded in either direction to a
499 * more convenient alighment for RPC efficiency or to make storage in the cache
500 * feasible.
501 *685 *
502 * The calling netfs must provide a table of operations, only one of which,686 * The calling netfs must provide a table of operations, only one of which,
503 * issue_op, is mandatory.687 * issue_read, is mandatory.
504 *688 *
505 * The check_write_begin() operation can be provided to check for and flush689 * The check_write_begin() operation can be provided to check for and flush
506 * conflicting writes once the folio is grabbed and locked. It is passed a690 * conflicting writes once the folio is grabbed and locked. It is passed a
@@ -528,8 +712,6 @@ int netfs_write_begin(struct netfs_inode *ctx, · 21 unmodified lines
528 pgoff_t index = pos >> PAGE_SHIFT;712 pgoff_t index = pos >> PAGE_SHIFT;
529 int ret;713 int ret;
530 714
531 DEFINE_READAHEAD(ractl, file, NULL, mapping, index);
532
533retry:715retry:
534 folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN,716 folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN,
535 mapping_gfp_mask(mapping));717 mapping_gfp_mask(mapping));
@@ -577,22 +759,13 @@ retry: · 41 unmodified lines
577 netfs_stat(&netfs_n_rh_write_begin);759 netfs_stat(&netfs_n_rh_write_begin);
578 trace_netfs_read(rreq, pos, len, netfs_read_trace_write_begin);760 trace_netfs_read(rreq, pos, len, netfs_read_trace_write_begin);
579 761
580 /* Expand the request to meet caching requirements and download
581 * preferences.
582 */
583 ractl._nr_pages = folio_nr_pages(folio);
584 netfs_rreq_expand(rreq, &ractl);
585
586 /* Set up the output buffer */762 /* Set up the output buffer */
587 iov_iter_xarray(&rreq->iter, ITER_DEST, &mapping->i_pages,763 ret = netfs_create_singular_buffer(rreq, folio);
588 rreq->start, rreq->len);764 if (ret < 0)
589 765 goto error_put;
590 /* We hold the folio locks, so we can drop the references */
591 folio_get(folio);
592 while (readahead_folio(&ractl))
593 ;
594 766
595 ret = netfs_begin_read(rreq, true);767 netfs_read_to_pagecache(rreq);
768 ret = netfs_wait_for_read(rreq);
596 if (ret < 0)769 if (ret < 0)
597 goto error;770 goto error;
598 netfs_put_request(rreq, false, netfs_rreq_trace_put_return);771 netfs_put_request(rreq, false, netfs_rreq_trace_put_return);
@@ -652,10 +825,13 @@ int netfs_prefetch_for_write(struct file *file, struct folio *folio, · 53 unmodified lines
652 trace_netfs_read(rreq, start, flen, netfs_read_trace_prefetch_for_write);825 trace_netfs_read(rreq, start, flen, netfs_read_trace_prefetch_for_write);
653 826
654 /* Set up the output buffer */827 /* Set up the output buffer */
655 iov_iter_xarray(&rreq->iter, ITER_DEST, &mapping->i_pages,828 ret = netfs_create_singular_buffer(rreq, folio);
656 rreq->start, rreq->len);829 if (ret < 0)
830 goto error_put;
657 831
658 ret = netfs_begin_read(rreq, true);832 folioq_mark2(rreq->buffer, 0);
833 netfs_read_to_pagecache(rreq);
834 ret = netfs_wait_for_read(rreq);
659 netfs_put_request(rreq, false, netfs_rreq_trace_put_return);835 netfs_put_request(rreq, false, netfs_rreq_trace_put_return);
660 return ret;836 return ret;
661 837
fs/netfs/direct_read.c+141−6
@@ -16,6 +16,143 @@ · 15 unmodified lines
16#include <linux/netfs.h>16#include <linux/netfs.h>
17#include "internal.h"17#include "internal.h"
18 18
19static void netfs_prepare_dio_read_iterator(struct netfs_io_subrequest *subreq)
20{
21 struct netfs_io_request *rreq = subreq->rreq;
22 size_t rsize;
23
24 rsize = umin(subreq->len, rreq->io_streams[0].sreq_max_len);
25 subreq->len = rsize;
26
27 if (unlikely(rreq->io_streams[0].sreq_max_segs)) {
28 size_t limit = netfs_limit_iter(&rreq->iter, 0, rsize,
29 rreq->io_streams[0].sreq_max_segs);
30
31 if (limit < rsize) {
32 subreq->len = limit;
33 trace_netfs_sreq(subreq, netfs_sreq_trace_limited);
34 }
35 }
36
37 trace_netfs_sreq(subreq, netfs_sreq_trace_prepare);
38
39 subreq->io_iter = rreq->iter;
40 iov_iter_truncate(&subreq->io_iter, subreq->len);
41 iov_iter_advance(&rreq->iter, subreq->len);
42}
43
44/*
45 * Perform a read to a buffer from the server, slicing up the region to be read
46 * according to the network rsize.
47 */
48static int netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq)
49{
50 unsigned long long start = rreq->start;
51 ssize_t size = rreq->len;
52 int ret = 0;
53
54 atomic_set(&rreq->nr_outstanding, 1);
55
56 do {
57 struct netfs_io_subrequest *subreq;
58 ssize_t slice;
59
60 subreq = netfs_alloc_subrequest(rreq);
61 if (!subreq) {
62 ret = -ENOMEM;
63 break;
64 }
65
66 subreq->source = NETFS_DOWNLOAD_FROM_SERVER;
67 subreq->start = start;
68 subreq->len = size;
69
70 atomic_inc(&rreq->nr_outstanding);
71 spin_lock_bh(&rreq->lock);
72 list_add_tail(&subreq->rreq_link, &rreq->subrequests);
73 subreq->prev_donated = rreq->prev_donated;
74 rreq->prev_donated = 0;
75 trace_netfs_sreq(subreq, netfs_sreq_trace_added);
76 spin_unlock_bh(&rreq->lock);
77
78 netfs_stat(&netfs_n_rh_download);
79 if (rreq->netfs_ops->prepare_read) {
80 ret = rreq->netfs_ops->prepare_read(subreq);
81 if (ret < 0) {
82 atomic_dec(&rreq->nr_outstanding);
83 netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_cancel);
84 break;
85 }
86 }
87
88 netfs_prepare_dio_read_iterator(subreq);
89 slice = subreq->len;
90 rreq->netfs_ops->issue_read(subreq);
91
92 size -= slice;
93 start += slice;
94 rreq->submitted += slice;
95
96 if (test_bit(NETFS_RREQ_BLOCKED, &rreq->flags) &&
97 test_bit(NETFS_RREQ_NONBLOCK, &rreq->flags))
98 break;
99 cond_resched();
100 } while (size > 0);
101
102 if (atomic_dec_and_test(&rreq->nr_outstanding))
103 netfs_rreq_terminated(rreq, false);
104 return ret;
105}
106
107/*
108 * Perform a read to an application buffer, bypassing the pagecache and the
109 * local disk cache.
110 */
111static int netfs_unbuffered_read(struct netfs_io_request *rreq, bool sync)
112{
113 int ret;
114
115 _enter("R=%x %llx-%llx",
116 rreq->debug_id, rreq->start, rreq->start + rreq->len - 1);
117
118 if (rreq->len == 0) {
119 pr_err("Zero-sized read [R=%x]\n", rreq->debug_id);
120 return -EIO;
121 }
122
123 // TODO: Use bounce buffer if requested
124
125 inode_dio_begin(rreq->inode);
126
127 ret = netfs_dispatch_unbuffered_reads(rreq);
128
129 if (!rreq->submitted) {
130 netfs_put_request(rreq, false, netfs_rreq_trace_put_no_submit);
131 inode_dio_end(rreq->inode);
132 ret = 0;
133 goto out;
134 }
135
136 if (sync) {
137 trace_netfs_rreq(rreq, netfs_rreq_trace_wait_ip);
138 wait_on_bit(&rreq->flags, NETFS_RREQ_IN_PROGRESS,
139 TASK_UNINTERRUPTIBLE);
140
141 ret = rreq->error;
142 if (ret == 0 && rreq->submitted < rreq->len &&
143 rreq->origin != NETFS_DIO_READ) {
144 trace_netfs_failure(rreq, NULL, ret, netfs_fail_short_read);
145 ret = -EIO;
146 }
147 } else {
148 ret = -EIOCBQUEUED;
149 }
150
151out:
152 _leave(" = %d", ret);
153 return ret;
154}
155
19/**156/**
20 * netfs_unbuffered_read_iter_locked - Perform an unbuffered or direct I/O read157 * netfs_unbuffered_read_iter_locked - Perform an unbuffered or direct I/O read
21 * @iocb: The I/O control descriptor describing the read158 * @iocb: The I/O control descriptor describing the read
@@ -31,7 +168,7 @@ ssize_t netfs_unbuffered_read_iter_locked(struct kiocb *iocb, struct iov_iter *i · 9 unmodified lines
31 struct netfs_io_request *rreq;168 struct netfs_io_request *rreq;
32 ssize_t ret;169 ssize_t ret;
33 size_t orig_count = iov_iter_count(iter);170 size_t orig_count = iov_iter_count(iter);
34 bool async = !is_sync_kiocb(iocb);171 bool sync = is_sync_kiocb(iocb);
35 172
36 _enter("");173 _enter("");
37 174
@@ -78,13 +215,13 @@ ssize_t netfs_unbuffered_read_iter_locked(struct kiocb *iocb, struct iov_iter *i · 40 unmodified lines
78 215
79 // TODO: Set up bounce buffer if needed216 // TODO: Set up bounce buffer if needed
80 217
81 if (async)218 if (!sync)
82 rreq->iocb = iocb;219 rreq->iocb = iocb;
83 220
84 ret = netfs_begin_read(rreq, is_sync_kiocb(iocb));221 ret = netfs_unbuffered_read(rreq, sync);
85 if (ret < 0)222 if (ret < 0)
86 goto out; /* May be -EIOCBQUEUED */223 goto out; /* May be -EIOCBQUEUED */
87 if (!async) {224 if (sync) {
88 // TODO: Copy from bounce buffer225 // TODO: Copy from bounce buffer
89 iocb->ki_pos += rreq->transferred;226 iocb->ki_pos += rreq->transferred;
90 ret = rreq->transferred;227 ret = rreq->transferred;
@@ -94,8 +231,6 @@ out: · 3 unmodified lines
94 netfs_put_request(rreq, false, netfs_rreq_trace_put_return);231 netfs_put_request(rreq, false, netfs_rreq_trace_put_return);
95 if (ret > 0)232 if (ret > 0)
96 orig_count -= ret;233 orig_count -= ret;
97 if (ret != -EIOCBQUEUED)
98 iov_iter_revert(iter, orig_count - iov_iter_count(iter));
99 return ret;234 return ret;
100}235}
101EXPORT_SYMBOL(netfs_unbuffered_read_iter_locked);236EXPORT_SYMBOL(netfs_unbuffered_read_iter_locked);
fs/netfs/internal.h+28−7
@@ -23,16 +23,9 @@ · 22 unmodified lines
23/*23/*
24 * buffered_read.c24 * buffered_read.c
25 */25 */
26void netfs_rreq_unlock_folios(struct netfs_io_request *rreq);
27int netfs_prefetch_for_write(struct file *file, struct folio *folio,26int netfs_prefetch_for_write(struct file *file, struct folio *folio,
28 size_t offset, size_t len);27 size_t offset, size_t len);
29 28
30/*
31 * io.c
32 */
33void netfs_rreq_work(struct work_struct *work);
34int netfs_begin_read(struct netfs_io_request *rreq, bool sync);
35
36/*29/*
37 * main.c30 * main.c
38 */31 */
@@ -90,6 +83,28 @@ static inline void netfs_see_request(struct netfs_io_request *rreq, · 51 unmodified lines
90 trace_netfs_rreq_ref(rreq->debug_id, refcount_read(&rreq->ref), what);83 trace_netfs_rreq_ref(rreq->debug_id, refcount_read(&rreq->ref), what);
91}84}
92 85
86/*
87 * read_collect.c
88 */
89void netfs_read_termination_worker(struct work_struct *work);
90void netfs_rreq_terminated(struct netfs_io_request *rreq, bool was_async);
91
92/*
93 * read_pgpriv2.c
94 */
95void netfs_pgpriv2_mark_copy_to_cache(struct netfs_io_subrequest *subreq,
96 struct netfs_io_request *rreq,
97 struct folio_queue *folioq,
98 int slot);
99void netfs_pgpriv2_write_to_the_cache(struct netfs_io_request *rreq);
100bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *wreq);
101
102/*
103 * read_retry.c
104 */
105void netfs_retry_reads(struct netfs_io_request *rreq);
106void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq);
107
93/*108/*
94 * stats.c109 * stats.c
95 */110 */
@@ -117,6 +132,7 @@ extern atomic_t netfs_n_wh_buffered_write; · 21 unmodified lines
117extern atomic_t netfs_n_wh_writethrough;132extern atomic_t netfs_n_wh_writethrough;
118extern atomic_t netfs_n_wh_dio_write;133extern atomic_t netfs_n_wh_dio_write;
119extern atomic_t netfs_n_wh_writepages;134extern atomic_t netfs_n_wh_writepages;
135extern atomic_t netfs_n_wh_copy_to_cache;
120extern atomic_t netfs_n_wh_wstream_conflict;136extern atomic_t netfs_n_wh_wstream_conflict;
121extern atomic_t netfs_n_wh_upload;137extern atomic_t netfs_n_wh_upload;
122extern atomic_t netfs_n_wh_upload_done;138extern atomic_t netfs_n_wh_upload_done;
@@ -162,6 +178,11 @@ struct netfs_io_request *netfs_create_write_req(struct address_space *mapping, · 39 unmodified lines
162void netfs_reissue_write(struct netfs_io_stream *stream,178void netfs_reissue_write(struct netfs_io_stream *stream,
163 struct netfs_io_subrequest *subreq,179 struct netfs_io_subrequest *subreq,
164 struct iov_iter *source);180 struct iov_iter *source);
181void netfs_issue_write(struct netfs_io_request *wreq,
182 struct netfs_io_stream *stream);
183int netfs_advance_write(struct netfs_io_request *wreq,
184 struct netfs_io_stream *stream,
185 loff_t start, size_t len, bool to_eof);
165struct netfs_io_request *netfs_begin_writethrough(struct kiocb *iocb, size_t len);186struct netfs_io_request *netfs_begin_writethrough(struct kiocb *iocb, size_t len);
166int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc,187int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc,
167 struct folio *folio, size_t copied, bool to_page_end,188 struct folio *folio, size_t copied, bool to_page_end,
fs/netfs/iterator.c+50−0
@@ -188,9 +188,59 @@ static size_t netfs_limit_xarray(const struct iov_iter *iter, size_t start_offse · 187 unmodified lines
188 return min(span, max_size);188 return min(span, max_size);
189}189}
190 190
191/*
192 * Select the span of a folio queue iterator we're going to use. Limit it by
193 * both maximum size and maximum number of segments. Returns the size of the
194 * span in bytes.
195 */
196static size_t netfs_limit_folioq(const struct iov_iter *iter, size_t start_offset,
197 size_t max_size, size_t max_segs)
198{
199 const struct folio_queue *folioq = iter->folioq;
200 unsigned int nsegs = 0;
201 unsigned int slot = iter->folioq_slot;
202 size_t span = 0, n = iter->count;
203
204 if (WARN_ON(!iov_iter_is_folioq(iter)) ||
205 WARN_ON(start_offset > n) ||
206 n == 0)
207 return 0;
208 max_size = umin(max_size, n - start_offset);
209
210 if (slot >= folioq_nr_slots(folioq)) {
211 folioq = folioq->next;
212 slot = 0;
213 }
214
215 start_offset += iter->iov_offset;
216 do {
217 size_t flen = folioq_folio_size(folioq, slot);
218
219 if (start_offset < flen) {
220 span += flen - start_offset;
221 nsegs++;
222 start_offset = 0;
223 } else {
224 start_offset -= flen;
225 }
226 if (span >= max_size || nsegs >= max_segs)
227 break;
228
229 slot++;
230 if (slot >= folioq_nr_slots(folioq)) {
231 folioq = folioq->next;
232 slot = 0;
233 }
234 } while (folioq);
235
236 return umin(span, max_size);
237}
238
191size_t netfs_limit_iter(const struct iov_iter *iter, size_t start_offset,239size_t netfs_limit_iter(const struct iov_iter *iter, size_t start_offset,
192 size_t max_size, size_t max_segs)240 size_t max_size, size_t max_segs)
193{241{
242 if (iov_iter_is_folioq(iter))
243 return netfs_limit_folioq(iter, start_offset, max_size, max_segs);
194 if (iov_iter_is_bvec(iter))244 if (iov_iter_is_bvec(iter))
195 return netfs_limit_bvec(iter, start_offset, max_size, max_segs);245 return netfs_limit_bvec(iter, start_offset, max_size, max_segs);
196 if (iov_iter_is_xarray(iter))246 if (iov_iter_is_xarray(iter))
fs/netfs/main.c+3−1
@@ -36,12 +36,14 @@ DEFINE_SPINLOCK(netfs_proc_lock); · 35 unmodified lines
36static const char *netfs_origins[nr__netfs_io_origin] = {36static const char *netfs_origins[nr__netfs_io_origin] = {
37 [NETFS_READAHEAD] = "RA",37 [NETFS_READAHEAD] = "RA",
38 [NETFS_READPAGE] = "RP",38 [NETFS_READPAGE] = "RP",
39 [NETFS_READ_GAPS] = "RG",
39 [NETFS_READ_FOR_WRITE] = "RW",40 [NETFS_READ_FOR_WRITE] = "RW",
40 [NETFS_DIO_READ] = "DR",41 [NETFS_DIO_READ] = "DR",
41 [NETFS_WRITEBACK] = "WB",42 [NETFS_WRITEBACK] = "WB",
42 [NETFS_WRITETHROUGH] = "WT",43 [NETFS_WRITETHROUGH] = "WT",
43 [NETFS_UNBUFFERED_WRITE] = "UW",44 [NETFS_UNBUFFERED_WRITE] = "UW",
44 [NETFS_DIO_WRITE] = "DW",45 [NETFS_DIO_WRITE] = "DW",
46 [NETFS_PGPRIV2_COPY_TO_CACHE] = "2C",
45};47};
46 48
47/*49/*
@@ -61,7 +63,7 @@ static int netfs_requests_seq_show(struct seq_file *m, void *v) · 13 unmodified lines
61 63
62 rreq = list_entry(v, struct netfs_io_request, proc_link);64 rreq = list_entry(v, struct netfs_io_request, proc_link);
63 seq_printf(m,65 seq_printf(m,
64 "%08x %s %3d %2lx %4d %3d @%04llx %llx/%llx",66 "%08x %s %3d %2lx %4ld %3d @%04llx %llx/%llx",
65 rreq->debug_id,67 rreq->debug_id,
66 netfs_origins[rreq->origin],68 netfs_origins[rreq->origin],
67 refcount_read(&rreq->ref),69 refcount_read(&rreq->ref),
fs/netfs/objects.c+5−3
@@ -36,7 +36,6 @@ struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, · 35 unmodified lines
36 memset(rreq, 0, kmem_cache_size(cache));36 memset(rreq, 0, kmem_cache_size(cache));
37 rreq->start = start;37 rreq->start = start;
38 rreq->len = len;38 rreq->len = len;
39 rreq->upper_len = len;
40 rreq->origin = origin;39 rreq->origin = origin;
41 rreq->netfs_ops = ctx->ops;40 rreq->netfs_ops = ctx->ops;
42 rreq->mapping = mapping;41 rreq->mapping = mapping;
@@ -44,6 +43,8 @@ struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, · 1 unmodified line
44 rreq->i_size = i_size_read(inode);43 rreq->i_size = i_size_read(inode);
45 rreq->debug_id = atomic_inc_return(&debug_ids);44 rreq->debug_id = atomic_inc_return(&debug_ids);
46 rreq->wsize = INT_MAX;45 rreq->wsize = INT_MAX;
46 rreq->io_streams[0].sreq_max_len = ULONG_MAX;
47 rreq->io_streams[0].sreq_max_segs = 0;
47 spin_lock_init(&rreq->lock);48 spin_lock_init(&rreq->lock);
48 INIT_LIST_HEAD(&rreq->io_streams[0].subrequests);49 INIT_LIST_HEAD(&rreq->io_streams[0].subrequests);
49 INIT_LIST_HEAD(&rreq->io_streams[1].subrequests);50 INIT_LIST_HEAD(&rreq->io_streams[1].subrequests);
@@ -52,9 +53,10 @@ struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, · 2 unmodified lines
52 53
53 if (origin == NETFS_READAHEAD ||54 if (origin == NETFS_READAHEAD ||
54 origin == NETFS_READPAGE ||55 origin == NETFS_READPAGE ||
56 origin == NETFS_READ_GAPS ||
55 origin == NETFS_READ_FOR_WRITE ||57 origin == NETFS_READ_FOR_WRITE ||
56 origin == NETFS_DIO_READ)58 origin == NETFS_DIO_READ)
57 INIT_WORK(&rreq->work, netfs_rreq_work);59 INIT_WORK(&rreq->work, netfs_read_termination_worker);
58 else60 else
59 INIT_WORK(&rreq->work, netfs_write_collection_worker);61 INIT_WORK(&rreq->work, netfs_write_collection_worker);
60 62
@@ -163,7 +165,7 @@ void netfs_put_request(struct netfs_io_request *rreq, bool was_async, · 102 unmodified lines
163 if (was_async) {165 if (was_async) {
164 rreq->work.func = netfs_free_request;166 rreq->work.func = netfs_free_request;
165 if (!queue_work(system_unbound_wq, &rreq->work))167 if (!queue_work(system_unbound_wq, &rreq->work))
166 BUG();168 WARN_ON(1);
167 } else {169 } else {
168 netfs_free_request(&rreq->work);170 netfs_free_request(&rreq->work);
169 }171 }
fs/netfs/read_collect.c+544−0
@@ -0,0 +1,544 @@
1// SPDX-License-Identifier: GPL-2.0-only
2/* Network filesystem read subrequest result collection, assessment and
3 * retrying.
4 *
5 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
6 * Written by David Howells (dhowells@redhat.com)
7 */
8
9#include <linux/export.h>
10#include <linux/fs.h>
11#include <linux/mm.h>
12#include <linux/pagemap.h>
13#include <linux/slab.h>
14#include <linux/task_io_accounting_ops.h>
15#include "internal.h"
16
17/*
18 * Clear the unread part of an I/O request.
19 */
20static void netfs_clear_unread(struct netfs_io_subrequest *subreq)
21{
22 netfs_reset_iter(subreq);
23 WARN_ON_ONCE(subreq->len - subreq->transferred != iov_iter_count(&subreq->io_iter));
24 iov_iter_zero(iov_iter_count(&subreq->io_iter), &subreq->io_iter);
25 if (subreq->start + subreq->transferred >= subreq->rreq->i_size)
26 __set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags);
27}
28
29/*
30 * Flush, mark and unlock a folio that's now completely read. If we want to
31 * cache the folio, we set the group to NETFS_FOLIO_COPY_TO_CACHE, mark it
32 * dirty and let writeback handle it.
33 */
34static void netfs_unlock_read_folio(struct netfs_io_subrequest *subreq,
35 struct netfs_io_request *rreq,
36 struct folio_queue *folioq,
37 int slot)
38{
39 struct netfs_folio *finfo;
40 struct folio *folio = folioq_folio(folioq, slot);
41
42 flush_dcache_folio(folio);
43 folio_mark_uptodate(folio);
44
45 if (!test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)) {
46 finfo = netfs_folio_info(folio);
47 if (finfo) {
48 trace_netfs_folio(folio, netfs_folio_trace_filled_gaps);
49 if (finfo->netfs_group)
50 folio_change_private(folio, finfo->netfs_group);
51 else
52 folio_detach_private(folio);
53 kfree(finfo);
54 }
55
56 if (test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags)) {
57 if (!WARN_ON_ONCE(folio_get_private(folio) != NULL)) {
58 trace_netfs_folio(folio, netfs_folio_trace_copy_to_cache);
59 folio_attach_private(folio, NETFS_FOLIO_COPY_TO_CACHE);
60 folio_mark_dirty(folio);
61 }
62 } else {
63 trace_netfs_folio(folio, netfs_folio_trace_read_done);
64 }
65 } else {
66 // TODO: Use of PG_private_2 is deprecated.
67 if (test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags))
68 netfs_pgpriv2_mark_copy_to_cache(subreq, rreq, folioq, slot);
69 }
70
71 if (!test_bit(NETFS_RREQ_DONT_UNLOCK_FOLIOS, &rreq->flags)) {
72 if (folio->index == rreq->no_unlock_folio &&
73 test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags)) {
74 _debug("no unlock");
75 } else {
76 trace_netfs_folio(folio, netfs_folio_trace_read_unlock);
77 folio_unlock(folio);
78 }
79 }
80}
81
82/*
83 * Unlock any folios that are now completely read. Returns true if the
84 * subrequest is removed from the list.
85 */
86static bool netfs_consume_read_data(struct netfs_io_subrequest *subreq, bool was_async)
87{
88 struct netfs_io_subrequest *prev, *next;
89 struct netfs_io_request *rreq = subreq->rreq;
90 struct folio_queue *folioq = subreq->curr_folioq;
91 size_t avail, prev_donated, next_donated, fsize, part, excess;
92 loff_t fpos, start;
93 loff_t fend;
94 int slot = subreq->curr_folioq_slot;
95
96 if (WARN(subreq->transferred > subreq->len,
97 "Subreq overread: R%x[%x] %zu > %zu",
98 rreq->debug_id, subreq->debug_index,
99 subreq->transferred, subreq->len))
100 subreq->transferred = subreq->len;
101
102next_folio:
103 fsize = PAGE_SIZE << subreq->curr_folio_order;
104 fpos = round_down(subreq->start + subreq->consumed, fsize);
105 fend = fpos + fsize;
106
107 if (WARN_ON_ONCE(!folioq) ||
108 WARN_ON_ONCE(!folioq_folio(folioq, slot)) ||
109 WARN_ON_ONCE(folioq_folio(folioq, slot)->index != fpos / PAGE_SIZE)) {
110 pr_err("R=%08x[%x] s=%llx-%llx ctl=%zx/%zx/%zx sl=%u\n",
111 rreq->debug_id, subreq->debug_index,
112 subreq->start, subreq->start + subreq->transferred - 1,
113 subreq->consumed, subreq->transferred, subreq->len,
114 slot);
115 if (folioq) {
116 struct folio *folio = folioq_folio(folioq, slot);
117
118 pr_err("folioq: orders=%02x%02x%02x%02x\n",
119 folioq->orders[0], folioq->orders[1],
120 folioq->orders[2], folioq->orders[3]);
121 if (folio)
122 pr_err("folio: %llx-%llx ix=%llx o=%u qo=%u\n",
123 fpos, fend - 1, folio_pos(folio), folio_order(folio),
124 folioq_folio_order(folioq, slot));
125 }
126 }
127
128donation_changed:
129 /* Try to consume the current folio if we've hit or passed the end of
130 * it. There's a possibility that this subreq doesn't start at the
131 * beginning of the folio, in which case we need to donate to/from the
132 * preceding subreq.
133 *
134 * We also need to include any potential donation back from the
135 * following subreq.
136 */
137 prev_donated = READ_ONCE(subreq->prev_donated);
138 next_donated = READ_ONCE(subreq->next_donated);
139 if (prev_donated || next_donated) {
140 spin_lock_bh(&rreq->lock);
141 prev_donated = subreq->prev_donated;
142 next_donated = subreq->next_donated;
143 subreq->start -= prev_donated;
144 subreq->len += prev_donated;
145 subreq->transferred += prev_donated;
146 prev_donated = subreq->prev_donated = 0;
147 if (subreq->transferred == subreq->len) {
148 subreq->len += next_donated;
149 subreq->transferred += next_donated;
150 next_donated = subreq->next_donated = 0;
151 }
152 trace_netfs_sreq(subreq, netfs_sreq_trace_add_donations);
153 spin_unlock_bh(&rreq->lock);
154 }
155
156 avail = subreq->transferred;
157 if (avail == subreq->len)
158 avail += next_donated;
159 start = subreq->start;
160 if (subreq->consumed == 0) {
161 start -= prev_donated;
162 avail += prev_donated;
163 } else {
164 start += subreq->consumed;
165 avail -= subreq->consumed;
166 }
167 part = umin(avail, fsize);
168
169 trace_netfs_progress(subreq, start, avail, part);
170
171 if (start + avail >= fend) {
172 if (fpos == start) {
173 /* Flush, unlock and mark for caching any folio we've just read. */
174 subreq->consumed = fend - subreq->start;
175 netfs_unlock_read_folio(subreq, rreq, folioq, slot);
176 folioq_mark2(folioq, slot);
177 if (subreq->consumed >= subreq->len)
178 goto remove_subreq;
179 } else if (fpos < start) {
180 excess = fend - subreq->start;
181
182 spin_lock_bh(&rreq->lock);
183 /* If we complete first on a folio split with the
184 * preceding subreq, donate to that subreq - otherwise
185 * we get the responsibility.
186 */
187 if (subreq->prev_donated != prev_donated) {
188 spin_unlock_bh(&rreq->lock);
189 goto donation_changed;
190 }
191
192 if (list_is_first(&subreq->rreq_link, &rreq->subrequests)) {
193 spin_unlock_bh(&rreq->lock);
194 pr_err("Can't donate prior to front\n");
195 goto bad;
196 }
197
198 prev = list_prev_entry(subreq, rreq_link);
199 WRITE_ONCE(prev->next_donated, prev->next_donated + excess);
200 subreq->start += excess;
201 subreq->len -= excess;
202 subreq->transferred -= excess;
203 trace_netfs_donate(rreq, subreq, prev, excess,
204 netfs_trace_donate_tail_to_prev);
205 trace_netfs_sreq(subreq, netfs_sreq_trace_donate_to_prev);
206
207 if (subreq->consumed >= subreq->len)
208 goto remove_subreq_locked;
209 spin_unlock_bh(&rreq->lock);
210 } else {
211 pr_err("fpos > start\n");
212 goto bad;
213 }
214
215 /* Advance the rolling buffer to the next folio. */
216 slot++;
217 if (slot >= folioq_nr_slots(folioq)) {
218 slot = 0;
219 folioq = folioq->next;
220 subreq->curr_folioq = folioq;
221 }
222 subreq->curr_folioq_slot = slot;
223 if (folioq && folioq_folio(folioq, slot))
224 subreq->curr_folio_order = folioq->orders[slot];
225 if (!was_async)
226 cond_resched();
227 goto next_folio;
228 }
229
230 /* Deal with partial progress. */
231 if (subreq->transferred < subreq->len)
232 return false;
233
234 /* Donate the remaining downloaded data to one of the neighbouring
235 * subrequests. Note that we may race with them doing the same thing.
236 */
237 spin_lock_bh(&rreq->lock);
238
239 if (subreq->prev_donated != prev_donated ||
240 subreq->next_donated != next_donated) {
241 spin_unlock_bh(&rreq->lock);
242 cond_resched();
243 goto donation_changed;
244 }
245
246 /* Deal with the trickiest case: that this subreq is in the middle of a
247 * folio, not touching either edge, but finishes first. In such a
248 * case, we donate to the previous subreq, if there is one, so that the
249 * donation is only handled when that completes - and remove this
250 * subreq from the list.
251 *
252 * If the previous subreq finished first, we will have acquired their
253 * donation and should be able to unlock folios and/or donate nextwards.
254 */
255 if (!subreq->consumed &&
256 !prev_donated &&
257 !list_is_first(&subreq->rreq_link, &rreq->subrequests)) {
258 prev = list_prev_entry(subreq, rreq_link);
259 WRITE_ONCE(prev->next_donated, prev->next_donated + subreq->len);
260 subreq->start += subreq->len;
261 subreq->len = 0;
262 subreq->transferred = 0;
263 trace_netfs_donate(rreq, subreq, prev, subreq->len,
264 netfs_trace_donate_to_prev);
265 trace_netfs_sreq(subreq, netfs_sreq_trace_donate_to_prev);
266 goto remove_subreq_locked;
267 }
268
269 /* If we can't donate down the chain, donate up the chain instead. */
270 excess = subreq->len - subreq->consumed + next_donated;
271
272 if (!subreq->consumed)
273 excess += prev_donated;
274
275 if (list_is_last(&subreq->rreq_link, &rreq->subrequests)) {
276 rreq->prev_donated = excess;
277 trace_netfs_donate(rreq, subreq, NULL, excess,
278 netfs_trace_donate_to_deferred_next);
279 } else {
280 next = list_next_entry(subreq, rreq_link);
281 WRITE_ONCE(next->prev_donated, excess);
282 trace_netfs_donate(rreq, subreq, next, excess,
283 netfs_trace_donate_to_next);
284 }
285 trace_netfs_sreq(subreq, netfs_sreq_trace_donate_to_next);
286 subreq->len = subreq->consumed;
287 subreq->transferred = subreq->consumed;
288 goto remove_subreq_locked;
289
290remove_subreq:
291 spin_lock_bh(&rreq->lock);
292remove_subreq_locked:
293 subreq->consumed = subreq->len;
294 list_del(&subreq->rreq_link);
295 spin_unlock_bh(&rreq->lock);
296 netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_consumed);
297 return true;
298
299bad:
300 /* Errr... prev and next both donated to us, but insufficient to finish
301 * the folio.
302 */
303 printk("R=%08x[%x] s=%llx-%llx %zx/%zx/%zx\n",
304 rreq->debug_id, subreq->debug_index,
305 subreq->start, subreq->start + subreq->transferred - 1,
306 subreq->consumed, subreq->transferred, subreq->len);
307 printk("folio: %llx-%llx\n", fpos, fend - 1);
308 printk("donated: prev=%zx next=%zx\n", prev_donated, next_donated);
309 printk("s=%llx av=%zx part=%zx\n", start, avail, part);
310 BUG();
311}
312
313/*
314 * Do page flushing and suchlike after DIO.
315 */
316static void netfs_rreq_assess_dio(struct netfs_io_request *rreq)
317{
318 struct netfs_io_subrequest *subreq;
319 unsigned int i;
320
321 /* Collect unbuffered reads and direct reads, adding up the transfer
322 * sizes until we find the first short or failed subrequest.
323 */
324 list_for_each_entry(subreq, &rreq->subrequests, rreq_link) {
325 rreq->transferred += subreq->transferred;
326
327 if (subreq->transferred < subreq->len ||
328 test_bit(NETFS_SREQ_FAILED, &subreq->flags)) {
329 rreq->error = subreq->error;
330 break;
331 }
332 }
333
334 if (rreq->origin == NETFS_DIO_READ) {
335 for (i = 0; i < rreq->direct_bv_count; i++) {
336 flush_dcache_page(rreq->direct_bv[i].bv_page);
337 // TODO: cifs marks pages in the destination buffer
338 // dirty under some circumstances after a read. Do we
339 // need to do that too?
340 set_page_dirty(rreq->direct_bv[i].bv_page);
341 }
342 }
343
344 if (rreq->iocb) {
345 rreq->iocb->ki_pos += rreq->transferred;
346 if (rreq->iocb->ki_complete)
347 rreq->iocb->ki_complete(
348 rreq->iocb, rreq->error ? rreq->error : rreq->transferred);
349 }
350 if (rreq->netfs_ops->done)
351 rreq->netfs_ops->done(rreq);
352 if (rreq->origin == NETFS_DIO_READ)
353 inode_dio_end(rreq->inode);
354}
355
356/*
357 * Assess the state of a read request and decide what to do next.
358 *
359 * Note that we're in normal kernel thread context at this point, possibly
360 * running on a workqueue.
361 */
362static void netfs_rreq_assess(struct netfs_io_request *rreq)
363{
364 trace_netfs_rreq(rreq, netfs_rreq_trace_assess);
365
366 //netfs_rreq_is_still_valid(rreq);
367
368 if (test_and_clear_bit(NETFS_RREQ_NEED_RETRY, &rreq->flags)) {
369 netfs_retry_reads(rreq);
370 return;
371 }
372
373 if (rreq->origin == NETFS_DIO_READ ||
374 rreq->origin == NETFS_READ_GAPS)
375 netfs_rreq_assess_dio(rreq);
376 task_io_account_read(rreq->transferred);
377
378 trace_netfs_rreq(rreq, netfs_rreq_trace_wake_ip);
379 clear_bit_unlock(NETFS_RREQ_IN_PROGRESS, &rreq->flags);
380 wake_up_bit(&rreq->flags, NETFS_RREQ_IN_PROGRESS);
381
382 trace_netfs_rreq(rreq, netfs_rreq_trace_done);
383 netfs_clear_subrequests(rreq, false);
384 netfs_unlock_abandoned_read_pages(rreq);
385 if (unlikely(test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)))
386 netfs_pgpriv2_write_to_the_cache(rreq);
387}
388
389void netfs_read_termination_worker(struct work_struct *work)
390{
391 struct netfs_io_request *rreq =
392 container_of(work, struct netfs_io_request, work);
393 netfs_see_request(rreq, netfs_rreq_trace_see_work);
394 netfs_rreq_assess(rreq);
395 netfs_put_request(rreq, false, netfs_rreq_trace_put_work_complete);
396}
397
398/*
399 * Handle the completion of all outstanding I/O operations on a read request.
400 * We inherit a ref from the caller.
401 */
402void netfs_rreq_terminated(struct netfs_io_request *rreq, bool was_async)
403{
404 if (!was_async)
405 return netfs_rreq_assess(rreq);
406 if (!work_pending(&rreq->work)) {
407 netfs_get_request(rreq, netfs_rreq_trace_get_work);
408 if (!queue_work(system_unbound_wq, &rreq->work))
409 netfs_put_request(rreq, was_async, netfs_rreq_trace_put_work_nq);
410 }
411}
412
413/**
414 * netfs_read_subreq_progress - Note progress of a read operation.
415 * @subreq: The read request that has terminated.
416 * @was_async: True if we're in an asynchronous context.
417 *
418 * This tells the read side of netfs lib that a contributory I/O operation has
419 * made some progress and that it may be possible to unlock some folios.
420 *
421 * Before calling, the filesystem should update subreq->transferred to track
422 * the amount of data copied into the output buffer.
423 *
424 * If @was_async is true, the caller might be running in softirq or interrupt
425 * context and we can't sleep.
426 */
427void netfs_read_subreq_progress(struct netfs_io_subrequest *subreq,
428 bool was_async)
429{
430 struct netfs_io_request *rreq = subreq->rreq;
431
432 trace_netfs_sreq(subreq, netfs_sreq_trace_progress);
433
434 if (subreq->transferred > subreq->consumed &&
435 (rreq->origin == NETFS_READAHEAD ||
436 rreq->origin == NETFS_READPAGE ||
437 rreq->origin == NETFS_READ_FOR_WRITE)) {
438 netfs_consume_read_data(subreq, was_async);
439 __clear_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags);
440 }
441}
442EXPORT_SYMBOL(netfs_read_subreq_progress);
443
444/**
445 * netfs_read_subreq_terminated - Note the termination of an I/O operation.
446 * @subreq: The I/O request that has terminated.
447 * @error: Error code indicating type of completion.
448 * @was_async: The termination was asynchronous
449 *
450 * This tells the read helper that a contributory I/O operation has terminated,
451 * one way or another, and that it should integrate the results.
452 *
453 * The caller indicates the outcome of the operation through @error, supplying
454 * 0 to indicate a successful or retryable transfer (if NETFS_SREQ_NEED_RETRY
455 * is set) or a negative error code. The helper will look after reissuing I/O
456 * operations as appropriate and writing downloaded data to the cache.
457 *
458 * Before calling, the filesystem should update subreq->transferred to track
459 * the amount of data copied into the output buffer.
460 *
461 * If @was_async is true, the caller might be running in softirq or interrupt
462 * context and we can't sleep.
463 */
464void netfs_read_subreq_terminated(struct netfs_io_subrequest *subreq,
465 int error, bool was_async)
466{
467 struct netfs_io_request *rreq = subreq->rreq;
468
469 switch (subreq->source) {
470 case NETFS_READ_FROM_CACHE:
471 netfs_stat(&netfs_n_rh_read_done);
472 break;
473 case NETFS_DOWNLOAD_FROM_SERVER:
474 netfs_stat(&netfs_n_rh_download_done);
475 break;
476 default:
477 break;
478 }
479
480 if (rreq->origin != NETFS_DIO_READ) {
481 /* Collect buffered reads.
482 *
483 * If the read completed validly short, then we can clear the
484 * tail before going on to unlock the folios.
485 */
486 if (error == 0 && subreq->transferred < subreq->len &&
487 (test_bit(NETFS_SREQ_HIT_EOF, &subreq->flags) ||
488 test_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags))) {
489 netfs_clear_unread(subreq);
490 subreq->transferred = subreq->len;
491 trace_netfs_sreq(subreq, netfs_sreq_trace_clear);
492 }
493 if (subreq->transferred > subreq->consumed &&
494 (rreq->origin == NETFS_READAHEAD ||
495 rreq->origin == NETFS_READPAGE ||
496 rreq->origin == NETFS_READ_FOR_WRITE)) {
497 netfs_consume_read_data(subreq, was_async);
498 __clear_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags);
499 }
500 rreq->transferred += subreq->transferred;
501 }
502
503 /* Deal with retry requests, short reads and errors. If we retry
504 * but don't make progress, we abandon the attempt.
505 */
506 if (!error && subreq->transferred < subreq->len) {
507 if (test_bit(NETFS_SREQ_HIT_EOF, &subreq->flags)) {
508 trace_netfs_sreq(subreq, netfs_sreq_trace_hit_eof);
509 } else {
510 trace_netfs_sreq(subreq, netfs_sreq_trace_short);
511 if (subreq->transferred > subreq->consumed) {
512 __set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
513 __clear_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags);
514 set_bit(NETFS_RREQ_NEED_RETRY, &rreq->flags);
515 } else if (!__test_and_set_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags)) {
516 __set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
517 set_bit(NETFS_RREQ_NEED_RETRY, &rreq->flags);
518 } else {
519 __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
520 error = -ENODATA;
521 }
522 }
523 }
524
525 subreq->error = error;
526 trace_netfs_sreq(subreq, netfs_sreq_trace_terminated);
527
528 if (unlikely(error < 0)) {
529 trace_netfs_failure(rreq, subreq, error, netfs_fail_read);
530 if (subreq->source == NETFS_READ_FROM_CACHE) {
531 netfs_stat(&netfs_n_rh_read_failed);
532 } else {
533 netfs_stat(&netfs_n_rh_download_failed);
534 set_bit(NETFS_RREQ_FAILED, &rreq->flags);
535 rreq->error = subreq->error;
536 }
537 }
538
539 if (atomic_dec_and_test(&rreq->nr_outstanding))
540 netfs_rreq_terminated(rreq, was_async);
541
542 netfs_put_subrequest(subreq, was_async, netfs_sreq_trace_put_terminated);
543}
544EXPORT_SYMBOL(netfs_read_subreq_terminated);
fs/netfs/read_pgpriv2.c+264−0
@@ -0,0 +1,264 @@
1// SPDX-License-Identifier: GPL-2.0-only
2/* Read with PG_private_2 [DEPRECATED].
3 *
4 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 */
7
8#include <linux/export.h>
9#include <linux/fs.h>
10#include <linux/mm.h>
11#include <linux/pagemap.h>
12#include <linux/slab.h>
13#include <linux/task_io_accounting_ops.h>
14#include "internal.h"
15
16/*
17 * [DEPRECATED] Mark page as requiring copy-to-cache using PG_private_2. The
18 * third mark in the folio queue is used to indicate that this folio needs
19 * writing.
20 */
21void netfs_pgpriv2_mark_copy_to_cache(struct netfs_io_subrequest *subreq,
22 struct netfs_io_request *rreq,
23 struct folio_queue *folioq,
24 int slot)
25{
26 struct folio *folio = folioq_folio(folioq, slot);
27
28 trace_netfs_folio(folio, netfs_folio_trace_copy_to_cache);
29 folio_start_private_2(folio);
30 folioq_mark3(folioq, slot);
31}
32
33/*
34 * [DEPRECATED] Cancel PG_private_2 on all marked folios in the event of an
35 * unrecoverable error.
36 */
37static void netfs_pgpriv2_cancel(struct folio_queue *folioq)
38{
39 struct folio *folio;
40 int slot;
41
42 while (folioq) {
43 if (!folioq->marks3) {
44 folioq = folioq->next;
45 continue;
46 }
47
48 slot = __ffs(folioq->marks3);
49 folio = folioq_folio(folioq, slot);
50
51 trace_netfs_folio(folio, netfs_folio_trace_cancel_copy);
52 folio_end_private_2(folio);
53 folioq_unmark3(folioq, slot);
54 }
55}
56
57/*
58 * [DEPRECATED] Copy a folio to the cache with PG_private_2 set.
59 */
60static int netfs_pgpriv2_copy_folio(struct netfs_io_request *wreq, struct folio *folio)
61{
62 struct netfs_io_stream *cache = &wreq->io_streams[1];
63 size_t fsize = folio_size(folio), flen = fsize;
64 loff_t fpos = folio_pos(folio), i_size;
65 bool to_eof = false;
66
67 _enter("");
68
69 /* netfs_perform_write() may shift i_size around the page or from out
70 * of the page to beyond it, but cannot move i_size into or through the
71 * page since we have it locked.
72 */
73 i_size = i_size_read(wreq->inode);
74
75 if (fpos >= i_size) {
76 /* mmap beyond eof. */
77 _debug("beyond eof");
78 folio_end_private_2(folio);
79 return 0;
80 }
81
82 if (fpos + fsize > wreq->i_size)
83 wreq->i_size = i_size;
84
85 if (flen > i_size - fpos) {
86 flen = i_size - fpos;
87 to_eof = true;
88 } else if (flen == i_size - fpos) {
89 to_eof = true;
90 }
91
92 _debug("folio %zx %zx", flen, fsize);
93
94 trace_netfs_folio(folio, netfs_folio_trace_store_copy);
95
96 /* Attach the folio to the rolling buffer. */
97 if (netfs_buffer_append_folio(wreq, folio, false) < 0)
98 return -ENOMEM;
99
100 cache->submit_max_len = fsize;
101 cache->submit_off = 0;
102 cache->submit_len = flen;
103
104 /* Attach the folio to one or more subrequests. For a big folio, we
105 * could end up with thousands of subrequests if the wsize is small -
106 * but we might need to wait during the creation of subrequests for
107 * network resources (eg. SMB credits).
108 */
109 do {
110 ssize_t part;
111
112 wreq->io_iter.iov_offset = cache->submit_off;
113
114 atomic64_set(&wreq->issued_to, fpos + cache->submit_off);
115 part = netfs_advance_write(wreq, cache, fpos + cache->submit_off,
116 cache->submit_len, to_eof);
117 cache->submit_off += part;
118 cache->submit_max_len -= part;
119 if (part > cache->submit_len)
120 cache->submit_len = 0;
121 else
122 cache->submit_len -= part;
123 } while (cache->submit_len > 0);
124
125 wreq->io_iter.iov_offset = 0;
126 iov_iter_advance(&wreq->io_iter, fsize);
127 atomic64_set(&wreq->issued_to, fpos + fsize);
128
129 if (flen < fsize)
130 netfs_issue_write(wreq, cache);
131
132 _leave(" = 0");
133 return 0;
134}
135
136/*
137 * [DEPRECATED] Go through the buffer and write any folios that are marked with
138 * the third mark to the cache.
139 */
140void netfs_pgpriv2_write_to_the_cache(struct netfs_io_request *rreq)
141{
142 struct netfs_io_request *wreq;
143 struct folio_queue *folioq;
144 struct folio *folio;
145 int error = 0;
146 int slot = 0;
147
148 _enter("");
149
150 if (!fscache_resources_valid(&rreq->cache_resources))
151 goto couldnt_start;
152
153 /* Need the first folio to be able to set up the op. */
154 for (folioq = rreq->buffer; folioq; folioq = folioq->next) {
155 if (folioq->marks3) {
156 slot = __ffs(folioq->marks3);
157 break;
158 }
159 }
160 if (!folioq)
161 return;
162 folio = folioq_folio(folioq, slot);
163
164 wreq = netfs_create_write_req(rreq->mapping, NULL, folio_pos(folio),
165 NETFS_PGPRIV2_COPY_TO_CACHE);
166 if (IS_ERR(wreq)) {
167 kleave(" [create %ld]", PTR_ERR(wreq));
168 goto couldnt_start;
169 }
170
171 trace_netfs_write(wreq, netfs_write_trace_copy_to_cache);
172 netfs_stat(&netfs_n_wh_copy_to_cache);
173
174 for (;;) {
175 error = netfs_pgpriv2_copy_folio(wreq, folio);
176 if (error < 0)
177 break;
178
179 folioq_unmark3(folioq, slot);
180 if (!folioq->marks3) {
181 folioq = folioq->next;
182 if (!folioq)
183 break;
184 }
185
186 slot = __ffs(folioq->marks3);
187 folio = folioq_folio(folioq, slot);
188 }
189
190 netfs_issue_write(wreq, &wreq->io_streams[1]);
191 smp_wmb(); /* Write lists before ALL_QUEUED. */
192 set_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags);
193
194 netfs_put_request(wreq, false, netfs_rreq_trace_put_return);
195 _leave(" = %d", error);
196couldnt_start:
197 netfs_pgpriv2_cancel(rreq->buffer);
198}
199
200/*
201 * [DEPRECATED] Remove the PG_private_2 mark from any folios we've finished
202 * copying.
203 */
204bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *wreq)
205{
206 struct folio_queue *folioq = wreq->buffer;
207 unsigned long long collected_to = wreq->collected_to;
208 unsigned int slot = wreq->buffer_head_slot;
209 bool made_progress = false;
210
211 if (slot >= folioq_nr_slots(folioq)) {
212 folioq = netfs_delete_buffer_head(wreq);
213 slot = 0;
214 }
215
216 for (;;) {
217 struct folio *folio;
218 unsigned long long fpos, fend;
219 size_t fsize, flen;
220
221 folio = folioq_folio(folioq, slot);
222 if (WARN_ONCE(!folio_test_private_2(folio),
223 "R=%08x: folio %lx is not marked private_2\n",
224 wreq->debug_id, folio->index))
225 trace_netfs_folio(folio, netfs_folio_trace_not_under_wback);
226
227 fpos = folio_pos(folio);
228 fsize = folio_size(folio);
229 flen = fsize;
230
231 fend = min_t(unsigned long long, fpos + flen, wreq->i_size);
232
233 trace_netfs_collect_folio(wreq, folio, fend, collected_to);
234
235 /* Unlock any folio we've transferred all of. */
236 if (collected_to < fend)
237 break;
238
239 trace_netfs_folio(folio, netfs_folio_trace_end_copy);
240 folio_end_private_2(folio);
241 wreq->cleaned_to = fpos + fsize;
242 made_progress = true;
243
244 /* Clean up the head folioq. If we clear an entire folioq, then
245 * we can get rid of it provided it's not also the tail folioq
246 * being filled by the issuer.
247 */
248 folioq_clear(folioq, slot);
249 slot++;
250 if (slot >= folioq_nr_slots(folioq)) {
251 if (READ_ONCE(wreq->buffer_tail) == folioq)
252 break;
253 folioq = netfs_delete_buffer_head(wreq);
254 slot = 0;
255 }
256
257 if (fpos + fsize >= collected_to)
258 break;
259 }
260
261 wreq->buffer = folioq;
262 wreq->buffer_head_slot = slot;
263 return made_progress;
264}
fs/netfs/read_retry.c+256−0
@@ -0,0 +1,256 @@
1// SPDX-License-Identifier: GPL-2.0-only
2/* Network filesystem read subrequest retrying.
3 *
4 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 */
7
8#include <linux/fs.h>
9#include <linux/slab.h>
10#include "internal.h"
11
12static void netfs_reissue_read(struct netfs_io_request *rreq,
13 struct netfs_io_subrequest *subreq)
14{
15 struct iov_iter *io_iter = &subreq->io_iter;
16
17 if (iov_iter_is_folioq(io_iter)) {
18 subreq->curr_folioq = (struct folio_queue *)io_iter->folioq;
19 subreq->curr_folioq_slot = io_iter->folioq_slot;
20 subreq->curr_folio_order = subreq->curr_folioq->orders[subreq->curr_folioq_slot];
21 }
22
23 atomic_inc(&rreq->nr_outstanding);
24 __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
25 netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit);
26 subreq->rreq->netfs_ops->issue_read(subreq);
27}
28
29/*
30 * Go through the list of failed/short reads, retrying all retryable ones. We
31 * need to switch failed cache reads to network downloads.
32 */
33static void netfs_retry_read_subrequests(struct netfs_io_request *rreq)
34{
35 struct netfs_io_subrequest *subreq;
36 struct netfs_io_stream *stream0 = &rreq->io_streams[0];
37 LIST_HEAD(sublist);
38 LIST_HEAD(queue);
39
40 _enter("R=%x", rreq->debug_id);
41
42 if (list_empty(&rreq->subrequests))
43 return;
44
45 if (rreq->netfs_ops->retry_request)
46 rreq->netfs_ops->retry_request(rreq, NULL);
47
48 /* If there's no renegotiation to do, just resend each retryable subreq
49 * up to the first permanently failed one.
50 */
51 if (!rreq->netfs_ops->prepare_read &&
52 !test_bit(NETFS_RREQ_COPY_TO_CACHE, &rreq->flags)) {
53 struct netfs_io_subrequest *subreq;
54
55 list_for_each_entry(subreq, &rreq->subrequests, rreq_link) {
56 if (test_bit(NETFS_SREQ_FAILED, &subreq->flags))
57 break;
58 if (__test_and_clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) {
59 netfs_reset_iter(subreq);
60 netfs_reissue_read(rreq, subreq);
61 }
62 }
63 return;
64 }
65
66 /* Okay, we need to renegotiate all the download requests and flip any
67 * failed cache reads over to being download requests and negotiate
68 * those also. All fully successful subreqs have been removed from the
69 * list and any spare data from those has been donated.
70 *
71 * What we do is decant the list and rebuild it one subreq at a time so
72 * that we don't end up with donations jumping over a gap we're busy
73 * populating with smaller subrequests. In the event that the subreq
74 * we just launched finishes before we insert the next subreq, it'll
75 * fill in rreq->prev_donated instead.
76
77 * Note: Alternatively, we could split the tail subrequest right before
78 * we reissue it and fix up the donations under lock.
79 */
80 list_splice_init(&rreq->subrequests, &queue);
81
82 do {
83 struct netfs_io_subrequest *from;
84 struct iov_iter source;
85 unsigned long long start, len;
86 size_t part, deferred_next_donated = 0;
87 bool boundary = false;
88
89 /* Go through the subreqs and find the next span of contiguous
90 * buffer that we then rejig (cifs, for example, needs the
91 * rsize renegotiating) and reissue.
92 */
93 from = list_first_entry(&queue, struct netfs_io_subrequest, rreq_link);
94 list_move_tail(&from->rreq_link, &sublist);
95 start = from->start + from->transferred;
96 len = from->len - from->transferred;
97
98 _debug("from R=%08x[%x] s=%llx ctl=%zx/%zx/%zx",
99 rreq->debug_id, from->debug_index,
100 from->start, from->consumed, from->transferred, from->len);
101
102 if (test_bit(NETFS_SREQ_FAILED, &from->flags) ||
103 !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags))
104 goto abandon;
105
106 deferred_next_donated = from->next_donated;
107 while ((subreq = list_first_entry_or_null(
108 &queue, struct netfs_io_subrequest, rreq_link))) {
109 if (subreq->start != start + len ||
110 subreq->transferred > 0 ||
111 !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags))
112 break;
113 list_move_tail(&subreq->rreq_link, &sublist);
114 len += subreq->len;
115 deferred_next_donated = subreq->next_donated;
116 if (test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags))
117 break;
118 }
119
120 _debug(" - range: %llx-%llx %llx", start, start + len - 1, len);
121
122 /* Determine the set of buffers we're going to use. Each
123 * subreq gets a subset of a single overall contiguous buffer.
124 */
125 netfs_reset_iter(from);
126 source = from->io_iter;
127 source.count = len;
128
129 /* Work through the sublist. */
130 while ((subreq = list_first_entry_or_null(
131 &sublist, struct netfs_io_subrequest, rreq_link))) {
132 list_del(&subreq->rreq_link);
133
134 subreq->source = NETFS_DOWNLOAD_FROM_SERVER;
135 subreq->start = start - subreq->transferred;
136 subreq->len = len + subreq->transferred;
137 stream0->sreq_max_len = subreq->len;
138
139 __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
140 __set_bit(NETFS_SREQ_RETRYING, &subreq->flags);
141
142 spin_lock_bh(&rreq->lock);
143 list_add_tail(&subreq->rreq_link, &rreq->subrequests);
144 subreq->prev_donated += rreq->prev_donated;
145 rreq->prev_donated = 0;
146 trace_netfs_sreq(subreq, netfs_sreq_trace_retry);
147 spin_unlock_bh(&rreq->lock);
148
149 BUG_ON(!len);
150
151 /* Renegotiate max_len (rsize) */
152 if (rreq->netfs_ops->prepare_read(subreq) < 0) {
153 trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed);
154 __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
155 }
156
157 part = umin(len, stream0->sreq_max_len);
158 if (unlikely(rreq->io_streams[0].sreq_max_segs))
159 part = netfs_limit_iter(&source, 0, part, stream0->sreq_max_segs);
160 subreq->len = subreq->transferred + part;
161 subreq->io_iter = source;
162 iov_iter_truncate(&subreq->io_iter, part);
163 iov_iter_advance(&source, part);
164 len -= part;
165 start += part;
166 if (!len) {
167 if (boundary)
168 __set_bit(NETFS_SREQ_BOUNDARY, &subreq->flags);
169 subreq->next_donated = deferred_next_donated;
170 } else {
171 __clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags);
172 subreq->next_donated = 0;
173 }
174
175 netfs_reissue_read(rreq, subreq);
176 if (!len)
177 break;
178
179 /* If we ran out of subrequests, allocate another. */
180 if (list_empty(&sublist)) {
181 subreq = netfs_alloc_subrequest(rreq);
182 if (!subreq)
183 goto abandon;
184 subreq->source = NETFS_DOWNLOAD_FROM_SERVER;
185 subreq->start = start;
186
187 /* We get two refs, but need just one. */
188 netfs_put_subrequest(subreq, false, netfs_sreq_trace_new);
189 trace_netfs_sreq(subreq, netfs_sreq_trace_split);
190 list_add_tail(&subreq->rreq_link, &sublist);
191 }
192 }
193
194 /* If we managed to use fewer subreqs, we can discard the
195 * excess.
196 */
197 while ((subreq = list_first_entry_or_null(
198 &sublist, struct netfs_io_subrequest, rreq_link))) {
199 trace_netfs_sreq(subreq, netfs_sreq_trace_discard);
200 list_del(&subreq->rreq_link);
201 netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_done);
202 }
203
204 } while (!list_empty(&queue));
205
206 return;
207
208 /* If we hit ENOMEM, fail all remaining subrequests */
209abandon:
210 list_splice_init(&sublist, &queue);
211 list_for_each_entry(subreq, &queue, rreq_link) {
212 if (!subreq->error)
213 subreq->error = -ENOMEM;
214 __clear_bit(NETFS_SREQ_FAILED, &subreq->flags);
215 __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
216 __clear_bit(NETFS_SREQ_RETRYING, &subreq->flags);
217 }
218 spin_lock_bh(&rreq->lock);
219 list_splice_tail_init(&queue, &rreq->subrequests);
220 spin_unlock_bh(&rreq->lock);
221}
222
223/*
224 * Retry reads.
225 */
226void netfs_retry_reads(struct netfs_io_request *rreq)
227{
228 trace_netfs_rreq(rreq, netfs_rreq_trace_resubmit);
229
230 atomic_inc(&rreq->nr_outstanding);
231
232 netfs_retry_read_subrequests(rreq);
233
234 if (atomic_dec_and_test(&rreq->nr_outstanding))
235 netfs_rreq_terminated(rreq, false);
236}
237
238/*
239 * Unlock any the pages that haven't been unlocked yet due to abandoned
240 * subrequests.
241 */
242void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq)
243{
244 struct folio_queue *p;
245
246 for (p = rreq->buffer; p; p = p->next) {
247 for (int slot = 0; slot < folioq_count(p); slot++) {
248 struct folio *folio = folioq_folio(p, slot);
249
250 if (folio && !folioq_is_marked2(p, slot)) {
251 trace_netfs_folio(folio, netfs_folio_trace_abandon);
252 folio_unlock(folio);
253 }
254 }
255 }
256}
fs/netfs/stats.c+4−2
@@ -32,6 +32,7 @@ atomic_t netfs_n_wh_buffered_write; · 31 unmodified lines
32atomic_t netfs_n_wh_writethrough;32atomic_t netfs_n_wh_writethrough;
33atomic_t netfs_n_wh_dio_write;33atomic_t netfs_n_wh_dio_write;
34atomic_t netfs_n_wh_writepages;34atomic_t netfs_n_wh_writepages;
35atomic_t netfs_n_wh_copy_to_cache;
35atomic_t netfs_n_wh_wstream_conflict;36atomic_t netfs_n_wh_wstream_conflict;
36atomic_t netfs_n_wh_upload;37atomic_t netfs_n_wh_upload;
37atomic_t netfs_n_wh_upload_done;38atomic_t netfs_n_wh_upload_done;
@@ -51,11 +52,12 @@ int netfs_stats_show(struct seq_file *m, void *v) · 13 unmodified lines
51 atomic_read(&netfs_n_rh_read_folio),52 atomic_read(&netfs_n_rh_read_folio),
52 atomic_read(&netfs_n_rh_write_begin),53 atomic_read(&netfs_n_rh_write_begin),
53 atomic_read(&netfs_n_rh_write_zskip));54 atomic_read(&netfs_n_rh_write_zskip));
54 seq_printf(m, "Writes : BW=%u WT=%u DW=%u WP=%u\n",55 seq_printf(m, "Writes : BW=%u WT=%u DW=%u WP=%u 2C=%u\n",
55 atomic_read(&netfs_n_wh_buffered_write),56 atomic_read(&netfs_n_wh_buffered_write),
56 atomic_read(&netfs_n_wh_writethrough),57 atomic_read(&netfs_n_wh_writethrough),
57 atomic_read(&netfs_n_wh_dio_write),58 atomic_read(&netfs_n_wh_dio_write),
58 atomic_read(&netfs_n_wh_writepages));59 atomic_read(&netfs_n_wh_writepages),
60 atomic_read(&netfs_n_wh_copy_to_cache));
59 seq_printf(m, "ZeroOps: ZR=%u sh=%u sk=%u\n",61 seq_printf(m, "ZeroOps: ZR=%u sh=%u sk=%u\n",
60 atomic_read(&netfs_n_rh_zero),62 atomic_read(&netfs_n_rh_zero),
61 atomic_read(&netfs_n_rh_short_read),63 atomic_read(&netfs_n_rh_short_read),
fs/netfs/write_collect.c+8−1
@@ -87,6 +87,12 @@ static void netfs_writeback_unlock_folios(struct netfs_io_request *wreq, · 86 unmodified lines
87 unsigned long long collected_to = wreq->collected_to;87 unsigned long long collected_to = wreq->collected_to;
88 unsigned int slot = wreq->buffer_head_slot;88 unsigned int slot = wreq->buffer_head_slot;
89 89
90 if (wreq->origin == NETFS_PGPRIV2_COPY_TO_CACHE) {
91 if (netfs_pgpriv2_unlock_copied_folios(wreq))
92 *notes |= MADE_PROGRESS;
93 return;
94 }
95
90 if (slot >= folioq_nr_slots(folioq)) {96 if (slot >= folioq_nr_slots(folioq)) {
91 folioq = netfs_delete_buffer_head(wreq);97 folioq = netfs_delete_buffer_head(wreq);
92 slot = 0;98 slot = 0;
@@ -383,7 +389,8 @@ reassess_streams: · 290 unmodified lines
383 smp_rmb();389 smp_rmb();
384 collected_to = ULLONG_MAX;390 collected_to = ULLONG_MAX;
385 if (wreq->origin == NETFS_WRITEBACK ||391 if (wreq->origin == NETFS_WRITEBACK ||
386 wreq->origin == NETFS_WRITETHROUGH)392 wreq->origin == NETFS_WRITETHROUGH ||
393 wreq->origin == NETFS_PGPRIV2_COPY_TO_CACHE)
387 notes = BUFFERED;394 notes = BUFFERED;
388 else395 else
389 notes = 0;396 notes = 0;
fs/netfs/write_issue.c+7−10
@@ -95,7 +95,8 @@ struct netfs_io_request *netfs_create_write_req(struct address_space *mapping, · 94 unmodified lines
95 struct netfs_io_request *wreq;95 struct netfs_io_request *wreq;
96 struct netfs_inode *ictx;96 struct netfs_inode *ictx;
97 bool is_buffered = (origin == NETFS_WRITEBACK ||97 bool is_buffered = (origin == NETFS_WRITEBACK ||
98 origin == NETFS_WRITETHROUGH);98 origin == NETFS_WRITETHROUGH ||
99 origin == NETFS_PGPRIV2_COPY_TO_CACHE);
99 100
100 wreq = netfs_alloc_request(mapping, file, start, 0, origin);101 wreq = netfs_alloc_request(mapping, file, start, 0, origin);
101 if (IS_ERR(wreq))102 if (IS_ERR(wreq))
@@ -161,10 +162,6 @@ static void netfs_prepare_write(struct netfs_io_request *wreq, · 59 unmodified lines
161 162
162 _enter("R=%x[%x]", wreq->debug_id, subreq->debug_index);163 _enter("R=%x[%x]", wreq->debug_id, subreq->debug_index);
163 164
164 trace_netfs_sreq_ref(wreq->debug_id, subreq->debug_index,
165 refcount_read(&subreq->ref),
166 netfs_sreq_trace_new);
167
168 trace_netfs_sreq(subreq, netfs_sreq_trace_prepare);165 trace_netfs_sreq(subreq, netfs_sreq_trace_prepare);
169 166
170 stream->sreq_max_len = UINT_MAX;167 stream->sreq_max_len = UINT_MAX;
@@ -241,8 +238,8 @@ void netfs_reissue_write(struct netfs_io_stream *stream, · 70 unmodified lines
241 netfs_do_issue_write(stream, subreq);238 netfs_do_issue_write(stream, subreq);
242}239}
243 240
244static void netfs_issue_write(struct netfs_io_request *wreq,241void netfs_issue_write(struct netfs_io_request *wreq,
245 struct netfs_io_stream *stream)242 struct netfs_io_stream *stream)
246{243{
247 struct netfs_io_subrequest *subreq = stream->construct;244 struct netfs_io_subrequest *subreq = stream->construct;
248 245
@@ -259,9 +256,9 @@ static void netfs_issue_write(struct netfs_io_request *wreq, · 10 unmodified lines
259 * we can avoid overrunning the credits obtained (cifs) and try to parallelise256 * we can avoid overrunning the credits obtained (cifs) and try to parallelise
260 * content-crypto preparation with network writes.257 * content-crypto preparation with network writes.
261 */258 */
262static int netfs_advance_write(struct netfs_io_request *wreq,259int netfs_advance_write(struct netfs_io_request *wreq,
263 struct netfs_io_stream *stream,260 struct netfs_io_stream *stream,
264 loff_t start, size_t len, bool to_eof)261 loff_t start, size_t len, bool to_eof)
265{262{
266 struct netfs_io_subrequest *subreq = stream->construct;263 struct netfs_io_subrequest *subreq = stream->construct;
267 size_t part;264 size_t part;
fs/nfs/fscache.c+6−13
@@ -267,6 +267,7 @@ static int nfs_netfs_init_request(struct netfs_io_request *rreq, struct file *fi · 266 unmodified lines
267 rreq->debug_id = atomic_inc_return(&nfs_netfs_debug_id);267 rreq->debug_id = atomic_inc_return(&nfs_netfs_debug_id);
268 /* [DEPRECATED] Use PG_private_2 to mark folio being written to the cache. */268 /* [DEPRECATED] Use PG_private_2 to mark folio being written to the cache. */
269 __set_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags);269 __set_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags);
270 rreq->io_streams[0].sreq_max_len = NFS_SB(rreq->inode->i_sb)->rsize;
270 271
271 return 0;272 return 0;
272}273}
@@ -288,14 +289,6 @@ static struct nfs_netfs_io_data *nfs_netfs_alloc(struct netfs_io_subrequest *sre · 15 unmodified lines
288 return netfs;289 return netfs;
289}290}
290 291
291static bool nfs_netfs_clamp_length(struct netfs_io_subrequest *sreq)
292{
293 size_t rsize = NFS_SB(sreq->rreq->inode->i_sb)->rsize;
294
295 sreq->len = min(sreq->len, rsize);
296 return true;
297}
298
299static void nfs_netfs_issue_read(struct netfs_io_subrequest *sreq)292static void nfs_netfs_issue_read(struct netfs_io_subrequest *sreq)
300{293{
301 struct nfs_netfs_io_data *netfs;294 struct nfs_netfs_io_data *netfs;
@@ -304,17 +297,18 @@ static void nfs_netfs_issue_read(struct netfs_io_subrequest *sreq) · 2 unmodified lines
304 struct nfs_open_context *ctx = sreq->rreq->netfs_priv;297 struct nfs_open_context *ctx = sreq->rreq->netfs_priv;
305 struct page *page;298 struct page *page;
306 unsigned long idx;299 unsigned long idx;
300 pgoff_t start, last;
307 int err;301 int err;
308 pgoff_t start = (sreq->start + sreq->transferred) >> PAGE_SHIFT;302
309 pgoff_t last = ((sreq->start + sreq->len -303 start = (sreq->start + sreq->transferred) >> PAGE_SHIFT;
310 sreq->transferred - 1) >> PAGE_SHIFT);304 last = ((sreq->start + sreq->len - sreq->transferred - 1) >> PAGE_SHIFT);
311 305
312 nfs_pageio_init_read(&pgio, inode, false,306 nfs_pageio_init_read(&pgio, inode, false,
313 &nfs_async_read_completion_ops);307 &nfs_async_read_completion_ops);
314 308
315 netfs = nfs_netfs_alloc(sreq);309 netfs = nfs_netfs_alloc(sreq);
316 if (!netfs)310 if (!netfs)
317 return netfs_subreq_terminated(sreq, -ENOMEM, false);311 return netfs_read_subreq_terminated(sreq, -ENOMEM, false);
318 312
319 pgio.pg_netfs = netfs; /* used in completion */313 pgio.pg_netfs = netfs; /* used in completion */
320 314
@@ -380,5 +374,4 @@ const struct netfs_request_ops nfs_netfs_ops = { · 59 unmodified lines
380 .init_request = nfs_netfs_init_request,374 .init_request = nfs_netfs_init_request,
381 .free_request = nfs_netfs_free_request,375 .free_request = nfs_netfs_free_request,
382 .issue_read = nfs_netfs_issue_read,376 .issue_read = nfs_netfs_issue_read,
383 .clamp_length = nfs_netfs_clamp_length
384};377};
fs/nfs/fscache.h+3−4
@@ -60,8 +60,6 @@ static inline void nfs_netfs_get(struct nfs_netfs_io_data *netfs) · 59 unmodified lines
60 60
61static inline void nfs_netfs_put(struct nfs_netfs_io_data *netfs)61static inline void nfs_netfs_put(struct nfs_netfs_io_data *netfs)
62{62{
63 ssize_t final_len;
64
65 /* Only the last RPC completion should call netfs_subreq_terminated() */63 /* Only the last RPC completion should call netfs_subreq_terminated() */
66 if (!refcount_dec_and_test(&netfs->refcount))64 if (!refcount_dec_and_test(&netfs->refcount))
67 return;65 return;
@@ -74,8 +72,9 @@ static inline void nfs_netfs_put(struct nfs_netfs_io_data *netfs) · 6 unmodified lines
74 * Correct the final length here to be no larger than the netfs subrequest72 * Correct the final length here to be no larger than the netfs subrequest
75 * length, and thus avoid netfs's "Subreq overread" warning message.73 * length, and thus avoid netfs's "Subreq overread" warning message.
76 */74 */
77 final_len = min_t(s64, netfs->sreq->len, atomic64_read(&netfs->transferred));75 netfs->sreq->transferred = min_t(s64, netfs->sreq->len,
78 netfs_subreq_terminated(netfs->sreq, netfs->error ?: final_len, false);76 atomic64_read(&netfs->transferred));
77 netfs_read_subreq_terminated(netfs->sreq, netfs->error, false);
79 kfree(netfs);78 kfree(netfs);
80}79}
81static inline void nfs_netfs_inode_init(struct nfs_inode *nfsi)80static inline void nfs_netfs_inode_init(struct nfs_inode *nfsi)
fs/smb/client/cifsglob.h+0−1
@@ -1485,7 +1485,6 @@ struct cifs_io_subrequest { · 1484 unmodified lines
1485 struct cifs_io_request *req;1485 struct cifs_io_request *req;
1486 };1486 };
1487 ssize_t got_bytes;1487 ssize_t got_bytes;
1488 size_t actual_len;
1489 unsigned int xid;1488 unsigned int xid;
1490 int result;1489 int result;
1491 bool have_xid;1490 bool have_xid;
fs/smb/client/cifssmb.c+2−4
@@ -1309,10 +1309,8 @@ cifs_readv_callback(struct mid_q_entry *mid) · 1308 unmodified lines
1309 if (rdata->result == 0 || rdata->result == -EAGAIN)1309 if (rdata->result == 0 || rdata->result == -EAGAIN)
1310 iov_iter_advance(&rdata->subreq.io_iter, rdata->got_bytes);1310 iov_iter_advance(&rdata->subreq.io_iter, rdata->got_bytes);
1311 rdata->credits.value = 0;1311 rdata->credits.value = 0;
1312 netfs_subreq_terminated(&rdata->subreq,1312 rdata->subreq.transferred += rdata->got_bytes;
1313 (rdata->result == 0 || rdata->result == -EAGAIN) ?1313 netfs_read_subreq_terminated(&rdata->subreq, rdata->result, false);
1314 rdata->got_bytes : rdata->result,
1315 false);
1316 release_mid(mid);1314 release_mid(mid);
1317 add_credits(server, &credits, 0);1315 add_credits(server, &credits, 0);
1318}1316}
fs/smb/client/file.c+28−54
@@ -112,7 +112,6 @@ static void cifs_issue_write(struct netfs_io_subrequest *subreq) · 111 unmodified lines
112 goto fail;112 goto fail;
113 }113 }
114 114
115 wdata->actual_len = wdata->subreq.len;
116 rc = adjust_credits(wdata->server, wdata, cifs_trace_rw_credits_issue_write_adjust);115 rc = adjust_credits(wdata->server, wdata, cifs_trace_rw_credits_issue_write_adjust);
117 if (rc)116 if (rc)
118 goto fail;117 goto fail;
@@ -141,25 +140,22 @@ static void cifs_netfs_invalidate_cache(struct netfs_io_request *wreq) · 22 unmodified lines
141}140}
142 141
143/*142/*
144 * Split the read up according to how many credits we can get for each piece.143 * Negotiate the size of a read operation on behalf of the netfs library.
145 * It's okay to sleep here if we need to wait for more credit to become
146 * available.
147 *
148 * We also choose the server and allocate an operation ID to be cleaned up
149 * later.
150 */144 */
151static bool cifs_clamp_length(struct netfs_io_subrequest *subreq)145static int cifs_prepare_read(struct netfs_io_subrequest *subreq)
152{146{
153 struct netfs_io_request *rreq = subreq->rreq;147 struct netfs_io_request *rreq = subreq->rreq;
154 struct netfs_io_stream *stream = &rreq->io_streams[subreq->stream_nr];
155 struct cifs_io_subrequest *rdata = container_of(subreq, struct cifs_io_subrequest, subreq);148 struct cifs_io_subrequest *rdata = container_of(subreq, struct cifs_io_subrequest, subreq);
156 struct cifs_io_request *req = container_of(subreq->rreq, struct cifs_io_request, rreq);149 struct cifs_io_request *req = container_of(subreq->rreq, struct cifs_io_request, rreq);
157 struct TCP_Server_Info *server = req->server;150 struct TCP_Server_Info *server = req->server;
158 struct cifs_sb_info *cifs_sb = CIFS_SB(rreq->inode->i_sb);151 struct cifs_sb_info *cifs_sb = CIFS_SB(rreq->inode->i_sb);
159 int rc;152 size_t size;
153 int rc = 0;
160 154
161 rdata->xid = get_xid();155 if (!rdata->have_xid) {
162 rdata->have_xid = true;156 rdata->xid = get_xid();
157 rdata->have_xid = true;
158 }
163 rdata->server = server;159 rdata->server = server;
164 160
165 if (cifs_sb->ctx->rsize == 0)161 if (cifs_sb->ctx->rsize == 0)
@@ -167,13 +163,12 @@ static bool cifs_clamp_length(struct netfs_io_subrequest *subreq) · 1 unmodified line
167 server->ops->negotiate_rsize(tlink_tcon(req->cfile->tlink),163 server->ops->negotiate_rsize(tlink_tcon(req->cfile->tlink),
168 cifs_sb->ctx);164 cifs_sb->ctx);
169 165
170
171 rc = server->ops->wait_mtu_credits(server, cifs_sb->ctx->rsize,166 rc = server->ops->wait_mtu_credits(server, cifs_sb->ctx->rsize,
172 &stream->sreq_max_len, &rdata->credits);167 &size, &rdata->credits);
173 if (rc) {168 if (rc)
174 subreq->error = rc;169 return rc;
175 return false;170
176 }171 rreq->io_streams[0].sreq_max_len = size;
177 172
178 rdata->credits.in_flight_check = 1;173 rdata->credits.in_flight_check = 1;
179 rdata->credits.rreq_debug_id = rreq->debug_id;174 rdata->credits.rreq_debug_id = rreq->debug_id;
@@ -185,14 +180,11 @@ static bool cifs_clamp_length(struct netfs_io_subrequest *subreq) · 5 unmodified lines
185 server->credits, server->in_flight, 0,180 server->credits, server->in_flight, 0,
186 cifs_trace_rw_credits_read_submit);181 cifs_trace_rw_credits_read_submit);
187 182
188 subreq->len = umin(subreq->len, stream->sreq_max_len);
189 rdata->actual_len = subreq->len;
190
191#ifdef CONFIG_CIFS_SMB_DIRECT183#ifdef CONFIG_CIFS_SMB_DIRECT
192 if (server->smbd_conn)184 if (server->smbd_conn)
193 stream->sreq_max_segs = server->smbd_conn->max_frmr_depth;185 rreq->io_streams[0].sreq_max_segs = server->smbd_conn->max_frmr_depth;
194#endif186#endif
195 return true;187 return 0;
196}188}
197 189
198/*190/*
@@ -201,59 +193,41 @@ static bool cifs_clamp_length(struct netfs_io_subrequest *subreq) · 2 unmodified lines
201 * to only read a portion of that, but as long as we read something, the netfs193 * to only read a portion of that, but as long as we read something, the netfs
202 * helper will call us again so that we can issue another read.194 * helper will call us again so that we can issue another read.
203 */195 */
204static void cifs_req_issue_read(struct netfs_io_subrequest *subreq)196static void cifs_issue_read(struct netfs_io_subrequest *subreq)
205{197{
206 struct netfs_io_request *rreq = subreq->rreq;198 struct netfs_io_request *rreq = subreq->rreq;
207 struct cifs_io_subrequest *rdata = container_of(subreq, struct cifs_io_subrequest, subreq);199 struct cifs_io_subrequest *rdata = container_of(subreq, struct cifs_io_subrequest, subreq);
208 struct cifs_io_request *req = container_of(subreq->rreq, struct cifs_io_request, rreq);200 struct cifs_io_request *req = container_of(subreq->rreq, struct cifs_io_request, rreq);
209 struct TCP_Server_Info *server = req->server;201 struct TCP_Server_Info *server = req->server;
210 struct cifs_sb_info *cifs_sb = CIFS_SB(rreq->inode->i_sb);
211 int rc = 0;202 int rc = 0;
212 203
213 cifs_dbg(FYI, "%s: op=%08x[%x] mapping=%p len=%zu/%zu\n",204 cifs_dbg(FYI, "%s: op=%08x[%x] mapping=%p len=%zu/%zu\n",
214 __func__, rreq->debug_id, subreq->debug_index, rreq->mapping,205 __func__, rreq->debug_id, subreq->debug_index, rreq->mapping,
215 subreq->transferred, subreq->len);206 subreq->transferred, subreq->len);
216 207
217 if (test_bit(NETFS_SREQ_RETRYING, &subreq->flags)) {208 rc = adjust_credits(server, rdata, cifs_trace_rw_credits_issue_read_adjust);
218 /*209 if (rc)
219 * As we're issuing a retry, we need to negotiate some new210 goto failed;
220 * credits otherwise the server may reject the op with
221 * INVALID_PARAMETER. Note, however, we may get back less
222 * credit than we need to complete the op, in which case, we
223 * shorten the op and rely on additional rounds of retry.
224 */
225 size_t rsize = umin(subreq->len - subreq->transferred,
226 cifs_sb->ctx->rsize);
227
228 rc = server->ops->wait_mtu_credits(server, rsize, &rdata->actual_len,
229 &rdata->credits);
230 if (rc)
231 goto out;
232
233 rdata->credits.in_flight_check = 1;
234
235 trace_smb3_rw_credits(rdata->rreq->debug_id,
236 rdata->subreq.debug_index,
237 rdata->credits.value,
238 server->credits, server->in_flight, 0,
239 cifs_trace_rw_credits_read_resubmit);
240 }
241 211
242 if (req->cfile->invalidHandle) {212 if (req->cfile->invalidHandle) {
243 do {213 do {
244 rc = cifs_reopen_file(req->cfile, true);214 rc = cifs_reopen_file(req->cfile, true);
245 } while (rc == -EAGAIN);215 } while (rc == -EAGAIN);
246 if (rc)216 if (rc)
247 goto out;217 goto failed;
248 }218 }
249 219
250 if (subreq->rreq->origin != NETFS_DIO_READ)220 if (subreq->rreq->origin != NETFS_DIO_READ)
251 __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags);221 __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags);
252 222
223 trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
253 rc = rdata->server->ops->async_readv(rdata);224 rc = rdata->server->ops->async_readv(rdata);
254out:
255 if (rc)225 if (rc)
256 netfs_subreq_terminated(subreq, rc, false);226 goto failed;
227 return;
228
229failed:
230 netfs_read_subreq_terminated(subreq, rc, false);
257}231}
258 232
259/*233/*
@@ -364,8 +338,8 @@ const struct netfs_request_ops cifs_req_ops = { · 104 unmodified lines
364 .init_request = cifs_init_request,338 .init_request = cifs_init_request,
365 .free_request = cifs_free_request,339 .free_request = cifs_free_request,
366 .free_subrequest = cifs_free_subrequest,340 .free_subrequest = cifs_free_subrequest,
367 .clamp_length = cifs_clamp_length,341 .prepare_read = cifs_prepare_read,
368 .issue_read = cifs_req_issue_read,342 .issue_read = cifs_issue_read,
369 .done = cifs_rreq_done,343 .done = cifs_rreq_done,
370 .begin_writeback = cifs_begin_writeback,344 .begin_writeback = cifs_begin_writeback,
371 .prepare_write = cifs_prepare_write,345 .prepare_write = cifs_prepare_write,
fs/smb/client/smb2ops.c+2−1
@@ -301,7 +301,8 @@ smb2_adjust_credits(struct TCP_Server_Info *server, · 300 unmodified lines
301 unsigned int /*enum smb3_rw_credits_trace*/ trace)301 unsigned int /*enum smb3_rw_credits_trace*/ trace)
302{302{
303 struct cifs_credits *credits = &subreq->credits;303 struct cifs_credits *credits = &subreq->credits;
304 int new_val = DIV_ROUND_UP(subreq->actual_len, SMB2_MAX_BUFFER_SIZE);304 int new_val = DIV_ROUND_UP(subreq->subreq.len - subreq->subreq.transferred,
305 SMB2_MAX_BUFFER_SIZE);
305 int scredits, in_flight;306 int scredits, in_flight;
306 307
307 if (!credits->value || credits->value == new_val)308 if (!credits->value || credits->value == new_val)
fs/smb/client/smb2pdu.c+16−11
@@ -4498,9 +4498,7 @@ static void smb2_readv_worker(struct work_struct *work) · 4497 unmodified lines
4498 struct cifs_io_subrequest *rdata =4498 struct cifs_io_subrequest *rdata =
4499 container_of(work, struct cifs_io_subrequest, subreq.work);4499 container_of(work, struct cifs_io_subrequest, subreq.work);
4500 4500
4501 netfs_subreq_terminated(&rdata->subreq,4501 netfs_read_subreq_terminated(&rdata->subreq, rdata->result, false);
4502 (rdata->result == 0 || rdata->result == -EAGAIN) ?
4503 rdata->got_bytes : rdata->result, true);
4504}4502}
4505 4503
4506static void4504static void
@@ -4532,7 +4530,7 @@ smb2_readv_callback(struct mid_q_entry *mid) · 25 unmodified lines
4532 4530
4533 cifs_dbg(FYI, "%s: mid=%llu state=%d result=%d bytes=%zu/%zu\n",4531 cifs_dbg(FYI, "%s: mid=%llu state=%d result=%d bytes=%zu/%zu\n",
4534 __func__, mid->mid, mid->mid_state, rdata->result,4532 __func__, mid->mid, mid->mid_state, rdata->result,
4535 rdata->actual_len, rdata->subreq.len - rdata->subreq.transferred);4533 rdata->got_bytes, rdata->subreq.len - rdata->subreq.transferred);
4536 4534
4537 switch (mid->mid_state) {4535 switch (mid->mid_state) {
4538 case MID_RESPONSE_RECEIVED:4536 case MID_RESPONSE_RECEIVED:
@@ -4554,6 +4552,7 @@ smb2_readv_callback(struct mid_q_entry *mid) · 15 unmodified lines
4554 break;4552 break;
4555 case MID_REQUEST_SUBMITTED:4553 case MID_REQUEST_SUBMITTED:
4556 case MID_RETRY_NEEDED:4554 case MID_RETRY_NEEDED:
4555 __set_bit(NETFS_SREQ_NEED_RETRY, &rdata->subreq.flags);
4557 rdata->result = -EAGAIN;4556 rdata->result = -EAGAIN;
4558 if (server->sign && rdata->got_bytes)4557 if (server->sign && rdata->got_bytes)
4559 /* reset bytes number since we can not check a sign */4558 /* reset bytes number since we can not check a sign */
@@ -4588,7 +4587,7 @@ smb2_readv_callback(struct mid_q_entry *mid) · 28 unmodified lines
4588 rdata->req->cfile->fid.persistent_fid,4587 rdata->req->cfile->fid.persistent_fid,
4589 tcon->tid, tcon->ses->Suid,4588 tcon->tid, tcon->ses->Suid,
4590 rdata->subreq.start + rdata->subreq.transferred,4589 rdata->subreq.start + rdata->subreq.transferred,
4591 rdata->actual_len,4590 rdata->subreq.len - rdata->subreq.transferred,
4592 rdata->result);4591 rdata->result);
4593 } else4592 } else
4594 trace_smb3_read_done(rdata->rreq->debug_id,4593 trace_smb3_read_done(rdata->rreq->debug_id,
@@ -4603,9 +4602,9 @@ smb2_readv_callback(struct mid_q_entry *mid) · 8 unmodified lines
4603 __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags);4602 __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags);
4604 rdata->result = 0;4603 rdata->result = 0;
4605 } else {4604 } else {
4606 if (rdata->got_bytes < rdata->actual_len &&4605 size_t trans = rdata->subreq.transferred + rdata->got_bytes;
4607 rdata->subreq.start + rdata->subreq.transferred + rdata->got_bytes ==4606 if (trans < rdata->subreq.len &&
4608 ictx->remote_i_size) {4607 rdata->subreq.start + trans == ictx->remote_i_size) {
4609 __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags);4608 __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags);
4610 rdata->result = 0;4609 rdata->result = 0;
4611 }4610 }
@@ -4614,6 +4613,10 @@ smb2_readv_callback(struct mid_q_entry *mid) · 2 unmodified lines
4614 server->credits, server->in_flight,4613 server->credits, server->in_flight,
4615 0, cifs_trace_rw_credits_read_response_clear);4614 0, cifs_trace_rw_credits_read_response_clear);
4616 rdata->credits.value = 0;4615 rdata->credits.value = 0;
4616 rdata->subreq.transferred += rdata->got_bytes;
4617 if (rdata->subreq.start + rdata->subreq.transferred >= rdata->subreq.rreq->i_size)
4618 __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags);
4619 trace_netfs_sreq(&rdata->subreq, netfs_sreq_trace_io_progress);
4617 INIT_WORK(&rdata->subreq.work, smb2_readv_worker);4620 INIT_WORK(&rdata->subreq.work, smb2_readv_worker);
4618 queue_work(cifsiod_wq, &rdata->subreq.work);4621 queue_work(cifsiod_wq, &rdata->subreq.work);
4619 release_mid(mid);4622 release_mid(mid);
@@ -4648,7 +4651,7 @@ smb2_async_readv(struct cifs_io_subrequest *rdata) · 28 unmodified lines
4648 io_parms.tcon = tlink_tcon(rdata->req->cfile->tlink);4651 io_parms.tcon = tlink_tcon(rdata->req->cfile->tlink);
4649 io_parms.server = server = rdata->server;4652 io_parms.server = server = rdata->server;
4650 io_parms.offset = subreq->start + subreq->transferred;4653 io_parms.offset = subreq->start + subreq->transferred;
4651 io_parms.length = rdata->actual_len;4654 io_parms.length = subreq->len - subreq->transferred;
4652 io_parms.persistent_fid = rdata->req->cfile->fid.persistent_fid;4655 io_parms.persistent_fid = rdata->req->cfile->fid.persistent_fid;
4653 io_parms.volatile_fid = rdata->req->cfile->fid.volatile_fid;4656 io_parms.volatile_fid = rdata->req->cfile->fid.volatile_fid;
4654 io_parms.pid = rdata->req->pid;4657 io_parms.pid = rdata->req->pid;
@@ -4669,7 +4672,7 @@ smb2_async_readv(struct cifs_io_subrequest *rdata) · 14 unmodified lines
4669 shdr = (struct smb2_hdr *)buf;4672 shdr = (struct smb2_hdr *)buf;
4670 4673
4671 if (rdata->credits.value > 0) {4674 if (rdata->credits.value > 0) {
4672 shdr->CreditCharge = cpu_to_le16(DIV_ROUND_UP(rdata->actual_len,4675 shdr->CreditCharge = cpu_to_le16(DIV_ROUND_UP(io_parms.length,
4673 SMB2_MAX_BUFFER_SIZE));4676 SMB2_MAX_BUFFER_SIZE));
4674 credit_request = le16_to_cpu(shdr->CreditCharge) + 8;4677 credit_request = le16_to_cpu(shdr->CreditCharge) + 8;
4675 if (server->credits >= server->max_credits)4678 if (server->credits >= server->max_credits)
@@ -4697,7 +4700,8 @@ smb2_async_readv(struct cifs_io_subrequest *rdata) · 21 unmodified lines
4697 rdata->xid, io_parms.persistent_fid,4700 rdata->xid, io_parms.persistent_fid,
4698 io_parms.tcon->tid,4701 io_parms.tcon->tid,
4699 io_parms.tcon->ses->Suid,4702 io_parms.tcon->ses->Suid,
4700 io_parms.offset, rdata->actual_len, rc);4703 io_parms.offset,
4704 subreq->len - subreq->transferred, rc);
4701 }4705 }
4702 4706
4703async_readv_out:4707async_readv_out:
@@ -4880,6 +4884,7 @@ smb2_writev_callback(struct mid_q_entry *mid) · 176 unmodified lines
4880 server->credits, server->in_flight,4884 server->credits, server->in_flight,
4881 0, cifs_trace_rw_credits_write_response_clear);4885 0, cifs_trace_rw_credits_write_response_clear);
4882 wdata->credits.value = 0;4886 wdata->credits.value = 0;
4887 trace_netfs_sreq(&wdata->subreq, netfs_sreq_trace_io_progress);
4883 cifs_write_subrequest_terminated(wdata, result ?: written, true);4888 cifs_write_subrequest_terminated(wdata, result ?: written, true);
4884 release_mid(mid);4889 release_mid(mid);
4885 trace_smb3_rw_credits(rreq_debug_id, subreq_debug_index, 0,4890 trace_smb3_rw_credits(rreq_debug_id, subreq_debug_index, 0,
include/linux/folio_queue.h+18−0
@@ -27,6 +27,7 @@ struct folio_queue { · 26 unmodified lines
27 struct folio_queue *prev; /* Previous queue segment of NULL */27 struct folio_queue *prev; /* Previous queue segment of NULL */
28 unsigned long marks; /* 1-bit mark per folio */28 unsigned long marks; /* 1-bit mark per folio */
29 unsigned long marks2; /* Second 1-bit mark per folio */29 unsigned long marks2; /* Second 1-bit mark per folio */
30 unsigned long marks3; /* Third 1-bit mark per folio */
30#if PAGEVEC_SIZE > BITS_PER_LONG31#if PAGEVEC_SIZE > BITS_PER_LONG
31#error marks is not big enough32#error marks is not big enough
32#endif33#endif
@@ -39,6 +40,7 @@ static inline void folioq_init(struct folio_queue *folioq) · 6 unmodified lines
39 folioq->prev = NULL;40 folioq->prev = NULL;
40 folioq->marks = 0;41 folioq->marks = 0;
41 folioq->marks2 = 0;42 folioq->marks2 = 0;
43 folioq->marks3 = 0;
42}44}
43 45
44static inline unsigned int folioq_nr_slots(const struct folio_queue *folioq)46static inline unsigned int folioq_nr_slots(const struct folio_queue *folioq)
@@ -87,6 +89,21 @@ static inline void folioq_unmark2(struct folio_queue *folioq, unsigned int slot) · 42 unmodified lines
87 clear_bit(slot, &folioq->marks2);89 clear_bit(slot, &folioq->marks2);
88}90}
89 91
92static inline bool folioq_is_marked3(const struct folio_queue *folioq, unsigned int slot)
93{
94 return test_bit(slot, &folioq->marks3);
95}
96
97static inline void folioq_mark3(struct folio_queue *folioq, unsigned int slot)
98{
99 set_bit(slot, &folioq->marks3);
100}
101
102static inline void folioq_unmark3(struct folio_queue *folioq, unsigned int slot)
103{
104 clear_bit(slot, &folioq->marks3);
105}
106
90static inline unsigned int __folio_order(struct folio *folio)107static inline unsigned int __folio_order(struct folio *folio)
91{108{
92 if (!folio_test_large(folio))109 if (!folio_test_large(folio))
@@ -133,6 +150,7 @@ static inline void folioq_clear(struct folio_queue *folioq, unsigned int slot) · 40 unmodified lines
133 folioq->vec.folios[slot] = NULL;150 folioq->vec.folios[slot] = NULL;
134 folioq_unmark(folioq, slot);151 folioq_unmark(folioq, slot);
135 folioq_unmark2(folioq, slot);152 folioq_unmark2(folioq, slot);
153 folioq_unmark3(folioq, slot);
136}154}
137 155
138#endif /* _LINUX_FOLIO_QUEUE_H */156#endif /* _LINUX_FOLIO_QUEUE_H */
include/linux/netfs.h+18−8
@@ -178,36 +178,43 @@ struct netfs_io_subrequest { · 177 unmodified lines
178 unsigned long long start; /* Where to start the I/O */178 unsigned long long start; /* Where to start the I/O */
179 size_t len; /* Size of the I/O */179 size_t len; /* Size of the I/O */
180 size_t transferred; /* Amount of data transferred */180 size_t transferred; /* Amount of data transferred */
181 size_t consumed; /* Amount of read data consumed */
182 size_t prev_donated; /* Amount of data donated from previous subreq */
183 size_t next_donated; /* Amount of data donated from next subreq */
181 refcount_t ref;184 refcount_t ref;
182 short error; /* 0 or error that occurred */185 short error; /* 0 or error that occurred */
183 unsigned short debug_index; /* Index in list (for debugging output) */186 unsigned short debug_index; /* Index in list (for debugging output) */
184 unsigned int nr_segs; /* Number of segs in io_iter */187 unsigned int nr_segs; /* Number of segs in io_iter */
185 enum netfs_io_source source; /* Where to read from/write to */188 enum netfs_io_source source; /* Where to read from/write to */
186 unsigned char stream_nr; /* I/O stream this belongs to */189 unsigned char stream_nr; /* I/O stream this belongs to */
190 unsigned char curr_folioq_slot; /* Folio currently being read */
191 unsigned char curr_folio_order; /* Order of folio */
192 struct folio_queue *curr_folioq; /* Queue segment in which current folio resides */
187 unsigned long flags;193 unsigned long flags;
188#define NETFS_SREQ_COPY_TO_CACHE 0 /* Set if should copy the data to the cache */194#define NETFS_SREQ_COPY_TO_CACHE 0 /* Set if should copy the data to the cache */
189#define NETFS_SREQ_CLEAR_TAIL 1 /* Set if the rest of the read should be cleared */195#define NETFS_SREQ_CLEAR_TAIL 1 /* Set if the rest of the read should be cleared */
190#define NETFS_SREQ_SHORT_IO 2 /* Set if the I/O was short */
191#define NETFS_SREQ_SEEK_DATA_READ 3 /* Set if ->read() should SEEK_DATA first */196#define NETFS_SREQ_SEEK_DATA_READ 3 /* Set if ->read() should SEEK_DATA first */
192#define NETFS_SREQ_NO_PROGRESS 4 /* Set if we didn't manage to read any data */197#define NETFS_SREQ_NO_PROGRESS 4 /* Set if we didn't manage to read any data */
193#define NETFS_SREQ_ONDEMAND 5 /* Set if it's from on-demand read mode */198#define NETFS_SREQ_ONDEMAND 5 /* Set if it's from on-demand read mode */
194#define NETFS_SREQ_BOUNDARY 6 /* Set if ends on hard boundary (eg. ceph object) */199#define NETFS_SREQ_BOUNDARY 6 /* Set if ends on hard boundary (eg. ceph object) */
200#define NETFS_SREQ_HIT_EOF 7 /* Set if short due to EOF */
195#define NETFS_SREQ_IN_PROGRESS 8 /* Unlocked when the subrequest completes */201#define NETFS_SREQ_IN_PROGRESS 8 /* Unlocked when the subrequest completes */
196#define NETFS_SREQ_NEED_RETRY 9 /* Set if the filesystem requests a retry */202#define NETFS_SREQ_NEED_RETRY 9 /* Set if the filesystem requests a retry */
197#define NETFS_SREQ_RETRYING 10 /* Set if we're retrying */203#define NETFS_SREQ_RETRYING 10 /* Set if we're retrying */
198#define NETFS_SREQ_FAILED 11 /* Set if the subreq failed unretryably */204#define NETFS_SREQ_FAILED 11 /* Set if the subreq failed unretryably */
199#define NETFS_SREQ_HIT_EOF 12 /* Set if we hit the EOF */
200};205};
201 206
202enum netfs_io_origin {207enum netfs_io_origin {
203 NETFS_READAHEAD, /* This read was triggered by readahead */208 NETFS_READAHEAD, /* This read was triggered by readahead */
204 NETFS_READPAGE, /* This read is a synchronous read */209 NETFS_READPAGE, /* This read is a synchronous read */
210 NETFS_READ_GAPS, /* This read is a synchronous read to fill gaps */
205 NETFS_READ_FOR_WRITE, /* This read is to prepare a write */211 NETFS_READ_FOR_WRITE, /* This read is to prepare a write */
206 NETFS_DIO_READ, /* This is a direct I/O read */212 NETFS_DIO_READ, /* This is a direct I/O read */
207 NETFS_WRITEBACK, /* This write was triggered by writepages */213 NETFS_WRITEBACK, /* This write was triggered by writepages */
208 NETFS_WRITETHROUGH, /* This write was made by netfs_perform_write() */214 NETFS_WRITETHROUGH, /* This write was made by netfs_perform_write() */
209 NETFS_UNBUFFERED_WRITE, /* This is an unbuffered write */215 NETFS_UNBUFFERED_WRITE, /* This is an unbuffered write */
210 NETFS_DIO_WRITE, /* This is a direct I/O write */216 NETFS_DIO_WRITE, /* This is a direct I/O write */
217 NETFS_PGPRIV2_COPY_TO_CACHE, /* [DEPRECATED] This is writing read data to the cache */
211 nr__netfs_io_origin218 nr__netfs_io_origin
212} __mode(byte);219} __mode(byte);
213 220
@@ -224,6 +231,7 @@ struct netfs_io_request { · 10 unmodified lines
224 struct address_space *mapping; /* The mapping being accessed */231 struct address_space *mapping; /* The mapping being accessed */
225 struct kiocb *iocb; /* AIO completion vector */232 struct kiocb *iocb; /* AIO completion vector */
226 struct netfs_cache_resources cache_resources;233 struct netfs_cache_resources cache_resources;
234 struct readahead_control *ractl; /* Readahead descriptor */
227 struct list_head proc_link; /* Link in netfs_iorequests */235 struct list_head proc_link; /* Link in netfs_iorequests */
228 struct list_head subrequests; /* Contributory I/O operations */236 struct list_head subrequests; /* Contributory I/O operations */
229 struct netfs_io_stream io_streams[2]; /* Streams of parallel I/O operations */237 struct netfs_io_stream io_streams[2]; /* Streams of parallel I/O operations */
@@ -244,12 +252,10 @@ struct netfs_io_request { · 14 unmodified lines
244 unsigned int nr_group_rel; /* Number of refs to release on ->group */252 unsigned int nr_group_rel; /* Number of refs to release on ->group */
245 spinlock_t lock; /* Lock for queuing subreqs */253 spinlock_t lock; /* Lock for queuing subreqs */
246 atomic_t nr_outstanding; /* Number of ops in progress */254 atomic_t nr_outstanding; /* Number of ops in progress */
247 atomic_t nr_copy_ops; /* Number of copy-to-cache ops in progress */
248 size_t upper_len; /* Length can be extended to here */
249 unsigned long long submitted; /* Amount submitted for I/O so far */255 unsigned long long submitted; /* Amount submitted for I/O so far */
250 unsigned long long len; /* Length of the request */256 unsigned long long len; /* Length of the request */
251 size_t transferred; /* Amount to be indicated as transferred */257 size_t transferred; /* Amount to be indicated as transferred */
252 short error; /* 0 or error that occurred */258 long error; /* 0 or error that occurred */
253 enum netfs_io_origin origin; /* Origin of the request */259 enum netfs_io_origin origin; /* Origin of the request */
254 bool direct_bv_unpin; /* T if direct_bv[] must be unpinned */260 bool direct_bv_unpin; /* T if direct_bv[] must be unpinned */
255 u8 buffer_head_slot; /* First slot in ->buffer */261 u8 buffer_head_slot; /* First slot in ->buffer */
@@ -260,9 +266,9 @@ struct netfs_io_request { · 4 unmodified lines
260 unsigned long long collected_to; /* Point we've collected to */266 unsigned long long collected_to; /* Point we've collected to */
261 unsigned long long cleaned_to; /* Position we've cleaned folios to */267 unsigned long long cleaned_to; /* Position we've cleaned folios to */
262 pgoff_t no_unlock_folio; /* Don't unlock this folio after read */268 pgoff_t no_unlock_folio; /* Don't unlock this folio after read */
269 size_t prev_donated; /* Fallback for subreq->prev_donated */
263 refcount_t ref;270 refcount_t ref;
264 unsigned long flags;271 unsigned long flags;
265#define NETFS_RREQ_INCOMPLETE_IO 0 /* Some ioreqs terminated short or with error */
266#define NETFS_RREQ_COPY_TO_CACHE 1 /* Need to write to the cache */272#define NETFS_RREQ_COPY_TO_CACHE 1 /* Need to write to the cache */
267#define NETFS_RREQ_NO_UNLOCK_FOLIO 2 /* Don't unlock no_unlock_folio on completion */273#define NETFS_RREQ_NO_UNLOCK_FOLIO 2 /* Don't unlock no_unlock_folio on completion */
268#define NETFS_RREQ_DONT_UNLOCK_FOLIOS 3 /* Don't unlock the folios on completion */274#define NETFS_RREQ_DONT_UNLOCK_FOLIOS 3 /* Don't unlock the folios on completion */
@@ -274,6 +280,7 @@ struct netfs_io_request { · 5 unmodified lines
274#define NETFS_RREQ_PAUSE 11 /* Pause subrequest generation */280#define NETFS_RREQ_PAUSE 11 /* Pause subrequest generation */
275#define NETFS_RREQ_USE_IO_ITER 12 /* Use ->io_iter rather than ->i_pages */281#define NETFS_RREQ_USE_IO_ITER 12 /* Use ->io_iter rather than ->i_pages */
276#define NETFS_RREQ_ALL_QUEUED 13 /* All subreqs are now queued */282#define NETFS_RREQ_ALL_QUEUED 13 /* All subreqs are now queued */
283#define NETFS_RREQ_NEED_RETRY 14 /* Need to try retrying */
277#define NETFS_RREQ_USE_PGPRIV2 31 /* [DEPRECATED] Use PG_private_2 to mark284#define NETFS_RREQ_USE_PGPRIV2 31 /* [DEPRECATED] Use PG_private_2 to mark
278 * write to cache on read */285 * write to cache on read */
279 const struct netfs_request_ops *netfs_ops;286 const struct netfs_request_ops *netfs_ops;
@@ -292,7 +299,7 @@ struct netfs_request_ops { · 12 unmodified lines
292 299
293 /* Read request handling */300 /* Read request handling */
294 void (*expand_readahead)(struct netfs_io_request *rreq);301 void (*expand_readahead)(struct netfs_io_request *rreq);
295 bool (*clamp_length)(struct netfs_io_subrequest *subreq);302 int (*prepare_read)(struct netfs_io_subrequest *subreq);
296 void (*issue_read)(struct netfs_io_subrequest *subreq);303 void (*issue_read)(struct netfs_io_subrequest *subreq);
297 bool (*is_still_valid)(struct netfs_io_request *rreq);304 bool (*is_still_valid)(struct netfs_io_request *rreq);
298 int (*check_write_begin)(struct file *file, loff_t pos, unsigned len,305 int (*check_write_begin)(struct file *file, loff_t pos, unsigned len,
@@ -422,7 +429,10 @@ bool netfs_release_folio(struct folio *folio, gfp_t gfp); · 123 unmodified lines
422vm_fault_t netfs_page_mkwrite(struct vm_fault *vmf, struct netfs_group *netfs_group);429vm_fault_t netfs_page_mkwrite(struct vm_fault *vmf, struct netfs_group *netfs_group);
423 430
424/* (Sub)request management API. */431/* (Sub)request management API. */
425void netfs_subreq_terminated(struct netfs_io_subrequest *, ssize_t, bool);432void netfs_read_subreq_progress(struct netfs_io_subrequest *subreq,
433 bool was_async);
434void netfs_read_subreq_terminated(struct netfs_io_subrequest *subreq,
435 int error, bool was_async);
426void netfs_get_subrequest(struct netfs_io_subrequest *subreq,436void netfs_get_subrequest(struct netfs_io_subrequest *subreq,
427 enum netfs_sreq_ref_trace what);437 enum netfs_sreq_ref_trace what);
428void netfs_put_subrequest(struct netfs_io_subrequest *subreq,438void netfs_put_subrequest(struct netfs_io_subrequest *subreq,
include/trace/events/netfs.h+98−5
@@ -20,6 +20,7 @@ · 19 unmodified lines
20 EM(netfs_read_trace_expanded, "EXPANDED ") \20 EM(netfs_read_trace_expanded, "EXPANDED ") \
21 EM(netfs_read_trace_readahead, "READAHEAD") \21 EM(netfs_read_trace_readahead, "READAHEAD") \
22 EM(netfs_read_trace_readpage, "READPAGE ") \22 EM(netfs_read_trace_readpage, "READPAGE ") \
23 EM(netfs_read_trace_read_gaps, "READ-GAPS") \
23 EM(netfs_read_trace_prefetch_for_write, "PREFETCHW") \24 EM(netfs_read_trace_prefetch_for_write, "PREFETCHW") \
24 E_(netfs_read_trace_write_begin, "WRITEBEGN")25 E_(netfs_read_trace_write_begin, "WRITEBEGN")
25 26
@@ -33,12 +34,14 @@ · 7 unmodified lines
33#define netfs_rreq_origins \34#define netfs_rreq_origins \
34 EM(NETFS_READAHEAD, "RA") \35 EM(NETFS_READAHEAD, "RA") \
35 EM(NETFS_READPAGE, "RP") \36 EM(NETFS_READPAGE, "RP") \
37 EM(NETFS_READ_GAPS, "RG") \
36 EM(NETFS_READ_FOR_WRITE, "RW") \38 EM(NETFS_READ_FOR_WRITE, "RW") \
37 EM(NETFS_DIO_READ, "DR") \39 EM(NETFS_DIO_READ, "DR") \
38 EM(NETFS_WRITEBACK, "WB") \40 EM(NETFS_WRITEBACK, "WB") \
39 EM(NETFS_WRITETHROUGH, "WT") \41 EM(NETFS_WRITETHROUGH, "WT") \
40 EM(NETFS_UNBUFFERED_WRITE, "UW") \42 EM(NETFS_UNBUFFERED_WRITE, "UW") \
41 E_(NETFS_DIO_WRITE, "DW")43 EM(NETFS_DIO_WRITE, "DW") \
44 E_(NETFS_PGPRIV2_COPY_TO_CACHE, "2C")
42 45
43#define netfs_rreq_traces \46#define netfs_rreq_traces \
44 EM(netfs_rreq_trace_assess, "ASSESS ") \47 EM(netfs_rreq_trace_assess, "ASSESS ") \
@@ -69,15 +72,25 @@ · 24 unmodified lines
69 E_(NETFS_INVALID_WRITE, "INVL")72 E_(NETFS_INVALID_WRITE, "INVL")
70 73
71#define netfs_sreq_traces \74#define netfs_sreq_traces \
75 EM(netfs_sreq_trace_add_donations, "+DON ") \
76 EM(netfs_sreq_trace_added, "ADD ") \
77 EM(netfs_sreq_trace_clear, "CLEAR") \
72 EM(netfs_sreq_trace_discard, "DSCRD") \78 EM(netfs_sreq_trace_discard, "DSCRD") \
79 EM(netfs_sreq_trace_donate_to_prev, "DON-P") \
80 EM(netfs_sreq_trace_donate_to_next, "DON-N") \
73 EM(netfs_sreq_trace_download_instead, "RDOWN") \81 EM(netfs_sreq_trace_download_instead, "RDOWN") \
74 EM(netfs_sreq_trace_fail, "FAIL ") \82 EM(netfs_sreq_trace_fail, "FAIL ") \
75 EM(netfs_sreq_trace_free, "FREE ") \83 EM(netfs_sreq_trace_free, "FREE ") \
84 EM(netfs_sreq_trace_hit_eof, "EOF ") \
85 EM(netfs_sreq_trace_io_progress, "IO ") \
76 EM(netfs_sreq_trace_limited, "LIMIT") \86 EM(netfs_sreq_trace_limited, "LIMIT") \
77 EM(netfs_sreq_trace_prepare, "PREP ") \87 EM(netfs_sreq_trace_prepare, "PREP ") \
78 EM(netfs_sreq_trace_prep_failed, "PRPFL") \88 EM(netfs_sreq_trace_prep_failed, "PRPFL") \
79 EM(netfs_sreq_trace_resubmit_short, "SHORT") \89 EM(netfs_sreq_trace_progress, "PRGRS") \
90 EM(netfs_sreq_trace_reprep_failed, "REPFL") \
80 EM(netfs_sreq_trace_retry, "RETRY") \91 EM(netfs_sreq_trace_retry, "RETRY") \
92 EM(netfs_sreq_trace_short, "SHORT") \
93 EM(netfs_sreq_trace_split, "SPLIT") \
81 EM(netfs_sreq_trace_submit, "SUBMT") \94 EM(netfs_sreq_trace_submit, "SUBMT") \
82 EM(netfs_sreq_trace_terminated, "TERM ") \95 EM(netfs_sreq_trace_terminated, "TERM ") \
83 EM(netfs_sreq_trace_write, "WRITE") \96 EM(netfs_sreq_trace_write, "WRITE") \
@@ -118,7 +131,7 @@ · 34 unmodified lines
118 EM(netfs_sreq_trace_new, "NEW ") \131 EM(netfs_sreq_trace_new, "NEW ") \
119 EM(netfs_sreq_trace_put_cancel, "PUT CANCEL ") \132 EM(netfs_sreq_trace_put_cancel, "PUT CANCEL ") \
120 EM(netfs_sreq_trace_put_clear, "PUT CLEAR ") \133 EM(netfs_sreq_trace_put_clear, "PUT CLEAR ") \
121 EM(netfs_sreq_trace_put_discard, "PUT DISCARD") \134 EM(netfs_sreq_trace_put_consumed, "PUT CONSUME") \
122 EM(netfs_sreq_trace_put_done, "PUT DONE ") \135 EM(netfs_sreq_trace_put_done, "PUT DONE ") \
123 EM(netfs_sreq_trace_put_failed, "PUT FAILED ") \136 EM(netfs_sreq_trace_put_failed, "PUT FAILED ") \
124 EM(netfs_sreq_trace_put_merged, "PUT MERGED ") \137 EM(netfs_sreq_trace_put_merged, "PUT MERGED ") \
@@ -138,6 +151,7 @@ · 13 unmodified lines
138 EM(netfs_flush_content, "flush") \151 EM(netfs_flush_content, "flush") \
139 EM(netfs_streaming_filled_page, "mod-streamw-f") \152 EM(netfs_streaming_filled_page, "mod-streamw-f") \
140 EM(netfs_streaming_cont_filled_page, "mod-streamw-f+") \153 EM(netfs_streaming_cont_filled_page, "mod-streamw-f+") \
154 EM(netfs_folio_trace_abandon, "abandon") \
141 EM(netfs_folio_trace_cancel_copy, "cancel-copy") \155 EM(netfs_folio_trace_cancel_copy, "cancel-copy") \
142 EM(netfs_folio_trace_clear, "clear") \156 EM(netfs_folio_trace_clear, "clear") \
143 EM(netfs_folio_trace_clear_cc, "clear-cc") \157 EM(netfs_folio_trace_clear_cc, "clear-cc") \
@@ -154,7 +168,11 @@ · 10 unmodified lines
154 EM(netfs_folio_trace_mkwrite_plus, "mkwrite+") \168 EM(netfs_folio_trace_mkwrite_plus, "mkwrite+") \
155 EM(netfs_folio_trace_not_under_wback, "!wback") \169 EM(netfs_folio_trace_not_under_wback, "!wback") \
156 EM(netfs_folio_trace_put, "put") \170 EM(netfs_folio_trace_put, "put") \
171 EM(netfs_folio_trace_read, "read") \
172 EM(netfs_folio_trace_read_done, "read-done") \
157 EM(netfs_folio_trace_read_gaps, "read-gaps") \173 EM(netfs_folio_trace_read_gaps, "read-gaps") \
174 EM(netfs_folio_trace_read_put, "read-put") \
175 EM(netfs_folio_trace_read_unlock, "read-unlock") \
158 EM(netfs_folio_trace_redirtied, "redirtied") \176 EM(netfs_folio_trace_redirtied, "redirtied") \
159 EM(netfs_folio_trace_store, "store") \177 EM(netfs_folio_trace_store, "store") \
160 EM(netfs_folio_trace_store_copy, "store-copy") \178 EM(netfs_folio_trace_store_copy, "store-copy") \
@@ -167,6 +185,12 @@ · 6 unmodified lines
167 EM(netfs_contig_trace_jump, "-->JUMP-->") \185 EM(netfs_contig_trace_jump, "-->JUMP-->") \
168 E_(netfs_contig_trace_unlock, "Unlock")186 E_(netfs_contig_trace_unlock, "Unlock")
169 187
188#define netfs_donate_traces \
189 EM(netfs_trace_donate_tail_to_prev, "tail-to-prev") \
190 EM(netfs_trace_donate_to_prev, "to-prev") \
191 EM(netfs_trace_donate_to_next, "to-next") \
192 E_(netfs_trace_donate_to_deferred_next, "defer-next")
193
170#ifndef __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY194#ifndef __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY
171#define __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY195#define __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY
172 196
@@ -184,6 +208,7 @@ enum netfs_rreq_ref_trace { netfs_rreq_ref_traces } __mode(byte); · 11 unmodified lines
184enum netfs_sreq_ref_trace { netfs_sreq_ref_traces } __mode(byte);208enum netfs_sreq_ref_trace { netfs_sreq_ref_traces } __mode(byte);
185enum netfs_folio_trace { netfs_folio_traces } __mode(byte);209enum netfs_folio_trace { netfs_folio_traces } __mode(byte);
186enum netfs_collect_contig_trace { netfs_collect_contig_traces } __mode(byte);210enum netfs_collect_contig_trace { netfs_collect_contig_traces } __mode(byte);
211enum netfs_donate_trace { netfs_donate_traces } __mode(byte);
187 212
188#endif213#endif
189 214
@@ -206,6 +231,7 @@ netfs_rreq_ref_traces; · 16 unmodified lines
206netfs_sreq_ref_traces;231netfs_sreq_ref_traces;
207netfs_folio_traces;232netfs_folio_traces;
208netfs_collect_contig_traces;233netfs_collect_contig_traces;
234netfs_donate_traces;
209 235
210/*236/*
211 * Now redefine the EM() and E_() macros to map the enums to the strings that237 * Now redefine the EM() and E_() macros to map the enums to the strings that
@@ -226,6 +252,7 @@ TRACE_EVENT(netfs_read, · 14 unmodified lines
226 TP_STRUCT__entry(252 TP_STRUCT__entry(
227 __field(unsigned int, rreq )253 __field(unsigned int, rreq )
228 __field(unsigned int, cookie )254 __field(unsigned int, cookie )
255 __field(loff_t, i_size )
229 __field(loff_t, start )256 __field(loff_t, start )
230 __field(size_t, len )257 __field(size_t, len )
231 __field(enum netfs_read_trace, what )258 __field(enum netfs_read_trace, what )
@@ -235,18 +262,19 @@ TRACE_EVENT(netfs_read, · 3 unmodified lines
235 TP_fast_assign(262 TP_fast_assign(
236 __entry->rreq = rreq->debug_id;263 __entry->rreq = rreq->debug_id;
237 __entry->cookie = rreq->cache_resources.debug_id;264 __entry->cookie = rreq->cache_resources.debug_id;
265 __entry->i_size = rreq->i_size;
238 __entry->start = start;266 __entry->start = start;
239 __entry->len = len;267 __entry->len = len;
240 __entry->what = what;268 __entry->what = what;
241 __entry->netfs_inode = rreq->inode->i_ino;269 __entry->netfs_inode = rreq->inode->i_ino;
242 ),270 ),
243 271
244 TP_printk("R=%08x %s c=%08x ni=%x s=%llx %zx",272 TP_printk("R=%08x %s c=%08x ni=%x s=%llx l=%zx sz=%llx",
245 __entry->rreq,273 __entry->rreq,
246 __print_symbolic(__entry->what, netfs_read_traces),274 __print_symbolic(__entry->what, netfs_read_traces),
247 __entry->cookie,275 __entry->cookie,
248 __entry->netfs_inode,276 __entry->netfs_inode,
249 __entry->start, __entry->len)277 __entry->start, __entry->len, __entry->i_size)
250 );278 );
251 279
252TRACE_EVENT(netfs_rreq,280TRACE_EVENT(netfs_rreq,
@@ -651,6 +679,71 @@ TRACE_EVENT(netfs_collect_stream, · 398 unmodified lines
651 __entry->collected_to, __entry->front)679 __entry->collected_to, __entry->front)
652 );680 );
653 681
682TRACE_EVENT(netfs_progress,
683 TP_PROTO(const struct netfs_io_subrequest *subreq,
684 unsigned long long start, size_t avail, size_t part),
685
686 TP_ARGS(subreq, start, avail, part),
687
688 TP_STRUCT__entry(
689 __field(unsigned int, rreq)
690 __field(unsigned int, subreq)
691 __field(unsigned int, consumed)
692 __field(unsigned int, transferred)
693 __field(unsigned long long, f_start)
694 __field(unsigned int, f_avail)
695 __field(unsigned int, f_part)
696 __field(unsigned char, slot)
697 ),
698
699 TP_fast_assign(
700 __entry->rreq = subreq->rreq->debug_id;
701 __entry->subreq = subreq->debug_index;
702 __entry->consumed = subreq->consumed;
703 __entry->transferred = subreq->transferred;
704 __entry->f_start = start;
705 __entry->f_avail = avail;
706 __entry->f_part = part;
707 __entry->slot = subreq->curr_folioq_slot;
708 ),
709
710 TP_printk("R=%08x[%02x] s=%llx ct=%x/%x pa=%x/%x sl=%x",
711 __entry->rreq, __entry->subreq, __entry->f_start,
712 __entry->consumed, __entry->transferred,
713 __entry->f_part, __entry->f_avail, __entry->slot)
714 );
715
716TRACE_EVENT(netfs_donate,
717 TP_PROTO(const struct netfs_io_request *rreq,
718 const struct netfs_io_subrequest *from,
719 const struct netfs_io_subrequest *to,
720 size_t amount,
721 enum netfs_donate_trace trace),
722
723 TP_ARGS(rreq, from, to, amount, trace),
724
725 TP_STRUCT__entry(
726 __field(unsigned int, rreq)
727 __field(unsigned int, from)
728 __field(unsigned int, to)
729 __field(unsigned int, amount)
730 __field(enum netfs_donate_trace, trace)
731 ),
732
733 TP_fast_assign(
734 __entry->rreq = rreq->debug_id;
735 __entry->from = from->debug_index;
736 __entry->to = to ? to->debug_index : -1;
737 __entry->amount = amount;
738 __entry->trace = trace;
739 ),
740
741 TP_printk("R=%08x[%02x] -> [%02x] %s am=%x",
742 __entry->rreq, __entry->from, __entry->to,
743 __print_symbolic(__entry->trace, netfs_donate_traces),
744 __entry->amount)
745 );
746
654#undef EM747#undef EM
655#undef E_748#undef E_
656#endif /* _TRACE_NETFS_H */749#endif /* _TRACE_NETFS_H */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t