Improve the efficiency of buffered reads in a number of ways: (1) Overhaul the algorithm in general so that it's a lot more compact and split the read submission code between buffered and unbuffered versions. The unbuffered version can be vastly simplified. (2) Read-result collection is handed off to a work queue rather than being done in the I/O thread. Multiple subrequests can be processes simultaneously. (3) When a subrequest is collected, any folios it fully spans are collected and "spare" data on either side is donated to either the previous or the next subrequest in the sequence. Notes: (*) Readahead expansion is massively slows down fio, presumably because it causes a load of extra allocations, both folio and xarray, up front before RPC requests can be transmitted. (*) RDMA with cifs does appear to work, both with SIW and RXE. (*) PG_private_2-based reading and copy-to-cache is split out into its own file and altered to use folio_queue. Note that the copy to the cache now creates a new write transaction against the cache and adds the folios to be copied into it. This allows it to use part of the writeback I/O code.
Unified · Split
| @@ -68,17 +68,22 @@ static void v9fs_issue_read(struct netfs_io_subrequest *subreq) · 67 unmodified lines | |||
| 68 | { | 68 | { |
| 69 | struct netfs_io_request *rreq = subreq->rreq; | 69 | struct netfs_io_request *rreq = subreq->rreq; |
| 70 | struct p9_fid *fid = rreq->netfs_priv; | 70 | struct p9_fid *fid = rreq->netfs_priv; |
| 71 | unsigned long long pos = subreq->start + subreq->transferred; | ||
| 71 | int total, err; | 72 | int total, err; |
| 72 | 73 | ||
| 73 | total = p9_client_read(fid, subreq->start + subreq->transferred, | 74 | total = p9_client_read(fid, pos, &subreq->io_iter, &err); |
| 74 | &subreq->io_iter, &err); | ||
| 75 | 75 | ||
| 76 | /* if we just extended the file size, any portion not in | 76 | /* if we just extended the file size, any portion not in |
| 77 | * cache won't be on server and is zeroes */ | 77 | * cache won't be on server and is zeroes */ |
| 78 | if (subreq->rreq->origin != NETFS_DIO_READ) | 78 | if (subreq->rreq->origin != NETFS_DIO_READ) |
| 79 | __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags); | 79 | __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags); |
| 80 | if (pos + total >= i_size_read(rreq->inode)) | ||
| 81 | __set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags); | ||
| 80 | 82 | ||
| 81 | netfs_subreq_terminated(subreq, err ?: total, false); | 83 | if (!err) |
| 84 | subreq->transferred += total; | ||
| 85 | |||
| 86 | netfs_read_subreq_terminated(subreq, err, false); | ||
| 82 | } | 87 | } |
| 83 | 88 | ||
| 84 | /** | 89 | /** |
| @@ -16,6 +16,7 @@ · 15 unmodified lines | |||
| 16 | #include <linux/mm.h> | 16 | #include <linux/mm.h> |
| 17 | #include <linux/swap.h> | 17 | #include <linux/swap.h> |
| 18 | #include <linux/netfs.h> | 18 | #include <linux/netfs.h> |
| 19 | #include <trace/events/netfs.h> | ||
| 19 | #include "internal.h" | 20 | #include "internal.h" |
| 20 | 21 | ||
| 21 | static int afs_file_mmap(struct file *file, struct vm_area_struct *vma); | 22 | static int afs_file_mmap(struct file *file, struct vm_area_struct *vma); |
| @@ -242,9 +243,10 @@ static void afs_fetch_data_notify(struct afs_operation *op) · 220 unmodified lines | |||
| 242 | 243 | ||
| 243 | req->error = error; | 244 | req->error = error; |
| 244 | if (subreq) { | 245 | if (subreq) { |
| 245 | if (subreq->rreq->origin != NETFS_DIO_READ) | 246 | subreq->rreq->i_size = req->file_size; |
| 246 | __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags); | 247 | if (req->pos + req->actual_len >= req->file_size) |
| 247 | netfs_subreq_terminated(subreq, error ?: req->actual_len, false); | 248 | __set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags); |
| 249 | netfs_read_subreq_terminated(subreq, error, false); | ||
| 248 | req->subreq = NULL; | 250 | req->subreq = NULL; |
| 249 | } else if (req->done) { | 251 | } else if (req->done) { |
| 250 | req->done(req); | 252 | req->done(req); |
| @@ -262,6 +264,12 @@ static void afs_fetch_data_success(struct afs_operation *op) · 11 unmodified lines | |||
| 262 | afs_fetch_data_notify(op); | 264 | afs_fetch_data_notify(op); |
| 263 | } | 265 | } |
| 264 | 266 | ||
| 267 | static void afs_fetch_data_aborted(struct afs_operation *op) | ||
| 268 | { | ||
| 269 | afs_check_for_remote_deletion(op); | ||
| 270 | afs_fetch_data_notify(op); | ||
| 271 | } | ||
| 272 | |||
| 265 | static void afs_fetch_data_put(struct afs_operation *op) | 273 | static void afs_fetch_data_put(struct afs_operation *op) |
| 266 | { | 274 | { |
| 267 | op->fetch.req->error = afs_op_error(op); | 275 | op->fetch.req->error = afs_op_error(op); |
| @@ -272,7 +280,7 @@ static const struct afs_operation_ops afs_fetch_data_operation = { · 4 unmodified lines | |||
| 272 | .issue_afs_rpc = afs_fs_fetch_data, | 280 | .issue_afs_rpc = afs_fs_fetch_data, |
| 273 | .issue_yfs_rpc = yfs_fs_fetch_data, | 281 | .issue_yfs_rpc = yfs_fs_fetch_data, |
| 274 | .success = afs_fetch_data_success, | 282 | .success = afs_fetch_data_success, |
| 275 | .aborted = afs_check_for_remote_deletion, | 283 | .aborted = afs_fetch_data_aborted, |
| 276 | .failed = afs_fetch_data_notify, | 284 | .failed = afs_fetch_data_notify, |
| 277 | .put = afs_fetch_data_put, | 285 | .put = afs_fetch_data_put, |
| 278 | }; | 286 | }; |
| @@ -294,7 +302,7 @@ int afs_fetch_data(struct afs_vnode *vnode, struct afs_read *req) · 15 unmodified lines | |||
| 294 | op = afs_alloc_operation(req->key, vnode->volume); | 302 | op = afs_alloc_operation(req->key, vnode->volume); |
| 295 | if (IS_ERR(op)) { | 303 | if (IS_ERR(op)) { |
| 296 | if (req->subreq) | 304 | if (req->subreq) |
| 297 | netfs_subreq_terminated(req->subreq, PTR_ERR(op), false); | 305 | netfs_read_subreq_terminated(req->subreq, PTR_ERR(op), false); |
| 298 | return PTR_ERR(op); | 306 | return PTR_ERR(op); |
| 299 | } | 307 | } |
| 300 | 308 | ||
| @@ -313,7 +321,7 @@ static void afs_read_worker(struct work_struct *work) · 12 unmodified lines | |||
| 313 | 321 | ||
| 314 | fsreq = afs_alloc_read(GFP_NOFS); | 322 | fsreq = afs_alloc_read(GFP_NOFS); |
| 315 | if (!fsreq) | 323 | if (!fsreq) |
| 316 | return netfs_subreq_terminated(subreq, -ENOMEM, false); | 324 | return netfs_read_subreq_terminated(subreq, -ENOMEM, false); |
| 317 | 325 | ||
| 318 | fsreq->subreq = subreq; | 326 | fsreq->subreq = subreq; |
| 319 | fsreq->pos = subreq->start + subreq->transferred; | 327 | fsreq->pos = subreq->start + subreq->transferred; |
| @@ -322,6 +330,7 @@ static void afs_read_worker(struct work_struct *work) · 2 unmodified lines | |||
| 322 | fsreq->vnode = vnode; | 330 | fsreq->vnode = vnode; |
| 323 | fsreq->iter = &subreq->io_iter; | 331 | fsreq->iter = &subreq->io_iter; |
| 324 | 332 | ||
| 333 | trace_netfs_sreq(subreq, netfs_sreq_trace_submit); | ||
| 325 | afs_fetch_data(fsreq->vnode, fsreq); | 334 | afs_fetch_data(fsreq->vnode, fsreq); |
| 326 | afs_put_read(fsreq); | 335 | afs_put_read(fsreq); |
| 327 | } | 336 | } |
| @@ -304,6 +304,7 @@ static int afs_deliver_fs_fetch_data(struct afs_call *call) · 303 unmodified lines | |||
| 304 | struct afs_vnode_param *vp = &op->file[0]; | 304 | struct afs_vnode_param *vp = &op->file[0]; |
| 305 | struct afs_read *req = op->fetch.req; | 305 | struct afs_read *req = op->fetch.req; |
| 306 | const __be32 *bp; | 306 | const __be32 *bp; |
| 307 | size_t count_before; | ||
| 307 | int ret; | 308 | int ret; |
| 308 | 309 | ||
| 309 | _enter("{%u,%zu,%zu/%llu}", | 310 | _enter("{%u,%zu,%zu/%llu}", |
| @@ -345,10 +346,14 @@ static int afs_deliver_fs_fetch_data(struct afs_call *call) · 35 unmodified lines | |||
| 345 | 346 | ||
| 346 | /* extract the returned data */ | 347 | /* extract the returned data */ |
| 347 | case 2: | 348 | case 2: |
| 348 | _debug("extract data %zu/%llu", | 349 | count_before = call->iov_len; |
| 349 | iov_iter_count(call->iter), req->actual_len); | 350 | _debug("extract data %zu/%llu", count_before, req->actual_len); |
| 350 | 351 | ||
| 351 | ret = afs_extract_data(call, true); | 352 | ret = afs_extract_data(call, true); |
| 353 | if (req->subreq) { | ||
| 354 | req->subreq->transferred += count_before - call->iov_len; | ||
| 355 | netfs_read_subreq_progress(req->subreq, false); | ||
| 356 | } | ||
| 352 | if (ret < 0) | 357 | if (ret < 0) |
| 353 | return ret; | 358 | return ret; |
| 354 | 359 | ||
| @@ -355,6 +355,7 @@ static int yfs_deliver_fs_fetch_data64(struct afs_call *call) · 354 unmodified lines | |||
| 355 | struct afs_vnode_param *vp = &op->file[0]; | 355 | struct afs_vnode_param *vp = &op->file[0]; |
| 356 | struct afs_read *req = op->fetch.req; | 356 | struct afs_read *req = op->fetch.req; |
| 357 | const __be32 *bp; | 357 | const __be32 *bp; |
| 358 | size_t count_before; | ||
| 358 | int ret; | 359 | int ret; |
| 359 | 360 | ||
| 360 | _enter("{%u,%zu, %zu/%llu}", | 361 | _enter("{%u,%zu, %zu/%llu}", |
| @@ -391,10 +392,14 @@ static int yfs_deliver_fs_fetch_data64(struct afs_call *call) · 30 unmodified lines | |||
| 391 | 392 | ||
| 392 | /* extract the returned data */ | 393 | /* extract the returned data */ |
| 393 | case 2: | 394 | case 2: |
| 394 | _debug("extract data %zu/%llu", | 395 | count_before = call->iov_len; |
| 395 | iov_iter_count(call->iter), req->actual_len); | 396 | _debug("extract data %zu/%llu", count_before, req->actual_len); |
| 396 | 397 | ||
| 397 | ret = afs_extract_data(call, true); | 398 | ret = afs_extract_data(call, true); |
| 399 | if (req->subreq) { | ||
| 400 | req->subreq->transferred += count_before - call->iov_len; | ||
| 401 | netfs_read_subreq_progress(req->subreq, false); | ||
| 402 | } | ||
| 398 | if (ret < 0) | 403 | if (ret < 0) |
| 399 | return ret; | 404 | return ret; |
| 400 | 405 | ||
| @@ -13,6 +13,7 @@ · 12 unmodified lines | |||
| 13 | #include <linux/iversion.h> | 13 | #include <linux/iversion.h> |
| 14 | #include <linux/ktime.h> | 14 | #include <linux/ktime.h> |
| 15 | #include <linux/netfs.h> | 15 | #include <linux/netfs.h> |
| 16 | #include <trace/events/netfs.h> | ||
| 16 | 17 | ||
| 17 | #include "super.h" | 18 | #include "super.h" |
| 18 | #include "mds_client.h" | 19 | #include "mds_client.h" |
| @@ -205,21 +206,6 @@ static void ceph_netfs_expand_readahead(struct netfs_io_request *rreq) · 186 unmodified lines | |||
| 205 | } | 206 | } |
| 206 | } | 207 | } |
| 207 | 208 | ||
| 208 | static bool ceph_netfs_clamp_length(struct netfs_io_subrequest *subreq) | ||
| 209 | { | ||
| 210 | struct inode *inode = subreq->rreq->inode; | ||
| 211 | struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); | ||
| 212 | struct ceph_inode_info *ci = ceph_inode(inode); | ||
| 213 | u64 objno, objoff; | ||
| 214 | u32 xlen; | ||
| 215 | |||
| 216 | /* Truncate the extent at the end of the current block */ | ||
| 217 | ceph_calc_file_object_mapping(&ci->i_layout, subreq->start, subreq->len, | ||
| 218 | &objno, &objoff, &xlen); | ||
| 219 | subreq->len = min(xlen, fsc->mount_options->rsize); | ||
| 220 | return true; | ||
| 221 | } | ||
| 222 | |||
| 223 | static void finish_netfs_read(struct ceph_osd_request *req) | 209 | static void finish_netfs_read(struct ceph_osd_request *req) |
| 224 | { | 210 | { |
| 225 | struct inode *inode = req->r_inode; | 211 | struct inode *inode = req->r_inode; |
| @@ -264,7 +250,12 @@ static void finish_netfs_read(struct ceph_osd_request *req) · 38 unmodified lines | |||
| 264 | calc_pages_for(osd_data->alignment, | 250 | calc_pages_for(osd_data->alignment, |
| 265 | osd_data->length), false); | 251 | osd_data->length), false); |
| 266 | } | 252 | } |
| 267 | netfs_subreq_terminated(subreq, err, false); | 253 | if (err > 0) { |
| 254 | subreq->transferred = err; | ||
| 255 | err = 0; | ||
| 256 | } | ||
| 257 | trace_netfs_sreq(subreq, netfs_sreq_trace_io_progress); | ||
| 258 | netfs_read_subreq_terminated(subreq, err, false); | ||
| 268 | iput(req->r_inode); | 259 | iput(req->r_inode); |
| 269 | ceph_dec_osd_stopping_blocker(fsc->mdsc); | 260 | ceph_dec_osd_stopping_blocker(fsc->mdsc); |
| 270 | } | 261 | } |
| @@ -278,7 +269,6 @@ static bool ceph_netfs_issue_op_inline(struct netfs_io_subrequest *subreq) · 7 unmodified lines | |||
| 278 | struct ceph_mds_request *req; | 269 | struct ceph_mds_request *req; |
| 279 | struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb); | 270 | struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb); |
| 280 | struct ceph_inode_info *ci = ceph_inode(inode); | 271 | struct ceph_inode_info *ci = ceph_inode(inode); |
| 281 | struct iov_iter iter; | ||
| 282 | ssize_t err = 0; | 272 | ssize_t err = 0; |
| 283 | size_t len; | 273 | size_t len; |
| 284 | int mode; | 274 | int mode; |
| @@ -301,6 +291,7 @@ static bool ceph_netfs_issue_op_inline(struct netfs_io_subrequest *subreq) · 16 unmodified lines | |||
| 301 | req->r_args.getattr.mask = cpu_to_le32(CEPH_STAT_CAP_INLINE_DATA); | 291 | req->r_args.getattr.mask = cpu_to_le32(CEPH_STAT_CAP_INLINE_DATA); |
| 302 | req->r_num_caps = 2; | 292 | req->r_num_caps = 2; |
| 303 | 293 | ||
| 294 | trace_netfs_sreq(subreq, netfs_sreq_trace_submit); | ||
| 304 | err = ceph_mdsc_do_request(mdsc, NULL, req); | 295 | err = ceph_mdsc_do_request(mdsc, NULL, req); |
| 305 | if (err < 0) | 296 | if (err < 0) |
| 306 | goto out; | 297 | goto out; |
| @@ -314,17 +305,36 @@ static bool ceph_netfs_issue_op_inline(struct netfs_io_subrequest *subreq) · 7 unmodified lines | |||
| 314 | } | 305 | } |
| 315 | 306 | ||
| 316 | len = min_t(size_t, iinfo->inline_len - subreq->start, subreq->len); | 307 | len = min_t(size_t, iinfo->inline_len - subreq->start, subreq->len); |
| 317 | iov_iter_xarray(&iter, ITER_DEST, &rreq->mapping->i_pages, subreq->start, len); | 308 | err = copy_to_iter(iinfo->inline_data + subreq->start, len, &subreq->io_iter); |
| 318 | err = copy_to_iter(iinfo->inline_data + subreq->start, len, &iter); | 309 | if (err == 0) { |
| 319 | if (err == 0) | ||
| 320 | err = -EFAULT; | 310 | err = -EFAULT; |
| 311 | } else { | ||
| 312 | subreq->transferred += err; | ||
| 313 | err = 0; | ||
| 314 | } | ||
| 321 | 315 | ||
| 322 | ceph_mdsc_put_request(req); | 316 | ceph_mdsc_put_request(req); |
| 323 | out: | 317 | out: |
| 324 | netfs_subreq_terminated(subreq, err, false); | 318 | netfs_read_subreq_terminated(subreq, err, false); |
| 325 | return true; | 319 | return true; |
| 326 | } | 320 | } |
| 327 | 321 | ||
| 322 | static int ceph_netfs_prepare_read(struct netfs_io_subrequest *subreq) | ||
| 323 | { | ||
| 324 | struct netfs_io_request *rreq = subreq->rreq; | ||
| 325 | struct inode *inode = rreq->inode; | ||
| 326 | struct ceph_inode_info *ci = ceph_inode(inode); | ||
| 327 | struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); | ||
| 328 | u64 objno, objoff; | ||
| 329 | u32 xlen; | ||
| 330 | |||
| 331 | /* Truncate the extent at the end of the current block */ | ||
| 332 | ceph_calc_file_object_mapping(&ci->i_layout, subreq->start, subreq->len, | ||
| 333 | &objno, &objoff, &xlen); | ||
| 334 | rreq->io_streams[0].sreq_max_len = umin(xlen, fsc->mount_options->rsize); | ||
| 335 | return 0; | ||
| 336 | } | ||
| 337 | |||
| 328 | static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) | 338 | static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) |
| 329 | { | 339 | { |
| 330 | struct netfs_io_request *rreq = subreq->rreq; | 340 | struct netfs_io_request *rreq = subreq->rreq; |
| @@ -334,9 +344,8 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) · 3 unmodified lines | |||
| 334 | struct ceph_client *cl = fsc->client; | 344 | struct ceph_client *cl = fsc->client; |
| 335 | struct ceph_osd_request *req = NULL; | 345 | struct ceph_osd_request *req = NULL; |
| 336 | struct ceph_vino vino = ceph_vino(inode); | 346 | struct ceph_vino vino = ceph_vino(inode); |
| 337 | struct iov_iter iter; | 347 | int err; |
| 338 | int err = 0; | 348 | u64 len; |
| 339 | u64 len = subreq->len; | ||
| 340 | bool sparse = IS_ENCRYPTED(inode) || ceph_test_mount_opt(fsc, SPARSEREAD); | 349 | bool sparse = IS_ENCRYPTED(inode) || ceph_test_mount_opt(fsc, SPARSEREAD); |
| 341 | u64 off = subreq->start; | 350 | u64 off = subreq->start; |
| 342 | int extent_cnt; | 351 | int extent_cnt; |
| @@ -349,6 +358,12 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) · 6 unmodified lines | |||
| 349 | if (ceph_has_inline_data(ci) && ceph_netfs_issue_op_inline(subreq)) | 358 | if (ceph_has_inline_data(ci) && ceph_netfs_issue_op_inline(subreq)) |
| 350 | return; | 359 | return; |
| 351 | 360 | ||
| 361 | // TODO: This rounding here is slightly dodgy. It *should* work, for | ||
| 362 | // now, as the cache only deals in blocks that are a multiple of | ||
| 363 | // PAGE_SIZE and fscrypt blocks are at most PAGE_SIZE. What needs to | ||
| 364 | // happen is for the fscrypt driving to be moved into netfslib and the | ||
| 365 | // data in the cache also to be stored encrypted. | ||
| 366 | len = subreq->len; | ||
| 352 | ceph_fscrypt_adjust_off_and_len(inode, &off, &len); | 367 | ceph_fscrypt_adjust_off_and_len(inode, &off, &len); |
| 353 | 368 | ||
| 354 | req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, vino, | 369 | req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, vino, |
| @@ -371,8 +386,6 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) · 16 unmodified lines | |||
| 371 | doutc(cl, "%llx.%llx pos=%llu orig_len=%zu len=%llu\n", | 386 | doutc(cl, "%llx.%llx pos=%llu orig_len=%zu len=%llu\n", |
| 372 | ceph_vinop(inode), subreq->start, subreq->len, len); | 387 | ceph_vinop(inode), subreq->start, subreq->len, len); |
| 373 | 388 | ||
| 374 | iov_iter_xarray(&iter, ITER_DEST, &rreq->mapping->i_pages, subreq->start, len); | ||
| 375 | |||
| 376 | /* | 389 | /* |
| 377 | * FIXME: For now, use CEPH_OSD_DATA_TYPE_PAGES instead of _ITER for | 390 | * FIXME: For now, use CEPH_OSD_DATA_TYPE_PAGES instead of _ITER for |
| 378 | * encrypted inodes. We'd need infrastructure that handles an iov_iter | 391 | * encrypted inodes. We'd need infrastructure that handles an iov_iter |
| @@ -384,7 +397,7 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) · 5 unmodified lines | |||
| 384 | struct page **pages; | 397 | struct page **pages; |
| 385 | size_t page_off; | 398 | size_t page_off; |
| 386 | 399 | ||
| 387 | err = iov_iter_get_pages_alloc2(&iter, &pages, len, &page_off); | 400 | err = iov_iter_get_pages_alloc2(&subreq->io_iter, &pages, len, &page_off); |
| 388 | if (err < 0) { | 401 | if (err < 0) { |
| 389 | doutc(cl, "%llx.%llx failed to allocate pages, %d\n", | 402 | doutc(cl, "%llx.%llx failed to allocate pages, %d\n", |
| 390 | ceph_vinop(inode), err); | 403 | ceph_vinop(inode), err); |
| @@ -399,7 +412,7 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) · 8 unmodified lines | |||
| 399 | osd_req_op_extent_osd_data_pages(req, 0, pages, len, 0, false, | 412 | osd_req_op_extent_osd_data_pages(req, 0, pages, len, 0, false, |
| 400 | false); | 413 | false); |
| 401 | } else { | 414 | } else { |
| 402 | osd_req_op_extent_osd_iter(req, 0, &iter); | 415 | osd_req_op_extent_osd_iter(req, 0, &subreq->io_iter); |
| 403 | } | 416 | } |
| 404 | if (!ceph_inc_osd_stopping_blocker(fsc->mdsc)) { | 417 | if (!ceph_inc_osd_stopping_blocker(fsc->mdsc)) { |
| 405 | err = -EIO; | 418 | err = -EIO; |
| @@ -410,17 +423,19 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) · 4 unmodified lines | |||
| 410 | req->r_inode = inode; | 423 | req->r_inode = inode; |
| 411 | ihold(inode); | 424 | ihold(inode); |
| 412 | 425 | ||
| 426 | trace_netfs_sreq(subreq, netfs_sreq_trace_submit); | ||
| 413 | ceph_osdc_start_request(req->r_osdc, req); | 427 | ceph_osdc_start_request(req->r_osdc, req); |
| 414 | out: | 428 | out: |
| 415 | ceph_osdc_put_request(req); | 429 | ceph_osdc_put_request(req); |
| 416 | if (err) | 430 | if (err) |
| 417 | netfs_subreq_terminated(subreq, err, false); | 431 | netfs_read_subreq_terminated(subreq, err, false); |
| 418 | doutc(cl, "%llx.%llx result %d\n", ceph_vinop(inode), err); | 432 | doutc(cl, "%llx.%llx result %d\n", ceph_vinop(inode), err); |
| 419 | } | 433 | } |
| 420 | 434 | ||
| 421 | static int ceph_init_request(struct netfs_io_request *rreq, struct file *file) | 435 | static int ceph_init_request(struct netfs_io_request *rreq, struct file *file) |
| 422 | { | 436 | { |
| 423 | struct inode *inode = rreq->inode; | 437 | struct inode *inode = rreq->inode; |
| 438 | struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); | ||
| 424 | struct ceph_client *cl = ceph_inode_to_client(inode); | 439 | struct ceph_client *cl = ceph_inode_to_client(inode); |
| 425 | int got = 0, want = CEPH_CAP_FILE_CACHE; | 440 | int got = 0, want = CEPH_CAP_FILE_CACHE; |
| 426 | struct ceph_netfs_request_data *priv; | 441 | struct ceph_netfs_request_data *priv; |
| @@ -472,6 +487,7 @@ static int ceph_init_request(struct netfs_io_request *rreq, struct file *file) · 45 unmodified lines | |||
| 472 | 487 | ||
| 473 | priv->caps = got; | 488 | priv->caps = got; |
| 474 | rreq->netfs_priv = priv; | 489 | rreq->netfs_priv = priv; |
| 490 | rreq->io_streams[0].sreq_max_len = fsc->mount_options->rsize; | ||
| 475 | 491 | ||
| 476 | out: | 492 | out: |
| 477 | if (ret < 0) | 493 | if (ret < 0) |
| @@ -496,9 +512,9 @@ static void ceph_netfs_free_request(struct netfs_io_request *rreq) · 18 unmodified lines | |||
| 496 | const struct netfs_request_ops ceph_netfs_ops = { | 512 | const struct netfs_request_ops ceph_netfs_ops = { |
| 497 | .init_request = ceph_init_request, | 513 | .init_request = ceph_init_request, |
| 498 | .free_request = ceph_netfs_free_request, | 514 | .free_request = ceph_netfs_free_request, |
| 515 | .prepare_read = ceph_netfs_prepare_read, | ||
| 499 | .issue_read = ceph_netfs_issue_read, | 516 | .issue_read = ceph_netfs_issue_read, |
| 500 | .expand_readahead = ceph_netfs_expand_readahead, | 517 | .expand_readahead = ceph_netfs_expand_readahead, |
| 501 | .clamp_length = ceph_netfs_clamp_length, | ||
| 502 | .check_write_begin = ceph_netfs_check_write_begin, | 518 | .check_write_begin = ceph_netfs_check_write_begin, |
| 503 | }; | 519 | }; |
| 504 | 520 | ||
| @@ -5,12 +5,14 @@ netfs-y := \ · 4 unmodified lines | |||
| 5 | buffered_write.o \ | 5 | buffered_write.o \ |
| 6 | direct_read.o \ | 6 | direct_read.o \ |
| 7 | direct_write.o \ | 7 | direct_write.o \ |
| 8 | io.o \ | ||
| 9 | iterator.o \ | 8 | iterator.o \ |
| 10 | locking.o \ | 9 | locking.o \ |
| 11 | main.o \ | 10 | main.o \ |
| 12 | misc.o \ | 11 | misc.o \ |
| 13 | objects.o \ | 12 | objects.o \ |
| 13 | read_collect.o \ | ||
| 14 | read_pgpriv2.o \ | ||
| 15 | read_retry.o \ | ||
| 14 | write_collect.o \ | 16 | write_collect.o \ |
| 15 | write_issue.o | 17 | write_issue.o |
| 16 | 18 | ||
| @@ -9,266 +9,388 @@ · 8 unmodified lines | |||
| 9 | #include <linux/task_io_accounting_ops.h> | 9 | #include <linux/task_io_accounting_ops.h> |
| 10 | #include "internal.h" | 10 | #include "internal.h" |
| 11 | 11 | ||
| 12 | /* | 12 | static void netfs_cache_expand_readahead(struct netfs_io_request *rreq, |
| 13 | * [DEPRECATED] Unlock the folios in a read operation for when the filesystem | 13 | unsigned long long *_start, |
| 14 | * is using PG_private_2 and direct writing to the cache from here rather than | 14 | unsigned long long *_len, |
| 15 | * marking the page for writeback. | 15 | unsigned long long i_size) |
| 16 | * | ||
| 17 | * Note that we don't touch folio->private in this code. | ||
| 18 | */ | ||
| 19 | static void netfs_rreq_unlock_folios_pgpriv2(struct netfs_io_request *rreq, | ||
| 20 | size_t *account) | ||
| 21 | { | 16 | { |
| 22 | struct netfs_io_subrequest *subreq; | 17 | struct netfs_cache_resources *cres = &rreq->cache_resources; |
| 23 | struct folio *folio; | ||
| 24 | pgoff_t start_page = rreq->start / PAGE_SIZE; | ||
| 25 | pgoff_t last_page = ((rreq->start + rreq->len) / PAGE_SIZE) - 1; | ||
| 26 | bool subreq_failed = false; | ||
| 27 | 18 | ||
| 28 | XA_STATE(xas, &rreq->mapping->i_pages, start_page); | 19 | if (cres->ops && cres->ops->expand_readahead) |
| 20 | cres->ops->expand_readahead(cres, _start, _len, i_size); | ||
| 21 | } | ||
| 29 | 22 | ||
| 30 | /* Walk through the pagecache and the I/O request lists simultaneously. | 23 | static void netfs_rreq_expand(struct netfs_io_request *rreq, |
| 31 | * We may have a mixture of cached and uncached sections and we only | 24 | struct readahead_control *ractl) |
| 32 | * really want to write out the uncached sections. This is slightly | 25 | { |
| 33 | * complicated by the possibility that we might have huge pages with a | 26 | /* Give the cache a chance to change the request parameters. The |
| 34 | * mixture inside. | 27 | * resultant request must contain the original region. |
| 35 | */ | 28 | */ |
| 36 | subreq = list_first_entry(&rreq->subrequests, | 29 | netfs_cache_expand_readahead(rreq, &rreq->start, &rreq->len, rreq->i_size); |
| 37 | struct netfs_io_subrequest, rreq_link); | ||
| 38 | subreq_failed = (subreq->error < 0); | ||
| 39 | 30 | ||
| 40 | trace_netfs_rreq(rreq, netfs_rreq_trace_unlock_pgpriv2); | 31 | /* Give the netfs a chance to change the request parameters. The |
| 32 | * resultant request must contain the original region. | ||
| 33 | */ | ||
| 34 | if (rreq->netfs_ops->expand_readahead) | ||
| 35 | rreq->netfs_ops->expand_readahead(rreq); | ||
| 41 | 36 | ||
| 42 | rcu_read_lock(); | 37 | /* Expand the request if the cache wants it to start earlier. Note |
| 43 | xas_for_each(&xas, folio, last_page) { | 38 | * that the expansion may get further extended if the VM wishes to |
| 44 | loff_t pg_end; | 39 | * insert THPs and the preferred start and/or end wind up in the middle |
| 45 | bool pg_failed = false; | 40 | * of THPs. |
| 46 | bool folio_started = false; | 41 | * |
| 42 | * If this is the case, however, the THP size should be an integer | ||
| 43 | * multiple of the cache granule size, so we get a whole number of | ||
| 44 | * granules to deal with. | ||
| 45 | */ | ||
| 46 | if (rreq->start != readahead_pos(ractl) || | ||
| 47 | rreq->len != readahead_length(ractl)) { | ||
| 48 | readahead_expand(ractl, rreq->start, rreq->len); | ||
| 49 | rreq->start = readahead_pos(ractl); | ||
| 50 | rreq->len = readahead_length(ractl); | ||
| 47 | 51 | ||
| 48 | if (xas_retry(&xas, folio)) | 52 | trace_netfs_read(rreq, readahead_pos(ractl), readahead_length(ractl), |
| 49 | continue; | 53 | netfs_read_trace_expanded); |
| 54 | } | ||
| 55 | } | ||
| 50 | 56 | ||
| 51 | pg_end = folio_pos(folio) + folio_size(folio) - 1; | 57 | /* |
| 58 | * Begin an operation, and fetch the stored zero point value from the cookie if | ||
| 59 | * available. | ||
| 60 | */ | ||
| 61 | static int netfs_begin_cache_read(struct netfs_io_request *rreq, struct netfs_inode *ctx) | ||
| 62 | { | ||
| 63 | return fscache_begin_read_operation(&rreq->cache_resources, netfs_i_cookie(ctx)); | ||
| 64 | } | ||
| 52 | 65 | ||
| 53 | for (;;) { | 66 | /* |
| 54 | loff_t sreq_end; | 67 | * Decant the list of folios to read into a rolling buffer. |
| 68 | */ | ||
| 69 | static size_t netfs_load_buffer_from_ra(struct netfs_io_request *rreq, | ||
| 70 | struct folio_queue *folioq) | ||
| 71 | { | ||
| 72 | unsigned int order, nr; | ||
| 73 | size_t size = 0; | ||
| 74 | |||
| 75 | nr = __readahead_batch(rreq->ractl, (struct page **)folioq->vec.folios, | ||
| 76 | ARRAY_SIZE(folioq->vec.folios)); | ||
| 77 | folioq->vec.nr = nr; | ||
| 78 | for (int i = 0; i < nr; i++) { | ||
| 79 | struct folio *folio = folioq_folio(folioq, i); | ||
| 80 | |||
| 81 | trace_netfs_folio(folio, netfs_folio_trace_read); | ||
| 82 | order = folio_order(folio); | ||
| 83 | folioq->orders[i] = order; | ||
| 84 | size += PAGE_SIZE << order; | ||
| 85 | } | ||
| 55 | 86 | ||
| 56 | if (!subreq) { | 87 | for (int i = nr; i < folioq_nr_slots(folioq); i++) |
| 57 | pg_failed = true; | 88 | folioq_clear(folioq, i); |
| 58 | break; | ||
| 59 | } | ||
| 60 | 89 | ||
| 61 | if (!folio_started && | 90 | return size; |
| 62 | test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags) && | 91 | } |
| 63 | fscache_operation_valid(&rreq->cache_resources)) { | ||
| 64 | trace_netfs_folio(folio, netfs_folio_trace_copy_to_cache); | ||
| 65 | folio_start_private_2(folio); | ||
| 66 | folio_started = true; | ||
| 67 | } | ||
| 68 | 92 | ||
| 69 | pg_failed |= subreq_failed; | 93 | /* |
| 70 | sreq_end = subreq->start + subreq->len - 1; | 94 | * netfs_prepare_read_iterator - Prepare the subreq iterator for I/O |
| 71 | if (pg_end < sreq_end) | 95 | * @subreq: The subrequest to be set up |
| 72 | break; | 96 | * |
| 97 | * Prepare the I/O iterator representing the read buffer on a subrequest for | ||
| 98 | * the filesystem to use for I/O (it can be passed directly to a socket). This | ||
| 99 | * is intended to be called from the ->issue_read() method once the filesystem | ||
| 100 | * has trimmed the request to the size it wants. | ||
| 101 | * | ||
| 102 | * Returns the limited size if successful and -ENOMEM if insufficient memory | ||
| 103 | * available. | ||
| 104 | * | ||
| 105 | * [!] NOTE: This must be run in the same thread as ->issue_read() was called | ||
| 106 | * in as we access the readahead_control struct. | ||
| 107 | */ | ||
| 108 | static ssize_t netfs_prepare_read_iterator(struct netfs_io_subrequest *subreq) | ||
| 109 | { | ||
| 110 | struct netfs_io_request *rreq = subreq->rreq; | ||
| 111 | size_t rsize = subreq->len; | ||
| 112 | |||
| 113 | if (subreq->source == NETFS_DOWNLOAD_FROM_SERVER) | ||
| 114 | rsize = umin(rsize, rreq->io_streams[0].sreq_max_len); | ||
| 115 | |||
| 116 | if (rreq->ractl) { | ||
| 117 | /* If we don't have sufficient folios in the rolling buffer, | ||
| 118 | * extract a folioq's worth from the readahead region at a time | ||
| 119 | * into the buffer. Note that this acquires a ref on each page | ||
| 120 | * that we will need to release later - but we don't want to do | ||
| 121 | * that until after we've started the I/O. | ||
| 122 | */ | ||
| 123 | while (rreq->submitted < subreq->start + rsize) { | ||
| 124 | struct folio_queue *tail = rreq->buffer_tail, *new; | ||
| 125 | size_t added; | ||
| 126 | |||
| 127 | new = kmalloc(sizeof(*new), GFP_NOFS); | ||
| 128 | if (!new) | ||
| 129 | return -ENOMEM; | ||
| 130 | netfs_stat(&netfs_n_folioq); | ||
| 131 | folioq_init(new); | ||
| 132 | new->prev = tail; | ||
| 133 | tail->next = new; | ||
| 134 | rreq->buffer_tail = new; | ||
| 135 | added = netfs_load_buffer_from_ra(rreq, new); | ||
| 136 | rreq->iter.count += added; | ||
| 137 | rreq->submitted += added; | ||
| 138 | } | ||
| 139 | } | ||
| 73 | 140 | ||
| 74 | *account += subreq->transferred; | 141 | subreq->len = rsize; |
| 75 | if (!list_is_last(&subreq->rreq_link, &rreq->subrequests)) { | 142 | if (unlikely(rreq->io_streams[0].sreq_max_segs)) { |
| 76 | subreq = list_next_entry(subreq, rreq_link); | 143 | size_t limit = netfs_limit_iter(&rreq->iter, 0, rsize, |
| 77 | subreq_failed = (subreq->error < 0); | 144 | rreq->io_streams[0].sreq_max_segs); |
| 78 | } else { | ||
| 79 | subreq = NULL; | ||
| 80 | subreq_failed = false; | ||
| 81 | } | ||
| 82 | 145 | ||
| 83 | if (pg_end == sreq_end) | 146 | if (limit < rsize) { |
| 84 | break; | 147 | subreq->len = limit; |
| 148 | trace_netfs_sreq(subreq, netfs_sreq_trace_limited); | ||
| 85 | } | 149 | } |
| 150 | } | ||
| 86 | 151 | ||
| 87 | if (!pg_failed) { | 152 | subreq->io_iter = rreq->iter; |
| 88 | flush_dcache_folio(folio); | ||
| 89 | folio_mark_uptodate(folio); | ||
| 90 | } | ||
| 91 | 153 | ||
| 92 | if (!test_bit(NETFS_RREQ_DONT_UNLOCK_FOLIOS, &rreq->flags)) { | 154 | if (iov_iter_is_folioq(&subreq->io_iter)) { |
| 93 | if (folio->index == rreq->no_unlock_folio && | 155 | if (subreq->io_iter.folioq_slot >= folioq_nr_slots(subreq->io_iter.folioq)) { |
| 94 | test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags)) | 156 | subreq->io_iter.folioq = subreq->io_iter.folioq->next; |
| 95 | _debug("no unlock"); | 157 | subreq->io_iter.folioq_slot = 0; |
| 96 | else | ||
| 97 | folio_unlock(folio); | ||
| 98 | } | 158 | } |
| 159 | subreq->curr_folioq = (struct folio_queue *)subreq->io_iter.folioq; | ||
| 160 | subreq->curr_folioq_slot = subreq->io_iter.folioq_slot; | ||
| 161 | subreq->curr_folio_order = subreq->curr_folioq->orders[subreq->curr_folioq_slot]; | ||
| 99 | } | 162 | } |
| 100 | rcu_read_unlock(); | 163 | |
| 164 | iov_iter_truncate(&subreq->io_iter, subreq->len); | ||
| 165 | iov_iter_advance(&rreq->iter, subreq->len); | ||
| 166 | return subreq->len; | ||
| 101 | } | 167 | } |
| 102 | 168 | ||
| 103 | /* | 169 | static enum netfs_io_source netfs_cache_prepare_read(struct netfs_io_request *rreq, |
| 104 | * Unlock the folios in a read operation. We need to set PG_writeback on any | 170 | struct netfs_io_subrequest *subreq, |
| 105 | * folios we're going to write back before we unlock them. | 171 | loff_t i_size) |
| 106 | * | ||
| 107 | * Note that if the deprecated NETFS_RREQ_USE_PGPRIV2 is set then we use | ||
| 108 | * PG_private_2 and do a direct write to the cache from here instead. | ||
| 109 | */ | ||
| 110 | void netfs_rreq_unlock_folios(struct netfs_io_request *rreq) | ||
| 111 | { | 172 | { |
| 112 | struct netfs_io_subrequest *subreq; | 173 | struct netfs_cache_resources *cres = &rreq->cache_resources; |
| 113 | struct netfs_folio *finfo; | ||
| 114 | struct folio *folio; | ||
| 115 | pgoff_t start_page = rreq->start / PAGE_SIZE; | ||
| 116 | pgoff_t last_page = ((rreq->start + rreq->len) / PAGE_SIZE) - 1; | ||
| 117 | size_t account = 0; | ||
| 118 | bool subreq_failed = false; | ||
| 119 | 174 | ||
| 120 | XA_STATE(xas, &rreq->mapping->i_pages, start_page); | 175 | if (!cres->ops) |
| 176 | return NETFS_DOWNLOAD_FROM_SERVER; | ||
| 177 | return cres->ops->prepare_read(subreq, i_size); | ||
| 178 | } | ||
| 121 | 179 | ||
| 122 | if (test_bit(NETFS_RREQ_FAILED, &rreq->flags)) { | 180 | static void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error, |
| 123 | __clear_bit(NETFS_RREQ_COPY_TO_CACHE, &rreq->flags); | 181 | bool was_async) |
| 124 | list_for_each_entry(subreq, &rreq->subrequests, rreq_link) { | 182 | { |
| 125 | __clear_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags); | 183 | struct netfs_io_subrequest *subreq = priv; |
| 126 | } | ||
| 127 | } | ||
| 128 | 184 | ||
| 129 | /* Handle deprecated PG_private_2 case. */ | 185 | if (transferred_or_error < 0) { |
| 130 | if (test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)) { | 186 | netfs_read_subreq_terminated(subreq, transferred_or_error, was_async); |
| 131 | netfs_rreq_unlock_folios_pgpriv2(rreq, &account); | 187 | return; |
| 132 | goto out; | ||
| 133 | } | 188 | } |
| 134 | 189 | ||
| 135 | /* Walk through the pagecache and the I/O request lists simultaneously. | 190 | if (transferred_or_error > 0) |
| 136 | * We may have a mixture of cached and uncached sections and we only | 191 | subreq->transferred += transferred_or_error; |
| 137 | * really want to write out the uncached sections. This is slightly | 192 | netfs_read_subreq_terminated(subreq, 0, was_async); |
| 138 | * complicated by the possibility that we might have huge pages with a | 193 | } |
| 139 | * mixture inside. | ||
| 140 | */ | ||
| 141 | subreq = list_first_entry(&rreq->subrequests, | ||
| 142 | struct netfs_io_subrequest, rreq_link); | ||
| 143 | subreq_failed = (subreq->error < 0); | ||
| 144 | |||
| 145 | trace_netfs_rreq(rreq, netfs_rreq_trace_unlock); | ||
| 146 | 194 | ||
| 147 | rcu_read_lock(); | 195 | /* |
| 148 | xas_for_each(&xas, folio, last_page) { | 196 | * Issue a read against the cache. |
| 149 | loff_t pg_end; | 197 | * - Eats the caller's ref on subreq. |
| 150 | bool pg_failed = false; | 198 | */ |
| 151 | bool wback_to_cache = false; | 199 | static void netfs_read_cache_to_pagecache(struct netfs_io_request *rreq, |
| 200 | struct netfs_io_subrequest *subreq) | ||
| 201 | { | ||
| 202 | struct netfs_cache_resources *cres = &rreq->cache_resources; | ||
| 152 | 203 | ||
| 153 | if (xas_retry(&xas, folio)) | 204 | netfs_stat(&netfs_n_rh_read); |
| 154 | continue; | 205 | cres->ops->read(cres, subreq->start, &subreq->io_iter, NETFS_READ_HOLE_IGNORE, |
| 206 | netfs_cache_read_terminated, subreq); | ||
| 207 | } | ||
| 155 | 208 | ||
| 156 | pg_end = folio_pos(folio) + folio_size(folio) - 1; | 209 | /* |
| 210 | * Perform a read to the pagecache from a series of sources of different types, | ||
| 211 | * slicing up the region to be read according to available cache blocks and | ||
| 212 | * network rsize. | ||
| 213 | */ | ||
| 214 | static void netfs_read_to_pagecache(struct netfs_io_request *rreq) | ||
| 215 | { | ||
| 216 | struct netfs_inode *ictx = netfs_inode(rreq->inode); | ||
| 217 | unsigned long long start = rreq->start; | ||
| 218 | ssize_t size = rreq->len; | ||
| 219 | int ret = 0; | ||
| 220 | |||
| 221 | atomic_inc(&rreq->nr_outstanding); | ||
| 222 | |||
| 223 | do { | ||
| 224 | struct netfs_io_subrequest *subreq; | ||
| 225 | enum netfs_io_source source = NETFS_DOWNLOAD_FROM_SERVER; | ||
| 226 | ssize_t slice; | ||
| 227 | |||
| 228 | subreq = netfs_alloc_subrequest(rreq); | ||
| 229 | if (!subreq) { | ||
| 230 | ret = -ENOMEM; | ||
| 231 | break; | ||
| 232 | } | ||
| 157 | 233 | ||
| 158 | for (;;) { | 234 | subreq->start = start; |
| 159 | loff_t sreq_end; | 235 | subreq->len = size; |
| 236 | |||
| 237 | atomic_inc(&rreq->nr_outstanding); | ||
| 238 | spin_lock_bh(&rreq->lock); | ||
| 239 | list_add_tail(&subreq->rreq_link, &rreq->subrequests); | ||
| 240 | subreq->prev_donated = rreq->prev_donated; | ||
| 241 | rreq->prev_donated = 0; | ||
| 242 | trace_netfs_sreq(subreq, netfs_sreq_trace_added); | ||
| 243 | spin_unlock_bh(&rreq->lock); | ||
| 244 | |||
| 245 | source = netfs_cache_prepare_read(rreq, subreq, rreq->i_size); | ||
| 246 | subreq->source = source; | ||
| 247 | if (source == NETFS_DOWNLOAD_FROM_SERVER) { | ||
| 248 | unsigned long long zp = umin(ictx->zero_point, rreq->i_size); | ||
| 249 | size_t len = subreq->len; | ||
| 250 | |||
| 251 | if (subreq->start >= zp) { | ||
| 252 | subreq->source = source = NETFS_FILL_WITH_ZEROES; | ||
| 253 | goto fill_with_zeroes; | ||
| 254 | } | ||
| 160 | 255 | ||
| 161 | if (!subreq) { | 256 | if (len > zp - subreq->start) |
| 162 | pg_failed = true; | 257 | len = zp - subreq->start; |
| 258 | if (len == 0) { | ||
| 259 | pr_err("ZERO-LEN READ: R=%08x[%x] l=%zx/%zx s=%llx z=%llx i=%llx", | ||
| 260 | rreq->debug_id, subreq->debug_index, | ||
| 261 | subreq->len, size, | ||
| 262 | subreq->start, ictx->zero_point, rreq->i_size); | ||
| 163 | break; | 263 | break; |
| 164 | } | 264 | } |
| 265 | subreq->len = len; | ||
| 266 | |||
| 267 | netfs_stat(&netfs_n_rh_download); | ||
| 268 | if (rreq->netfs_ops->prepare_read) { | ||
| 269 | ret = rreq->netfs_ops->prepare_read(subreq); | ||
| 270 | if (ret < 0) { | ||
| 271 | atomic_dec(&rreq->nr_outstanding); | ||
| 272 | netfs_put_subrequest(subreq, false, | ||
| 273 | netfs_sreq_trace_put_cancel); | ||
| 274 | break; | ||
| 275 | } | ||
| 276 | trace_netfs_sreq(subreq, netfs_sreq_trace_prepare); | ||
| 277 | } | ||
| 165 | 278 | ||
| 166 | wback_to_cache |= test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags); | 279 | slice = netfs_prepare_read_iterator(subreq); |
| 167 | pg_failed |= subreq_failed; | 280 | if (slice < 0) { |
| 168 | sreq_end = subreq->start + subreq->len - 1; | 281 | atomic_dec(&rreq->nr_outstanding); |
| 169 | if (pg_end < sreq_end) | 282 | netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_cancel); |
| 283 | ret = slice; | ||
| 170 | break; | 284 | break; |
| 171 | |||
| 172 | account += subreq->transferred; | ||
| 173 | if (!list_is_last(&subreq->rreq_link, &rreq->subrequests)) { | ||
| 174 | subreq = list_next_entry(subreq, rreq_link); | ||
| 175 | subreq_failed = (subreq->error < 0); | ||
| 176 | } else { | ||
| 177 | subreq = NULL; | ||
| 178 | subreq_failed = false; | ||
| 179 | } | 285 | } |
| 180 | 286 | ||
| 181 | if (pg_end == sreq_end) | 287 | rreq->netfs_ops->issue_read(subreq); |
| 182 | break; | 288 | goto done; |
| 183 | } | 289 | } |
| 184 | 290 | ||
| 185 | if (!pg_failed) { | 291 | fill_with_zeroes: |
| 186 | flush_dcache_folio(folio); | 292 | if (source == NETFS_FILL_WITH_ZEROES) { |
| 187 | finfo = netfs_folio_info(folio); | 293 | subreq->source = NETFS_FILL_WITH_ZEROES; |
| 188 | if (finfo) { | 294 | trace_netfs_sreq(subreq, netfs_sreq_trace_submit); |
| 189 | trace_netfs_folio(folio, netfs_folio_trace_filled_gaps); | 295 | netfs_stat(&netfs_n_rh_zero); |
| 190 | if (finfo->netfs_group) | 296 | slice = netfs_prepare_read_iterator(subreq); |
| 191 | folio_change_private(folio, finfo->netfs_group); | 297 | __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags); |
| 192 | else | 298 | netfs_read_subreq_terminated(subreq, 0, false); |
| 193 | folio_detach_private(folio); | 299 | goto done; |
| 194 | kfree(finfo); | ||
| 195 | } | ||
| 196 | folio_mark_uptodate(folio); | ||
| 197 | if (wback_to_cache && !WARN_ON_ONCE(folio_get_private(folio) != NULL)) { | ||
| 198 | trace_netfs_folio(folio, netfs_folio_trace_copy_to_cache); | ||
| 199 | folio_attach_private(folio, NETFS_FOLIO_COPY_TO_CACHE); | ||
| 200 | filemap_dirty_folio(folio->mapping, folio); | ||
| 201 | } | ||
| 202 | } | 300 | } |
| 203 | 301 | ||
| 204 | if (!test_bit(NETFS_RREQ_DONT_UNLOCK_FOLIOS, &rreq->flags)) { | 302 | if (source == NETFS_READ_FROM_CACHE) { |
| 205 | if (folio->index == rreq->no_unlock_folio && | 303 | trace_netfs_sreq(subreq, netfs_sreq_trace_submit); |
| 206 | test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags)) | 304 | slice = netfs_prepare_read_iterator(subreq); |
| 207 | _debug("no unlock"); | 305 | netfs_read_cache_to_pagecache(rreq, subreq); |
| 208 | else | 306 | goto done; |
| 209 | folio_unlock(folio); | ||
| 210 | } | 307 | } |
| 211 | } | ||
| 212 | rcu_read_unlock(); | ||
| 213 | 308 | ||
| 214 | out: | 309 | pr_err("Unexpected read source %u\n", source); |
| 215 | task_io_account_read(account); | 310 | WARN_ON_ONCE(1); |
| 216 | if (rreq->netfs_ops->done) | 311 | break; |
| 217 | rreq->netfs_ops->done(rreq); | ||
| 218 | } | ||
| 219 | 312 | ||
| 220 | static void netfs_cache_expand_readahead(struct netfs_io_request *rreq, | 313 | done: |
| 221 | unsigned long long *_start, | 314 | size -= slice; |
| 222 | unsigned long long *_len, | 315 | start += slice; |
| 223 | unsigned long long i_size) | 316 | cond_resched(); |
| 224 | { | 317 | } while (size > 0); |
| 225 | struct netfs_cache_resources *cres = &rreq->cache_resources; | ||
| 226 | 318 | ||
| 227 | if (cres->ops && cres->ops->expand_readahead) | 319 | if (atomic_dec_and_test(&rreq->nr_outstanding)) |
| 228 | cres->ops->expand_readahead(cres, _start, _len, i_size); | 320 | netfs_rreq_terminated(rreq, false); |
| 321 | |||
| 322 | /* Defer error return as we may need to wait for outstanding I/O. */ | ||
| 323 | cmpxchg(&rreq->error, 0, ret); | ||
| 229 | } | 324 | } |
| 230 | 325 | ||
| 231 | static void netfs_rreq_expand(struct netfs_io_request *rreq, | 326 | /* |
| 232 | struct readahead_control *ractl) | 327 | * Wait for the read operation to complete, successfully or otherwise. |
| 328 | */ | ||
| 329 | static int netfs_wait_for_read(struct netfs_io_request *rreq) | ||
| 233 | { | 330 | { |
| 234 | /* Give the cache a chance to change the request parameters. The | 331 | int ret; |
| 235 | * resultant request must contain the original region. | ||
| 236 | */ | ||
| 237 | netfs_cache_expand_readahead(rreq, &rreq->start, &rreq->len, rreq->i_size); | ||
| 238 | 332 | ||
| 239 | /* Give the netfs a chance to change the request parameters. The | 333 | trace_netfs_rreq(rreq, netfs_rreq_trace_wait_ip); |
| 240 | * resultant request must contain the original region. | 334 | wait_on_bit(&rreq->flags, NETFS_RREQ_IN_PROGRESS, TASK_UNINTERRUPTIBLE); |
| 241 | */ | 335 | ret = rreq->error; |
| 242 | if (rreq->netfs_ops->expand_readahead) | 336 | if (ret == 0 && rreq->submitted < rreq->len) { |
| 243 | rreq->netfs_ops->expand_readahead(rreq); | 337 | trace_netfs_failure(rreq, NULL, ret, netfs_fail_short_read); |
| 338 | ret = -EIO; | ||
| 339 | } | ||
| 244 | 340 | ||
| 245 | /* Expand the request if the cache wants it to start earlier. Note | 341 | return ret; |
| 246 | * that the expansion may get further extended if the VM wishes to | 342 | } |
| 247 | * insert THPs and the preferred start and/or end wind up in the middle | ||
| 248 | * of THPs. | ||
| 249 | * | ||
| 250 | * If this is the case, however, the THP size should be an integer | ||
| 251 | * multiple of the cache granule size, so we get a whole number of | ||
| 252 | * granules to deal with. | ||
| 253 | */ | ||
| 254 | if (rreq->start != readahead_pos(ractl) || | ||
| 255 | rreq->len != readahead_length(ractl)) { | ||
| 256 | readahead_expand(ractl, rreq->start, rreq->len); | ||
| 257 | rreq->start = readahead_pos(ractl); | ||
| 258 | rreq->len = readahead_length(ractl); | ||
| 259 | 343 | ||
| 260 | trace_netfs_read(rreq, readahead_pos(ractl), readahead_length(ractl), | 344 | /* |
| 261 | netfs_read_trace_expanded); | 345 | * Set up the initial folioq of buffer folios in the rolling buffer and set the |
| 262 | } | 346 | * iterator to refer to it. |
| 347 | */ | ||
| 348 | static int netfs_prime_buffer(struct netfs_io_request *rreq) | ||
| 349 | { | ||
| 350 | struct folio_queue *folioq; | ||
| 351 | size_t added; | ||
| 352 | |||
| 353 | folioq = kmalloc(sizeof(*folioq), GFP_KERNEL); | ||
| 354 | if (!folioq) | ||
| 355 | return -ENOMEM; | ||
| 356 | netfs_stat(&netfs_n_folioq); | ||
| 357 | folioq_init(folioq); | ||
| 358 | rreq->buffer = folioq; | ||
| 359 | rreq->buffer_tail = folioq; | ||
| 360 | rreq->submitted = rreq->start; | ||
| 361 | iov_iter_folio_queue(&rreq->iter, ITER_DEST, folioq, 0, 0, 0); | ||
| 362 | |||
| 363 | added = netfs_load_buffer_from_ra(rreq, folioq); | ||
| 364 | rreq->iter.count += added; | ||
| 365 | rreq->submitted += added; | ||
| 366 | return 0; | ||
| 263 | } | 367 | } |
| 264 | 368 | ||
| 265 | /* | 369 | /* |
| 266 | * Begin an operation, and fetch the stored zero point value from the cookie if | 370 | * Drop the ref on each folio that we inherited from the VM readahead code. We |
| 267 | * available. | 371 | * still have the folio locks to pin the page until we complete the I/O. |
| 372 | * | ||
| 373 | * Note that we can't just release the batch in each queue struct as we use the | ||
| 374 | * occupancy count in other places. | ||
| 268 | */ | 375 | */ |
| 269 | static int netfs_begin_cache_read(struct netfs_io_request *rreq, struct netfs_inode *ctx) | 376 | static void netfs_put_ra_refs(struct folio_queue *folioq) |
| 270 | { | 377 | { |
| 271 | return fscache_begin_read_operation(&rreq->cache_resources, netfs_i_cookie(ctx)); | 378 | struct folio_batch fbatch; |
| 379 | |||
| 380 | folio_batch_init(&fbatch); | ||
| 381 | while (folioq) { | ||
| 382 | for (unsigned int slot = 0; slot < folioq_count(folioq); slot++) { | ||
| 383 | struct folio *folio = folioq_folio(folioq, slot); | ||
| 384 | if (!folio) | ||
| 385 | continue; | ||
| 386 | trace_netfs_folio(folio, netfs_folio_trace_read_put); | ||
| 387 | if (!folio_batch_add(&fbatch, folio)) | ||
| 388 | folio_batch_release(&fbatch); | ||
| 389 | } | ||
| 390 | folioq = folioq->next; | ||
| 391 | } | ||
| 392 | |||
| 393 | folio_batch_release(&fbatch); | ||
| 272 | } | 394 | } |
| 273 | 395 | ||
| 274 | /** | 396 | /** |
| @@ -289,22 +411,17 @@ static int netfs_begin_cache_read(struct netfs_io_request *rreq, struct netfs_in · 14 unmodified lines | |||
| 289 | void netfs_readahead(struct readahead_control *ractl) | 411 | void netfs_readahead(struct readahead_control *ractl) |
| 290 | { | 412 | { |
| 291 | struct netfs_io_request *rreq; | 413 | struct netfs_io_request *rreq; |
| 292 | struct netfs_inode *ctx = netfs_inode(ractl->mapping->host); | 414 | struct netfs_inode *ictx = netfs_inode(ractl->mapping->host); |
| 415 | unsigned long long start = readahead_pos(ractl); | ||
| 416 | size_t size = readahead_length(ractl); | ||
| 293 | int ret; | 417 | int ret; |
| 294 | 418 | ||
| 295 | _enter("%lx,%x", readahead_index(ractl), readahead_count(ractl)); | 419 | rreq = netfs_alloc_request(ractl->mapping, ractl->file, start, size, |
| 296 | |||
| 297 | if (readahead_count(ractl) == 0) | ||
| 298 | return; | ||
| 299 | |||
| 300 | rreq = netfs_alloc_request(ractl->mapping, ractl->file, | ||
| 301 | readahead_pos(ractl), | ||
| 302 | readahead_length(ractl), | ||
| 303 | NETFS_READAHEAD); | 420 | NETFS_READAHEAD); |
| 304 | if (IS_ERR(rreq)) | 421 | if (IS_ERR(rreq)) |
| 305 | return; | 422 | return; |
| 306 | 423 | ||
| 307 | ret = netfs_begin_cache_read(rreq, ctx); | 424 | ret = netfs_begin_cache_read(rreq, ictx); |
| 308 | if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS) | 425 | if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS) |
| 309 | goto cleanup_free; | 426 | goto cleanup_free; |
| 310 | 427 | ||
| @@ -314,18 +431,15 @@ void netfs_readahead(struct readahead_control *ractl) · 3 unmodified lines | |||
| 314 | 431 | ||
| 315 | netfs_rreq_expand(rreq, ractl); | 432 | netfs_rreq_expand(rreq, ractl); |
| 316 | 433 | ||
| 317 | /* Set up the output buffer */ | 434 | rreq->ractl = ractl; |
| 318 | iov_iter_xarray(&rreq->iter, ITER_DEST, &ractl->mapping->i_pages, | 435 | if (netfs_prime_buffer(rreq) < 0) |
| 319 | rreq->start, rreq->len); | 436 | goto cleanup_free; |
| 437 | netfs_read_to_pagecache(rreq); | ||
| 320 | 438 | ||
| 321 | /* Drop the refs on the folios here rather than in the cache or | 439 | /* Release the folio refs whilst we're waiting for the I/O. */ |
| 322 | * filesystem. The locks will be dropped in netfs_rreq_unlock(). | 440 | netfs_put_ra_refs(rreq->buffer); |
| 323 | */ | ||
| 324 | while (readahead_folio(ractl)) | ||
| 325 | ; | ||
| 326 | 441 | ||
| 327 | netfs_begin_read(rreq, false); | 442 | netfs_put_request(rreq, true, netfs_rreq_trace_put_return); |
| 328 | netfs_put_request(rreq, false, netfs_rreq_trace_put_return); | ||
| 329 | return; | 443 | return; |
| 330 | 444 | ||
| 331 | cleanup_free: | 445 | cleanup_free: |
| @@ -334,6 +448,117 @@ cleanup_free: · 2 unmodified lines | |||
| 334 | } | 448 | } |
| 335 | EXPORT_SYMBOL(netfs_readahead); | 449 | EXPORT_SYMBOL(netfs_readahead); |
| 336 | 450 | ||
| 451 | /* | ||
| 452 | * Create a rolling buffer with a single occupying folio. | ||
| 453 | */ | ||
| 454 | static int netfs_create_singular_buffer(struct netfs_io_request *rreq, struct folio *folio) | ||
| 455 | { | ||
| 456 | struct folio_queue *folioq; | ||
| 457 | |||
| 458 | folioq = kmalloc(sizeof(*folioq), GFP_KERNEL); | ||
| 459 | if (!folioq) | ||
| 460 | return -ENOMEM; | ||
| 461 | |||
| 462 | netfs_stat(&netfs_n_folioq); | ||
| 463 | folioq_init(folioq); | ||
| 464 | folioq_append(folioq, folio); | ||
| 465 | BUG_ON(folioq_folio(folioq, 0) != folio); | ||
| 466 | BUG_ON(folioq_folio_order(folioq, 0) != folio_order(folio)); | ||
| 467 | rreq->buffer = folioq; | ||
| 468 | rreq->buffer_tail = folioq; | ||
| 469 | rreq->submitted = rreq->start + rreq->len; | ||
| 470 | iov_iter_folio_queue(&rreq->iter, ITER_DEST, folioq, 0, 0, rreq->len); | ||
| 471 | rreq->ractl = (struct readahead_control *)1UL; | ||
| 472 | return 0; | ||
| 473 | } | ||
| 474 | |||
| 475 | /* | ||
| 476 | * Read into gaps in a folio partially filled by a streaming write. | ||
| 477 | */ | ||
| 478 | static int netfs_read_gaps(struct file *file, struct folio *folio) | ||
| 479 | { | ||
| 480 | struct netfs_io_request *rreq; | ||
| 481 | struct address_space *mapping = folio->mapping; | ||
| 482 | struct netfs_folio *finfo = netfs_folio_info(folio); | ||
| 483 | struct netfs_inode *ctx = netfs_inode(mapping->host); | ||
| 484 | struct folio *sink = NULL; | ||
| 485 | struct bio_vec *bvec; | ||
| 486 | unsigned int from = finfo->dirty_offset; | ||
| 487 | unsigned int to = from + finfo->dirty_len; | ||
| 488 | unsigned int off = 0, i = 0; | ||
| 489 | size_t flen = folio_size(folio); | ||
| 490 | size_t nr_bvec = flen / PAGE_SIZE + 2; | ||
| 491 | size_t part; | ||
| 492 | int ret; | ||
| 493 | |||
| 494 | _enter("%lx", folio->index); | ||
| 495 | |||
| 496 | rreq = netfs_alloc_request(mapping, file, folio_pos(folio), flen, NETFS_READ_GAPS); | ||
| 497 | if (IS_ERR(rreq)) { | ||
| 498 | ret = PTR_ERR(rreq); | ||
| 499 | goto alloc_error; | ||
| 500 | } | ||
| 501 | |||
| 502 | ret = netfs_begin_cache_read(rreq, ctx); | ||
| 503 | if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS) | ||
| 504 | goto discard; | ||
| 505 | |||
| 506 | netfs_stat(&netfs_n_rh_read_folio); | ||
| 507 | trace_netfs_read(rreq, rreq->start, rreq->len, netfs_read_trace_read_gaps); | ||
| 508 | |||
| 509 | /* Fiddle the buffer so that a gap at the beginning and/or a gap at the | ||
| 510 | * end get copied to, but the middle is discarded. | ||
| 511 | */ | ||
| 512 | ret = -ENOMEM; | ||
| 513 | bvec = kmalloc_array(nr_bvec, sizeof(*bvec), GFP_KERNEL); | ||
| 514 | if (!bvec) | ||
| 515 | goto discard; | ||
| 516 | |||
| 517 | sink = folio_alloc(GFP_KERNEL, 0); | ||
| 518 | if (!sink) { | ||
| 519 | kfree(bvec); | ||
| 520 | goto discard; | ||
| 521 | } | ||
| 522 | |||
| 523 | trace_netfs_folio(folio, netfs_folio_trace_read_gaps); | ||
| 524 | |||
| 525 | rreq->direct_bv = bvec; | ||
| 526 | rreq->direct_bv_count = nr_bvec; | ||
| 527 | if (from > 0) { | ||
| 528 | bvec_set_folio(&bvec[i++], folio, from, 0); | ||
| 529 | off = from; | ||
| 530 | } | ||
| 531 | while (off < to) { | ||
| 532 | part = min_t(size_t, to - off, PAGE_SIZE); | ||
| 533 | bvec_set_folio(&bvec[i++], sink, part, 0); | ||
| 534 | off += part; | ||
| 535 | } | ||
| 536 | if (to < flen) | ||
| 537 | bvec_set_folio(&bvec[i++], folio, flen - to, to); | ||
| 538 | iov_iter_bvec(&rreq->iter, ITER_DEST, bvec, i, rreq->len); | ||
| 539 | rreq->submitted = rreq->start + flen; | ||
| 540 | |||
| 541 | netfs_read_to_pagecache(rreq); | ||
| 542 | |||
| 543 | if (sink) | ||
| 544 | folio_put(sink); | ||
| 545 | |||
| 546 | ret = netfs_wait_for_read(rreq); | ||
| 547 | if (ret == 0) { | ||
| 548 | flush_dcache_folio(folio); | ||
| 549 | folio_mark_uptodate(folio); | ||
| 550 | } | ||
| 551 | folio_unlock(folio); | ||
| 552 | netfs_put_request(rreq, false, netfs_rreq_trace_put_return); | ||
| 553 | return ret < 0 ? ret : 0; | ||
| 554 | |||
| 555 | discard: | ||
| 556 | netfs_put_request(rreq, false, netfs_rreq_trace_put_discard); | ||
| 557 | alloc_error: | ||
| 558 | folio_unlock(folio); | ||
| 559 | return ret; | ||
| 560 | } | ||
| 561 | |||
| 337 | /** | 562 | /** |
| 338 | * netfs_read_folio - Helper to manage a read_folio request | 563 | * netfs_read_folio - Helper to manage a read_folio request |
| 339 | * @file: The file to read from | 564 | * @file: The file to read from |
| @@ -353,9 +578,13 @@ int netfs_read_folio(struct file *file, struct folio *folio) · 13 unmodified lines | |||
| 353 | struct address_space *mapping = folio->mapping; | 578 | struct address_space *mapping = folio->mapping; |
| 354 | struct netfs_io_request *rreq; | 579 | struct netfs_io_request *rreq; |
| 355 | struct netfs_inode *ctx = netfs_inode(mapping->host); | 580 | struct netfs_inode *ctx = netfs_inode(mapping->host); |
| 356 | struct folio *sink = NULL; | ||
| 357 | int ret; | 581 | int ret; |
| 358 | 582 | ||
| 583 | if (folio_test_dirty(folio)) { | ||
| 584 | trace_netfs_folio(folio, netfs_folio_trace_read_gaps); | ||
| 585 | return netfs_read_gaps(file, folio); | ||
| 586 | } | ||
| 587 | |||
| 359 | _enter("%lx", folio->index); | 588 | _enter("%lx", folio->index); |
| 360 | 589 | ||
| 361 | rreq = netfs_alloc_request(mapping, file, | 590 | rreq = netfs_alloc_request(mapping, file, |
| @@ -374,54 +603,12 @@ int netfs_read_folio(struct file *file, struct folio *folio) · 12 unmodified lines | |||
| 374 | trace_netfs_read(rreq, rreq->start, rreq->len, netfs_read_trace_readpage); | 603 | trace_netfs_read(rreq, rreq->start, rreq->len, netfs_read_trace_readpage); |
| 375 | 604 | ||
| 376 | /* Set up the output buffer */ | 605 | /* Set up the output buffer */ |
| 377 | if (folio_test_dirty(folio)) { | 606 | ret = netfs_create_singular_buffer(rreq, folio); |
| 378 | /* Handle someone trying to read from an unflushed streaming | 607 | if (ret < 0) |
| 379 | * write. We fiddle the buffer so that a gap at the beginning | 608 | goto discard; |
| 380 | * and/or a gap at the end get copied to, but the middle is | ||
| 381 | * discarded. | ||
| 382 | */ | ||
| 383 | struct netfs_folio *finfo = netfs_folio_info(folio); | ||
| 384 | struct bio_vec *bvec; | ||
| 385 | unsigned int from = finfo->dirty_offset; | ||
| 386 | unsigned int to = from + finfo->dirty_len; | ||
| 387 | unsigned int off = 0, i = 0; | ||
| 388 | size_t flen = folio_size(folio); | ||
| 389 | size_t nr_bvec = flen / PAGE_SIZE + 2; | ||
| 390 | size_t part; | ||
| 391 | |||
| 392 | ret = -ENOMEM; | ||
| 393 | bvec = kmalloc_array(nr_bvec, sizeof(*bvec), GFP_KERNEL); | ||
| 394 | if (!bvec) | ||
| 395 | goto discard; | ||
| 396 | |||
| 397 | sink = folio_alloc(GFP_KERNEL, 0); | ||
| 398 | if (!sink) | ||
| 399 | goto discard; | ||
| 400 | |||
| 401 | trace_netfs_folio(folio, netfs_folio_trace_read_gaps); | ||
| 402 | |||
| 403 | rreq->direct_bv = bvec; | ||
| 404 | rreq->direct_bv_count = nr_bvec; | ||
| 405 | if (from > 0) { | ||
| 406 | bvec_set_folio(&bvec[i++], folio, from, 0); | ||
| 407 | off = from; | ||
| 408 | } | ||
| 409 | while (off < to) { | ||
| 410 | part = min_t(size_t, to - off, PAGE_SIZE); | ||
| 411 | bvec_set_folio(&bvec[i++], sink, part, 0); | ||
| 412 | off += part; | ||
| 413 | } | ||
| 414 | if (to < flen) | ||
| 415 | bvec_set_folio(&bvec[i++], folio, flen - to, to); | ||
| 416 | iov_iter_bvec(&rreq->iter, ITER_DEST, bvec, i, rreq->len); | ||
| 417 | } else { | ||
| 418 | iov_iter_xarray(&rreq->iter, ITER_DEST, &mapping->i_pages, | ||
| 419 | rreq->start, rreq->len); | ||
| 420 | } | ||
| 421 | 609 | ||
| 422 | ret = netfs_begin_read(rreq, true); | 610 | netfs_read_to_pagecache(rreq); |
| 423 | if (sink) | 611 | ret = netfs_wait_for_read(rreq); |
| 424 | folio_put(sink); | ||
| 425 | netfs_put_request(rreq, false, netfs_rreq_trace_put_return); | 612 | netfs_put_request(rreq, false, netfs_rreq_trace_put_return); |
| 426 | return ret < 0 ? ret : 0; | 613 | return ret < 0 ? ret : 0; |
| 427 | 614 | ||
| @@ -494,13 +681,10 @@ zero_out: · 66 unmodified lines | |||
| 494 | * | 681 | * |
| 495 | * Pre-read data for a write-begin request by drawing data from the cache if | 682 | * Pre-read data for a write-begin request by drawing data from the cache if |
| 496 | * possible, or the netfs if not. Space beyond the EOF is zero-filled. | 683 | * possible, or the netfs if not. Space beyond the EOF is zero-filled. |
| 497 | * Multiple I/O requests from different sources will get munged together. If | 684 | * Multiple I/O requests from different sources will get munged together. |
| 498 | * necessary, the readahead window can be expanded in either direction to a | ||
| 499 | * more convenient alighment for RPC efficiency or to make storage in the cache | ||
| 500 | * feasible. | ||
| 501 | * | 685 | * |
| 502 | * The calling netfs must provide a table of operations, only one of which, | 686 | * The calling netfs must provide a table of operations, only one of which, |
| 503 | * issue_op, is mandatory. | 687 | * issue_read, is mandatory. |
| 504 | * | 688 | * |
| 505 | * The check_write_begin() operation can be provided to check for and flush | 689 | * The check_write_begin() operation can be provided to check for and flush |
| 506 | * conflicting writes once the folio is grabbed and locked. It is passed a | 690 | * conflicting writes once the folio is grabbed and locked. It is passed a |
| @@ -528,8 +712,6 @@ int netfs_write_begin(struct netfs_inode *ctx, · 21 unmodified lines | |||
| 528 | pgoff_t index = pos >> PAGE_SHIFT; | 712 | pgoff_t index = pos >> PAGE_SHIFT; |
| 529 | int ret; | 713 | int ret; |
| 530 | 714 | ||
| 531 | DEFINE_READAHEAD(ractl, file, NULL, mapping, index); | ||
| 532 | |||
| 533 | retry: | 715 | retry: |
| 534 | folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN, | 716 | folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN, |
| 535 | mapping_gfp_mask(mapping)); | 717 | mapping_gfp_mask(mapping)); |
| @@ -577,22 +759,13 @@ retry: · 41 unmodified lines | |||
| 577 | netfs_stat(&netfs_n_rh_write_begin); | 759 | netfs_stat(&netfs_n_rh_write_begin); |
| 578 | trace_netfs_read(rreq, pos, len, netfs_read_trace_write_begin); | 760 | trace_netfs_read(rreq, pos, len, netfs_read_trace_write_begin); |
| 579 | 761 | ||
| 580 | /* Expand the request to meet caching requirements and download | ||
| 581 | * preferences. | ||
| 582 | */ | ||
| 583 | ractl._nr_pages = folio_nr_pages(folio); | ||
| 584 | netfs_rreq_expand(rreq, &ractl); | ||
| 585 | |||
| 586 | /* Set up the output buffer */ | 762 | /* Set up the output buffer */ |
| 587 | iov_iter_xarray(&rreq->iter, ITER_DEST, &mapping->i_pages, | 763 | ret = netfs_create_singular_buffer(rreq, folio); |
| 588 | rreq->start, rreq->len); | 764 | if (ret < 0) |
| 589 | 765 | goto error_put; | |
| 590 | /* We hold the folio locks, so we can drop the references */ | ||
| 591 | folio_get(folio); | ||
| 592 | while (readahead_folio(&ractl)) | ||
| 593 | ; | ||
| 594 | 766 | ||
| 595 | ret = netfs_begin_read(rreq, true); | 767 | netfs_read_to_pagecache(rreq); |
| 768 | ret = netfs_wait_for_read(rreq); | ||
| 596 | if (ret < 0) | 769 | if (ret < 0) |
| 597 | goto error; | 770 | goto error; |
| 598 | netfs_put_request(rreq, false, netfs_rreq_trace_put_return); | 771 | netfs_put_request(rreq, false, netfs_rreq_trace_put_return); |
| @@ -652,10 +825,13 @@ int netfs_prefetch_for_write(struct file *file, struct folio *folio, · 53 unmodified lines | |||
| 652 | trace_netfs_read(rreq, start, flen, netfs_read_trace_prefetch_for_write); | 825 | trace_netfs_read(rreq, start, flen, netfs_read_trace_prefetch_for_write); |
| 653 | 826 | ||
| 654 | /* Set up the output buffer */ | 827 | /* Set up the output buffer */ |
| 655 | iov_iter_xarray(&rreq->iter, ITER_DEST, &mapping->i_pages, | 828 | ret = netfs_create_singular_buffer(rreq, folio); |
| 656 | rreq->start, rreq->len); | 829 | if (ret < 0) |
| 830 | goto error_put; | ||
| 657 | 831 | ||
| 658 | ret = netfs_begin_read(rreq, true); | 832 | folioq_mark2(rreq->buffer, 0); |
| 833 | netfs_read_to_pagecache(rreq); | ||
| 834 | ret = netfs_wait_for_read(rreq); | ||
| 659 | netfs_put_request(rreq, false, netfs_rreq_trace_put_return); | 835 | netfs_put_request(rreq, false, netfs_rreq_trace_put_return); |
| 660 | return ret; | 836 | return ret; |
| 661 | 837 | ||
| @@ -16,6 +16,143 @@ · 15 unmodified lines | |||
| 16 | #include <linux/netfs.h> | 16 | #include <linux/netfs.h> |
| 17 | #include "internal.h" | 17 | #include "internal.h" |
| 18 | 18 | ||
| 19 | static void netfs_prepare_dio_read_iterator(struct netfs_io_subrequest *subreq) | ||
| 20 | { | ||
| 21 | struct netfs_io_request *rreq = subreq->rreq; | ||
| 22 | size_t rsize; | ||
| 23 | |||
| 24 | rsize = umin(subreq->len, rreq->io_streams[0].sreq_max_len); | ||
| 25 | subreq->len = rsize; | ||
| 26 | |||
| 27 | if (unlikely(rreq->io_streams[0].sreq_max_segs)) { | ||
| 28 | size_t limit = netfs_limit_iter(&rreq->iter, 0, rsize, | ||
| 29 | rreq->io_streams[0].sreq_max_segs); | ||
| 30 | |||
| 31 | if (limit < rsize) { | ||
| 32 | subreq->len = limit; | ||
| 33 | trace_netfs_sreq(subreq, netfs_sreq_trace_limited); | ||
| 34 | } | ||
| 35 | } | ||
| 36 | |||
| 37 | trace_netfs_sreq(subreq, netfs_sreq_trace_prepare); | ||
| 38 | |||
| 39 | subreq->io_iter = rreq->iter; | ||
| 40 | iov_iter_truncate(&subreq->io_iter, subreq->len); | ||
| 41 | iov_iter_advance(&rreq->iter, subreq->len); | ||
| 42 | } | ||
| 43 | |||
| 44 | /* | ||
| 45 | * Perform a read to a buffer from the server, slicing up the region to be read | ||
| 46 | * according to the network rsize. | ||
| 47 | */ | ||
| 48 | static int netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq) | ||
| 49 | { | ||
| 50 | unsigned long long start = rreq->start; | ||
| 51 | ssize_t size = rreq->len; | ||
| 52 | int ret = 0; | ||
| 53 | |||
| 54 | atomic_set(&rreq->nr_outstanding, 1); | ||
| 55 | |||
| 56 | do { | ||
| 57 | struct netfs_io_subrequest *subreq; | ||
| 58 | ssize_t slice; | ||
| 59 | |||
| 60 | subreq = netfs_alloc_subrequest(rreq); | ||
| 61 | if (!subreq) { | ||
| 62 | ret = -ENOMEM; | ||
| 63 | break; | ||
| 64 | } | ||
| 65 | |||
| 66 | subreq->source = NETFS_DOWNLOAD_FROM_SERVER; | ||
| 67 | subreq->start = start; | ||
| 68 | subreq->len = size; | ||
| 69 | |||
| 70 | atomic_inc(&rreq->nr_outstanding); | ||
| 71 | spin_lock_bh(&rreq->lock); | ||
| 72 | list_add_tail(&subreq->rreq_link, &rreq->subrequests); | ||
| 73 | subreq->prev_donated = rreq->prev_donated; | ||
| 74 | rreq->prev_donated = 0; | ||
| 75 | trace_netfs_sreq(subreq, netfs_sreq_trace_added); | ||
| 76 | spin_unlock_bh(&rreq->lock); | ||
| 77 | |||
| 78 | netfs_stat(&netfs_n_rh_download); | ||
| 79 | if (rreq->netfs_ops->prepare_read) { | ||
| 80 | ret = rreq->netfs_ops->prepare_read(subreq); | ||
| 81 | if (ret < 0) { | ||
| 82 | atomic_dec(&rreq->nr_outstanding); | ||
| 83 | netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_cancel); | ||
| 84 | break; | ||
| 85 | } | ||
| 86 | } | ||
| 87 | |||
| 88 | netfs_prepare_dio_read_iterator(subreq); | ||
| 89 | slice = subreq->len; | ||
| 90 | rreq->netfs_ops->issue_read(subreq); | ||
| 91 | |||
| 92 | size -= slice; | ||
| 93 | start += slice; | ||
| 94 | rreq->submitted += slice; | ||
| 95 | |||
| 96 | if (test_bit(NETFS_RREQ_BLOCKED, &rreq->flags) && | ||
| 97 | test_bit(NETFS_RREQ_NONBLOCK, &rreq->flags)) | ||
| 98 | break; | ||
| 99 | cond_resched(); | ||
| 100 | } while (size > 0); | ||
| 101 | |||
| 102 | if (atomic_dec_and_test(&rreq->nr_outstanding)) | ||
| 103 | netfs_rreq_terminated(rreq, false); | ||
| 104 | return ret; | ||
| 105 | } | ||
| 106 | |||
| 107 | /* | ||
| 108 | * Perform a read to an application buffer, bypassing the pagecache and the | ||
| 109 | * local disk cache. | ||
| 110 | */ | ||
| 111 | static int netfs_unbuffered_read(struct netfs_io_request *rreq, bool sync) | ||
| 112 | { | ||
| 113 | int ret; | ||
| 114 | |||
| 115 | _enter("R=%x %llx-%llx", | ||
| 116 | rreq->debug_id, rreq->start, rreq->start + rreq->len - 1); | ||
| 117 | |||
| 118 | if (rreq->len == 0) { | ||
| 119 | pr_err("Zero-sized read [R=%x]\n", rreq->debug_id); | ||
| 120 | return -EIO; | ||
| 121 | } | ||
| 122 | |||
| 123 | // TODO: Use bounce buffer if requested | ||
| 124 | |||
| 125 | inode_dio_begin(rreq->inode); | ||
| 126 | |||
| 127 | ret = netfs_dispatch_unbuffered_reads(rreq); | ||
| 128 | |||
| 129 | if (!rreq->submitted) { | ||
| 130 | netfs_put_request(rreq, false, netfs_rreq_trace_put_no_submit); | ||
| 131 | inode_dio_end(rreq->inode); | ||
| 132 | ret = 0; | ||
| 133 | goto out; | ||
| 134 | } | ||
| 135 | |||
| 136 | if (sync) { | ||
| 137 | trace_netfs_rreq(rreq, netfs_rreq_trace_wait_ip); | ||
| 138 | wait_on_bit(&rreq->flags, NETFS_RREQ_IN_PROGRESS, | ||
| 139 | TASK_UNINTERRUPTIBLE); | ||
| 140 | |||
| 141 | ret = rreq->error; | ||
| 142 | if (ret == 0 && rreq->submitted < rreq->len && | ||
| 143 | rreq->origin != NETFS_DIO_READ) { | ||
| 144 | trace_netfs_failure(rreq, NULL, ret, netfs_fail_short_read); | ||
| 145 | ret = -EIO; | ||
| 146 | } | ||
| 147 | } else { | ||
| 148 | ret = -EIOCBQUEUED; | ||
| 149 | } | ||
| 150 | |||
| 151 | out: | ||
| 152 | _leave(" = %d", ret); | ||
| 153 | return ret; | ||
| 154 | } | ||
| 155 | |||
| 19 | /** | 156 | /** |
| 20 | * netfs_unbuffered_read_iter_locked - Perform an unbuffered or direct I/O read | 157 | * netfs_unbuffered_read_iter_locked - Perform an unbuffered or direct I/O read |
| 21 | * @iocb: The I/O control descriptor describing the read | 158 | * @iocb: The I/O control descriptor describing the read |
| @@ -31,7 +168,7 @@ ssize_t netfs_unbuffered_read_iter_locked(struct kiocb *iocb, struct iov_iter *i · 9 unmodified lines | |||
| 31 | struct netfs_io_request *rreq; | 168 | struct netfs_io_request *rreq; |
| 32 | ssize_t ret; | 169 | ssize_t ret; |
| 33 | size_t orig_count = iov_iter_count(iter); | 170 | size_t orig_count = iov_iter_count(iter); |
| 34 | bool async = !is_sync_kiocb(iocb); | 171 | bool sync = is_sync_kiocb(iocb); |
| 35 | 172 | ||
| 36 | _enter(""); | 173 | _enter(""); |
| 37 | 174 | ||
| @@ -78,13 +215,13 @@ ssize_t netfs_unbuffered_read_iter_locked(struct kiocb *iocb, struct iov_iter *i · 40 unmodified lines | |||
| 78 | 215 | ||
| 79 | // TODO: Set up bounce buffer if needed | 216 | // TODO: Set up bounce buffer if needed |
| 80 | 217 | ||
| 81 | if (async) | 218 | if (!sync) |
| 82 | rreq->iocb = iocb; | 219 | rreq->iocb = iocb; |
| 83 | 220 | ||
| 84 | ret = netfs_begin_read(rreq, is_sync_kiocb(iocb)); | 221 | ret = netfs_unbuffered_read(rreq, sync); |
| 85 | if (ret < 0) | 222 | if (ret < 0) |
| 86 | goto out; /* May be -EIOCBQUEUED */ | 223 | goto out; /* May be -EIOCBQUEUED */ |
| 87 | if (!async) { | 224 | if (sync) { |
| 88 | // TODO: Copy from bounce buffer | 225 | // TODO: Copy from bounce buffer |
| 89 | iocb->ki_pos += rreq->transferred; | 226 | iocb->ki_pos += rreq->transferred; |
| 90 | ret = rreq->transferred; | 227 | ret = rreq->transferred; |
| @@ -94,8 +231,6 @@ out: · 3 unmodified lines | |||
| 94 | netfs_put_request(rreq, false, netfs_rreq_trace_put_return); | 231 | netfs_put_request(rreq, false, netfs_rreq_trace_put_return); |
| 95 | if (ret > 0) | 232 | if (ret > 0) |
| 96 | orig_count -= ret; | 233 | orig_count -= ret; |
| 97 | if (ret != -EIOCBQUEUED) | ||
| 98 | iov_iter_revert(iter, orig_count - iov_iter_count(iter)); | ||
| 99 | return ret; | 234 | return ret; |
| 100 | } | 235 | } |
| 101 | EXPORT_SYMBOL(netfs_unbuffered_read_iter_locked); | 236 | EXPORT_SYMBOL(netfs_unbuffered_read_iter_locked); |
| @@ -23,16 +23,9 @@ · 22 unmodified lines | |||
| 23 | /* | 23 | /* |
| 24 | * buffered_read.c | 24 | * buffered_read.c |
| 25 | */ | 25 | */ |
| 26 | void netfs_rreq_unlock_folios(struct netfs_io_request *rreq); | ||
| 27 | int netfs_prefetch_for_write(struct file *file, struct folio *folio, | 26 | int netfs_prefetch_for_write(struct file *file, struct folio *folio, |
| 28 | size_t offset, size_t len); | 27 | size_t offset, size_t len); |
| 29 | 28 | ||
| 30 | /* | ||
| 31 | * io.c | ||
| 32 | */ | ||
| 33 | void netfs_rreq_work(struct work_struct *work); | ||
| 34 | int netfs_begin_read(struct netfs_io_request *rreq, bool sync); | ||
| 35 | |||
| 36 | /* | 29 | /* |
| 37 | * main.c | 30 | * main.c |
| 38 | */ | 31 | */ |
| @@ -90,6 +83,28 @@ static inline void netfs_see_request(struct netfs_io_request *rreq, · 51 unmodified lines | |||
| 90 | trace_netfs_rreq_ref(rreq->debug_id, refcount_read(&rreq->ref), what); | 83 | trace_netfs_rreq_ref(rreq->debug_id, refcount_read(&rreq->ref), what); |
| 91 | } | 84 | } |
| 92 | 85 | ||
| 86 | /* | ||
| 87 | * read_collect.c | ||
| 88 | */ | ||
| 89 | void netfs_read_termination_worker(struct work_struct *work); | ||
| 90 | void netfs_rreq_terminated(struct netfs_io_request *rreq, bool was_async); | ||
| 91 | |||
| 92 | /* | ||
| 93 | * read_pgpriv2.c | ||
| 94 | */ | ||
| 95 | void netfs_pgpriv2_mark_copy_to_cache(struct netfs_io_subrequest *subreq, | ||
| 96 | struct netfs_io_request *rreq, | ||
| 97 | struct folio_queue *folioq, | ||
| 98 | int slot); | ||
| 99 | void netfs_pgpriv2_write_to_the_cache(struct netfs_io_request *rreq); | ||
| 100 | bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *wreq); | ||
| 101 | |||
| 102 | /* | ||
| 103 | * read_retry.c | ||
| 104 | */ | ||
| 105 | void netfs_retry_reads(struct netfs_io_request *rreq); | ||
| 106 | void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq); | ||
| 107 | |||
| 93 | /* | 108 | /* |
| 94 | * stats.c | 109 | * stats.c |
| 95 | */ | 110 | */ |
| @@ -117,6 +132,7 @@ extern atomic_t netfs_n_wh_buffered_write; · 21 unmodified lines | |||
| 117 | extern atomic_t netfs_n_wh_writethrough; | 132 | extern atomic_t netfs_n_wh_writethrough; |
| 118 | extern atomic_t netfs_n_wh_dio_write; | 133 | extern atomic_t netfs_n_wh_dio_write; |
| 119 | extern atomic_t netfs_n_wh_writepages; | 134 | extern atomic_t netfs_n_wh_writepages; |
| 135 | extern atomic_t netfs_n_wh_copy_to_cache; | ||
| 120 | extern atomic_t netfs_n_wh_wstream_conflict; | 136 | extern atomic_t netfs_n_wh_wstream_conflict; |
| 121 | extern atomic_t netfs_n_wh_upload; | 137 | extern atomic_t netfs_n_wh_upload; |
| 122 | extern atomic_t netfs_n_wh_upload_done; | 138 | extern atomic_t netfs_n_wh_upload_done; |
| @@ -162,6 +178,11 @@ struct netfs_io_request *netfs_create_write_req(struct address_space *mapping, · 39 unmodified lines | |||
| 162 | void netfs_reissue_write(struct netfs_io_stream *stream, | 178 | void netfs_reissue_write(struct netfs_io_stream *stream, |
| 163 | struct netfs_io_subrequest *subreq, | 179 | struct netfs_io_subrequest *subreq, |
| 164 | struct iov_iter *source); | 180 | struct iov_iter *source); |
| 181 | void netfs_issue_write(struct netfs_io_request *wreq, | ||
| 182 | struct netfs_io_stream *stream); | ||
| 183 | int netfs_advance_write(struct netfs_io_request *wreq, | ||
| 184 | struct netfs_io_stream *stream, | ||
| 185 | loff_t start, size_t len, bool to_eof); | ||
| 165 | struct netfs_io_request *netfs_begin_writethrough(struct kiocb *iocb, size_t len); | 186 | struct netfs_io_request *netfs_begin_writethrough(struct kiocb *iocb, size_t len); |
| 166 | int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc, | 187 | int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc, |
| 167 | struct folio *folio, size_t copied, bool to_page_end, | 188 | struct folio *folio, size_t copied, bool to_page_end, |
| @@ -188,9 +188,59 @@ static size_t netfs_limit_xarray(const struct iov_iter *iter, size_t start_offse · 187 unmodified lines | |||
| 188 | return min(span, max_size); | 188 | return min(span, max_size); |
| 189 | } | 189 | } |
| 190 | 190 | ||
| 191 | /* | ||
| 192 | * Select the span of a folio queue iterator we're going to use. Limit it by | ||
| 193 | * both maximum size and maximum number of segments. Returns the size of the | ||
| 194 | * span in bytes. | ||
| 195 | */ | ||
| 196 | static size_t netfs_limit_folioq(const struct iov_iter *iter, size_t start_offset, | ||
| 197 | size_t max_size, size_t max_segs) | ||
| 198 | { | ||
| 199 | const struct folio_queue *folioq = iter->folioq; | ||
| 200 | unsigned int nsegs = 0; | ||
| 201 | unsigned int slot = iter->folioq_slot; | ||
| 202 | size_t span = 0, n = iter->count; | ||
| 203 | |||
| 204 | if (WARN_ON(!iov_iter_is_folioq(iter)) || | ||
| 205 | WARN_ON(start_offset > n) || | ||
| 206 | n == 0) | ||
| 207 | return 0; | ||
| 208 | max_size = umin(max_size, n - start_offset); | ||
| 209 | |||
| 210 | if (slot >= folioq_nr_slots(folioq)) { | ||
| 211 | folioq = folioq->next; | ||
| 212 | slot = 0; | ||
| 213 | } | ||
| 214 | |||
| 215 | start_offset += iter->iov_offset; | ||
| 216 | do { | ||
| 217 | size_t flen = folioq_folio_size(folioq, slot); | ||
| 218 | |||
| 219 | if (start_offset < flen) { | ||
| 220 | span += flen - start_offset; | ||
| 221 | nsegs++; | ||
| 222 | start_offset = 0; | ||
| 223 | } else { | ||
| 224 | start_offset -= flen; | ||
| 225 | } | ||
| 226 | if (span >= max_size || nsegs >= max_segs) | ||
| 227 | break; | ||
| 228 | |||
| 229 | slot++; | ||
| 230 | if (slot >= folioq_nr_slots(folioq)) { | ||
| 231 | folioq = folioq->next; | ||
| 232 | slot = 0; | ||
| 233 | } | ||
| 234 | } while (folioq); | ||
| 235 | |||
| 236 | return umin(span, max_size); | ||
| 237 | } | ||
| 238 | |||
| 191 | size_t netfs_limit_iter(const struct iov_iter *iter, size_t start_offset, | 239 | size_t netfs_limit_iter(const struct iov_iter *iter, size_t start_offset, |
| 192 | size_t max_size, size_t max_segs) | 240 | size_t max_size, size_t max_segs) |
| 193 | { | 241 | { |
| 242 | if (iov_iter_is_folioq(iter)) | ||
| 243 | return netfs_limit_folioq(iter, start_offset, max_size, max_segs); | ||
| 194 | if (iov_iter_is_bvec(iter)) | 244 | if (iov_iter_is_bvec(iter)) |
| 195 | return netfs_limit_bvec(iter, start_offset, max_size, max_segs); | 245 | return netfs_limit_bvec(iter, start_offset, max_size, max_segs); |
| 196 | if (iov_iter_is_xarray(iter)) | 246 | if (iov_iter_is_xarray(iter)) |
| @@ -36,12 +36,14 @@ DEFINE_SPINLOCK(netfs_proc_lock); · 35 unmodified lines | |||
| 36 | static const char *netfs_origins[nr__netfs_io_origin] = { | 36 | static const char *netfs_origins[nr__netfs_io_origin] = { |
| 37 | [NETFS_READAHEAD] = "RA", | 37 | [NETFS_READAHEAD] = "RA", |
| 38 | [NETFS_READPAGE] = "RP", | 38 | [NETFS_READPAGE] = "RP", |
| 39 | [NETFS_READ_GAPS] = "RG", | ||
| 39 | [NETFS_READ_FOR_WRITE] = "RW", | 40 | [NETFS_READ_FOR_WRITE] = "RW", |
| 40 | [NETFS_DIO_READ] = "DR", | 41 | [NETFS_DIO_READ] = "DR", |
| 41 | [NETFS_WRITEBACK] = "WB", | 42 | [NETFS_WRITEBACK] = "WB", |
| 42 | [NETFS_WRITETHROUGH] = "WT", | 43 | [NETFS_WRITETHROUGH] = "WT", |
| 43 | [NETFS_UNBUFFERED_WRITE] = "UW", | 44 | [NETFS_UNBUFFERED_WRITE] = "UW", |
| 44 | [NETFS_DIO_WRITE] = "DW", | 45 | [NETFS_DIO_WRITE] = "DW", |
| 46 | [NETFS_PGPRIV2_COPY_TO_CACHE] = "2C", | ||
| 45 | }; | 47 | }; |
| 46 | 48 | ||
| 47 | /* | 49 | /* |
| @@ -61,7 +63,7 @@ static int netfs_requests_seq_show(struct seq_file *m, void *v) · 13 unmodified lines | |||
| 61 | 63 | ||
| 62 | rreq = list_entry(v, struct netfs_io_request, proc_link); | 64 | rreq = list_entry(v, struct netfs_io_request, proc_link); |
| 63 | seq_printf(m, | 65 | seq_printf(m, |
| 64 | "%08x %s %3d %2lx %4d %3d @%04llx %llx/%llx", | 66 | "%08x %s %3d %2lx %4ld %3d @%04llx %llx/%llx", |
| 65 | rreq->debug_id, | 67 | rreq->debug_id, |
| 66 | netfs_origins[rreq->origin], | 68 | netfs_origins[rreq->origin], |
| 67 | refcount_read(&rreq->ref), | 69 | refcount_read(&rreq->ref), |
| @@ -36,7 +36,6 @@ struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, · 35 unmodified lines | |||
| 36 | memset(rreq, 0, kmem_cache_size(cache)); | 36 | memset(rreq, 0, kmem_cache_size(cache)); |
| 37 | rreq->start = start; | 37 | rreq->start = start; |
| 38 | rreq->len = len; | 38 | rreq->len = len; |
| 39 | rreq->upper_len = len; | ||
| 40 | rreq->origin = origin; | 39 | rreq->origin = origin; |
| 41 | rreq->netfs_ops = ctx->ops; | 40 | rreq->netfs_ops = ctx->ops; |
| 42 | rreq->mapping = mapping; | 41 | rreq->mapping = mapping; |
| @@ -44,6 +43,8 @@ struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, · 1 unmodified line | |||
| 44 | rreq->i_size = i_size_read(inode); | 43 | rreq->i_size = i_size_read(inode); |
| 45 | rreq->debug_id = atomic_inc_return(&debug_ids); | 44 | rreq->debug_id = atomic_inc_return(&debug_ids); |
| 46 | rreq->wsize = INT_MAX; | 45 | rreq->wsize = INT_MAX; |
| 46 | rreq->io_streams[0].sreq_max_len = ULONG_MAX; | ||
| 47 | rreq->io_streams[0].sreq_max_segs = 0; | ||
| 47 | spin_lock_init(&rreq->lock); | 48 | spin_lock_init(&rreq->lock); |
| 48 | INIT_LIST_HEAD(&rreq->io_streams[0].subrequests); | 49 | INIT_LIST_HEAD(&rreq->io_streams[0].subrequests); |
| 49 | INIT_LIST_HEAD(&rreq->io_streams[1].subrequests); | 50 | INIT_LIST_HEAD(&rreq->io_streams[1].subrequests); |
| @@ -52,9 +53,10 @@ struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, · 2 unmodified lines | |||
| 52 | 53 | ||
| 53 | if (origin == NETFS_READAHEAD || | 54 | if (origin == NETFS_READAHEAD || |
| 54 | origin == NETFS_READPAGE || | 55 | origin == NETFS_READPAGE || |
| 56 | origin == NETFS_READ_GAPS || | ||
| 55 | origin == NETFS_READ_FOR_WRITE || | 57 | origin == NETFS_READ_FOR_WRITE || |
| 56 | origin == NETFS_DIO_READ) | 58 | origin == NETFS_DIO_READ) |
| 57 | INIT_WORK(&rreq->work, netfs_rreq_work); | 59 | INIT_WORK(&rreq->work, netfs_read_termination_worker); |
| 58 | else | 60 | else |
| 59 | INIT_WORK(&rreq->work, netfs_write_collection_worker); | 61 | INIT_WORK(&rreq->work, netfs_write_collection_worker); |
| 60 | 62 | ||
| @@ -163,7 +165,7 @@ void netfs_put_request(struct netfs_io_request *rreq, bool was_async, · 102 unmodified lines | |||
| 163 | if (was_async) { | 165 | if (was_async) { |
| 164 | rreq->work.func = netfs_free_request; | 166 | rreq->work.func = netfs_free_request; |
| 165 | if (!queue_work(system_unbound_wq, &rreq->work)) | 167 | if (!queue_work(system_unbound_wq, &rreq->work)) |
| 166 | BUG(); | 168 | WARN_ON(1); |
| 167 | } else { | 169 | } else { |
| 168 | netfs_free_request(&rreq->work); | 170 | netfs_free_request(&rreq->work); |
| 169 | } | 171 | } |
| @@ -0,0 +1,544 @@ | |||
| 1 | // SPDX-License-Identifier: GPL-2.0-only | ||
| 2 | /* Network filesystem read subrequest result collection, assessment and | ||
| 3 | * retrying. | ||
| 4 | * | ||
| 5 | * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. | ||
| 6 | * Written by David Howells (dhowells@redhat.com) | ||
| 7 | */ | ||
| 8 | |||
| 9 | #include <linux/export.h> | ||
| 10 | #include <linux/fs.h> | ||
| 11 | #include <linux/mm.h> | ||
| 12 | #include <linux/pagemap.h> | ||
| 13 | #include <linux/slab.h> | ||
| 14 | #include <linux/task_io_accounting_ops.h> | ||
| 15 | #include "internal.h" | ||
| 16 | |||
| 17 | /* | ||
| 18 | * Clear the unread part of an I/O request. | ||
| 19 | */ | ||
| 20 | static void netfs_clear_unread(struct netfs_io_subrequest *subreq) | ||
| 21 | { | ||
| 22 | netfs_reset_iter(subreq); | ||
| 23 | WARN_ON_ONCE(subreq->len - subreq->transferred != iov_iter_count(&subreq->io_iter)); | ||
| 24 | iov_iter_zero(iov_iter_count(&subreq->io_iter), &subreq->io_iter); | ||
| 25 | if (subreq->start + subreq->transferred >= subreq->rreq->i_size) | ||
| 26 | __set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags); | ||
| 27 | } | ||
| 28 | |||
| 29 | /* | ||
| 30 | * Flush, mark and unlock a folio that's now completely read. If we want to | ||
| 31 | * cache the folio, we set the group to NETFS_FOLIO_COPY_TO_CACHE, mark it | ||
| 32 | * dirty and let writeback handle it. | ||
| 33 | */ | ||
| 34 | static void netfs_unlock_read_folio(struct netfs_io_subrequest *subreq, | ||
| 35 | struct netfs_io_request *rreq, | ||
| 36 | struct folio_queue *folioq, | ||
| 37 | int slot) | ||
| 38 | { | ||
| 39 | struct netfs_folio *finfo; | ||
| 40 | struct folio *folio = folioq_folio(folioq, slot); | ||
| 41 | |||
| 42 | flush_dcache_folio(folio); | ||
| 43 | folio_mark_uptodate(folio); | ||
| 44 | |||
| 45 | if (!test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)) { | ||
| 46 | finfo = netfs_folio_info(folio); | ||
| 47 | if (finfo) { | ||
| 48 | trace_netfs_folio(folio, netfs_folio_trace_filled_gaps); | ||
| 49 | if (finfo->netfs_group) | ||
| 50 | folio_change_private(folio, finfo->netfs_group); | ||
| 51 | else | ||
| 52 | folio_detach_private(folio); | ||
| 53 | kfree(finfo); | ||
| 54 | } | ||
| 55 | |||
| 56 | if (test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags)) { | ||
| 57 | if (!WARN_ON_ONCE(folio_get_private(folio) != NULL)) { | ||
| 58 | trace_netfs_folio(folio, netfs_folio_trace_copy_to_cache); | ||
| 59 | folio_attach_private(folio, NETFS_FOLIO_COPY_TO_CACHE); | ||
| 60 | folio_mark_dirty(folio); | ||
| 61 | } | ||
| 62 | } else { | ||
| 63 | trace_netfs_folio(folio, netfs_folio_trace_read_done); | ||
| 64 | } | ||
| 65 | } else { | ||
| 66 | // TODO: Use of PG_private_2 is deprecated. | ||
| 67 | if (test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags)) | ||
| 68 | netfs_pgpriv2_mark_copy_to_cache(subreq, rreq, folioq, slot); | ||
| 69 | } | ||
| 70 | |||
| 71 | if (!test_bit(NETFS_RREQ_DONT_UNLOCK_FOLIOS, &rreq->flags)) { | ||
| 72 | if (folio->index == rreq->no_unlock_folio && | ||
| 73 | test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags)) { | ||
| 74 | _debug("no unlock"); | ||
| 75 | } else { | ||
| 76 | trace_netfs_folio(folio, netfs_folio_trace_read_unlock); | ||
| 77 | folio_unlock(folio); | ||
| 78 | } | ||
| 79 | } | ||
| 80 | } | ||
| 81 | |||
| 82 | /* | ||
| 83 | * Unlock any folios that are now completely read. Returns true if the | ||
| 84 | * subrequest is removed from the list. | ||
| 85 | */ | ||
| 86 | static bool netfs_consume_read_data(struct netfs_io_subrequest *subreq, bool was_async) | ||
| 87 | { | ||
| 88 | struct netfs_io_subrequest *prev, *next; | ||
| 89 | struct netfs_io_request *rreq = subreq->rreq; | ||
| 90 | struct folio_queue *folioq = subreq->curr_folioq; | ||
| 91 | size_t avail, prev_donated, next_donated, fsize, part, excess; | ||
| 92 | loff_t fpos, start; | ||
| 93 | loff_t fend; | ||
| 94 | int slot = subreq->curr_folioq_slot; | ||
| 95 | |||
| 96 | if (WARN(subreq->transferred > subreq->len, | ||
| 97 | "Subreq overread: R%x[%x] %zu > %zu", | ||
| 98 | rreq->debug_id, subreq->debug_index, | ||
| 99 | subreq->transferred, subreq->len)) | ||
| 100 | subreq->transferred = subreq->len; | ||
| 101 | |||
| 102 | next_folio: | ||
| 103 | fsize = PAGE_SIZE << subreq->curr_folio_order; | ||
| 104 | fpos = round_down(subreq->start + subreq->consumed, fsize); | ||
| 105 | fend = fpos + fsize; | ||
| 106 | |||
| 107 | if (WARN_ON_ONCE(!folioq) || | ||
| 108 | WARN_ON_ONCE(!folioq_folio(folioq, slot)) || | ||
| 109 | WARN_ON_ONCE(folioq_folio(folioq, slot)->index != fpos / PAGE_SIZE)) { | ||
| 110 | pr_err("R=%08x[%x] s=%llx-%llx ctl=%zx/%zx/%zx sl=%u\n", | ||
| 111 | rreq->debug_id, subreq->debug_index, | ||
| 112 | subreq->start, subreq->start + subreq->transferred - 1, | ||
| 113 | subreq->consumed, subreq->transferred, subreq->len, | ||
| 114 | slot); | ||
| 115 | if (folioq) { | ||
| 116 | struct folio *folio = folioq_folio(folioq, slot); | ||
| 117 | |||
| 118 | pr_err("folioq: orders=%02x%02x%02x%02x\n", | ||
| 119 | folioq->orders[0], folioq->orders[1], | ||
| 120 | folioq->orders[2], folioq->orders[3]); | ||
| 121 | if (folio) | ||
| 122 | pr_err("folio: %llx-%llx ix=%llx o=%u qo=%u\n", | ||
| 123 | fpos, fend - 1, folio_pos(folio), folio_order(folio), | ||
| 124 | folioq_folio_order(folioq, slot)); | ||
| 125 | } | ||
| 126 | } | ||
| 127 | |||
| 128 | donation_changed: | ||
| 129 | /* Try to consume the current folio if we've hit or passed the end of | ||
| 130 | * it. There's a possibility that this subreq doesn't start at the | ||
| 131 | * beginning of the folio, in which case we need to donate to/from the | ||
| 132 | * preceding subreq. | ||
| 133 | * | ||
| 134 | * We also need to include any potential donation back from the | ||
| 135 | * following subreq. | ||
| 136 | */ | ||
| 137 | prev_donated = READ_ONCE(subreq->prev_donated); | ||
| 138 | next_donated = READ_ONCE(subreq->next_donated); | ||
| 139 | if (prev_donated || next_donated) { | ||
| 140 | spin_lock_bh(&rreq->lock); | ||
| 141 | prev_donated = subreq->prev_donated; | ||
| 142 | next_donated = subreq->next_donated; | ||
| 143 | subreq->start -= prev_donated; | ||
| 144 | subreq->len += prev_donated; | ||
| 145 | subreq->transferred += prev_donated; | ||
| 146 | prev_donated = subreq->prev_donated = 0; | ||
| 147 | if (subreq->transferred == subreq->len) { | ||
| 148 | subreq->len += next_donated; | ||
| 149 | subreq->transferred += next_donated; | ||
| 150 | next_donated = subreq->next_donated = 0; | ||
| 151 | } | ||
| 152 | trace_netfs_sreq(subreq, netfs_sreq_trace_add_donations); | ||
| 153 | spin_unlock_bh(&rreq->lock); | ||
| 154 | } | ||
| 155 | |||
| 156 | avail = subreq->transferred; | ||
| 157 | if (avail == subreq->len) | ||
| 158 | avail += next_donated; | ||
| 159 | start = subreq->start; | ||
| 160 | if (subreq->consumed == 0) { | ||
| 161 | start -= prev_donated; | ||
| 162 | avail += prev_donated; | ||
| 163 | } else { | ||
| 164 | start += subreq->consumed; | ||
| 165 | avail -= subreq->consumed; | ||
| 166 | } | ||
| 167 | part = umin(avail, fsize); | ||
| 168 | |||
| 169 | trace_netfs_progress(subreq, start, avail, part); | ||
| 170 | |||
| 171 | if (start + avail >= fend) { | ||
| 172 | if (fpos == start) { | ||
| 173 | /* Flush, unlock and mark for caching any folio we've just read. */ | ||
| 174 | subreq->consumed = fend - subreq->start; | ||
| 175 | netfs_unlock_read_folio(subreq, rreq, folioq, slot); | ||
| 176 | folioq_mark2(folioq, slot); | ||
| 177 | if (subreq->consumed >= subreq->len) | ||
| 178 | goto remove_subreq; | ||
| 179 | } else if (fpos < start) { | ||
| 180 | excess = fend - subreq->start; | ||
| 181 | |||
| 182 | spin_lock_bh(&rreq->lock); | ||
| 183 | /* If we complete first on a folio split with the | ||
| 184 | * preceding subreq, donate to that subreq - otherwise | ||
| 185 | * we get the responsibility. | ||
| 186 | */ | ||
| 187 | if (subreq->prev_donated != prev_donated) { | ||
| 188 | spin_unlock_bh(&rreq->lock); | ||
| 189 | goto donation_changed; | ||
| 190 | } | ||
| 191 | |||
| 192 | if (list_is_first(&subreq->rreq_link, &rreq->subrequests)) { | ||
| 193 | spin_unlock_bh(&rreq->lock); | ||
| 194 | pr_err("Can't donate prior to front\n"); | ||
| 195 | goto bad; | ||
| 196 | } | ||
| 197 | |||
| 198 | prev = list_prev_entry(subreq, rreq_link); | ||
| 199 | WRITE_ONCE(prev->next_donated, prev->next_donated + excess); | ||
| 200 | subreq->start += excess; | ||
| 201 | subreq->len -= excess; | ||
| 202 | subreq->transferred -= excess; | ||
| 203 | trace_netfs_donate(rreq, subreq, prev, excess, | ||
| 204 | netfs_trace_donate_tail_to_prev); | ||
| 205 | trace_netfs_sreq(subreq, netfs_sreq_trace_donate_to_prev); | ||
| 206 | |||
| 207 | if (subreq->consumed >= subreq->len) | ||
| 208 | goto remove_subreq_locked; | ||
| 209 | spin_unlock_bh(&rreq->lock); | ||
| 210 | } else { | ||
| 211 | pr_err("fpos > start\n"); | ||
| 212 | goto bad; | ||
| 213 | } | ||
| 214 | |||
| 215 | /* Advance the rolling buffer to the next folio. */ | ||
| 216 | slot++; | ||
| 217 | if (slot >= folioq_nr_slots(folioq)) { | ||
| 218 | slot = 0; | ||
| 219 | folioq = folioq->next; | ||
| 220 | subreq->curr_folioq = folioq; | ||
| 221 | } | ||
| 222 | subreq->curr_folioq_slot = slot; | ||
| 223 | if (folioq && folioq_folio(folioq, slot)) | ||
| 224 | subreq->curr_folio_order = folioq->orders[slot]; | ||
| 225 | if (!was_async) | ||
| 226 | cond_resched(); | ||
| 227 | goto next_folio; | ||
| 228 | } | ||
| 229 | |||
| 230 | /* Deal with partial progress. */ | ||
| 231 | if (subreq->transferred < subreq->len) | ||
| 232 | return false; | ||
| 233 | |||
| 234 | /* Donate the remaining downloaded data to one of the neighbouring | ||
| 235 | * subrequests. Note that we may race with them doing the same thing. | ||
| 236 | */ | ||
| 237 | spin_lock_bh(&rreq->lock); | ||
| 238 | |||
| 239 | if (subreq->prev_donated != prev_donated || | ||
| 240 | subreq->next_donated != next_donated) { | ||
| 241 | spin_unlock_bh(&rreq->lock); | ||
| 242 | cond_resched(); | ||
| 243 | goto donation_changed; | ||
| 244 | } | ||
| 245 | |||
| 246 | /* Deal with the trickiest case: that this subreq is in the middle of a | ||
| 247 | * folio, not touching either edge, but finishes first. In such a | ||
| 248 | * case, we donate to the previous subreq, if there is one, so that the | ||
| 249 | * donation is only handled when that completes - and remove this | ||
| 250 | * subreq from the list. | ||
| 251 | * | ||
| 252 | * If the previous subreq finished first, we will have acquired their | ||
| 253 | * donation and should be able to unlock folios and/or donate nextwards. | ||
| 254 | */ | ||
| 255 | if (!subreq->consumed && | ||
| 256 | !prev_donated && | ||
| 257 | !list_is_first(&subreq->rreq_link, &rreq->subrequests)) { | ||
| 258 | prev = list_prev_entry(subreq, rreq_link); | ||
| 259 | WRITE_ONCE(prev->next_donated, prev->next_donated + subreq->len); | ||
| 260 | subreq->start += subreq->len; | ||
| 261 | subreq->len = 0; | ||
| 262 | subreq->transferred = 0; | ||
| 263 | trace_netfs_donate(rreq, subreq, prev, subreq->len, | ||
| 264 | netfs_trace_donate_to_prev); | ||
| 265 | trace_netfs_sreq(subreq, netfs_sreq_trace_donate_to_prev); | ||
| 266 | goto remove_subreq_locked; | ||
| 267 | } | ||
| 268 | |||
| 269 | /* If we can't donate down the chain, donate up the chain instead. */ | ||
| 270 | excess = subreq->len - subreq->consumed + next_donated; | ||
| 271 | |||
| 272 | if (!subreq->consumed) | ||
| 273 | excess += prev_donated; | ||
| 274 | |||
| 275 | if (list_is_last(&subreq->rreq_link, &rreq->subrequests)) { | ||
| 276 | rreq->prev_donated = excess; | ||
| 277 | trace_netfs_donate(rreq, subreq, NULL, excess, | ||
| 278 | netfs_trace_donate_to_deferred_next); | ||
| 279 | } else { | ||
| 280 | next = list_next_entry(subreq, rreq_link); | ||
| 281 | WRITE_ONCE(next->prev_donated, excess); | ||
| 282 | trace_netfs_donate(rreq, subreq, next, excess, | ||
| 283 | netfs_trace_donate_to_next); | ||
| 284 | } | ||
| 285 | trace_netfs_sreq(subreq, netfs_sreq_trace_donate_to_next); | ||
| 286 | subreq->len = subreq->consumed; | ||
| 287 | subreq->transferred = subreq->consumed; | ||
| 288 | goto remove_subreq_locked; | ||
| 289 | |||
| 290 | remove_subreq: | ||
| 291 | spin_lock_bh(&rreq->lock); | ||
| 292 | remove_subreq_locked: | ||
| 293 | subreq->consumed = subreq->len; | ||
| 294 | list_del(&subreq->rreq_link); | ||
| 295 | spin_unlock_bh(&rreq->lock); | ||
| 296 | netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_consumed); | ||
| 297 | return true; | ||
| 298 | |||
| 299 | bad: | ||
| 300 | /* Errr... prev and next both donated to us, but insufficient to finish | ||
| 301 | * the folio. | ||
| 302 | */ | ||
| 303 | printk("R=%08x[%x] s=%llx-%llx %zx/%zx/%zx\n", | ||
| 304 | rreq->debug_id, subreq->debug_index, | ||
| 305 | subreq->start, subreq->start + subreq->transferred - 1, | ||
| 306 | subreq->consumed, subreq->transferred, subreq->len); | ||
| 307 | printk("folio: %llx-%llx\n", fpos, fend - 1); | ||
| 308 | printk("donated: prev=%zx next=%zx\n", prev_donated, next_donated); | ||
| 309 | printk("s=%llx av=%zx part=%zx\n", start, avail, part); | ||
| 310 | BUG(); | ||
| 311 | } | ||
| 312 | |||
| 313 | /* | ||
| 314 | * Do page flushing and suchlike after DIO. | ||
| 315 | */ | ||
| 316 | static void netfs_rreq_assess_dio(struct netfs_io_request *rreq) | ||
| 317 | { | ||
| 318 | struct netfs_io_subrequest *subreq; | ||
| 319 | unsigned int i; | ||
| 320 | |||
| 321 | /* Collect unbuffered reads and direct reads, adding up the transfer | ||
| 322 | * sizes until we find the first short or failed subrequest. | ||
| 323 | */ | ||
| 324 | list_for_each_entry(subreq, &rreq->subrequests, rreq_link) { | ||
| 325 | rreq->transferred += subreq->transferred; | ||
| 326 | |||
| 327 | if (subreq->transferred < subreq->len || | ||
| 328 | test_bit(NETFS_SREQ_FAILED, &subreq->flags)) { | ||
| 329 | rreq->error = subreq->error; | ||
| 330 | break; | ||
| 331 | } | ||
| 332 | } | ||
| 333 | |||
| 334 | if (rreq->origin == NETFS_DIO_READ) { | ||
| 335 | for (i = 0; i < rreq->direct_bv_count; i++) { | ||
| 336 | flush_dcache_page(rreq->direct_bv[i].bv_page); | ||
| 337 | // TODO: cifs marks pages in the destination buffer | ||
| 338 | // dirty under some circumstances after a read. Do we | ||
| 339 | // need to do that too? | ||
| 340 | set_page_dirty(rreq->direct_bv[i].bv_page); | ||
| 341 | } | ||
| 342 | } | ||
| 343 | |||
| 344 | if (rreq->iocb) { | ||
| 345 | rreq->iocb->ki_pos += rreq->transferred; | ||
| 346 | if (rreq->iocb->ki_complete) | ||
| 347 | rreq->iocb->ki_complete( | ||
| 348 | rreq->iocb, rreq->error ? rreq->error : rreq->transferred); | ||
| 349 | } | ||
| 350 | if (rreq->netfs_ops->done) | ||
| 351 | rreq->netfs_ops->done(rreq); | ||
| 352 | if (rreq->origin == NETFS_DIO_READ) | ||
| 353 | inode_dio_end(rreq->inode); | ||
| 354 | } | ||
| 355 | |||
| 356 | /* | ||
| 357 | * Assess the state of a read request and decide what to do next. | ||
| 358 | * | ||
| 359 | * Note that we're in normal kernel thread context at this point, possibly | ||
| 360 | * running on a workqueue. | ||
| 361 | */ | ||
| 362 | static void netfs_rreq_assess(struct netfs_io_request *rreq) | ||
| 363 | { | ||
| 364 | trace_netfs_rreq(rreq, netfs_rreq_trace_assess); | ||
| 365 | |||
| 366 | //netfs_rreq_is_still_valid(rreq); | ||
| 367 | |||
| 368 | if (test_and_clear_bit(NETFS_RREQ_NEED_RETRY, &rreq->flags)) { | ||
| 369 | netfs_retry_reads(rreq); | ||
| 370 | return; | ||
| 371 | } | ||
| 372 | |||
| 373 | if (rreq->origin == NETFS_DIO_READ || | ||
| 374 | rreq->origin == NETFS_READ_GAPS) | ||
| 375 | netfs_rreq_assess_dio(rreq); | ||
| 376 | task_io_account_read(rreq->transferred); | ||
| 377 | |||
| 378 | trace_netfs_rreq(rreq, netfs_rreq_trace_wake_ip); | ||
| 379 | clear_bit_unlock(NETFS_RREQ_IN_PROGRESS, &rreq->flags); | ||
| 380 | wake_up_bit(&rreq->flags, NETFS_RREQ_IN_PROGRESS); | ||
| 381 | |||
| 382 | trace_netfs_rreq(rreq, netfs_rreq_trace_done); | ||
| 383 | netfs_clear_subrequests(rreq, false); | ||
| 384 | netfs_unlock_abandoned_read_pages(rreq); | ||
| 385 | if (unlikely(test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags))) | ||
| 386 | netfs_pgpriv2_write_to_the_cache(rreq); | ||
| 387 | } | ||
| 388 | |||
| 389 | void netfs_read_termination_worker(struct work_struct *work) | ||
| 390 | { | ||
| 391 | struct netfs_io_request *rreq = | ||
| 392 | container_of(work, struct netfs_io_request, work); | ||
| 393 | netfs_see_request(rreq, netfs_rreq_trace_see_work); | ||
| 394 | netfs_rreq_assess(rreq); | ||
| 395 | netfs_put_request(rreq, false, netfs_rreq_trace_put_work_complete); | ||
| 396 | } | ||
| 397 | |||
| 398 | /* | ||
| 399 | * Handle the completion of all outstanding I/O operations on a read request. | ||
| 400 | * We inherit a ref from the caller. | ||
| 401 | */ | ||
| 402 | void netfs_rreq_terminated(struct netfs_io_request *rreq, bool was_async) | ||
| 403 | { | ||
| 404 | if (!was_async) | ||
| 405 | return netfs_rreq_assess(rreq); | ||
| 406 | if (!work_pending(&rreq->work)) { | ||
| 407 | netfs_get_request(rreq, netfs_rreq_trace_get_work); | ||
| 408 | if (!queue_work(system_unbound_wq, &rreq->work)) | ||
| 409 | netfs_put_request(rreq, was_async, netfs_rreq_trace_put_work_nq); | ||
| 410 | } | ||
| 411 | } | ||
| 412 | |||
| 413 | /** | ||
| 414 | * netfs_read_subreq_progress - Note progress of a read operation. | ||
| 415 | * @subreq: The read request that has terminated. | ||
| 416 | * @was_async: True if we're in an asynchronous context. | ||
| 417 | * | ||
| 418 | * This tells the read side of netfs lib that a contributory I/O operation has | ||
| 419 | * made some progress and that it may be possible to unlock some folios. | ||
| 420 | * | ||
| 421 | * Before calling, the filesystem should update subreq->transferred to track | ||
| 422 | * the amount of data copied into the output buffer. | ||
| 423 | * | ||
| 424 | * If @was_async is true, the caller might be running in softirq or interrupt | ||
| 425 | * context and we can't sleep. | ||
| 426 | */ | ||
| 427 | void netfs_read_subreq_progress(struct netfs_io_subrequest *subreq, | ||
| 428 | bool was_async) | ||
| 429 | { | ||
| 430 | struct netfs_io_request *rreq = subreq->rreq; | ||
| 431 | |||
| 432 | trace_netfs_sreq(subreq, netfs_sreq_trace_progress); | ||
| 433 | |||
| 434 | if (subreq->transferred > subreq->consumed && | ||
| 435 | (rreq->origin == NETFS_READAHEAD || | ||
| 436 | rreq->origin == NETFS_READPAGE || | ||
| 437 | rreq->origin == NETFS_READ_FOR_WRITE)) { | ||
| 438 | netfs_consume_read_data(subreq, was_async); | ||
| 439 | __clear_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags); | ||
| 440 | } | ||
| 441 | } | ||
| 442 | EXPORT_SYMBOL(netfs_read_subreq_progress); | ||
| 443 | |||
| 444 | /** | ||
| 445 | * netfs_read_subreq_terminated - Note the termination of an I/O operation. | ||
| 446 | * @subreq: The I/O request that has terminated. | ||
| 447 | * @error: Error code indicating type of completion. | ||
| 448 | * @was_async: The termination was asynchronous | ||
| 449 | * | ||
| 450 | * This tells the read helper that a contributory I/O operation has terminated, | ||
| 451 | * one way or another, and that it should integrate the results. | ||
| 452 | * | ||
| 453 | * The caller indicates the outcome of the operation through @error, supplying | ||
| 454 | * 0 to indicate a successful or retryable transfer (if NETFS_SREQ_NEED_RETRY | ||
| 455 | * is set) or a negative error code. The helper will look after reissuing I/O | ||
| 456 | * operations as appropriate and writing downloaded data to the cache. | ||
| 457 | * | ||
| 458 | * Before calling, the filesystem should update subreq->transferred to track | ||
| 459 | * the amount of data copied into the output buffer. | ||
| 460 | * | ||
| 461 | * If @was_async is true, the caller might be running in softirq or interrupt | ||
| 462 | * context and we can't sleep. | ||
| 463 | */ | ||
| 464 | void netfs_read_subreq_terminated(struct netfs_io_subrequest *subreq, | ||
| 465 | int error, bool was_async) | ||
| 466 | { | ||
| 467 | struct netfs_io_request *rreq = subreq->rreq; | ||
| 468 | |||
| 469 | switch (subreq->source) { | ||
| 470 | case NETFS_READ_FROM_CACHE: | ||
| 471 | netfs_stat(&netfs_n_rh_read_done); | ||
| 472 | break; | ||
| 473 | case NETFS_DOWNLOAD_FROM_SERVER: | ||
| 474 | netfs_stat(&netfs_n_rh_download_done); | ||
| 475 | break; | ||
| 476 | default: | ||
| 477 | break; | ||
| 478 | } | ||
| 479 | |||
| 480 | if (rreq->origin != NETFS_DIO_READ) { | ||
| 481 | /* Collect buffered reads. | ||
| 482 | * | ||
| 483 | * If the read completed validly short, then we can clear the | ||
| 484 | * tail before going on to unlock the folios. | ||
| 485 | */ | ||
| 486 | if (error == 0 && subreq->transferred < subreq->len && | ||
| 487 | (test_bit(NETFS_SREQ_HIT_EOF, &subreq->flags) || | ||
| 488 | test_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags))) { | ||
| 489 | netfs_clear_unread(subreq); | ||
| 490 | subreq->transferred = subreq->len; | ||
| 491 | trace_netfs_sreq(subreq, netfs_sreq_trace_clear); | ||
| 492 | } | ||
| 493 | if (subreq->transferred > subreq->consumed && | ||
| 494 | (rreq->origin == NETFS_READAHEAD || | ||
| 495 | rreq->origin == NETFS_READPAGE || | ||
| 496 | rreq->origin == NETFS_READ_FOR_WRITE)) { | ||
| 497 | netfs_consume_read_data(subreq, was_async); | ||
| 498 | __clear_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags); | ||
| 499 | } | ||
| 500 | rreq->transferred += subreq->transferred; | ||
| 501 | } | ||
| 502 | |||
| 503 | /* Deal with retry requests, short reads and errors. If we retry | ||
| 504 | * but don't make progress, we abandon the attempt. | ||
| 505 | */ | ||
| 506 | if (!error && subreq->transferred < subreq->len) { | ||
| 507 | if (test_bit(NETFS_SREQ_HIT_EOF, &subreq->flags)) { | ||
| 508 | trace_netfs_sreq(subreq, netfs_sreq_trace_hit_eof); | ||
| 509 | } else { | ||
| 510 | trace_netfs_sreq(subreq, netfs_sreq_trace_short); | ||
| 511 | if (subreq->transferred > subreq->consumed) { | ||
| 512 | __set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); | ||
| 513 | __clear_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags); | ||
| 514 | set_bit(NETFS_RREQ_NEED_RETRY, &rreq->flags); | ||
| 515 | } else if (!__test_and_set_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags)) { | ||
| 516 | __set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); | ||
| 517 | set_bit(NETFS_RREQ_NEED_RETRY, &rreq->flags); | ||
| 518 | } else { | ||
| 519 | __set_bit(NETFS_SREQ_FAILED, &subreq->flags); | ||
| 520 | error = -ENODATA; | ||
| 521 | } | ||
| 522 | } | ||
| 523 | } | ||
| 524 | |||
| 525 | subreq->error = error; | ||
| 526 | trace_netfs_sreq(subreq, netfs_sreq_trace_terminated); | ||
| 527 | |||
| 528 | if (unlikely(error < 0)) { | ||
| 529 | trace_netfs_failure(rreq, subreq, error, netfs_fail_read); | ||
| 530 | if (subreq->source == NETFS_READ_FROM_CACHE) { | ||
| 531 | netfs_stat(&netfs_n_rh_read_failed); | ||
| 532 | } else { | ||
| 533 | netfs_stat(&netfs_n_rh_download_failed); | ||
| 534 | set_bit(NETFS_RREQ_FAILED, &rreq->flags); | ||
| 535 | rreq->error = subreq->error; | ||
| 536 | } | ||
| 537 | } | ||
| 538 | |||
| 539 | if (atomic_dec_and_test(&rreq->nr_outstanding)) | ||
| 540 | netfs_rreq_terminated(rreq, was_async); | ||
| 541 | |||
| 542 | netfs_put_subrequest(subreq, was_async, netfs_sreq_trace_put_terminated); | ||
| 543 | } | ||
| 544 | EXPORT_SYMBOL(netfs_read_subreq_terminated); | ||
| @@ -0,0 +1,264 @@ | |||
| 1 | // SPDX-License-Identifier: GPL-2.0-only | ||
| 2 | /* Read with PG_private_2 [DEPRECATED]. | ||
| 3 | * | ||
| 4 | * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. | ||
| 5 | * Written by David Howells (dhowells@redhat.com) | ||
| 6 | */ | ||
| 7 | |||
| 8 | #include <linux/export.h> | ||
| 9 | #include <linux/fs.h> | ||
| 10 | #include <linux/mm.h> | ||
| 11 | #include <linux/pagemap.h> | ||
| 12 | #include <linux/slab.h> | ||
| 13 | #include <linux/task_io_accounting_ops.h> | ||
| 14 | #include "internal.h" | ||
| 15 | |||
| 16 | /* | ||
| 17 | * [DEPRECATED] Mark page as requiring copy-to-cache using PG_private_2. The | ||
| 18 | * third mark in the folio queue is used to indicate that this folio needs | ||
| 19 | * writing. | ||
| 20 | */ | ||
| 21 | void netfs_pgpriv2_mark_copy_to_cache(struct netfs_io_subrequest *subreq, | ||
| 22 | struct netfs_io_request *rreq, | ||
| 23 | struct folio_queue *folioq, | ||
| 24 | int slot) | ||
| 25 | { | ||
| 26 | struct folio *folio = folioq_folio(folioq, slot); | ||
| 27 | |||
| 28 | trace_netfs_folio(folio, netfs_folio_trace_copy_to_cache); | ||
| 29 | folio_start_private_2(folio); | ||
| 30 | folioq_mark3(folioq, slot); | ||
| 31 | } | ||
| 32 | |||
| 33 | /* | ||
| 34 | * [DEPRECATED] Cancel PG_private_2 on all marked folios in the event of an | ||
| 35 | * unrecoverable error. | ||
| 36 | */ | ||
| 37 | static void netfs_pgpriv2_cancel(struct folio_queue *folioq) | ||
| 38 | { | ||
| 39 | struct folio *folio; | ||
| 40 | int slot; | ||
| 41 | |||
| 42 | while (folioq) { | ||
| 43 | if (!folioq->marks3) { | ||
| 44 | folioq = folioq->next; | ||
| 45 | continue; | ||
| 46 | } | ||
| 47 | |||
| 48 | slot = __ffs(folioq->marks3); | ||
| 49 | folio = folioq_folio(folioq, slot); | ||
| 50 | |||
| 51 | trace_netfs_folio(folio, netfs_folio_trace_cancel_copy); | ||
| 52 | folio_end_private_2(folio); | ||
| 53 | folioq_unmark3(folioq, slot); | ||
| 54 | } | ||
| 55 | } | ||
| 56 | |||
| 57 | /* | ||
| 58 | * [DEPRECATED] Copy a folio to the cache with PG_private_2 set. | ||
| 59 | */ | ||
| 60 | static int netfs_pgpriv2_copy_folio(struct netfs_io_request *wreq, struct folio *folio) | ||
| 61 | { | ||
| 62 | struct netfs_io_stream *cache = &wreq->io_streams[1]; | ||
| 63 | size_t fsize = folio_size(folio), flen = fsize; | ||
| 64 | loff_t fpos = folio_pos(folio), i_size; | ||
| 65 | bool to_eof = false; | ||
| 66 | |||
| 67 | _enter(""); | ||
| 68 | |||
| 69 | /* netfs_perform_write() may shift i_size around the page or from out | ||
| 70 | * of the page to beyond it, but cannot move i_size into or through the | ||
| 71 | * page since we have it locked. | ||
| 72 | */ | ||
| 73 | i_size = i_size_read(wreq->inode); | ||
| 74 | |||
| 75 | if (fpos >= i_size) { | ||
| 76 | /* mmap beyond eof. */ | ||
| 77 | _debug("beyond eof"); | ||
| 78 | folio_end_private_2(folio); | ||
| 79 | return 0; | ||
| 80 | } | ||
| 81 | |||
| 82 | if (fpos + fsize > wreq->i_size) | ||
| 83 | wreq->i_size = i_size; | ||
| 84 | |||
| 85 | if (flen > i_size - fpos) { | ||
| 86 | flen = i_size - fpos; | ||
| 87 | to_eof = true; | ||
| 88 | } else if (flen == i_size - fpos) { | ||
| 89 | to_eof = true; | ||
| 90 | } | ||
| 91 | |||
| 92 | _debug("folio %zx %zx", flen, fsize); | ||
| 93 | |||
| 94 | trace_netfs_folio(folio, netfs_folio_trace_store_copy); | ||
| 95 | |||
| 96 | /* Attach the folio to the rolling buffer. */ | ||
| 97 | if (netfs_buffer_append_folio(wreq, folio, false) < 0) | ||
| 98 | return -ENOMEM; | ||
| 99 | |||
| 100 | cache->submit_max_len = fsize; | ||
| 101 | cache->submit_off = 0; | ||
| 102 | cache->submit_len = flen; | ||
| 103 | |||
| 104 | /* Attach the folio to one or more subrequests. For a big folio, we | ||
| 105 | * could end up with thousands of subrequests if the wsize is small - | ||
| 106 | * but we might need to wait during the creation of subrequests for | ||
| 107 | * network resources (eg. SMB credits). | ||
| 108 | */ | ||
| 109 | do { | ||
| 110 | ssize_t part; | ||
| 111 | |||
| 112 | wreq->io_iter.iov_offset = cache->submit_off; | ||
| 113 | |||
| 114 | atomic64_set(&wreq->issued_to, fpos + cache->submit_off); | ||
| 115 | part = netfs_advance_write(wreq, cache, fpos + cache->submit_off, | ||
| 116 | cache->submit_len, to_eof); | ||
| 117 | cache->submit_off += part; | ||
| 118 | cache->submit_max_len -= part; | ||
| 119 | if (part > cache->submit_len) | ||
| 120 | cache->submit_len = 0; | ||
| 121 | else | ||
| 122 | cache->submit_len -= part; | ||
| 123 | } while (cache->submit_len > 0); | ||
| 124 | |||
| 125 | wreq->io_iter.iov_offset = 0; | ||
| 126 | iov_iter_advance(&wreq->io_iter, fsize); | ||
| 127 | atomic64_set(&wreq->issued_to, fpos + fsize); | ||
| 128 | |||
| 129 | if (flen < fsize) | ||
| 130 | netfs_issue_write(wreq, cache); | ||
| 131 | |||
| 132 | _leave(" = 0"); | ||
| 133 | return 0; | ||
| 134 | } | ||
| 135 | |||
| 136 | /* | ||
| 137 | * [DEPRECATED] Go through the buffer and write any folios that are marked with | ||
| 138 | * the third mark to the cache. | ||
| 139 | */ | ||
| 140 | void netfs_pgpriv2_write_to_the_cache(struct netfs_io_request *rreq) | ||
| 141 | { | ||
| 142 | struct netfs_io_request *wreq; | ||
| 143 | struct folio_queue *folioq; | ||
| 144 | struct folio *folio; | ||
| 145 | int error = 0; | ||
| 146 | int slot = 0; | ||
| 147 | |||
| 148 | _enter(""); | ||
| 149 | |||
| 150 | if (!fscache_resources_valid(&rreq->cache_resources)) | ||
| 151 | goto couldnt_start; | ||
| 152 | |||
| 153 | /* Need the first folio to be able to set up the op. */ | ||
| 154 | for (folioq = rreq->buffer; folioq; folioq = folioq->next) { | ||
| 155 | if (folioq->marks3) { | ||
| 156 | slot = __ffs(folioq->marks3); | ||
| 157 | break; | ||
| 158 | } | ||
| 159 | } | ||
| 160 | if (!folioq) | ||
| 161 | return; | ||
| 162 | folio = folioq_folio(folioq, slot); | ||
| 163 | |||
| 164 | wreq = netfs_create_write_req(rreq->mapping, NULL, folio_pos(folio), | ||
| 165 | NETFS_PGPRIV2_COPY_TO_CACHE); | ||
| 166 | if (IS_ERR(wreq)) { | ||
| 167 | kleave(" [create %ld]", PTR_ERR(wreq)); | ||
| 168 | goto couldnt_start; | ||
| 169 | } | ||
| 170 | |||
| 171 | trace_netfs_write(wreq, netfs_write_trace_copy_to_cache); | ||
| 172 | netfs_stat(&netfs_n_wh_copy_to_cache); | ||
| 173 | |||
| 174 | for (;;) { | ||
| 175 | error = netfs_pgpriv2_copy_folio(wreq, folio); | ||
| 176 | if (error < 0) | ||
| 177 | break; | ||
| 178 | |||
| 179 | folioq_unmark3(folioq, slot); | ||
| 180 | if (!folioq->marks3) { | ||
| 181 | folioq = folioq->next; | ||
| 182 | if (!folioq) | ||
| 183 | break; | ||
| 184 | } | ||
| 185 | |||
| 186 | slot = __ffs(folioq->marks3); | ||
| 187 | folio = folioq_folio(folioq, slot); | ||
| 188 | } | ||
| 189 | |||
| 190 | netfs_issue_write(wreq, &wreq->io_streams[1]); | ||
| 191 | smp_wmb(); /* Write lists before ALL_QUEUED. */ | ||
| 192 | set_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags); | ||
| 193 | |||
| 194 | netfs_put_request(wreq, false, netfs_rreq_trace_put_return); | ||
| 195 | _leave(" = %d", error); | ||
| 196 | couldnt_start: | ||
| 197 | netfs_pgpriv2_cancel(rreq->buffer); | ||
| 198 | } | ||
| 199 | |||
| 200 | /* | ||
| 201 | * [DEPRECATED] Remove the PG_private_2 mark from any folios we've finished | ||
| 202 | * copying. | ||
| 203 | */ | ||
| 204 | bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *wreq) | ||
| 205 | { | ||
| 206 | struct folio_queue *folioq = wreq->buffer; | ||
| 207 | unsigned long long collected_to = wreq->collected_to; | ||
| 208 | unsigned int slot = wreq->buffer_head_slot; | ||
| 209 | bool made_progress = false; | ||
| 210 | |||
| 211 | if (slot >= folioq_nr_slots(folioq)) { | ||
| 212 | folioq = netfs_delete_buffer_head(wreq); | ||
| 213 | slot = 0; | ||
| 214 | } | ||
| 215 | |||
| 216 | for (;;) { | ||
| 217 | struct folio *folio; | ||
| 218 | unsigned long long fpos, fend; | ||
| 219 | size_t fsize, flen; | ||
| 220 | |||
| 221 | folio = folioq_folio(folioq, slot); | ||
| 222 | if (WARN_ONCE(!folio_test_private_2(folio), | ||
| 223 | "R=%08x: folio %lx is not marked private_2\n", | ||
| 224 | wreq->debug_id, folio->index)) | ||
| 225 | trace_netfs_folio(folio, netfs_folio_trace_not_under_wback); | ||
| 226 | |||
| 227 | fpos = folio_pos(folio); | ||
| 228 | fsize = folio_size(folio); | ||
| 229 | flen = fsize; | ||
| 230 | |||
| 231 | fend = min_t(unsigned long long, fpos + flen, wreq->i_size); | ||
| 232 | |||
| 233 | trace_netfs_collect_folio(wreq, folio, fend, collected_to); | ||
| 234 | |||
| 235 | /* Unlock any folio we've transferred all of. */ | ||
| 236 | if (collected_to < fend) | ||
| 237 | break; | ||
| 238 | |||
| 239 | trace_netfs_folio(folio, netfs_folio_trace_end_copy); | ||
| 240 | folio_end_private_2(folio); | ||
| 241 | wreq->cleaned_to = fpos + fsize; | ||
| 242 | made_progress = true; | ||
| 243 | |||
| 244 | /* Clean up the head folioq. If we clear an entire folioq, then | ||
| 245 | * we can get rid of it provided it's not also the tail folioq | ||
| 246 | * being filled by the issuer. | ||
| 247 | */ | ||
| 248 | folioq_clear(folioq, slot); | ||
| 249 | slot++; | ||
| 250 | if (slot >= folioq_nr_slots(folioq)) { | ||
| 251 | if (READ_ONCE(wreq->buffer_tail) == folioq) | ||
| 252 | break; | ||
| 253 | folioq = netfs_delete_buffer_head(wreq); | ||
| 254 | slot = 0; | ||
| 255 | } | ||
| 256 | |||
| 257 | if (fpos + fsize >= collected_to) | ||
| 258 | break; | ||
| 259 | } | ||
| 260 | |||
| 261 | wreq->buffer = folioq; | ||
| 262 | wreq->buffer_head_slot = slot; | ||
| 263 | return made_progress; | ||
| 264 | } | ||
| @@ -0,0 +1,256 @@ | |||
| 1 | // SPDX-License-Identifier: GPL-2.0-only | ||
| 2 | /* Network filesystem read subrequest retrying. | ||
| 3 | * | ||
| 4 | * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. | ||
| 5 | * Written by David Howells (dhowells@redhat.com) | ||
| 6 | */ | ||
| 7 | |||
| 8 | #include <linux/fs.h> | ||
| 9 | #include <linux/slab.h> | ||
| 10 | #include "internal.h" | ||
| 11 | |||
| 12 | static void netfs_reissue_read(struct netfs_io_request *rreq, | ||
| 13 | struct netfs_io_subrequest *subreq) | ||
| 14 | { | ||
| 15 | struct iov_iter *io_iter = &subreq->io_iter; | ||
| 16 | |||
| 17 | if (iov_iter_is_folioq(io_iter)) { | ||
| 18 | subreq->curr_folioq = (struct folio_queue *)io_iter->folioq; | ||
| 19 | subreq->curr_folioq_slot = io_iter->folioq_slot; | ||
| 20 | subreq->curr_folio_order = subreq->curr_folioq->orders[subreq->curr_folioq_slot]; | ||
| 21 | } | ||
| 22 | |||
| 23 | atomic_inc(&rreq->nr_outstanding); | ||
| 24 | __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); | ||
| 25 | netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); | ||
| 26 | subreq->rreq->netfs_ops->issue_read(subreq); | ||
| 27 | } | ||
| 28 | |||
| 29 | /* | ||
| 30 | * Go through the list of failed/short reads, retrying all retryable ones. We | ||
| 31 | * need to switch failed cache reads to network downloads. | ||
| 32 | */ | ||
| 33 | static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) | ||
| 34 | { | ||
| 35 | struct netfs_io_subrequest *subreq; | ||
| 36 | struct netfs_io_stream *stream0 = &rreq->io_streams[0]; | ||
| 37 | LIST_HEAD(sublist); | ||
| 38 | LIST_HEAD(queue); | ||
| 39 | |||
| 40 | _enter("R=%x", rreq->debug_id); | ||
| 41 | |||
| 42 | if (list_empty(&rreq->subrequests)) | ||
| 43 | return; | ||
| 44 | |||
| 45 | if (rreq->netfs_ops->retry_request) | ||
| 46 | rreq->netfs_ops->retry_request(rreq, NULL); | ||
| 47 | |||
| 48 | /* If there's no renegotiation to do, just resend each retryable subreq | ||
| 49 | * up to the first permanently failed one. | ||
| 50 | */ | ||
| 51 | if (!rreq->netfs_ops->prepare_read && | ||
| 52 | !test_bit(NETFS_RREQ_COPY_TO_CACHE, &rreq->flags)) { | ||
| 53 | struct netfs_io_subrequest *subreq; | ||
| 54 | |||
| 55 | list_for_each_entry(subreq, &rreq->subrequests, rreq_link) { | ||
| 56 | if (test_bit(NETFS_SREQ_FAILED, &subreq->flags)) | ||
| 57 | break; | ||
| 58 | if (__test_and_clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { | ||
| 59 | netfs_reset_iter(subreq); | ||
| 60 | netfs_reissue_read(rreq, subreq); | ||
| 61 | } | ||
| 62 | } | ||
| 63 | return; | ||
| 64 | } | ||
| 65 | |||
| 66 | /* Okay, we need to renegotiate all the download requests and flip any | ||
| 67 | * failed cache reads over to being download requests and negotiate | ||
| 68 | * those also. All fully successful subreqs have been removed from the | ||
| 69 | * list and any spare data from those has been donated. | ||
| 70 | * | ||
| 71 | * What we do is decant the list and rebuild it one subreq at a time so | ||
| 72 | * that we don't end up with donations jumping over a gap we're busy | ||
| 73 | * populating with smaller subrequests. In the event that the subreq | ||
| 74 | * we just launched finishes before we insert the next subreq, it'll | ||
| 75 | * fill in rreq->prev_donated instead. | ||
| 76 | |||
| 77 | * Note: Alternatively, we could split the tail subrequest right before | ||
| 78 | * we reissue it and fix up the donations under lock. | ||
| 79 | */ | ||
| 80 | list_splice_init(&rreq->subrequests, &queue); | ||
| 81 | |||
| 82 | do { | ||
| 83 | struct netfs_io_subrequest *from; | ||
| 84 | struct iov_iter source; | ||
| 85 | unsigned long long start, len; | ||
| 86 | size_t part, deferred_next_donated = 0; | ||
| 87 | bool boundary = false; | ||
| 88 | |||
| 89 | /* Go through the subreqs and find the next span of contiguous | ||
| 90 | * buffer that we then rejig (cifs, for example, needs the | ||
| 91 | * rsize renegotiating) and reissue. | ||
| 92 | */ | ||
| 93 | from = list_first_entry(&queue, struct netfs_io_subrequest, rreq_link); | ||
| 94 | list_move_tail(&from->rreq_link, &sublist); | ||
| 95 | start = from->start + from->transferred; | ||
| 96 | len = from->len - from->transferred; | ||
| 97 | |||
| 98 | _debug("from R=%08x[%x] s=%llx ctl=%zx/%zx/%zx", | ||
| 99 | rreq->debug_id, from->debug_index, | ||
| 100 | from->start, from->consumed, from->transferred, from->len); | ||
| 101 | |||
| 102 | if (test_bit(NETFS_SREQ_FAILED, &from->flags) || | ||
| 103 | !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags)) | ||
| 104 | goto abandon; | ||
| 105 | |||
| 106 | deferred_next_donated = from->next_donated; | ||
| 107 | while ((subreq = list_first_entry_or_null( | ||
| 108 | &queue, struct netfs_io_subrequest, rreq_link))) { | ||
| 109 | if (subreq->start != start + len || | ||
| 110 | subreq->transferred > 0 || | ||
| 111 | !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) | ||
| 112 | break; | ||
| 113 | list_move_tail(&subreq->rreq_link, &sublist); | ||
| 114 | len += subreq->len; | ||
| 115 | deferred_next_donated = subreq->next_donated; | ||
| 116 | if (test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags)) | ||
| 117 | break; | ||
| 118 | } | ||
| 119 | |||
| 120 | _debug(" - range: %llx-%llx %llx", start, start + len - 1, len); | ||
| 121 | |||
| 122 | /* Determine the set of buffers we're going to use. Each | ||
| 123 | * subreq gets a subset of a single overall contiguous buffer. | ||
| 124 | */ | ||
| 125 | netfs_reset_iter(from); | ||
| 126 | source = from->io_iter; | ||
| 127 | source.count = len; | ||
| 128 | |||
| 129 | /* Work through the sublist. */ | ||
| 130 | while ((subreq = list_first_entry_or_null( | ||
| 131 | &sublist, struct netfs_io_subrequest, rreq_link))) { | ||
| 132 | list_del(&subreq->rreq_link); | ||
| 133 | |||
| 134 | subreq->source = NETFS_DOWNLOAD_FROM_SERVER; | ||
| 135 | subreq->start = start - subreq->transferred; | ||
| 136 | subreq->len = len + subreq->transferred; | ||
| 137 | stream0->sreq_max_len = subreq->len; | ||
| 138 | |||
| 139 | __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); | ||
| 140 | __set_bit(NETFS_SREQ_RETRYING, &subreq->flags); | ||
| 141 | |||
| 142 | spin_lock_bh(&rreq->lock); | ||
| 143 | list_add_tail(&subreq->rreq_link, &rreq->subrequests); | ||
| 144 | subreq->prev_donated += rreq->prev_donated; | ||
| 145 | rreq->prev_donated = 0; | ||
| 146 | trace_netfs_sreq(subreq, netfs_sreq_trace_retry); | ||
| 147 | spin_unlock_bh(&rreq->lock); | ||
| 148 | |||
| 149 | BUG_ON(!len); | ||
| 150 | |||
| 151 | /* Renegotiate max_len (rsize) */ | ||
| 152 | if (rreq->netfs_ops->prepare_read(subreq) < 0) { | ||
| 153 | trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed); | ||
| 154 | __set_bit(NETFS_SREQ_FAILED, &subreq->flags); | ||
| 155 | } | ||
| 156 | |||
| 157 | part = umin(len, stream0->sreq_max_len); | ||
| 158 | if (unlikely(rreq->io_streams[0].sreq_max_segs)) | ||
| 159 | part = netfs_limit_iter(&source, 0, part, stream0->sreq_max_segs); | ||
| 160 | subreq->len = subreq->transferred + part; | ||
| 161 | subreq->io_iter = source; | ||
| 162 | iov_iter_truncate(&subreq->io_iter, part); | ||
| 163 | iov_iter_advance(&source, part); | ||
| 164 | len -= part; | ||
| 165 | start += part; | ||
| 166 | if (!len) { | ||
| 167 | if (boundary) | ||
| 168 | __set_bit(NETFS_SREQ_BOUNDARY, &subreq->flags); | ||
| 169 | subreq->next_donated = deferred_next_donated; | ||
| 170 | } else { | ||
| 171 | __clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags); | ||
| 172 | subreq->next_donated = 0; | ||
| 173 | } | ||
| 174 | |||
| 175 | netfs_reissue_read(rreq, subreq); | ||
| 176 | if (!len) | ||
| 177 | break; | ||
| 178 | |||
| 179 | /* If we ran out of subrequests, allocate another. */ | ||
| 180 | if (list_empty(&sublist)) { | ||
| 181 | subreq = netfs_alloc_subrequest(rreq); | ||
| 182 | if (!subreq) | ||
| 183 | goto abandon; | ||
| 184 | subreq->source = NETFS_DOWNLOAD_FROM_SERVER; | ||
| 185 | subreq->start = start; | ||
| 186 | |||
| 187 | /* We get two refs, but need just one. */ | ||
| 188 | netfs_put_subrequest(subreq, false, netfs_sreq_trace_new); | ||
| 189 | trace_netfs_sreq(subreq, netfs_sreq_trace_split); | ||
| 190 | list_add_tail(&subreq->rreq_link, &sublist); | ||
| 191 | } | ||
| 192 | } | ||
| 193 | |||
| 194 | /* If we managed to use fewer subreqs, we can discard the | ||
| 195 | * excess. | ||
| 196 | */ | ||
| 197 | while ((subreq = list_first_entry_or_null( | ||
| 198 | &sublist, struct netfs_io_subrequest, rreq_link))) { | ||
| 199 | trace_netfs_sreq(subreq, netfs_sreq_trace_discard); | ||
| 200 | list_del(&subreq->rreq_link); | ||
| 201 | netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_done); | ||
| 202 | } | ||
| 203 | |||
| 204 | } while (!list_empty(&queue)); | ||
| 205 | |||
| 206 | return; | ||
| 207 | |||
| 208 | /* If we hit ENOMEM, fail all remaining subrequests */ | ||
| 209 | abandon: | ||
| 210 | list_splice_init(&sublist, &queue); | ||
| 211 | list_for_each_entry(subreq, &queue, rreq_link) { | ||
| 212 | if (!subreq->error) | ||
| 213 | subreq->error = -ENOMEM; | ||
| 214 | __clear_bit(NETFS_SREQ_FAILED, &subreq->flags); | ||
| 215 | __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); | ||
| 216 | __clear_bit(NETFS_SREQ_RETRYING, &subreq->flags); | ||
| 217 | } | ||
| 218 | spin_lock_bh(&rreq->lock); | ||
| 219 | list_splice_tail_init(&queue, &rreq->subrequests); | ||
| 220 | spin_unlock_bh(&rreq->lock); | ||
| 221 | } | ||
| 222 | |||
| 223 | /* | ||
| 224 | * Retry reads. | ||
| 225 | */ | ||
| 226 | void netfs_retry_reads(struct netfs_io_request *rreq) | ||
| 227 | { | ||
| 228 | trace_netfs_rreq(rreq, netfs_rreq_trace_resubmit); | ||
| 229 | |||
| 230 | atomic_inc(&rreq->nr_outstanding); | ||
| 231 | |||
| 232 | netfs_retry_read_subrequests(rreq); | ||
| 233 | |||
| 234 | if (atomic_dec_and_test(&rreq->nr_outstanding)) | ||
| 235 | netfs_rreq_terminated(rreq, false); | ||
| 236 | } | ||
| 237 | |||
| 238 | /* | ||
| 239 | * Unlock any the pages that haven't been unlocked yet due to abandoned | ||
| 240 | * subrequests. | ||
| 241 | */ | ||
| 242 | void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq) | ||
| 243 | { | ||
| 244 | struct folio_queue *p; | ||
| 245 | |||
| 246 | for (p = rreq->buffer; p; p = p->next) { | ||
| 247 | for (int slot = 0; slot < folioq_count(p); slot++) { | ||
| 248 | struct folio *folio = folioq_folio(p, slot); | ||
| 249 | |||
| 250 | if (folio && !folioq_is_marked2(p, slot)) { | ||
| 251 | trace_netfs_folio(folio, netfs_folio_trace_abandon); | ||
| 252 | folio_unlock(folio); | ||
| 253 | } | ||
| 254 | } | ||
| 255 | } | ||
| 256 | } | ||
| @@ -32,6 +32,7 @@ atomic_t netfs_n_wh_buffered_write; · 31 unmodified lines | |||
| 32 | atomic_t netfs_n_wh_writethrough; | 32 | atomic_t netfs_n_wh_writethrough; |
| 33 | atomic_t netfs_n_wh_dio_write; | 33 | atomic_t netfs_n_wh_dio_write; |
| 34 | atomic_t netfs_n_wh_writepages; | 34 | atomic_t netfs_n_wh_writepages; |
| 35 | atomic_t netfs_n_wh_copy_to_cache; | ||
| 35 | atomic_t netfs_n_wh_wstream_conflict; | 36 | atomic_t netfs_n_wh_wstream_conflict; |
| 36 | atomic_t netfs_n_wh_upload; | 37 | atomic_t netfs_n_wh_upload; |
| 37 | atomic_t netfs_n_wh_upload_done; | 38 | atomic_t netfs_n_wh_upload_done; |
| @@ -51,11 +52,12 @@ int netfs_stats_show(struct seq_file *m, void *v) · 13 unmodified lines | |||
| 51 | atomic_read(&netfs_n_rh_read_folio), | 52 | atomic_read(&netfs_n_rh_read_folio), |
| 52 | atomic_read(&netfs_n_rh_write_begin), | 53 | atomic_read(&netfs_n_rh_write_begin), |
| 53 | atomic_read(&netfs_n_rh_write_zskip)); | 54 | atomic_read(&netfs_n_rh_write_zskip)); |
| 54 | seq_printf(m, "Writes : BW=%u WT=%u DW=%u WP=%u\n", | 55 | seq_printf(m, "Writes : BW=%u WT=%u DW=%u WP=%u 2C=%u\n", |
| 55 | atomic_read(&netfs_n_wh_buffered_write), | 56 | atomic_read(&netfs_n_wh_buffered_write), |
| 56 | atomic_read(&netfs_n_wh_writethrough), | 57 | atomic_read(&netfs_n_wh_writethrough), |
| 57 | atomic_read(&netfs_n_wh_dio_write), | 58 | atomic_read(&netfs_n_wh_dio_write), |
| 58 | atomic_read(&netfs_n_wh_writepages)); | 59 | atomic_read(&netfs_n_wh_writepages), |
| 60 | atomic_read(&netfs_n_wh_copy_to_cache)); | ||
| 59 | seq_printf(m, "ZeroOps: ZR=%u sh=%u sk=%u\n", | 61 | seq_printf(m, "ZeroOps: ZR=%u sh=%u sk=%u\n", |
| 60 | atomic_read(&netfs_n_rh_zero), | 62 | atomic_read(&netfs_n_rh_zero), |
| 61 | atomic_read(&netfs_n_rh_short_read), | 63 | atomic_read(&netfs_n_rh_short_read), |
| @@ -87,6 +87,12 @@ static void netfs_writeback_unlock_folios(struct netfs_io_request *wreq, · 86 unmodified lines | |||
| 87 | unsigned long long collected_to = wreq->collected_to; | 87 | unsigned long long collected_to = wreq->collected_to; |
| 88 | unsigned int slot = wreq->buffer_head_slot; | 88 | unsigned int slot = wreq->buffer_head_slot; |
| 89 | 89 | ||
| 90 | if (wreq->origin == NETFS_PGPRIV2_COPY_TO_CACHE) { | ||
| 91 | if (netfs_pgpriv2_unlock_copied_folios(wreq)) | ||
| 92 | *notes |= MADE_PROGRESS; | ||
| 93 | return; | ||
| 94 | } | ||
| 95 | |||
| 90 | if (slot >= folioq_nr_slots(folioq)) { | 96 | if (slot >= folioq_nr_slots(folioq)) { |
| 91 | folioq = netfs_delete_buffer_head(wreq); | 97 | folioq = netfs_delete_buffer_head(wreq); |
| 92 | slot = 0; | 98 | slot = 0; |
| @@ -383,7 +389,8 @@ reassess_streams: · 290 unmodified lines | |||
| 383 | smp_rmb(); | 389 | smp_rmb(); |
| 384 | collected_to = ULLONG_MAX; | 390 | collected_to = ULLONG_MAX; |
| 385 | if (wreq->origin == NETFS_WRITEBACK || | 391 | if (wreq->origin == NETFS_WRITEBACK || |
| 386 | wreq->origin == NETFS_WRITETHROUGH) | 392 | wreq->origin == NETFS_WRITETHROUGH || |
| 393 | wreq->origin == NETFS_PGPRIV2_COPY_TO_CACHE) | ||
| 387 | notes = BUFFERED; | 394 | notes = BUFFERED; |
| 388 | else | 395 | else |
| 389 | notes = 0; | 396 | notes = 0; |
| @@ -95,7 +95,8 @@ struct netfs_io_request *netfs_create_write_req(struct address_space *mapping, · 94 unmodified lines | |||
| 95 | struct netfs_io_request *wreq; | 95 | struct netfs_io_request *wreq; |
| 96 | struct netfs_inode *ictx; | 96 | struct netfs_inode *ictx; |
| 97 | bool is_buffered = (origin == NETFS_WRITEBACK || | 97 | bool is_buffered = (origin == NETFS_WRITEBACK || |
| 98 | origin == NETFS_WRITETHROUGH); | 98 | origin == NETFS_WRITETHROUGH || |
| 99 | origin == NETFS_PGPRIV2_COPY_TO_CACHE); | ||
| 99 | 100 | ||
| 100 | wreq = netfs_alloc_request(mapping, file, start, 0, origin); | 101 | wreq = netfs_alloc_request(mapping, file, start, 0, origin); |
| 101 | if (IS_ERR(wreq)) | 102 | if (IS_ERR(wreq)) |
| @@ -161,10 +162,6 @@ static void netfs_prepare_write(struct netfs_io_request *wreq, · 59 unmodified lines | |||
| 161 | 162 | ||
| 162 | _enter("R=%x[%x]", wreq->debug_id, subreq->debug_index); | 163 | _enter("R=%x[%x]", wreq->debug_id, subreq->debug_index); |
| 163 | 164 | ||
| 164 | trace_netfs_sreq_ref(wreq->debug_id, subreq->debug_index, | ||
| 165 | refcount_read(&subreq->ref), | ||
| 166 | netfs_sreq_trace_new); | ||
| 167 | |||
| 168 | trace_netfs_sreq(subreq, netfs_sreq_trace_prepare); | 165 | trace_netfs_sreq(subreq, netfs_sreq_trace_prepare); |
| 169 | 166 | ||
| 170 | stream->sreq_max_len = UINT_MAX; | 167 | stream->sreq_max_len = UINT_MAX; |
| @@ -241,8 +238,8 @@ void netfs_reissue_write(struct netfs_io_stream *stream, · 70 unmodified lines | |||
| 241 | netfs_do_issue_write(stream, subreq); | 238 | netfs_do_issue_write(stream, subreq); |
| 242 | } | 239 | } |
| 243 | 240 | ||
| 244 | static void netfs_issue_write(struct netfs_io_request *wreq, | 241 | void netfs_issue_write(struct netfs_io_request *wreq, |
| 245 | struct netfs_io_stream *stream) | 242 | struct netfs_io_stream *stream) |
| 246 | { | 243 | { |
| 247 | struct netfs_io_subrequest *subreq = stream->construct; | 244 | struct netfs_io_subrequest *subreq = stream->construct; |
| 248 | 245 | ||
| @@ -259,9 +256,9 @@ static void netfs_issue_write(struct netfs_io_request *wreq, · 10 unmodified lines | |||
| 259 | * we can avoid overrunning the credits obtained (cifs) and try to parallelise | 256 | * we can avoid overrunning the credits obtained (cifs) and try to parallelise |
| 260 | * content-crypto preparation with network writes. | 257 | * content-crypto preparation with network writes. |
| 261 | */ | 258 | */ |
| 262 | static int netfs_advance_write(struct netfs_io_request *wreq, | 259 | int netfs_advance_write(struct netfs_io_request *wreq, |
| 263 | struct netfs_io_stream *stream, | 260 | struct netfs_io_stream *stream, |
| 264 | loff_t start, size_t len, bool to_eof) | 261 | loff_t start, size_t len, bool to_eof) |
| 265 | { | 262 | { |
| 266 | struct netfs_io_subrequest *subreq = stream->construct; | 263 | struct netfs_io_subrequest *subreq = stream->construct; |
| 267 | size_t part; | 264 | size_t part; |
| @@ -267,6 +267,7 @@ static int nfs_netfs_init_request(struct netfs_io_request *rreq, struct file *fi · 266 unmodified lines | |||
| 267 | rreq->debug_id = atomic_inc_return(&nfs_netfs_debug_id); | 267 | rreq->debug_id = atomic_inc_return(&nfs_netfs_debug_id); |
| 268 | /* [DEPRECATED] Use PG_private_2 to mark folio being written to the cache. */ | 268 | /* [DEPRECATED] Use PG_private_2 to mark folio being written to the cache. */ |
| 269 | __set_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags); | 269 | __set_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags); |
| 270 | rreq->io_streams[0].sreq_max_len = NFS_SB(rreq->inode->i_sb)->rsize; | ||
| 270 | 271 | ||
| 271 | return 0; | 272 | return 0; |
| 272 | } | 273 | } |
| @@ -288,14 +289,6 @@ static struct nfs_netfs_io_data *nfs_netfs_alloc(struct netfs_io_subrequest *sre · 15 unmodified lines | |||
| 288 | return netfs; | 289 | return netfs; |
| 289 | } | 290 | } |
| 290 | 291 | ||
| 291 | static bool nfs_netfs_clamp_length(struct netfs_io_subrequest *sreq) | ||
| 292 | { | ||
| 293 | size_t rsize = NFS_SB(sreq->rreq->inode->i_sb)->rsize; | ||
| 294 | |||
| 295 | sreq->len = min(sreq->len, rsize); | ||
| 296 | return true; | ||
| 297 | } | ||
| 298 | |||
| 299 | static void nfs_netfs_issue_read(struct netfs_io_subrequest *sreq) | 292 | static void nfs_netfs_issue_read(struct netfs_io_subrequest *sreq) |
| 300 | { | 293 | { |
| 301 | struct nfs_netfs_io_data *netfs; | 294 | struct nfs_netfs_io_data *netfs; |
| @@ -304,17 +297,18 @@ static void nfs_netfs_issue_read(struct netfs_io_subrequest *sreq) · 2 unmodified lines | |||
| 304 | struct nfs_open_context *ctx = sreq->rreq->netfs_priv; | 297 | struct nfs_open_context *ctx = sreq->rreq->netfs_priv; |
| 305 | struct page *page; | 298 | struct page *page; |
| 306 | unsigned long idx; | 299 | unsigned long idx; |
| 300 | pgoff_t start, last; | ||
| 307 | int err; | 301 | int err; |
| 308 | pgoff_t start = (sreq->start + sreq->transferred) >> PAGE_SHIFT; | 302 | |
| 309 | pgoff_t last = ((sreq->start + sreq->len - | 303 | start = (sreq->start + sreq->transferred) >> PAGE_SHIFT; |
| 310 | sreq->transferred - 1) >> PAGE_SHIFT); | 304 | last = ((sreq->start + sreq->len - sreq->transferred - 1) >> PAGE_SHIFT); |
| 311 | 305 | ||
| 312 | nfs_pageio_init_read(&pgio, inode, false, | 306 | nfs_pageio_init_read(&pgio, inode, false, |
| 313 | &nfs_async_read_completion_ops); | 307 | &nfs_async_read_completion_ops); |
| 314 | 308 | ||
| 315 | netfs = nfs_netfs_alloc(sreq); | 309 | netfs = nfs_netfs_alloc(sreq); |
| 316 | if (!netfs) | 310 | if (!netfs) |
| 317 | return netfs_subreq_terminated(sreq, -ENOMEM, false); | 311 | return netfs_read_subreq_terminated(sreq, -ENOMEM, false); |
| 318 | 312 | ||
| 319 | pgio.pg_netfs = netfs; /* used in completion */ | 313 | pgio.pg_netfs = netfs; /* used in completion */ |
| 320 | 314 | ||
| @@ -380,5 +374,4 @@ const struct netfs_request_ops nfs_netfs_ops = { · 59 unmodified lines | |||
| 380 | .init_request = nfs_netfs_init_request, | 374 | .init_request = nfs_netfs_init_request, |
| 381 | .free_request = nfs_netfs_free_request, | 375 | .free_request = nfs_netfs_free_request, |
| 382 | .issue_read = nfs_netfs_issue_read, | 376 | .issue_read = nfs_netfs_issue_read, |
| 383 | .clamp_length = nfs_netfs_clamp_length | ||
| 384 | }; | 377 | }; |
| @@ -60,8 +60,6 @@ static inline void nfs_netfs_get(struct nfs_netfs_io_data *netfs) · 59 unmodified lines | |||
| 60 | 60 | ||
| 61 | static inline void nfs_netfs_put(struct nfs_netfs_io_data *netfs) | 61 | static inline void nfs_netfs_put(struct nfs_netfs_io_data *netfs) |
| 62 | { | 62 | { |
| 63 | ssize_t final_len; | ||
| 64 | |||
| 65 | /* Only the last RPC completion should call netfs_subreq_terminated() */ | 63 | /* Only the last RPC completion should call netfs_subreq_terminated() */ |
| 66 | if (!refcount_dec_and_test(&netfs->refcount)) | 64 | if (!refcount_dec_and_test(&netfs->refcount)) |
| 67 | return; | 65 | return; |
| @@ -74,8 +72,9 @@ static inline void nfs_netfs_put(struct nfs_netfs_io_data *netfs) · 6 unmodified lines | |||
| 74 | * Correct the final length here to be no larger than the netfs subrequest | 72 | * Correct the final length here to be no larger than the netfs subrequest |
| 75 | * length, and thus avoid netfs's "Subreq overread" warning message. | 73 | * length, and thus avoid netfs's "Subreq overread" warning message. |
| 76 | */ | 74 | */ |
| 77 | final_len = min_t(s64, netfs->sreq->len, atomic64_read(&netfs->transferred)); | 75 | netfs->sreq->transferred = min_t(s64, netfs->sreq->len, |
| 78 | netfs_subreq_terminated(netfs->sreq, netfs->error ?: final_len, false); | 76 | atomic64_read(&netfs->transferred)); |
| 77 | netfs_read_subreq_terminated(netfs->sreq, netfs->error, false); | ||
| 79 | kfree(netfs); | 78 | kfree(netfs); |
| 80 | } | 79 | } |
| 81 | static inline void nfs_netfs_inode_init(struct nfs_inode *nfsi) | 80 | static inline void nfs_netfs_inode_init(struct nfs_inode *nfsi) |
| @@ -1485,7 +1485,6 @@ struct cifs_io_subrequest { · 1484 unmodified lines | |||
| 1485 | struct cifs_io_request *req; | 1485 | struct cifs_io_request *req; |
| 1486 | }; | 1486 | }; |
| 1487 | ssize_t got_bytes; | 1487 | ssize_t got_bytes; |
| 1488 | size_t actual_len; | ||
| 1489 | unsigned int xid; | 1488 | unsigned int xid; |
| 1490 | int result; | 1489 | int result; |
| 1491 | bool have_xid; | 1490 | bool have_xid; |
| @@ -1309,10 +1309,8 @@ cifs_readv_callback(struct mid_q_entry *mid) · 1308 unmodified lines | |||
| 1309 | if (rdata->result == 0 || rdata->result == -EAGAIN) | 1309 | if (rdata->result == 0 || rdata->result == -EAGAIN) |
| 1310 | iov_iter_advance(&rdata->subreq.io_iter, rdata->got_bytes); | 1310 | iov_iter_advance(&rdata->subreq.io_iter, rdata->got_bytes); |
| 1311 | rdata->credits.value = 0; | 1311 | rdata->credits.value = 0; |
| 1312 | netfs_subreq_terminated(&rdata->subreq, | 1312 | rdata->subreq.transferred += rdata->got_bytes; |
| 1313 | (rdata->result == 0 || rdata->result == -EAGAIN) ? | 1313 | netfs_read_subreq_terminated(&rdata->subreq, rdata->result, false); |
| 1314 | rdata->got_bytes : rdata->result, | ||
| 1315 | false); | ||
| 1316 | release_mid(mid); | 1314 | release_mid(mid); |
| 1317 | add_credits(server, &credits, 0); | 1315 | add_credits(server, &credits, 0); |
| 1318 | } | 1316 | } |
| @@ -112,7 +112,6 @@ static void cifs_issue_write(struct netfs_io_subrequest *subreq) · 111 unmodified lines | |||
| 112 | goto fail; | 112 | goto fail; |
| 113 | } | 113 | } |
| 114 | 114 | ||
| 115 | wdata->actual_len = wdata->subreq.len; | ||
| 116 | rc = adjust_credits(wdata->server, wdata, cifs_trace_rw_credits_issue_write_adjust); | 115 | rc = adjust_credits(wdata->server, wdata, cifs_trace_rw_credits_issue_write_adjust); |
| 117 | if (rc) | 116 | if (rc) |
| 118 | goto fail; | 117 | goto fail; |
| @@ -141,25 +140,22 @@ static void cifs_netfs_invalidate_cache(struct netfs_io_request *wreq) · 22 unmodified lines | |||
| 141 | } | 140 | } |
| 142 | 141 | ||
| 143 | /* | 142 | /* |
| 144 | * Split the read up according to how many credits we can get for each piece. | 143 | * Negotiate the size of a read operation on behalf of the netfs library. |
| 145 | * It's okay to sleep here if we need to wait for more credit to become | ||
| 146 | * available. | ||
| 147 | * | ||
| 148 | * We also choose the server and allocate an operation ID to be cleaned up | ||
| 149 | * later. | ||
| 150 | */ | 144 | */ |
| 151 | static bool cifs_clamp_length(struct netfs_io_subrequest *subreq) | 145 | static int cifs_prepare_read(struct netfs_io_subrequest *subreq) |
| 152 | { | 146 | { |
| 153 | struct netfs_io_request *rreq = subreq->rreq; | 147 | struct netfs_io_request *rreq = subreq->rreq; |
| 154 | struct netfs_io_stream *stream = &rreq->io_streams[subreq->stream_nr]; | ||
| 155 | struct cifs_io_subrequest *rdata = container_of(subreq, struct cifs_io_subrequest, subreq); | 148 | struct cifs_io_subrequest *rdata = container_of(subreq, struct cifs_io_subrequest, subreq); |
| 156 | struct cifs_io_request *req = container_of(subreq->rreq, struct cifs_io_request, rreq); | 149 | struct cifs_io_request *req = container_of(subreq->rreq, struct cifs_io_request, rreq); |
| 157 | struct TCP_Server_Info *server = req->server; | 150 | struct TCP_Server_Info *server = req->server; |
| 158 | struct cifs_sb_info *cifs_sb = CIFS_SB(rreq->inode->i_sb); | 151 | struct cifs_sb_info *cifs_sb = CIFS_SB(rreq->inode->i_sb); |
| 159 | int rc; | 152 | size_t size; |
| 153 | int rc = 0; | ||
| 160 | 154 | ||
| 161 | rdata->xid = get_xid(); | 155 | if (!rdata->have_xid) { |
| 162 | rdata->have_xid = true; | 156 | rdata->xid = get_xid(); |
| 157 | rdata->have_xid = true; | ||
| 158 | } | ||
| 163 | rdata->server = server; | 159 | rdata->server = server; |
| 164 | 160 | ||
| 165 | if (cifs_sb->ctx->rsize == 0) | 161 | if (cifs_sb->ctx->rsize == 0) |
| @@ -167,13 +163,12 @@ static bool cifs_clamp_length(struct netfs_io_subrequest *subreq) · 1 unmodified line | |||
| 167 | server->ops->negotiate_rsize(tlink_tcon(req->cfile->tlink), | 163 | server->ops->negotiate_rsize(tlink_tcon(req->cfile->tlink), |
| 168 | cifs_sb->ctx); | 164 | cifs_sb->ctx); |
| 169 | 165 | ||
| 170 | |||
| 171 | rc = server->ops->wait_mtu_credits(server, cifs_sb->ctx->rsize, | 166 | rc = server->ops->wait_mtu_credits(server, cifs_sb->ctx->rsize, |
| 172 | &stream->sreq_max_len, &rdata->credits); | 167 | &size, &rdata->credits); |
| 173 | if (rc) { | 168 | if (rc) |
| 174 | subreq->error = rc; | 169 | return rc; |
| 175 | return false; | 170 | |
| 176 | } | 171 | rreq->io_streams[0].sreq_max_len = size; |
| 177 | 172 | ||
| 178 | rdata->credits.in_flight_check = 1; | 173 | rdata->credits.in_flight_check = 1; |
| 179 | rdata->credits.rreq_debug_id = rreq->debug_id; | 174 | rdata->credits.rreq_debug_id = rreq->debug_id; |
| @@ -185,14 +180,11 @@ static bool cifs_clamp_length(struct netfs_io_subrequest *subreq) · 5 unmodified lines | |||
| 185 | server->credits, server->in_flight, 0, | 180 | server->credits, server->in_flight, 0, |
| 186 | cifs_trace_rw_credits_read_submit); | 181 | cifs_trace_rw_credits_read_submit); |
| 187 | 182 | ||
| 188 | subreq->len = umin(subreq->len, stream->sreq_max_len); | ||
| 189 | rdata->actual_len = subreq->len; | ||
| 190 | |||
| 191 | #ifdef CONFIG_CIFS_SMB_DIRECT | 183 | #ifdef CONFIG_CIFS_SMB_DIRECT |
| 192 | if (server->smbd_conn) | 184 | if (server->smbd_conn) |
| 193 | stream->sreq_max_segs = server->smbd_conn->max_frmr_depth; | 185 | rreq->io_streams[0].sreq_max_segs = server->smbd_conn->max_frmr_depth; |
| 194 | #endif | 186 | #endif |
| 195 | return true; | 187 | return 0; |
| 196 | } | 188 | } |
| 197 | 189 | ||
| 198 | /* | 190 | /* |
| @@ -201,59 +193,41 @@ static bool cifs_clamp_length(struct netfs_io_subrequest *subreq) · 2 unmodified lines | |||
| 201 | * to only read a portion of that, but as long as we read something, the netfs | 193 | * to only read a portion of that, but as long as we read something, the netfs |
| 202 | * helper will call us again so that we can issue another read. | 194 | * helper will call us again so that we can issue another read. |
| 203 | */ | 195 | */ |
| 204 | static void cifs_req_issue_read(struct netfs_io_subrequest *subreq) | 196 | static void cifs_issue_read(struct netfs_io_subrequest *subreq) |
| 205 | { | 197 | { |
| 206 | struct netfs_io_request *rreq = subreq->rreq; | 198 | struct netfs_io_request *rreq = subreq->rreq; |
| 207 | struct cifs_io_subrequest *rdata = container_of(subreq, struct cifs_io_subrequest, subreq); | 199 | struct cifs_io_subrequest *rdata = container_of(subreq, struct cifs_io_subrequest, subreq); |
| 208 | struct cifs_io_request *req = container_of(subreq->rreq, struct cifs_io_request, rreq); | 200 | struct cifs_io_request *req = container_of(subreq->rreq, struct cifs_io_request, rreq); |
| 209 | struct TCP_Server_Info *server = req->server; | 201 | struct TCP_Server_Info *server = req->server; |
| 210 | struct cifs_sb_info *cifs_sb = CIFS_SB(rreq->inode->i_sb); | ||
| 211 | int rc = 0; | 202 | int rc = 0; |
| 212 | 203 | ||
| 213 | cifs_dbg(FYI, "%s: op=%08x[%x] mapping=%p len=%zu/%zu\n", | 204 | cifs_dbg(FYI, "%s: op=%08x[%x] mapping=%p len=%zu/%zu\n", |
| 214 | __func__, rreq->debug_id, subreq->debug_index, rreq->mapping, | 205 | __func__, rreq->debug_id, subreq->debug_index, rreq->mapping, |
| 215 | subreq->transferred, subreq->len); | 206 | subreq->transferred, subreq->len); |
| 216 | 207 | ||
| 217 | if (test_bit(NETFS_SREQ_RETRYING, &subreq->flags)) { | 208 | rc = adjust_credits(server, rdata, cifs_trace_rw_credits_issue_read_adjust); |
| 218 | /* | 209 | if (rc) |
| 219 | * As we're issuing a retry, we need to negotiate some new | 210 | goto failed; |
| 220 | * credits otherwise the server may reject the op with | ||
| 221 | * INVALID_PARAMETER. Note, however, we may get back less | ||
| 222 | * credit than we need to complete the op, in which case, we | ||
| 223 | * shorten the op and rely on additional rounds of retry. | ||
| 224 | */ | ||
| 225 | size_t rsize = umin(subreq->len - subreq->transferred, | ||
| 226 | cifs_sb->ctx->rsize); | ||
| 227 | |||
| 228 | rc = server->ops->wait_mtu_credits(server, rsize, &rdata->actual_len, | ||
| 229 | &rdata->credits); | ||
| 230 | if (rc) | ||
| 231 | goto out; | ||
| 232 | |||
| 233 | rdata->credits.in_flight_check = 1; | ||
| 234 | |||
| 235 | trace_smb3_rw_credits(rdata->rreq->debug_id, | ||
| 236 | rdata->subreq.debug_index, | ||
| 237 | rdata->credits.value, | ||
| 238 | server->credits, server->in_flight, 0, | ||
| 239 | cifs_trace_rw_credits_read_resubmit); | ||
| 240 | } | ||
| 241 | 211 | ||
| 242 | if (req->cfile->invalidHandle) { | 212 | if (req->cfile->invalidHandle) { |
| 243 | do { | 213 | do { |
| 244 | rc = cifs_reopen_file(req->cfile, true); | 214 | rc = cifs_reopen_file(req->cfile, true); |
| 245 | } while (rc == -EAGAIN); | 215 | } while (rc == -EAGAIN); |
| 246 | if (rc) | 216 | if (rc) |
| 247 | goto out; | 217 | goto failed; |
| 248 | } | 218 | } |
| 249 | 219 | ||
| 250 | if (subreq->rreq->origin != NETFS_DIO_READ) | 220 | if (subreq->rreq->origin != NETFS_DIO_READ) |
| 251 | __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags); | 221 | __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags); |
| 252 | 222 | ||
| 223 | trace_netfs_sreq(subreq, netfs_sreq_trace_submit); | ||
| 253 | rc = rdata->server->ops->async_readv(rdata); | 224 | rc = rdata->server->ops->async_readv(rdata); |
| 254 | out: | ||
| 255 | if (rc) | 225 | if (rc) |
| 256 | netfs_subreq_terminated(subreq, rc, false); | 226 | goto failed; |
| 227 | return; | ||
| 228 | |||
| 229 | failed: | ||
| 230 | netfs_read_subreq_terminated(subreq, rc, false); | ||
| 257 | } | 231 | } |
| 258 | 232 | ||
| 259 | /* | 233 | /* |
| @@ -364,8 +338,8 @@ const struct netfs_request_ops cifs_req_ops = { · 104 unmodified lines | |||
| 364 | .init_request = cifs_init_request, | 338 | .init_request = cifs_init_request, |
| 365 | .free_request = cifs_free_request, | 339 | .free_request = cifs_free_request, |
| 366 | .free_subrequest = cifs_free_subrequest, | 340 | .free_subrequest = cifs_free_subrequest, |
| 367 | .clamp_length = cifs_clamp_length, | 341 | .prepare_read = cifs_prepare_read, |
| 368 | .issue_read = cifs_req_issue_read, | 342 | .issue_read = cifs_issue_read, |
| 369 | .done = cifs_rreq_done, | 343 | .done = cifs_rreq_done, |
| 370 | .begin_writeback = cifs_begin_writeback, | 344 | .begin_writeback = cifs_begin_writeback, |
| 371 | .prepare_write = cifs_prepare_write, | 345 | .prepare_write = cifs_prepare_write, |
| @@ -301,7 +301,8 @@ smb2_adjust_credits(struct TCP_Server_Info *server, · 300 unmodified lines | |||
| 301 | unsigned int /*enum smb3_rw_credits_trace*/ trace) | 301 | unsigned int /*enum smb3_rw_credits_trace*/ trace) |
| 302 | { | 302 | { |
| 303 | struct cifs_credits *credits = &subreq->credits; | 303 | struct cifs_credits *credits = &subreq->credits; |
| 304 | int new_val = DIV_ROUND_UP(subreq->actual_len, SMB2_MAX_BUFFER_SIZE); | 304 | int new_val = DIV_ROUND_UP(subreq->subreq.len - subreq->subreq.transferred, |
| 305 | SMB2_MAX_BUFFER_SIZE); | ||
| 305 | int scredits, in_flight; | 306 | int scredits, in_flight; |
| 306 | 307 | ||
| 307 | if (!credits->value || credits->value == new_val) | 308 | if (!credits->value || credits->value == new_val) |
| @@ -4498,9 +4498,7 @@ static void smb2_readv_worker(struct work_struct *work) · 4497 unmodified lines | |||
| 4498 | struct cifs_io_subrequest *rdata = | 4498 | struct cifs_io_subrequest *rdata = |
| 4499 | container_of(work, struct cifs_io_subrequest, subreq.work); | 4499 | container_of(work, struct cifs_io_subrequest, subreq.work); |
| 4500 | 4500 | ||
| 4501 | netfs_subreq_terminated(&rdata->subreq, | 4501 | netfs_read_subreq_terminated(&rdata->subreq, rdata->result, false); |
| 4502 | (rdata->result == 0 || rdata->result == -EAGAIN) ? | ||
| 4503 | rdata->got_bytes : rdata->result, true); | ||
| 4504 | } | 4502 | } |
| 4505 | 4503 | ||
| 4506 | static void | 4504 | static void |
| @@ -4532,7 +4530,7 @@ smb2_readv_callback(struct mid_q_entry *mid) · 25 unmodified lines | |||
| 4532 | 4530 | ||
| 4533 | cifs_dbg(FYI, "%s: mid=%llu state=%d result=%d bytes=%zu/%zu\n", | 4531 | cifs_dbg(FYI, "%s: mid=%llu state=%d result=%d bytes=%zu/%zu\n", |
| 4534 | __func__, mid->mid, mid->mid_state, rdata->result, | 4532 | __func__, mid->mid, mid->mid_state, rdata->result, |
| 4535 | rdata->actual_len, rdata->subreq.len - rdata->subreq.transferred); | 4533 | rdata->got_bytes, rdata->subreq.len - rdata->subreq.transferred); |
| 4536 | 4534 | ||
| 4537 | switch (mid->mid_state) { | 4535 | switch (mid->mid_state) { |
| 4538 | case MID_RESPONSE_RECEIVED: | 4536 | case MID_RESPONSE_RECEIVED: |
| @@ -4554,6 +4552,7 @@ smb2_readv_callback(struct mid_q_entry *mid) · 15 unmodified lines | |||
| 4554 | break; | 4552 | break; |
| 4555 | case MID_REQUEST_SUBMITTED: | 4553 | case MID_REQUEST_SUBMITTED: |
| 4556 | case MID_RETRY_NEEDED: | 4554 | case MID_RETRY_NEEDED: |
| 4555 | __set_bit(NETFS_SREQ_NEED_RETRY, &rdata->subreq.flags); | ||
| 4557 | rdata->result = -EAGAIN; | 4556 | rdata->result = -EAGAIN; |
| 4558 | if (server->sign && rdata->got_bytes) | 4557 | if (server->sign && rdata->got_bytes) |
| 4559 | /* reset bytes number since we can not check a sign */ | 4558 | /* reset bytes number since we can not check a sign */ |
| @@ -4588,7 +4587,7 @@ smb2_readv_callback(struct mid_q_entry *mid) · 28 unmodified lines | |||
| 4588 | rdata->req->cfile->fid.persistent_fid, | 4587 | rdata->req->cfile->fid.persistent_fid, |
| 4589 | tcon->tid, tcon->ses->Suid, | 4588 | tcon->tid, tcon->ses->Suid, |
| 4590 | rdata->subreq.start + rdata->subreq.transferred, | 4589 | rdata->subreq.start + rdata->subreq.transferred, |
| 4591 | rdata->actual_len, | 4590 | rdata->subreq.len - rdata->subreq.transferred, |
| 4592 | rdata->result); | 4591 | rdata->result); |
| 4593 | } else | 4592 | } else |
| 4594 | trace_smb3_read_done(rdata->rreq->debug_id, | 4593 | trace_smb3_read_done(rdata->rreq->debug_id, |
| @@ -4603,9 +4602,9 @@ smb2_readv_callback(struct mid_q_entry *mid) · 8 unmodified lines | |||
| 4603 | __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags); | 4602 | __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags); |
| 4604 | rdata->result = 0; | 4603 | rdata->result = 0; |
| 4605 | } else { | 4604 | } else { |
| 4606 | if (rdata->got_bytes < rdata->actual_len && | 4605 | size_t trans = rdata->subreq.transferred + rdata->got_bytes; |
| 4607 | rdata->subreq.start + rdata->subreq.transferred + rdata->got_bytes == | 4606 | if (trans < rdata->subreq.len && |
| 4608 | ictx->remote_i_size) { | 4607 | rdata->subreq.start + trans == ictx->remote_i_size) { |
| 4609 | __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags); | 4608 | __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags); |
| 4610 | rdata->result = 0; | 4609 | rdata->result = 0; |
| 4611 | } | 4610 | } |
| @@ -4614,6 +4613,10 @@ smb2_readv_callback(struct mid_q_entry *mid) · 2 unmodified lines | |||
| 4614 | server->credits, server->in_flight, | 4613 | server->credits, server->in_flight, |
| 4615 | 0, cifs_trace_rw_credits_read_response_clear); | 4614 | 0, cifs_trace_rw_credits_read_response_clear); |
| 4616 | rdata->credits.value = 0; | 4615 | rdata->credits.value = 0; |
| 4616 | rdata->subreq.transferred += rdata->got_bytes; | ||
| 4617 | if (rdata->subreq.start + rdata->subreq.transferred >= rdata->subreq.rreq->i_size) | ||
| 4618 | __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags); | ||
| 4619 | trace_netfs_sreq(&rdata->subreq, netfs_sreq_trace_io_progress); | ||
| 4617 | INIT_WORK(&rdata->subreq.work, smb2_readv_worker); | 4620 | INIT_WORK(&rdata->subreq.work, smb2_readv_worker); |
| 4618 | queue_work(cifsiod_wq, &rdata->subreq.work); | 4621 | queue_work(cifsiod_wq, &rdata->subreq.work); |
| 4619 | release_mid(mid); | 4622 | release_mid(mid); |
| @@ -4648,7 +4651,7 @@ smb2_async_readv(struct cifs_io_subrequest *rdata) · 28 unmodified lines | |||
| 4648 | io_parms.tcon = tlink_tcon(rdata->req->cfile->tlink); | 4651 | io_parms.tcon = tlink_tcon(rdata->req->cfile->tlink); |
| 4649 | io_parms.server = server = rdata->server; | 4652 | io_parms.server = server = rdata->server; |
| 4650 | io_parms.offset = subreq->start + subreq->transferred; | 4653 | io_parms.offset = subreq->start + subreq->transferred; |
| 4651 | io_parms.length = rdata->actual_len; | 4654 | io_parms.length = subreq->len - subreq->transferred; |
| 4652 | io_parms.persistent_fid = rdata->req->cfile->fid.persistent_fid; | 4655 | io_parms.persistent_fid = rdata->req->cfile->fid.persistent_fid; |
| 4653 | io_parms.volatile_fid = rdata->req->cfile->fid.volatile_fid; | 4656 | io_parms.volatile_fid = rdata->req->cfile->fid.volatile_fid; |
| 4654 | io_parms.pid = rdata->req->pid; | 4657 | io_parms.pid = rdata->req->pid; |
| @@ -4669,7 +4672,7 @@ smb2_async_readv(struct cifs_io_subrequest *rdata) · 14 unmodified lines | |||
| 4669 | shdr = (struct smb2_hdr *)buf; | 4672 | shdr = (struct smb2_hdr *)buf; |
| 4670 | 4673 | ||
| 4671 | if (rdata->credits.value > 0) { | 4674 | if (rdata->credits.value > 0) { |
| 4672 | shdr->CreditCharge = cpu_to_le16(DIV_ROUND_UP(rdata->actual_len, | 4675 | shdr->CreditCharge = cpu_to_le16(DIV_ROUND_UP(io_parms.length, |
| 4673 | SMB2_MAX_BUFFER_SIZE)); | 4676 | SMB2_MAX_BUFFER_SIZE)); |
| 4674 | credit_request = le16_to_cpu(shdr->CreditCharge) + 8; | 4677 | credit_request = le16_to_cpu(shdr->CreditCharge) + 8; |
| 4675 | if (server->credits >= server->max_credits) | 4678 | if (server->credits >= server->max_credits) |
| @@ -4697,7 +4700,8 @@ smb2_async_readv(struct cifs_io_subrequest *rdata) · 21 unmodified lines | |||
| 4697 | rdata->xid, io_parms.persistent_fid, | 4700 | rdata->xid, io_parms.persistent_fid, |
| 4698 | io_parms.tcon->tid, | 4701 | io_parms.tcon->tid, |
| 4699 | io_parms.tcon->ses->Suid, | 4702 | io_parms.tcon->ses->Suid, |
| 4700 | io_parms.offset, rdata->actual_len, rc); | 4703 | io_parms.offset, |
| 4704 | subreq->len - subreq->transferred, rc); | ||
| 4701 | } | 4705 | } |
| 4702 | 4706 | ||
| 4703 | async_readv_out: | 4707 | async_readv_out: |
| @@ -4880,6 +4884,7 @@ smb2_writev_callback(struct mid_q_entry *mid) · 176 unmodified lines | |||
| 4880 | server->credits, server->in_flight, | 4884 | server->credits, server->in_flight, |
| 4881 | 0, cifs_trace_rw_credits_write_response_clear); | 4885 | 0, cifs_trace_rw_credits_write_response_clear); |
| 4882 | wdata->credits.value = 0; | 4886 | wdata->credits.value = 0; |
| 4887 | trace_netfs_sreq(&wdata->subreq, netfs_sreq_trace_io_progress); | ||
| 4883 | cifs_write_subrequest_terminated(wdata, result ?: written, true); | 4888 | cifs_write_subrequest_terminated(wdata, result ?: written, true); |
| 4884 | release_mid(mid); | 4889 | release_mid(mid); |
| 4885 | trace_smb3_rw_credits(rreq_debug_id, subreq_debug_index, 0, | 4890 | trace_smb3_rw_credits(rreq_debug_id, subreq_debug_index, 0, |
| @@ -27,6 +27,7 @@ struct folio_queue { · 26 unmodified lines | |||
| 27 | struct folio_queue *prev; /* Previous queue segment of NULL */ | 27 | struct folio_queue *prev; /* Previous queue segment of NULL */ |
| 28 | unsigned long marks; /* 1-bit mark per folio */ | 28 | unsigned long marks; /* 1-bit mark per folio */ |
| 29 | unsigned long marks2; /* Second 1-bit mark per folio */ | 29 | unsigned long marks2; /* Second 1-bit mark per folio */ |
| 30 | unsigned long marks3; /* Third 1-bit mark per folio */ | ||
| 30 | #if PAGEVEC_SIZE > BITS_PER_LONG | 31 | #if PAGEVEC_SIZE > BITS_PER_LONG |
| 31 | #error marks is not big enough | 32 | #error marks is not big enough |
| 32 | #endif | 33 | #endif |
| @@ -39,6 +40,7 @@ static inline void folioq_init(struct folio_queue *folioq) · 6 unmodified lines | |||
| 39 | folioq->prev = NULL; | 40 | folioq->prev = NULL; |
| 40 | folioq->marks = 0; | 41 | folioq->marks = 0; |
| 41 | folioq->marks2 = 0; | 42 | folioq->marks2 = 0; |
| 43 | folioq->marks3 = 0; | ||
| 42 | } | 44 | } |
| 43 | 45 | ||
| 44 | static inline unsigned int folioq_nr_slots(const struct folio_queue *folioq) | 46 | static inline unsigned int folioq_nr_slots(const struct folio_queue *folioq) |
| @@ -87,6 +89,21 @@ static inline void folioq_unmark2(struct folio_queue *folioq, unsigned int slot) · 42 unmodified lines | |||
| 87 | clear_bit(slot, &folioq->marks2); | 89 | clear_bit(slot, &folioq->marks2); |
| 88 | } | 90 | } |
| 89 | 91 | ||
| 92 | static inline bool folioq_is_marked3(const struct folio_queue *folioq, unsigned int slot) | ||
| 93 | { | ||
| 94 | return test_bit(slot, &folioq->marks3); | ||
| 95 | } | ||
| 96 | |||
| 97 | static inline void folioq_mark3(struct folio_queue *folioq, unsigned int slot) | ||
| 98 | { | ||
| 99 | set_bit(slot, &folioq->marks3); | ||
| 100 | } | ||
| 101 | |||
| 102 | static inline void folioq_unmark3(struct folio_queue *folioq, unsigned int slot) | ||
| 103 | { | ||
| 104 | clear_bit(slot, &folioq->marks3); | ||
| 105 | } | ||
| 106 | |||
| 90 | static inline unsigned int __folio_order(struct folio *folio) | 107 | static inline unsigned int __folio_order(struct folio *folio) |
| 91 | { | 108 | { |
| 92 | if (!folio_test_large(folio)) | 109 | if (!folio_test_large(folio)) |
| @@ -133,6 +150,7 @@ static inline void folioq_clear(struct folio_queue *folioq, unsigned int slot) · 40 unmodified lines | |||
| 133 | folioq->vec.folios[slot] = NULL; | 150 | folioq->vec.folios[slot] = NULL; |
| 134 | folioq_unmark(folioq, slot); | 151 | folioq_unmark(folioq, slot); |
| 135 | folioq_unmark2(folioq, slot); | 152 | folioq_unmark2(folioq, slot); |
| 153 | folioq_unmark3(folioq, slot); | ||
| 136 | } | 154 | } |
| 137 | 155 | ||
| 138 | #endif /* _LINUX_FOLIO_QUEUE_H */ | 156 | #endif /* _LINUX_FOLIO_QUEUE_H */ |
| @@ -178,36 +178,43 @@ struct netfs_io_subrequest { · 177 unmodified lines | |||
| 178 | unsigned long long start; /* Where to start the I/O */ | 178 | unsigned long long start; /* Where to start the I/O */ |
| 179 | size_t len; /* Size of the I/O */ | 179 | size_t len; /* Size of the I/O */ |
| 180 | size_t transferred; /* Amount of data transferred */ | 180 | size_t transferred; /* Amount of data transferred */ |
| 181 | size_t consumed; /* Amount of read data consumed */ | ||
| 182 | size_t prev_donated; /* Amount of data donated from previous subreq */ | ||
| 183 | size_t next_donated; /* Amount of data donated from next subreq */ | ||
| 181 | refcount_t ref; | 184 | refcount_t ref; |
| 182 | short error; /* 0 or error that occurred */ | 185 | short error; /* 0 or error that occurred */ |
| 183 | unsigned short debug_index; /* Index in list (for debugging output) */ | 186 | unsigned short debug_index; /* Index in list (for debugging output) */ |
| 184 | unsigned int nr_segs; /* Number of segs in io_iter */ | 187 | unsigned int nr_segs; /* Number of segs in io_iter */ |
| 185 | enum netfs_io_source source; /* Where to read from/write to */ | 188 | enum netfs_io_source source; /* Where to read from/write to */ |
| 186 | unsigned char stream_nr; /* I/O stream this belongs to */ | 189 | unsigned char stream_nr; /* I/O stream this belongs to */ |
| 190 | unsigned char curr_folioq_slot; /* Folio currently being read */ | ||
| 191 | unsigned char curr_folio_order; /* Order of folio */ | ||
| 192 | struct folio_queue *curr_folioq; /* Queue segment in which current folio resides */ | ||
| 187 | unsigned long flags; | 193 | unsigned long flags; |
| 188 | #define NETFS_SREQ_COPY_TO_CACHE 0 /* Set if should copy the data to the cache */ | 194 | #define NETFS_SREQ_COPY_TO_CACHE 0 /* Set if should copy the data to the cache */ |
| 189 | #define NETFS_SREQ_CLEAR_TAIL 1 /* Set if the rest of the read should be cleared */ | 195 | #define NETFS_SREQ_CLEAR_TAIL 1 /* Set if the rest of the read should be cleared */ |
| 190 | #define NETFS_SREQ_SHORT_IO 2 /* Set if the I/O was short */ | ||
| 191 | #define NETFS_SREQ_SEEK_DATA_READ 3 /* Set if ->read() should SEEK_DATA first */ | 196 | #define NETFS_SREQ_SEEK_DATA_READ 3 /* Set if ->read() should SEEK_DATA first */ |
| 192 | #define NETFS_SREQ_NO_PROGRESS 4 /* Set if we didn't manage to read any data */ | 197 | #define NETFS_SREQ_NO_PROGRESS 4 /* Set if we didn't manage to read any data */ |
| 193 | #define NETFS_SREQ_ONDEMAND 5 /* Set if it's from on-demand read mode */ | 198 | #define NETFS_SREQ_ONDEMAND 5 /* Set if it's from on-demand read mode */ |
| 194 | #define NETFS_SREQ_BOUNDARY 6 /* Set if ends on hard boundary (eg. ceph object) */ | 199 | #define NETFS_SREQ_BOUNDARY 6 /* Set if ends on hard boundary (eg. ceph object) */ |
| 200 | #define NETFS_SREQ_HIT_EOF 7 /* Set if short due to EOF */ | ||
| 195 | #define NETFS_SREQ_IN_PROGRESS 8 /* Unlocked when the subrequest completes */ | 201 | #define NETFS_SREQ_IN_PROGRESS 8 /* Unlocked when the subrequest completes */ |
| 196 | #define NETFS_SREQ_NEED_RETRY 9 /* Set if the filesystem requests a retry */ | 202 | #define NETFS_SREQ_NEED_RETRY 9 /* Set if the filesystem requests a retry */ |
| 197 | #define NETFS_SREQ_RETRYING 10 /* Set if we're retrying */ | 203 | #define NETFS_SREQ_RETRYING 10 /* Set if we're retrying */ |
| 198 | #define NETFS_SREQ_FAILED 11 /* Set if the subreq failed unretryably */ | 204 | #define NETFS_SREQ_FAILED 11 /* Set if the subreq failed unretryably */ |
| 199 | #define NETFS_SREQ_HIT_EOF 12 /* Set if we hit the EOF */ | ||
| 200 | }; | 205 | }; |
| 201 | 206 | ||
| 202 | enum netfs_io_origin { | 207 | enum netfs_io_origin { |
| 203 | NETFS_READAHEAD, /* This read was triggered by readahead */ | 208 | NETFS_READAHEAD, /* This read was triggered by readahead */ |
| 204 | NETFS_READPAGE, /* This read is a synchronous read */ | 209 | NETFS_READPAGE, /* This read is a synchronous read */ |
| 210 | NETFS_READ_GAPS, /* This read is a synchronous read to fill gaps */ | ||
| 205 | NETFS_READ_FOR_WRITE, /* This read is to prepare a write */ | 211 | NETFS_READ_FOR_WRITE, /* This read is to prepare a write */ |
| 206 | NETFS_DIO_READ, /* This is a direct I/O read */ | 212 | NETFS_DIO_READ, /* This is a direct I/O read */ |
| 207 | NETFS_WRITEBACK, /* This write was triggered by writepages */ | 213 | NETFS_WRITEBACK, /* This write was triggered by writepages */ |
| 208 | NETFS_WRITETHROUGH, /* This write was made by netfs_perform_write() */ | 214 | NETFS_WRITETHROUGH, /* This write was made by netfs_perform_write() */ |
| 209 | NETFS_UNBUFFERED_WRITE, /* This is an unbuffered write */ | 215 | NETFS_UNBUFFERED_WRITE, /* This is an unbuffered write */ |
| 210 | NETFS_DIO_WRITE, /* This is a direct I/O write */ | 216 | NETFS_DIO_WRITE, /* This is a direct I/O write */ |
| 217 | NETFS_PGPRIV2_COPY_TO_CACHE, /* [DEPRECATED] This is writing read data to the cache */ | ||
| 211 | nr__netfs_io_origin | 218 | nr__netfs_io_origin |
| 212 | } __mode(byte); | 219 | } __mode(byte); |
| 213 | 220 | ||
| @@ -224,6 +231,7 @@ struct netfs_io_request { · 10 unmodified lines | |||
| 224 | struct address_space *mapping; /* The mapping being accessed */ | 231 | struct address_space *mapping; /* The mapping being accessed */ |
| 225 | struct kiocb *iocb; /* AIO completion vector */ | 232 | struct kiocb *iocb; /* AIO completion vector */ |
| 226 | struct netfs_cache_resources cache_resources; | 233 | struct netfs_cache_resources cache_resources; |
| 234 | struct readahead_control *ractl; /* Readahead descriptor */ | ||
| 227 | struct list_head proc_link; /* Link in netfs_iorequests */ | 235 | struct list_head proc_link; /* Link in netfs_iorequests */ |
| 228 | struct list_head subrequests; /* Contributory I/O operations */ | 236 | struct list_head subrequests; /* Contributory I/O operations */ |
| 229 | struct netfs_io_stream io_streams[2]; /* Streams of parallel I/O operations */ | 237 | struct netfs_io_stream io_streams[2]; /* Streams of parallel I/O operations */ |
| @@ -244,12 +252,10 @@ struct netfs_io_request { · 14 unmodified lines | |||
| 244 | unsigned int nr_group_rel; /* Number of refs to release on ->group */ | 252 | unsigned int nr_group_rel; /* Number of refs to release on ->group */ |
| 245 | spinlock_t lock; /* Lock for queuing subreqs */ | 253 | spinlock_t lock; /* Lock for queuing subreqs */ |
| 246 | atomic_t nr_outstanding; /* Number of ops in progress */ | 254 | atomic_t nr_outstanding; /* Number of ops in progress */ |
| 247 | atomic_t nr_copy_ops; /* Number of copy-to-cache ops in progress */ | ||
| 248 | size_t upper_len; /* Length can be extended to here */ | ||
| 249 | unsigned long long submitted; /* Amount submitted for I/O so far */ | 255 | unsigned long long submitted; /* Amount submitted for I/O so far */ |
| 250 | unsigned long long len; /* Length of the request */ | 256 | unsigned long long len; /* Length of the request */ |
| 251 | size_t transferred; /* Amount to be indicated as transferred */ | 257 | size_t transferred; /* Amount to be indicated as transferred */ |
| 252 | short error; /* 0 or error that occurred */ | 258 | long error; /* 0 or error that occurred */ |
| 253 | enum netfs_io_origin origin; /* Origin of the request */ | 259 | enum netfs_io_origin origin; /* Origin of the request */ |
| 254 | bool direct_bv_unpin; /* T if direct_bv[] must be unpinned */ | 260 | bool direct_bv_unpin; /* T if direct_bv[] must be unpinned */ |
| 255 | u8 buffer_head_slot; /* First slot in ->buffer */ | 261 | u8 buffer_head_slot; /* First slot in ->buffer */ |
| @@ -260,9 +266,9 @@ struct netfs_io_request { · 4 unmodified lines | |||
| 260 | unsigned long long collected_to; /* Point we've collected to */ | 266 | unsigned long long collected_to; /* Point we've collected to */ |
| 261 | unsigned long long cleaned_to; /* Position we've cleaned folios to */ | 267 | unsigned long long cleaned_to; /* Position we've cleaned folios to */ |
| 262 | pgoff_t no_unlock_folio; /* Don't unlock this folio after read */ | 268 | pgoff_t no_unlock_folio; /* Don't unlock this folio after read */ |
| 269 | size_t prev_donated; /* Fallback for subreq->prev_donated */ | ||
| 263 | refcount_t ref; | 270 | refcount_t ref; |
| 264 | unsigned long flags; | 271 | unsigned long flags; |
| 265 | #define NETFS_RREQ_INCOMPLETE_IO 0 /* Some ioreqs terminated short or with error */ | ||
| 266 | #define NETFS_RREQ_COPY_TO_CACHE 1 /* Need to write to the cache */ | 272 | #define NETFS_RREQ_COPY_TO_CACHE 1 /* Need to write to the cache */ |
| 267 | #define NETFS_RREQ_NO_UNLOCK_FOLIO 2 /* Don't unlock no_unlock_folio on completion */ | 273 | #define NETFS_RREQ_NO_UNLOCK_FOLIO 2 /* Don't unlock no_unlock_folio on completion */ |
| 268 | #define NETFS_RREQ_DONT_UNLOCK_FOLIOS 3 /* Don't unlock the folios on completion */ | 274 | #define NETFS_RREQ_DONT_UNLOCK_FOLIOS 3 /* Don't unlock the folios on completion */ |
| @@ -274,6 +280,7 @@ struct netfs_io_request { · 5 unmodified lines | |||
| 274 | #define NETFS_RREQ_PAUSE 11 /* Pause subrequest generation */ | 280 | #define NETFS_RREQ_PAUSE 11 /* Pause subrequest generation */ |
| 275 | #define NETFS_RREQ_USE_IO_ITER 12 /* Use ->io_iter rather than ->i_pages */ | 281 | #define NETFS_RREQ_USE_IO_ITER 12 /* Use ->io_iter rather than ->i_pages */ |
| 276 | #define NETFS_RREQ_ALL_QUEUED 13 /* All subreqs are now queued */ | 282 | #define NETFS_RREQ_ALL_QUEUED 13 /* All subreqs are now queued */ |
| 283 | #define NETFS_RREQ_NEED_RETRY 14 /* Need to try retrying */ | ||
| 277 | #define NETFS_RREQ_USE_PGPRIV2 31 /* [DEPRECATED] Use PG_private_2 to mark | 284 | #define NETFS_RREQ_USE_PGPRIV2 31 /* [DEPRECATED] Use PG_private_2 to mark |
| 278 | * write to cache on read */ | 285 | * write to cache on read */ |
| 279 | const struct netfs_request_ops *netfs_ops; | 286 | const struct netfs_request_ops *netfs_ops; |
| @@ -292,7 +299,7 @@ struct netfs_request_ops { · 12 unmodified lines | |||
| 292 | 299 | ||
| 293 | /* Read request handling */ | 300 | /* Read request handling */ |
| 294 | void (*expand_readahead)(struct netfs_io_request *rreq); | 301 | void (*expand_readahead)(struct netfs_io_request *rreq); |
| 295 | bool (*clamp_length)(struct netfs_io_subrequest *subreq); | 302 | int (*prepare_read)(struct netfs_io_subrequest *subreq); |
| 296 | void (*issue_read)(struct netfs_io_subrequest *subreq); | 303 | void (*issue_read)(struct netfs_io_subrequest *subreq); |
| 297 | bool (*is_still_valid)(struct netfs_io_request *rreq); | 304 | bool (*is_still_valid)(struct netfs_io_request *rreq); |
| 298 | int (*check_write_begin)(struct file *file, loff_t pos, unsigned len, | 305 | int (*check_write_begin)(struct file *file, loff_t pos, unsigned len, |
| @@ -422,7 +429,10 @@ bool netfs_release_folio(struct folio *folio, gfp_t gfp); · 123 unmodified lines | |||
| 422 | vm_fault_t netfs_page_mkwrite(struct vm_fault *vmf, struct netfs_group *netfs_group); | 429 | vm_fault_t netfs_page_mkwrite(struct vm_fault *vmf, struct netfs_group *netfs_group); |
| 423 | 430 | ||
| 424 | /* (Sub)request management API. */ | 431 | /* (Sub)request management API. */ |
| 425 | void netfs_subreq_terminated(struct netfs_io_subrequest *, ssize_t, bool); | 432 | void netfs_read_subreq_progress(struct netfs_io_subrequest *subreq, |
| 433 | bool was_async); | ||
| 434 | void netfs_read_subreq_terminated(struct netfs_io_subrequest *subreq, | ||
| 435 | int error, bool was_async); | ||
| 426 | void netfs_get_subrequest(struct netfs_io_subrequest *subreq, | 436 | void netfs_get_subrequest(struct netfs_io_subrequest *subreq, |
| 427 | enum netfs_sreq_ref_trace what); | 437 | enum netfs_sreq_ref_trace what); |
| 428 | void netfs_put_subrequest(struct netfs_io_subrequest *subreq, | 438 | void netfs_put_subrequest(struct netfs_io_subrequest *subreq, |
| @@ -20,6 +20,7 @@ · 19 unmodified lines | |||
| 20 | EM(netfs_read_trace_expanded, "EXPANDED ") \ | 20 | EM(netfs_read_trace_expanded, "EXPANDED ") \ |
| 21 | EM(netfs_read_trace_readahead, "READAHEAD") \ | 21 | EM(netfs_read_trace_readahead, "READAHEAD") \ |
| 22 | EM(netfs_read_trace_readpage, "READPAGE ") \ | 22 | EM(netfs_read_trace_readpage, "READPAGE ") \ |
| 23 | EM(netfs_read_trace_read_gaps, "READ-GAPS") \ | ||
| 23 | EM(netfs_read_trace_prefetch_for_write, "PREFETCHW") \ | 24 | EM(netfs_read_trace_prefetch_for_write, "PREFETCHW") \ |
| 24 | E_(netfs_read_trace_write_begin, "WRITEBEGN") | 25 | E_(netfs_read_trace_write_begin, "WRITEBEGN") |
| 25 | 26 | ||
| @@ -33,12 +34,14 @@ · 7 unmodified lines | |||
| 33 | #define netfs_rreq_origins \ | 34 | #define netfs_rreq_origins \ |
| 34 | EM(NETFS_READAHEAD, "RA") \ | 35 | EM(NETFS_READAHEAD, "RA") \ |
| 35 | EM(NETFS_READPAGE, "RP") \ | 36 | EM(NETFS_READPAGE, "RP") \ |
| 37 | EM(NETFS_READ_GAPS, "RG") \ | ||
| 36 | EM(NETFS_READ_FOR_WRITE, "RW") \ | 38 | EM(NETFS_READ_FOR_WRITE, "RW") \ |
| 37 | EM(NETFS_DIO_READ, "DR") \ | 39 | EM(NETFS_DIO_READ, "DR") \ |
| 38 | EM(NETFS_WRITEBACK, "WB") \ | 40 | EM(NETFS_WRITEBACK, "WB") \ |
| 39 | EM(NETFS_WRITETHROUGH, "WT") \ | 41 | EM(NETFS_WRITETHROUGH, "WT") \ |
| 40 | EM(NETFS_UNBUFFERED_WRITE, "UW") \ | 42 | EM(NETFS_UNBUFFERED_WRITE, "UW") \ |
| 41 | E_(NETFS_DIO_WRITE, "DW") | 43 | EM(NETFS_DIO_WRITE, "DW") \ |
| 44 | E_(NETFS_PGPRIV2_COPY_TO_CACHE, "2C") | ||
| 42 | 45 | ||
| 43 | #define netfs_rreq_traces \ | 46 | #define netfs_rreq_traces \ |
| 44 | EM(netfs_rreq_trace_assess, "ASSESS ") \ | 47 | EM(netfs_rreq_trace_assess, "ASSESS ") \ |
| @@ -69,15 +72,25 @@ · 24 unmodified lines | |||
| 69 | E_(NETFS_INVALID_WRITE, "INVL") | 72 | E_(NETFS_INVALID_WRITE, "INVL") |
| 70 | 73 | ||
| 71 | #define netfs_sreq_traces \ | 74 | #define netfs_sreq_traces \ |
| 75 | EM(netfs_sreq_trace_add_donations, "+DON ") \ | ||
| 76 | EM(netfs_sreq_trace_added, "ADD ") \ | ||
| 77 | EM(netfs_sreq_trace_clear, "CLEAR") \ | ||
| 72 | EM(netfs_sreq_trace_discard, "DSCRD") \ | 78 | EM(netfs_sreq_trace_discard, "DSCRD") \ |
| 79 | EM(netfs_sreq_trace_donate_to_prev, "DON-P") \ | ||
| 80 | EM(netfs_sreq_trace_donate_to_next, "DON-N") \ | ||
| 73 | EM(netfs_sreq_trace_download_instead, "RDOWN") \ | 81 | EM(netfs_sreq_trace_download_instead, "RDOWN") \ |
| 74 | EM(netfs_sreq_trace_fail, "FAIL ") \ | 82 | EM(netfs_sreq_trace_fail, "FAIL ") \ |
| 75 | EM(netfs_sreq_trace_free, "FREE ") \ | 83 | EM(netfs_sreq_trace_free, "FREE ") \ |
| 84 | EM(netfs_sreq_trace_hit_eof, "EOF ") \ | ||
| 85 | EM(netfs_sreq_trace_io_progress, "IO ") \ | ||
| 76 | EM(netfs_sreq_trace_limited, "LIMIT") \ | 86 | EM(netfs_sreq_trace_limited, "LIMIT") \ |
| 77 | EM(netfs_sreq_trace_prepare, "PREP ") \ | 87 | EM(netfs_sreq_trace_prepare, "PREP ") \ |
| 78 | EM(netfs_sreq_trace_prep_failed, "PRPFL") \ | 88 | EM(netfs_sreq_trace_prep_failed, "PRPFL") \ |
| 79 | EM(netfs_sreq_trace_resubmit_short, "SHORT") \ | 89 | EM(netfs_sreq_trace_progress, "PRGRS") \ |
| 90 | EM(netfs_sreq_trace_reprep_failed, "REPFL") \ | ||
| 80 | EM(netfs_sreq_trace_retry, "RETRY") \ | 91 | EM(netfs_sreq_trace_retry, "RETRY") \ |
| 92 | EM(netfs_sreq_trace_short, "SHORT") \ | ||
| 93 | EM(netfs_sreq_trace_split, "SPLIT") \ | ||
| 81 | EM(netfs_sreq_trace_submit, "SUBMT") \ | 94 | EM(netfs_sreq_trace_submit, "SUBMT") \ |
| 82 | EM(netfs_sreq_trace_terminated, "TERM ") \ | 95 | EM(netfs_sreq_trace_terminated, "TERM ") \ |
| 83 | EM(netfs_sreq_trace_write, "WRITE") \ | 96 | EM(netfs_sreq_trace_write, "WRITE") \ |
| @@ -118,7 +131,7 @@ · 34 unmodified lines | |||
| 118 | EM(netfs_sreq_trace_new, "NEW ") \ | 131 | EM(netfs_sreq_trace_new, "NEW ") \ |
| 119 | EM(netfs_sreq_trace_put_cancel, "PUT CANCEL ") \ | 132 | EM(netfs_sreq_trace_put_cancel, "PUT CANCEL ") \ |
| 120 | EM(netfs_sreq_trace_put_clear, "PUT CLEAR ") \ | 133 | EM(netfs_sreq_trace_put_clear, "PUT CLEAR ") \ |
| 121 | EM(netfs_sreq_trace_put_discard, "PUT DISCARD") \ | 134 | EM(netfs_sreq_trace_put_consumed, "PUT CONSUME") \ |
| 122 | EM(netfs_sreq_trace_put_done, "PUT DONE ") \ | 135 | EM(netfs_sreq_trace_put_done, "PUT DONE ") \ |
| 123 | EM(netfs_sreq_trace_put_failed, "PUT FAILED ") \ | 136 | EM(netfs_sreq_trace_put_failed, "PUT FAILED ") \ |
| 124 | EM(netfs_sreq_trace_put_merged, "PUT MERGED ") \ | 137 | EM(netfs_sreq_trace_put_merged, "PUT MERGED ") \ |
| @@ -138,6 +151,7 @@ · 13 unmodified lines | |||
| 138 | EM(netfs_flush_content, "flush") \ | 151 | EM(netfs_flush_content, "flush") \ |
| 139 | EM(netfs_streaming_filled_page, "mod-streamw-f") \ | 152 | EM(netfs_streaming_filled_page, "mod-streamw-f") \ |
| 140 | EM(netfs_streaming_cont_filled_page, "mod-streamw-f+") \ | 153 | EM(netfs_streaming_cont_filled_page, "mod-streamw-f+") \ |
| 154 | EM(netfs_folio_trace_abandon, "abandon") \ | ||
| 141 | EM(netfs_folio_trace_cancel_copy, "cancel-copy") \ | 155 | EM(netfs_folio_trace_cancel_copy, "cancel-copy") \ |
| 142 | EM(netfs_folio_trace_clear, "clear") \ | 156 | EM(netfs_folio_trace_clear, "clear") \ |
| 143 | EM(netfs_folio_trace_clear_cc, "clear-cc") \ | 157 | EM(netfs_folio_trace_clear_cc, "clear-cc") \ |
| @@ -154,7 +168,11 @@ · 10 unmodified lines | |||
| 154 | EM(netfs_folio_trace_mkwrite_plus, "mkwrite+") \ | 168 | EM(netfs_folio_trace_mkwrite_plus, "mkwrite+") \ |
| 155 | EM(netfs_folio_trace_not_under_wback, "!wback") \ | 169 | EM(netfs_folio_trace_not_under_wback, "!wback") \ |
| 156 | EM(netfs_folio_trace_put, "put") \ | 170 | EM(netfs_folio_trace_put, "put") \ |
| 171 | EM(netfs_folio_trace_read, "read") \ | ||
| 172 | EM(netfs_folio_trace_read_done, "read-done") \ | ||
| 157 | EM(netfs_folio_trace_read_gaps, "read-gaps") \ | 173 | EM(netfs_folio_trace_read_gaps, "read-gaps") \ |
| 174 | EM(netfs_folio_trace_read_put, "read-put") \ | ||
| 175 | EM(netfs_folio_trace_read_unlock, "read-unlock") \ | ||
| 158 | EM(netfs_folio_trace_redirtied, "redirtied") \ | 176 | EM(netfs_folio_trace_redirtied, "redirtied") \ |
| 159 | EM(netfs_folio_trace_store, "store") \ | 177 | EM(netfs_folio_trace_store, "store") \ |
| 160 | EM(netfs_folio_trace_store_copy, "store-copy") \ | 178 | EM(netfs_folio_trace_store_copy, "store-copy") \ |
| @@ -167,6 +185,12 @@ · 6 unmodified lines | |||
| 167 | EM(netfs_contig_trace_jump, "-->JUMP-->") \ | 185 | EM(netfs_contig_trace_jump, "-->JUMP-->") \ |
| 168 | E_(netfs_contig_trace_unlock, "Unlock") | 186 | E_(netfs_contig_trace_unlock, "Unlock") |
| 169 | 187 | ||
| 188 | #define netfs_donate_traces \ | ||
| 189 | EM(netfs_trace_donate_tail_to_prev, "tail-to-prev") \ | ||
| 190 | EM(netfs_trace_donate_to_prev, "to-prev") \ | ||
| 191 | EM(netfs_trace_donate_to_next, "to-next") \ | ||
| 192 | E_(netfs_trace_donate_to_deferred_next, "defer-next") | ||
| 193 | |||
| 170 | #ifndef __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY | 194 | #ifndef __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY |
| 171 | #define __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY | 195 | #define __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY |
| 172 | 196 | ||
| @@ -184,6 +208,7 @@ enum netfs_rreq_ref_trace { netfs_rreq_ref_traces } __mode(byte); · 11 unmodified lines | |||
| 184 | enum netfs_sreq_ref_trace { netfs_sreq_ref_traces } __mode(byte); | 208 | enum netfs_sreq_ref_trace { netfs_sreq_ref_traces } __mode(byte); |
| 185 | enum netfs_folio_trace { netfs_folio_traces } __mode(byte); | 209 | enum netfs_folio_trace { netfs_folio_traces } __mode(byte); |
| 186 | enum netfs_collect_contig_trace { netfs_collect_contig_traces } __mode(byte); | 210 | enum netfs_collect_contig_trace { netfs_collect_contig_traces } __mode(byte); |
| 211 | enum netfs_donate_trace { netfs_donate_traces } __mode(byte); | ||
| 187 | 212 | ||
| 188 | #endif | 213 | #endif |
| 189 | 214 | ||
| @@ -206,6 +231,7 @@ netfs_rreq_ref_traces; · 16 unmodified lines | |||
| 206 | netfs_sreq_ref_traces; | 231 | netfs_sreq_ref_traces; |
| 207 | netfs_folio_traces; | 232 | netfs_folio_traces; |
| 208 | netfs_collect_contig_traces; | 233 | netfs_collect_contig_traces; |
| 234 | netfs_donate_traces; | ||
| 209 | 235 | ||
| 210 | /* | 236 | /* |
| 211 | * Now redefine the EM() and E_() macros to map the enums to the strings that | 237 | * Now redefine the EM() and E_() macros to map the enums to the strings that |
| @@ -226,6 +252,7 @@ TRACE_EVENT(netfs_read, · 14 unmodified lines | |||
| 226 | TP_STRUCT__entry( | 252 | TP_STRUCT__entry( |
| 227 | __field(unsigned int, rreq ) | 253 | __field(unsigned int, rreq ) |
| 228 | __field(unsigned int, cookie ) | 254 | __field(unsigned int, cookie ) |
| 255 | __field(loff_t, i_size ) | ||
| 229 | __field(loff_t, start ) | 256 | __field(loff_t, start ) |
| 230 | __field(size_t, len ) | 257 | __field(size_t, len ) |
| 231 | __field(enum netfs_read_trace, what ) | 258 | __field(enum netfs_read_trace, what ) |
| @@ -235,18 +262,19 @@ TRACE_EVENT(netfs_read, · 3 unmodified lines | |||
| 235 | TP_fast_assign( | 262 | TP_fast_assign( |
| 236 | __entry->rreq = rreq->debug_id; | 263 | __entry->rreq = rreq->debug_id; |
| 237 | __entry->cookie = rreq->cache_resources.debug_id; | 264 | __entry->cookie = rreq->cache_resources.debug_id; |
| 265 | __entry->i_size = rreq->i_size; | ||
| 238 | __entry->start = start; | 266 | __entry->start = start; |
| 239 | __entry->len = len; | 267 | __entry->len = len; |
| 240 | __entry->what = what; | 268 | __entry->what = what; |
| 241 | __entry->netfs_inode = rreq->inode->i_ino; | 269 | __entry->netfs_inode = rreq->inode->i_ino; |
| 242 | ), | 270 | ), |
| 243 | 271 | ||
| 244 | TP_printk("R=%08x %s c=%08x ni=%x s=%llx %zx", | 272 | TP_printk("R=%08x %s c=%08x ni=%x s=%llx l=%zx sz=%llx", |
| 245 | __entry->rreq, | 273 | __entry->rreq, |
| 246 | __print_symbolic(__entry->what, netfs_read_traces), | 274 | __print_symbolic(__entry->what, netfs_read_traces), |
| 247 | __entry->cookie, | 275 | __entry->cookie, |
| 248 | __entry->netfs_inode, | 276 | __entry->netfs_inode, |
| 249 | __entry->start, __entry->len) | 277 | __entry->start, __entry->len, __entry->i_size) |
| 250 | ); | 278 | ); |
| 251 | 279 | ||
| 252 | TRACE_EVENT(netfs_rreq, | 280 | TRACE_EVENT(netfs_rreq, |
| @@ -651,6 +679,71 @@ TRACE_EVENT(netfs_collect_stream, · 398 unmodified lines | |||
| 651 | __entry->collected_to, __entry->front) | 679 | __entry->collected_to, __entry->front) |
| 652 | ); | 680 | ); |
| 653 | 681 | ||
| 682 | TRACE_EVENT(netfs_progress, | ||
| 683 | TP_PROTO(const struct netfs_io_subrequest *subreq, | ||
| 684 | unsigned long long start, size_t avail, size_t part), | ||
| 685 | |||
| 686 | TP_ARGS(subreq, start, avail, part), | ||
| 687 | |||
| 688 | TP_STRUCT__entry( | ||
| 689 | __field(unsigned int, rreq) | ||
| 690 | __field(unsigned int, subreq) | ||
| 691 | __field(unsigned int, consumed) | ||
| 692 | __field(unsigned int, transferred) | ||
| 693 | __field(unsigned long long, f_start) | ||
| 694 | __field(unsigned int, f_avail) | ||
| 695 | __field(unsigned int, f_part) | ||
| 696 | __field(unsigned char, slot) | ||
| 697 | ), | ||
| 698 | |||
| 699 | TP_fast_assign( | ||
| 700 | __entry->rreq = subreq->rreq->debug_id; | ||
| 701 | __entry->subreq = subreq->debug_index; | ||
| 702 | __entry->consumed = subreq->consumed; | ||
| 703 | __entry->transferred = subreq->transferred; | ||
| 704 | __entry->f_start = start; | ||
| 705 | __entry->f_avail = avail; | ||
| 706 | __entry->f_part = part; | ||
| 707 | __entry->slot = subreq->curr_folioq_slot; | ||
| 708 | ), | ||
| 709 | |||
| 710 | TP_printk("R=%08x[%02x] s=%llx ct=%x/%x pa=%x/%x sl=%x", | ||
| 711 | __entry->rreq, __entry->subreq, __entry->f_start, | ||
| 712 | __entry->consumed, __entry->transferred, | ||
| 713 | __entry->f_part, __entry->f_avail, __entry->slot) | ||
| 714 | ); | ||
| 715 | |||
| 716 | TRACE_EVENT(netfs_donate, | ||
| 717 | TP_PROTO(const struct netfs_io_request *rreq, | ||
| 718 | const struct netfs_io_subrequest *from, | ||
| 719 | const struct netfs_io_subrequest *to, | ||
| 720 | size_t amount, | ||
| 721 | enum netfs_donate_trace trace), | ||
| 722 | |||
| 723 | TP_ARGS(rreq, from, to, amount, trace), | ||
| 724 | |||
| 725 | TP_STRUCT__entry( | ||
| 726 | __field(unsigned int, rreq) | ||
| 727 | __field(unsigned int, from) | ||
| 728 | __field(unsigned int, to) | ||
| 729 | __field(unsigned int, amount) | ||
| 730 | __field(enum netfs_donate_trace, trace) | ||
| 731 | ), | ||
| 732 | |||
| 733 | TP_fast_assign( | ||
| 734 | __entry->rreq = rreq->debug_id; | ||
| 735 | __entry->from = from->debug_index; | ||
| 736 | __entry->to = to ? to->debug_index : -1; | ||
| 737 | __entry->amount = amount; | ||
| 738 | __entry->trace = trace; | ||
| 739 | ), | ||
| 740 | |||
| 741 | TP_printk("R=%08x[%02x] -> [%02x] %s am=%x", | ||
| 742 | __entry->rreq, __entry->from, __entry->to, | ||
| 743 | __print_symbolic(__entry->trace, netfs_donate_traces), | ||
| 744 | __entry->amount) | ||
| 745 | ); | ||
| 746 | |||
| 654 | #undef EM | 747 | #undef EM |
| 655 | #undef E_ | 748 | #undef E_ |
| 656 | #endif /* _TRACE_NETFS_H */ | 749 | #endif /* _TRACE_NETFS_H */ |