Improve the efficiency of buffered reads in a number of ways: (1) Overhaul the algorithm in general so that it's a lot more compact and split the read submission code between buffered and unbuffered versions. The unbuffered version can be vastly simplified. (2) Read-result collection is handed off to a work queue rather than being done in the I/O thread. Multiple subrequests can be processes simultaneously. (3) When a subrequest is collected, any folios it fully spans are collected and "spare" data on either side is donated to either the previous or the next subrequest in the sequence. Notes: (*) Readahead expansion is massively slows down fio, presumably because it causes a load of extra allocations, both folio and xarray, up front before RPC requests can be transmitted. (*) RDMA with cifs does appear to work, both with SIW and RXE. (*) PG_private_2-based reading and copy-to-cache is split out into its own file and altered to use folio_queue. Note that the copy to the cache now creates a new write transaction against the cache and adds the folios to be copied into it. This allows it to use part of the writeback I/O code.
Unified · Split
| @@ -68,17 +68,22 @@ static void v9fs_issue_read(struct netfs_io_subrequest *subreq) · 67 unmodified lines | ||
| 68 | 68 | { |
| 69 | 69 | struct netfs_io_request *rreq = subreq->rreq; |
| 70 | 70 | struct p9_fid *fid = rreq->netfs_priv; |
| 71 | + unsigned long long pos = subreq->start + subreq->transferred; | |
| 71 | 72 | int total, err; |
| 72 | 73 | |
| 73 | − total = p9_client_read(fid, subreq->start + subreq->transferred, | |
| 74 | − &subreq->io_iter, &err); | |
| 74 | + total = p9_client_read(fid, pos, &subreq->io_iter, &err); | |
| 75 | 75 | |
| 76 | 76 | /* if we just extended the file size, any portion not in |
| 77 | 77 | * cache won't be on server and is zeroes */ |
| 78 | 78 | if (subreq->rreq->origin != NETFS_DIO_READ) |
| 79 | 79 | __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags); |
| 80 | + if (pos + total >= i_size_read(rreq->inode)) | |
| 81 | + __set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags); | |
| 80 | 82 | |
| 81 | − netfs_subreq_terminated(subreq, err ?: total, false); | |
| 83 | + if (!err) | |
| 84 | + subreq->transferred += total; | |
| 85 | + | |
| 86 | + netfs_read_subreq_terminated(subreq, err, false); | |
| 82 | 87 | } |
| 83 | 88 | |
| 84 | 89 | /** |
| @@ -16,6 +16,7 @@ · 15 unmodified lines | ||
| 16 | 16 | #include <linux/mm.h> |
| 17 | 17 | #include <linux/swap.h> |
| 18 | 18 | #include <linux/netfs.h> |
| 19 | +#include <trace/events/netfs.h> | |
| 19 | 20 | #include "internal.h" |
| 20 | 21 | |
| 21 | 22 | static int afs_file_mmap(struct file *file, struct vm_area_struct *vma); |
| @@ -242,9 +243,10 @@ static void afs_fetch_data_notify(struct afs_operation *op) · 220 unmodified lines | ||
| 242 | 243 | |
| 243 | 244 | req->error = error; |
| 244 | 245 | if (subreq) { |
| 245 | − if (subreq->rreq->origin != NETFS_DIO_READ) | |
| 246 | − __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags); | |
| 247 | − netfs_subreq_terminated(subreq, error ?: req->actual_len, false); | |
| 246 | + subreq->rreq->i_size = req->file_size; | |
| 247 | + if (req->pos + req->actual_len >= req->file_size) | |
| 248 | + __set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags); | |
| 249 | + netfs_read_subreq_terminated(subreq, error, false); | |
| 248 | 250 | req->subreq = NULL; |
| 249 | 251 | } else if (req->done) { |
| 250 | 252 | req->done(req); |
| @@ -262,6 +264,12 @@ static void afs_fetch_data_success(struct afs_operation *op) · 11 unmodified lines | ||
| 262 | 264 | afs_fetch_data_notify(op); |
| 263 | 265 | } |
| 264 | 266 | |
| 267 | +static void afs_fetch_data_aborted(struct afs_operation *op) | |
| 268 | +{ | |
| 269 | + afs_check_for_remote_deletion(op); | |
| 270 | + afs_fetch_data_notify(op); | |
| 271 | +} | |
| 272 | + | |
| 265 | 273 | static void afs_fetch_data_put(struct afs_operation *op) |
| 266 | 274 | { |
| 267 | 275 | op->fetch.req->error = afs_op_error(op); |
| @@ -272,7 +280,7 @@ static const struct afs_operation_ops afs_fetch_data_operation = { · 4 unmodified lines | ||
| 272 | 280 | .issue_afs_rpc = afs_fs_fetch_data, |
| 273 | 281 | .issue_yfs_rpc = yfs_fs_fetch_data, |
| 274 | 282 | .success = afs_fetch_data_success, |
| 275 | − .aborted = afs_check_for_remote_deletion, | |
| 283 | + .aborted = afs_fetch_data_aborted, | |
| 276 | 284 | .failed = afs_fetch_data_notify, |
| 277 | 285 | .put = afs_fetch_data_put, |
| 278 | 286 | }; |
| @@ -294,7 +302,7 @@ int afs_fetch_data(struct afs_vnode *vnode, struct afs_read *req) · 15 unmodified lines | ||
| 294 | 302 | op = afs_alloc_operation(req->key, vnode->volume); |
| 295 | 303 | if (IS_ERR(op)) { |
| 296 | 304 | if (req->subreq) |
| 297 | − netfs_subreq_terminated(req->subreq, PTR_ERR(op), false); | |
| 305 | + netfs_read_subreq_terminated(req->subreq, PTR_ERR(op), false); | |
| 298 | 306 | return PTR_ERR(op); |
| 299 | 307 | } |
| 300 | 308 | |
| @@ -313,7 +321,7 @@ static void afs_read_worker(struct work_struct *work) · 12 unmodified lines | ||
| 313 | 321 | |
| 314 | 322 | fsreq = afs_alloc_read(GFP_NOFS); |
| 315 | 323 | if (!fsreq) |
| 316 | − return netfs_subreq_terminated(subreq, -ENOMEM, false); | |
| 324 | + return netfs_read_subreq_terminated(subreq, -ENOMEM, false); | |
| 317 | 325 | |
| 318 | 326 | fsreq->subreq = subreq; |
| 319 | 327 | fsreq->pos = subreq->start + subreq->transferred; |
| @@ -322,6 +330,7 @@ static void afs_read_worker(struct work_struct *work) · 2 unmodified lines | ||
| 322 | 330 | fsreq->vnode = vnode; |
| 323 | 331 | fsreq->iter = &subreq->io_iter; |
| 324 | 332 | |
| 333 | + trace_netfs_sreq(subreq, netfs_sreq_trace_submit); | |
| 325 | 334 | afs_fetch_data(fsreq->vnode, fsreq); |
| 326 | 335 | afs_put_read(fsreq); |
| 327 | 336 | } |
| @@ -304,6 +304,7 @@ static int afs_deliver_fs_fetch_data(struct afs_call *call) · 303 unmodified lines | ||
| 304 | 304 | struct afs_vnode_param *vp = &op->file[0]; |
| 305 | 305 | struct afs_read *req = op->fetch.req; |
| 306 | 306 | const __be32 *bp; |
| 307 | + size_t count_before; | |
| 307 | 308 | int ret; |
| 308 | 309 | |
| 309 | 310 | _enter("{%u,%zu,%zu/%llu}", |
| @@ -345,10 +346,14 @@ static int afs_deliver_fs_fetch_data(struct afs_call *call) · 35 unmodified lines | ||
| 345 | 346 | |
| 346 | 347 | /* extract the returned data */ |
| 347 | 348 | case 2: |
| 348 | − _debug("extract data %zu/%llu", | |
| 349 | − iov_iter_count(call->iter), req->actual_len); | |
| 349 | + count_before = call->iov_len; | |
| 350 | + _debug("extract data %zu/%llu", count_before, req->actual_len); | |
| 350 | 351 | |
| 351 | 352 | ret = afs_extract_data(call, true); |
| 353 | + if (req->subreq) { | |
| 354 | + req->subreq->transferred += count_before - call->iov_len; | |
| 355 | + netfs_read_subreq_progress(req->subreq, false); | |
| 356 | + } | |
| 352 | 357 | if (ret < 0) |
| 353 | 358 | return ret; |
| 354 | 359 | |
| @@ -355,6 +355,7 @@ static int yfs_deliver_fs_fetch_data64(struct afs_call *call) · 354 unmodified lines | ||
| 355 | 355 | struct afs_vnode_param *vp = &op->file[0]; |
| 356 | 356 | struct afs_read *req = op->fetch.req; |
| 357 | 357 | const __be32 *bp; |
| 358 | + size_t count_before; | |
| 358 | 359 | int ret; |
| 359 | 360 | |
| 360 | 361 | _enter("{%u,%zu, %zu/%llu}", |
| @@ -391,10 +392,14 @@ static int yfs_deliver_fs_fetch_data64(struct afs_call *call) · 30 unmodified lines | ||
| 391 | 392 | |
| 392 | 393 | /* extract the returned data */ |
| 393 | 394 | case 2: |
| 394 | − _debug("extract data %zu/%llu", | |
| 395 | − iov_iter_count(call->iter), req->actual_len); | |
| 395 | + count_before = call->iov_len; | |
| 396 | + _debug("extract data %zu/%llu", count_before, req->actual_len); | |
| 396 | 397 | |
| 397 | 398 | ret = afs_extract_data(call, true); |
| 399 | + if (req->subreq) { | |
| 400 | + req->subreq->transferred += count_before - call->iov_len; | |
| 401 | + netfs_read_subreq_progress(req->subreq, false); | |
| 402 | + } | |
| 398 | 403 | if (ret < 0) |
| 399 | 404 | return ret; |
| 400 | 405 | |
| @@ -13,6 +13,7 @@ · 12 unmodified lines | ||
| 13 | 13 | #include <linux/iversion.h> |
| 14 | 14 | #include <linux/ktime.h> |
| 15 | 15 | #include <linux/netfs.h> |
| 16 | +#include <trace/events/netfs.h> | |
| 16 | 17 | |
| 17 | 18 | #include "super.h" |
| 18 | 19 | #include "mds_client.h" |
| @@ -205,21 +206,6 @@ static void ceph_netfs_expand_readahead(struct netfs_io_request *rreq) · 186 unmodified lines | ||
| 205 | 206 | } |
| 206 | 207 | } |
| 207 | 208 | |
| 208 | −static bool ceph_netfs_clamp_length(struct netfs_io_subrequest *subreq) | |
| 209 | −{ | |
| 210 | − struct inode *inode = subreq->rreq->inode; | |
| 211 | − struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); | |
| 212 | − struct ceph_inode_info *ci = ceph_inode(inode); | |
| 213 | − u64 objno, objoff; | |
| 214 | − u32 xlen; | |
| 215 | − | |
| 216 | − /* Truncate the extent at the end of the current block */ | |
| 217 | − ceph_calc_file_object_mapping(&ci->i_layout, subreq->start, subreq->len, | |
| 218 | − &objno, &objoff, &xlen); | |
| 219 | − subreq->len = min(xlen, fsc->mount_options->rsize); | |
| 220 | − return true; | |
| 221 | −} | |
| 222 | − | |
| 223 | 209 | static void finish_netfs_read(struct ceph_osd_request *req) |
| 224 | 210 | { |
| 225 | 211 | struct inode *inode = req->r_inode; |
| @@ -264,7 +250,12 @@ static void finish_netfs_read(struct ceph_osd_request *req) · 38 unmodified lines | ||
| 264 | 250 | calc_pages_for(osd_data->alignment, |
| 265 | 251 | osd_data->length), false); |
| 266 | 252 | } |
| 267 | − netfs_subreq_terminated(subreq, err, false); | |
| 253 | + if (err > 0) { | |
| 254 | + subreq->transferred = err; | |
| 255 | + err = 0; | |
| 256 | + } | |
| 257 | + trace_netfs_sreq(subreq, netfs_sreq_trace_io_progress); | |
| 258 | + netfs_read_subreq_terminated(subreq, err, false); | |
| 268 | 259 | iput(req->r_inode); |
| 269 | 260 | ceph_dec_osd_stopping_blocker(fsc->mdsc); |
| 270 | 261 | } |
| @@ -278,7 +269,6 @@ static bool ceph_netfs_issue_op_inline(struct netfs_io_subrequest *subreq) · 7 unmodified lines | ||
| 278 | 269 | struct ceph_mds_request *req; |
| 279 | 270 | struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb); |
| 280 | 271 | struct ceph_inode_info *ci = ceph_inode(inode); |
| 281 | − struct iov_iter iter; | |
| 282 | 272 | ssize_t err = 0; |
| 283 | 273 | size_t len; |
| 284 | 274 | int mode; |
| @@ -301,6 +291,7 @@ static bool ceph_netfs_issue_op_inline(struct netfs_io_subrequest *subreq) · 16 unmodified lines | ||
| 301 | 291 | req->r_args.getattr.mask = cpu_to_le32(CEPH_STAT_CAP_INLINE_DATA); |
| 302 | 292 | req->r_num_caps = 2; |
| 303 | 293 | |
| 294 | + trace_netfs_sreq(subreq, netfs_sreq_trace_submit); | |
| 304 | 295 | err = ceph_mdsc_do_request(mdsc, NULL, req); |
| 305 | 296 | if (err < 0) |
| 306 | 297 | goto out; |
| @@ -314,17 +305,36 @@ static bool ceph_netfs_issue_op_inline(struct netfs_io_subrequest *subreq) · 7 unmodified lines | ||
| 314 | 305 | } |
| 315 | 306 | |
| 316 | 307 | len = min_t(size_t, iinfo->inline_len - subreq->start, subreq->len); |
| 317 | − iov_iter_xarray(&iter, ITER_DEST, &rreq->mapping->i_pages, subreq->start, len); | |
| 318 | − err = copy_to_iter(iinfo->inline_data + subreq->start, len, &iter); | |
| 319 | − if (err == 0) | |
| 308 | + err = copy_to_iter(iinfo->inline_data + subreq->start, len, &subreq->io_iter); | |
| 309 | + if (err == 0) { | |
| 320 | 310 | err = -EFAULT; |
| 311 | + } else { | |
| 312 | + subreq->transferred += err; | |
| 313 | + err = 0; | |
| 314 | + } | |
| 321 | 315 | |
| 322 | 316 | ceph_mdsc_put_request(req); |
| 323 | 317 | out: |
| 324 | − netfs_subreq_terminated(subreq, err, false); | |
| 318 | + netfs_read_subreq_terminated(subreq, err, false); | |
| 325 | 319 | return true; |
| 326 | 320 | } |
| 327 | 321 | |
| 322 | +static int ceph_netfs_prepare_read(struct netfs_io_subrequest *subreq) | |
| 323 | +{ | |
| 324 | + struct netfs_io_request *rreq = subreq->rreq; | |
| 325 | + struct inode *inode = rreq->inode; | |
| 326 | + struct ceph_inode_info *ci = ceph_inode(inode); | |
| 327 | + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); | |
| 328 | + u64 objno, objoff; | |
| 329 | + u32 xlen; | |
| 330 | + | |
| 331 | + /* Truncate the extent at the end of the current block */ | |
| 332 | + ceph_calc_file_object_mapping(&ci->i_layout, subreq->start, subreq->len, | |
| 333 | + &objno, &objoff, &xlen); | |
| 334 | + rreq->io_streams[0].sreq_max_len = umin(xlen, fsc->mount_options->rsize); | |
| 335 | + return 0; | |
| 336 | +} | |
| 337 | + | |
| 328 | 338 | static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) |
| 329 | 339 | { |
| 330 | 340 | struct netfs_io_request *rreq = subreq->rreq; |
| @@ -334,9 +344,8 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) · 3 unmodified lines | ||
| 334 | 344 | struct ceph_client *cl = fsc->client; |
| 335 | 345 | struct ceph_osd_request *req = NULL; |
| 336 | 346 | struct ceph_vino vino = ceph_vino(inode); |
| 337 | − struct iov_iter iter; | |
| 338 | − int err = 0; | |
| 339 | − u64 len = subreq->len; | |
| 347 | + int err; | |
| 348 | + u64 len; | |
| 340 | 349 | bool sparse = IS_ENCRYPTED(inode) || ceph_test_mount_opt(fsc, SPARSEREAD); |
| 341 | 350 | u64 off = subreq->start; |
| 342 | 351 | int extent_cnt; |
| @@ -349,6 +358,12 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) · 6 unmodified lines | ||
| 349 | 358 | if (ceph_has_inline_data(ci) && ceph_netfs_issue_op_inline(subreq)) |
| 350 | 359 | return; |
| 351 | 360 | |
| 361 | + // TODO: This rounding here is slightly dodgy. It *should* work, for | |
| 362 | + // now, as the cache only deals in blocks that are a multiple of | |
| 363 | + // PAGE_SIZE and fscrypt blocks are at most PAGE_SIZE. What needs to | |
| 364 | + // happen is for the fscrypt driving to be moved into netfslib and the | |
| 365 | + // data in the cache also to be stored encrypted. | |
| 366 | + len = subreq->len; | |
| 352 | 367 | ceph_fscrypt_adjust_off_and_len(inode, &off, &len); |
| 353 | 368 | |
| 354 | 369 | req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, vino, |
| @@ -371,8 +386,6 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) · 16 unmodified lines | ||
| 371 | 386 | doutc(cl, "%llx.%llx pos=%llu orig_len=%zu len=%llu\n", |
| 372 | 387 | ceph_vinop(inode), subreq->start, subreq->len, len); |
| 373 | 388 | |
| 374 | − iov_iter_xarray(&iter, ITER_DEST, &rreq->mapping->i_pages, subreq->start, len); | |
| 375 | − | |
| 376 | 389 | /* |
| 377 | 390 | * FIXME: For now, use CEPH_OSD_DATA_TYPE_PAGES instead of _ITER for |
| 378 | 391 | * encrypted inodes. We'd need infrastructure that handles an iov_iter |
| @@ -384,7 +397,7 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) · 5 unmodified lines | ||
| 384 | 397 | struct page **pages; |
| 385 | 398 | size_t page_off; |
| 386 | 399 | |
| 387 | − err = iov_iter_get_pages_alloc2(&iter, &pages, len, &page_off); | |
| 400 | + err = iov_iter_get_pages_alloc2(&subreq->io_iter, &pages, len, &page_off); | |
| 388 | 401 | if (err < 0) { |
| 389 | 402 | doutc(cl, "%llx.%llx failed to allocate pages, %d\n", |
| 390 | 403 | ceph_vinop(inode), err); |
| @@ -399,7 +412,7 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) · 8 unmodified lines | ||
| 399 | 412 | osd_req_op_extent_osd_data_pages(req, 0, pages, len, 0, false, |
| 400 | 413 | false); |
| 401 | 414 | } else { |
| 402 | − osd_req_op_extent_osd_iter(req, 0, &iter); | |
| 415 | + osd_req_op_extent_osd_iter(req, 0, &subreq->io_iter); | |
| 403 | 416 | } |
| 404 | 417 | if (!ceph_inc_osd_stopping_blocker(fsc->mdsc)) { |
| 405 | 418 | err = -EIO; |
| @@ -410,17 +423,19 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) · 4 unmodified lines | ||
| 410 | 423 | req->r_inode = inode; |
| 411 | 424 | ihold(inode); |
| 412 | 425 | |
| 426 | + trace_netfs_sreq(subreq, netfs_sreq_trace_submit); | |
| 413 | 427 | ceph_osdc_start_request(req->r_osdc, req); |
| 414 | 428 | out: |
| 415 | 429 | ceph_osdc_put_request(req); |
| 416 | 430 | if (err) |
| 417 | − netfs_subreq_terminated(subreq, err, false); | |
| 431 | + netfs_read_subreq_terminated(subreq, err, false); | |
| 418 | 432 | doutc(cl, "%llx.%llx result %d\n", ceph_vinop(inode), err); |
| 419 | 433 | } |
| 420 | 434 | |
| 421 | 435 | static int ceph_init_request(struct netfs_io_request *rreq, struct file *file) |
| 422 | 436 | { |
| 423 | 437 | struct inode *inode = rreq->inode; |
| 438 | + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); | |
| 424 | 439 | struct ceph_client *cl = ceph_inode_to_client(inode); |
| 425 | 440 | int got = 0, want = CEPH_CAP_FILE_CACHE; |
| 426 | 441 | struct ceph_netfs_request_data *priv; |
| @@ -472,6 +487,7 @@ static int ceph_init_request(struct netfs_io_request *rreq, struct file *file) · 45 unmodified lines | ||
| 472 | 487 | |
| 473 | 488 | priv->caps = got; |
| 474 | 489 | rreq->netfs_priv = priv; |
| 490 | + rreq->io_streams[0].sreq_max_len = fsc->mount_options->rsize; | |
| 475 | 491 | |
| 476 | 492 | out: |
| 477 | 493 | if (ret < 0) |
| @@ -496,9 +512,9 @@ static void ceph_netfs_free_request(struct netfs_io_request *rreq) · 18 unmodified lines | ||
| 496 | 512 | const struct netfs_request_ops ceph_netfs_ops = { |
| 497 | 513 | .init_request = ceph_init_request, |
| 498 | 514 | .free_request = ceph_netfs_free_request, |
| 515 | + .prepare_read = ceph_netfs_prepare_read, | |
| 499 | 516 | .issue_read = ceph_netfs_issue_read, |
| 500 | 517 | .expand_readahead = ceph_netfs_expand_readahead, |
| 501 | − .clamp_length = ceph_netfs_clamp_length, | |
| 502 | 518 | .check_write_begin = ceph_netfs_check_write_begin, |
| 503 | 519 | }; |
| 504 | 520 | |
| @@ -5,12 +5,14 @@ netfs-y := \ · 4 unmodified lines | ||
| 5 | 5 | buffered_write.o \ |
| 6 | 6 | direct_read.o \ |
| 7 | 7 | direct_write.o \ |
| 8 | − io.o \ | |
| 9 | 8 | iterator.o \ |
| 10 | 9 | locking.o \ |
| 11 | 10 | main.o \ |
| 12 | 11 | misc.o \ |
| 13 | 12 | objects.o \ |
| 13 | + read_collect.o \ | |
| 14 | + read_pgpriv2.o \ | |
| 15 | + read_retry.o \ | |
| 14 | 16 | write_collect.o \ |
| 15 | 17 | write_issue.o |
| 16 | 18 | |
| @@ -9,266 +9,388 @@ · 8 unmodified lines | ||
| 9 | 9 | #include <linux/task_io_accounting_ops.h> |
| 10 | 10 | #include "internal.h" |
| 11 | 11 | |
| 12 | −/* | |
| 13 | − * [DEPRECATED] Unlock the folios in a read operation for when the filesystem | |
| 14 | − * is using PG_private_2 and direct writing to the cache from here rather than | |
| 15 | − * marking the page for writeback. | |
| 16 | − * | |
| 17 | − * Note that we don't touch folio->private in this code. | |
| 18 | − */ | |
| 19 | −static void netfs_rreq_unlock_folios_pgpriv2(struct netfs_io_request *rreq, | |
| 20 | − size_t *account) | |
| 12 | +static void netfs_cache_expand_readahead(struct netfs_io_request *rreq, | |
| 13 | + unsigned long long *_start, | |
| 14 | + unsigned long long *_len, | |
| 15 | + unsigned long long i_size) | |
| 21 | 16 | { |
| 22 | − struct netfs_io_subrequest *subreq; | |
| 23 | − struct folio *folio; | |
| 24 | − pgoff_t start_page = rreq->start / PAGE_SIZE; | |
| 25 | − pgoff_t last_page = ((rreq->start + rreq->len) / PAGE_SIZE) - 1; | |
| 26 | − bool subreq_failed = false; | |
| 17 | + struct netfs_cache_resources *cres = &rreq->cache_resources; | |
| 27 | 18 | |
| 28 | − XA_STATE(xas, &rreq->mapping->i_pages, start_page); | |
| 19 | + if (cres->ops && cres->ops->expand_readahead) | |
| 20 | + cres->ops->expand_readahead(cres, _start, _len, i_size); | |
| 21 | +} | |
| 29 | 22 | |
| 30 | − /* Walk through the pagecache and the I/O request lists simultaneously. | |
| 31 | − * We may have a mixture of cached and uncached sections and we only | |
| 32 | − * really want to write out the uncached sections. This is slightly | |
| 33 | − * complicated by the possibility that we might have huge pages with a | |
| 34 | − * mixture inside. | |
| 23 | +static void netfs_rreq_expand(struct netfs_io_request *rreq, | |
| 24 | + struct readahead_control *ractl) | |
| 25 | +{ | |
| 26 | + /* Give the cache a chance to change the request parameters. The | |
| 27 | + * resultant request must contain the original region. | |
| 35 | 28 | */ |
| 36 | − subreq = list_first_entry(&rreq->subrequests, | |
| 37 | − struct netfs_io_subrequest, rreq_link); | |
| 38 | − subreq_failed = (subreq->error < 0); | |
| 29 | + netfs_cache_expand_readahead(rreq, &rreq->start, &rreq->len, rreq->i_size); | |
| 39 | 30 | |
| 40 | − trace_netfs_rreq(rreq, netfs_rreq_trace_unlock_pgpriv2); | |
| 31 | + /* Give the netfs a chance to change the request parameters. The | |
| 32 | + * resultant request must contain the original region. | |
| 33 | + */ | |
| 34 | + if (rreq->netfs_ops->expand_readahead) | |
| 35 | + rreq->netfs_ops->expand_readahead(rreq); | |
| 41 | 36 | |
| 42 | − rcu_read_lock(); | |
| 43 | − xas_for_each(&xas, folio, last_page) { | |
| 44 | − loff_t pg_end; | |
| 45 | − bool pg_failed = false; | |
| 46 | − bool folio_started = false; | |
| 37 | + /* Expand the request if the cache wants it to start earlier. Note | |
| 38 | + * that the expansion may get further extended if the VM wishes to | |
| 39 | + * insert THPs and the preferred start and/or end wind up in the middle | |
| 40 | + * of THPs. | |
| 41 | + * | |
| 42 | + * If this is the case, however, the THP size should be an integer | |
| 43 | + * multiple of the cache granule size, so we get a whole number of | |
| 44 | + * granules to deal with. | |
| 45 | + */ | |
| 46 | + if (rreq->start != readahead_pos(ractl) || | |
| 47 | + rreq->len != readahead_length(ractl)) { | |
| 48 | + readahead_expand(ractl, rreq->start, rreq->len); | |
| 49 | + rreq->start = readahead_pos(ractl); | |
| 50 | + rreq->len = readahead_length(ractl); | |
| 47 | 51 | |
| 48 | − if (xas_retry(&xas, folio)) | |
| 49 | − continue; | |
| 52 | + trace_netfs_read(rreq, readahead_pos(ractl), readahead_length(ractl), | |
| 53 | + netfs_read_trace_expanded); | |
| 54 | + } | |
| 55 | +} | |
| 50 | 56 | |
| 51 | − pg_end = folio_pos(folio) + folio_size(folio) - 1; | |
| 57 | +/* | |
| 58 | + * Begin an operation, and fetch the stored zero point value from the cookie if | |
| 59 | + * available. | |
| 60 | + */ | |
| 61 | +static int netfs_begin_cache_read(struct netfs_io_request *rreq, struct netfs_inode *ctx) | |
| 62 | +{ | |
| 63 | + return fscache_begin_read_operation(&rreq->cache_resources, netfs_i_cookie(ctx)); | |
| 64 | +} | |
| 52 | 65 | |
| 53 | − for (;;) { | |
| 54 | − loff_t sreq_end; | |
| 66 | +/* | |
| 67 | + * Decant the list of folios to read into a rolling buffer. | |
| 68 | + */ | |
| 69 | +static size_t netfs_load_buffer_from_ra(struct netfs_io_request *rreq, | |
| 70 | + struct folio_queue *folioq) | |
| 71 | +{ | |
| 72 | + unsigned int order, nr; | |
| 73 | + size_t size = 0; | |
| 74 | + | |
| 75 | + nr = __readahead_batch(rreq->ractl, (struct page **)folioq->vec.folios, | |
| 76 | + ARRAY_SIZE(folioq->vec.folios)); | |
| 77 | + folioq->vec.nr = nr; | |
| 78 | + for (int i = 0; i < nr; i++) { | |
| 79 | + struct folio *folio = folioq_folio(folioq, i); | |
| 80 | + | |
| 81 | + trace_netfs_folio(folio, netfs_folio_trace_read); | |
| 82 | + order = folio_order(folio); | |
| 83 | + folioq->orders[i] = order; | |
| 84 | + size += PAGE_SIZE << order; | |
| 85 | + } | |
| 55 | 86 | |
| 56 | − if (!subreq) { | |
| 57 | − pg_failed = true; | |
| 58 | − break; | |
| 59 | − } | |
| 87 | + for (int i = nr; i < folioq_nr_slots(folioq); i++) | |
| 88 | + folioq_clear(folioq, i); | |
| 60 | 89 | |
| 61 | − if (!folio_started && | |
| 62 | − test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags) && | |
| 63 | − fscache_operation_valid(&rreq->cache_resources)) { | |
| 64 | − trace_netfs_folio(folio, netfs_folio_trace_copy_to_cache); | |
| 65 | − folio_start_private_2(folio); | |
| 66 | − folio_started = true; | |
| 67 | − } | |
| 90 | + return size; | |
| 91 | +} | |
| 68 | 92 | |
| 69 | − pg_failed |= subreq_failed; | |
| 70 | − sreq_end = subreq->start + subreq->len - 1; | |
| 71 | − if (pg_end < sreq_end) | |
| 72 | − break; | |
| 93 | +/* | |
| 94 | + * netfs_prepare_read_iterator - Prepare the subreq iterator for I/O | |
| 95 | + * @subreq: The subrequest to be set up | |
| 96 | + * | |
| 97 | + * Prepare the I/O iterator representing the read buffer on a subrequest for | |
| 98 | + * the filesystem to use for I/O (it can be passed directly to a socket). This | |
| 99 | + * is intended to be called from the ->issue_read() method once the filesystem | |
| 100 | + * has trimmed the request to the size it wants. | |
| 101 | + * | |
| 102 | + * Returns the limited size if successful and -ENOMEM if insufficient memory | |
| 103 | + * available. | |
| 104 | + * | |
| 105 | + * [!] NOTE: This must be run in the same thread as ->issue_read() was called | |
| 106 | + * in as we access the readahead_control struct. | |
| 107 | + */ | |
| 108 | +static ssize_t netfs_prepare_read_iterator(struct netfs_io_subrequest *subreq) | |
| 109 | +{ | |
| 110 | + struct netfs_io_request *rreq = subreq->rreq; | |
| 111 | + size_t rsize = subreq->len; | |
| 112 | + | |
| 113 | + if (subreq->source == NETFS_DOWNLOAD_FROM_SERVER) | |
| 114 | + rsize = umin(rsize, rreq->io_streams[0].sreq_max_len); | |
| 115 | + | |
| 116 | + if (rreq->ractl) { | |
| 117 | + /* If we don't have sufficient folios in the rolling buffer, | |
| 118 | + * extract a folioq's worth from the readahead region at a time | |
| 119 | + * into the buffer. Note that this acquires a ref on each page | |
| 120 | + * that we will need to release later - but we don't want to do | |
| 121 | + * that until after we've started the I/O. | |
| 122 | + */ | |
| 123 | + while (rreq->submitted < subreq->start + rsize) { | |
| 124 | + struct folio_queue *tail = rreq->buffer_tail, *new; | |
| 125 | + size_t added; | |
| 126 | + | |
| 127 | + new = kmalloc(sizeof(*new), GFP_NOFS); | |
| 128 | + if (!new) | |
| 129 | + return -ENOMEM; | |
| 130 | + netfs_stat(&netfs_n_folioq); | |
| 131 | + folioq_init(new); | |
| 132 | + new->prev = tail; | |
| 133 | + tail->next = new; | |
| 134 | + rreq->buffer_tail = new; | |
| 135 | + added = netfs_load_buffer_from_ra(rreq, new); | |
| 136 | + rreq->iter.count += added; | |
| 137 | + rreq->submitted += added; | |
| 138 | + } | |
| 139 | + } | |
| 73 | 140 | |
| 74 | − *account += subreq->transferred; | |
| 75 | − if (!list_is_last(&subreq->rreq_link, &rreq->subrequests)) { | |
| 76 | − subreq = list_next_entry(subreq, rreq_link); | |
| 77 | − subreq_failed = (subreq->error < 0); | |
| 78 | − } else { | |
| 79 | − subreq = NULL; | |
| 80 | − subreq_failed = false; | |
| 81 | − } | |
| 141 | + subreq->len = rsize; | |
| 142 | + if (unlikely(rreq->io_streams[0].sreq_max_segs)) { | |
| 143 | + size_t limit = netfs_limit_iter(&rreq->iter, 0, rsize, | |
| 144 | + rreq->io_streams[0].sreq_max_segs); | |
| 82 | 145 | |
| 83 | − if (pg_end == sreq_end) | |
| 84 | − break; | |
| 146 | + if (limit < rsize) { | |
| 147 | + subreq->len = limit; | |
| 148 | + trace_netfs_sreq(subreq, netfs_sreq_trace_limited); | |
| 85 | 149 | } |
| 150 | + } | |
| 86 | 151 | |
| 87 | − if (!pg_failed) { | |
| 88 | − flush_dcache_folio(folio); | |
| 89 | − folio_mark_uptodate(folio); | |
| 90 | − } | |
| 152 | + subreq->io_iter = rreq->iter; | |
| 91 | 153 | |
| 92 | − if (!test_bit(NETFS_RREQ_DONT_UNLOCK_FOLIOS, &rreq->flags)) { | |
| 93 | − if (folio->index == rreq->no_unlock_folio && | |
| 94 | − test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags)) | |
| 95 | − _debug("no unlock"); | |
| 96 | − else | |
| 97 | − folio_unlock(folio); | |
| 154 | + if (iov_iter_is_folioq(&subreq->io_iter)) { | |
| 155 | + if (subreq->io_iter.folioq_slot >= folioq_nr_slots(subreq->io_iter.folioq)) { | |
| 156 | + subreq->io_iter.folioq = subreq->io_iter.folioq->next; | |
| 157 | + subreq->io_iter.folioq_slot = 0; | |
| 98 | 158 | } |
| 159 | + subreq->curr_folioq = (struct folio_queue *)subreq->io_iter.folioq; | |
| 160 | + subreq->curr_folioq_slot = subreq->io_iter.folioq_slot; | |
| 161 | + subreq->curr_folio_order = subreq->curr_folioq->orders[subreq->curr_folioq_slot]; | |
| 99 | 162 | } |
| 100 | − rcu_read_unlock(); | |
| 163 | + | |
| 164 | + iov_iter_truncate(&subreq->io_iter, subreq->len); | |
| 165 | + iov_iter_advance(&rreq->iter, subreq->len); | |
| 166 | + return subreq->len; | |
| 101 | 167 | } |
| 102 | 168 | |
| 103 | −/* | |
| 104 | − * Unlock the folios in a read operation. We need to set PG_writeback on any | |
| 105 | − * folios we're going to write back before we unlock them. | |
| 106 | − * | |
| 107 | − * Note that if the deprecated NETFS_RREQ_USE_PGPRIV2 is set then we use | |
| 108 | − * PG_private_2 and do a direct write to the cache from here instead. | |
| 109 | − */ | |
| 110 | −void netfs_rreq_unlock_folios(struct netfs_io_request *rreq) | |
| 169 | +static enum netfs_io_source netfs_cache_prepare_read(struct netfs_io_request *rreq, | |
| 170 | + struct netfs_io_subrequest *subreq, | |
| 171 | + loff_t i_size) | |
| 111 | 172 | { |
| 112 | − struct netfs_io_subrequest *subreq; | |
| 113 | − struct netfs_folio *finfo; | |
| 114 | − struct folio *folio; | |
| 115 | − pgoff_t start_page = rreq->start / PAGE_SIZE; | |
| 116 | − pgoff_t last_page = ((rreq->start + rreq->len) / PAGE_SIZE) - 1; | |
| 117 | − size_t account = 0; | |
| 118 | − bool subreq_failed = false; | |
| 173 | + struct netfs_cache_resources *cres = &rreq->cache_resources; | |
| 119 | 174 | |
| 120 | − XA_STATE(xas, &rreq->mapping->i_pages, start_page); | |
| 175 | + if (!cres->ops) | |
| 176 | + return NETFS_DOWNLOAD_FROM_SERVER; | |
| 177 | + return cres->ops->prepare_read(subreq, i_size); | |
| 178 | +} | |
| 121 | 179 | |
| 122 | − if (test_bit(NETFS_RREQ_FAILED, &rreq->flags)) { | |
| 123 | − __clear_bit(NETFS_RREQ_COPY_TO_CACHE, &rreq->flags); | |
| 124 | − list_for_each_entry(subreq, &rreq->subrequests, rreq_link) { | |
| 125 | − __clear_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags); | |
| 126 | − } | |
| 127 | − } | |
| 180 | +static void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error, | |
| 181 | + bool was_async) | |
| 182 | +{ | |
| 183 | + struct netfs_io_subrequest *subreq = priv; | |
| 128 | 184 | |
| 129 | − /* Handle deprecated PG_private_2 case. */ | |
| 130 | − if (test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)) { | |
| 131 | − netfs_rreq_unlock_folios_pgpriv2(rreq, &account); | |
| 132 | − goto out; | |
| 185 | + if (transferred_or_error < 0) { | |
| 186 | + netfs_read_subreq_terminated(subreq, transferred_or_error, was_async); | |
| 187 | + return; | |
| 133 | 188 | } |
| 134 | 189 | |
| 135 | − /* Walk through the pagecache and the I/O request lists simultaneously. | |
| 136 | − * We may have a mixture of cached and uncached sections and we only | |
| 137 | − * really want to write out the uncached sections. This is slightly | |
| 138 | − * complicated by the possibility that we might have huge pages with a | |
| 139 | − * mixture inside. | |
| 140 | − */ | |
| 141 | − subreq = list_first_entry(&rreq->subrequests, | |
| 142 | − struct netfs_io_subrequest, rreq_link); | |
| 143 | − subreq_failed = (subreq->error < 0); | |
| 144 | − | |
| 145 | − trace_netfs_rreq(rreq, netfs_rreq_trace_unlock); | |
| 190 | + if (transferred_or_error > 0) | |
| 191 | + subreq->transferred += transferred_or_error; | |
| 192 | + netfs_read_subreq_terminated(subreq, 0, was_async); | |
| 193 | +} | |
| 146 | 194 | |
| 147 | − rcu_read_lock(); | |
| 148 | − xas_for_each(&xas, folio, last_page) { | |
| 149 | − loff_t pg_end; | |
| 150 | − bool pg_failed = false; | |
| 151 | − bool wback_to_cache = false; | |
| 195 | +/* | |
| 196 | + * Issue a read against the cache. | |
| 197 | + * - Eats the caller's ref on subreq. | |
| 198 | + */ | |
| 199 | +static void netfs_read_cache_to_pagecache(struct netfs_io_request *rreq, | |
| 200 | + struct netfs_io_subrequest *subreq) | |
| 201 | +{ | |
| 202 | + struct netfs_cache_resources *cres = &rreq->cache_resources; | |
| 152 | 203 | |
| 153 | − if (xas_retry(&xas, folio)) | |
| 154 | − continue; | |
| 204 | + netfs_stat(&netfs_n_rh_read); | |
| 205 | + cres->ops->read(cres, subreq->start, &subreq->io_iter, NETFS_READ_HOLE_IGNORE, | |
| 206 | + netfs_cache_read_terminated, subreq); | |
| 207 | +} | |
| 155 | 208 | |
| 156 | − pg_end = folio_pos(folio) + folio_size(folio) - 1; | |
| 209 | +/* | |
| 210 | + * Perform a read to the pagecache from a series of sources of different types, | |
| 211 | + * slicing up the region to be read according to available cache blocks and | |
| 212 | + * network rsize. | |
| 213 | + */ | |
| 214 | +static void netfs_read_to_pagecache(struct netfs_io_request *rreq) | |
| 215 | +{ | |
| 216 | + struct netfs_inode *ictx = netfs_inode(rreq->inode); | |
| 217 | + unsigned long long start = rreq->start; | |
| 218 | + ssize_t size = rreq->len; | |
| 219 | + int ret = 0; | |
| 220 | + | |
| 221 | + atomic_inc(&rreq->nr_outstanding); | |
| 222 | + | |
| 223 | + do { | |
| 224 | + struct netfs_io_subrequest *subreq; | |
| 225 | + enum netfs_io_source source = NETFS_DOWNLOAD_FROM_SERVER; | |
| 226 | + ssize_t slice; | |
| 227 | + | |
| 228 | + subreq = netfs_alloc_subrequest(rreq); | |
| 229 | + if (!subreq) { | |
| 230 | + ret = -ENOMEM; | |
| 231 | + break; | |
| 232 | + } | |
| 157 | 233 | |
| 158 | − for (;;) { | |
| 159 | − loff_t sreq_end; | |
| 234 | + subreq->start = start; | |
| 235 | + subreq->len = size; | |
| 236 | + | |
| 237 | + atomic_inc(&rreq->nr_outstanding); | |
| 238 | + spin_lock_bh(&rreq->lock); | |
| 239 | + list_add_tail(&subreq->rreq_link, &rreq->subrequests); | |
| 240 | + subreq->prev_donated = rreq->prev_donated; | |
| 241 | + rreq->prev_donated = 0; | |
| 242 | + trace_netfs_sreq(subreq, netfs_sreq_trace_added); | |
| 243 | + spin_unlock_bh(&rreq->lock); | |
| 244 | + | |
| 245 | + source = netfs_cache_prepare_read(rreq, subreq, rreq->i_size); | |
| 246 | + subreq->source = source; | |
| 247 | + if (source == NETFS_DOWNLOAD_FROM_SERVER) { | |
| 248 | + unsigned long long zp = umin(ictx->zero_point, rreq->i_size); | |
| 249 | + size_t len = subreq->len; | |
| 250 | + | |
| 251 | + if (subreq->start >= zp) { | |
| 252 | + subreq->source = source = NETFS_FILL_WITH_ZEROES; | |
| 253 | + goto fill_with_zeroes; | |
| 254 | + } | |
| 160 | 255 | |
| 161 | − if (!subreq) { | |
| 162 | − pg_failed = true; | |
| 256 | + if (len > zp - subreq->start) | |
| 257 | + len = zp - subreq->start; | |
| 258 | + if (len == 0) { | |
| 259 | + pr_err("ZERO-LEN READ: R=%08x[%x] l=%zx/%zx s=%llx z=%llx i=%llx", | |
| 260 | + rreq->debug_id, subreq->debug_index, | |
| 261 | + subreq->len, size, | |
| 262 | + subreq->start, ictx->zero_point, rreq->i_size); | |
| 163 | 263 | break; |
| 164 | 264 | } |
| 265 | + subreq->len = len; | |
| 266 | + | |
| 267 | + netfs_stat(&netfs_n_rh_download); | |
| 268 | + if (rreq->netfs_ops->prepare_read) { | |
| 269 | + ret = rreq->netfs_ops->prepare_read(subreq); | |
| 270 | + if (ret < 0) { | |
| 271 | + atomic_dec(&rreq->nr_outstanding); | |
| 272 | + netfs_put_subrequest(subreq, false, | |
| 273 | + netfs_sreq_trace_put_cancel); | |
| 274 | + break; | |
| 275 | + } | |
| 276 | + trace_netfs_sreq(subreq, netfs_sreq_trace_prepare); | |
| 277 | + } | |
| 165 | 278 | |
| 166 | − wback_to_cache |= test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags); | |
| 167 | − pg_failed |= subreq_failed; | |
| 168 | − sreq_end = subreq->start + subreq->len - 1; | |
| 169 | − if (pg_end < sreq_end) | |
| 279 | + slice = netfs_prepare_read_iterator(subreq); | |
| 280 | + if (slice < 0) { | |
| 281 | + atomic_dec(&rreq->nr_outstanding); | |
| 282 | + netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_cancel); | |
| 283 | + ret = slice; | |
| 170 | 284 | break; |
| 171 | − | |
| 172 | − account += subreq->transferred; | |
| 173 | − if (!list_is_last(&subreq->rreq_link, &rreq->subrequests)) { | |
| 174 | − subreq = list_next_entry(subreq, rreq_link); | |
| 175 | − subreq_failed = (subreq->error < 0); | |
| 176 | − } else { | |
| 177 | − subreq = NULL; | |
| 178 | − subreq_failed = false; | |
| 179 | 285 | } |
| 180 | 286 | |
| 181 | − if (pg_end == sreq_end) | |
| 182 | − break; | |
| 287 | + rreq->netfs_ops->issue_read(subreq); | |
| 288 | + goto done; | |
| 183 | 289 | } |
| 184 | 290 | |
| 185 | − if (!pg_failed) { | |
| 186 | − flush_dcache_folio(folio); | |
| 187 | − finfo = netfs_folio_info(folio); | |
| 188 | − if (finfo) { | |
| 189 | − trace_netfs_folio(folio, netfs_folio_trace_filled_gaps); | |
| 190 | − if (finfo->netfs_group) | |
| 191 | − folio_change_private(folio, finfo->netfs_group); | |
| 192 | − else | |
| 193 | − folio_detach_private(folio); | |
| 194 | − kfree(finfo); | |
| 195 | − } | |
| 196 | − folio_mark_uptodate(folio); | |
| 197 | − if (wback_to_cache && !WARN_ON_ONCE(folio_get_private(folio) != NULL)) { | |
| 198 | − trace_netfs_folio(folio, netfs_folio_trace_copy_to_cache); | |
| 199 | − folio_attach_private(folio, NETFS_FOLIO_COPY_TO_CACHE); | |
| 200 | − filemap_dirty_folio(folio->mapping, folio); | |
| 201 | − } | |
| 291 | + fill_with_zeroes: | |
| 292 | + if (source == NETFS_FILL_WITH_ZEROES) { | |
| 293 | + subreq->source = NETFS_FILL_WITH_ZEROES; | |
| 294 | + trace_netfs_sreq(subreq, netfs_sreq_trace_submit); | |
| 295 | + netfs_stat(&netfs_n_rh_zero); | |
| 296 | + slice = netfs_prepare_read_iterator(subreq); | |
| 297 | + __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags); | |
| 298 | + netfs_read_subreq_terminated(subreq, 0, false); | |
| 299 | + goto done; | |
| 202 | 300 | } |
| 203 | 301 | |
| 204 | − if (!test_bit(NETFS_RREQ_DONT_UNLOCK_FOLIOS, &rreq->flags)) { | |
| 205 | − if (folio->index == rreq->no_unlock_folio && | |
| 206 | − test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags)) | |
| 207 | − _debug("no unlock"); | |
| 208 | − else | |
| 209 | − folio_unlock(folio); | |
| 302 | + if (source == NETFS_READ_FROM_CACHE) { | |
| 303 | + trace_netfs_sreq(subreq, netfs_sreq_trace_submit); | |
| 304 | + slice = netfs_prepare_read_iterator(subreq); | |
| 305 | + netfs_read_cache_to_pagecache(rreq, subreq); | |
| 306 | + goto done; | |
| 210 | 307 | } |
| 211 | − } | |
| 212 | − rcu_read_unlock(); | |
| 213 | 308 | |
| 214 | −out: | |
| 215 | − task_io_account_read(account); | |
| 216 | − if (rreq->netfs_ops->done) | |
| 217 | − rreq->netfs_ops->done(rreq); | |
| 218 | −} | |
| 309 | + pr_err("Unexpected read source %u\n", source); | |
| 310 | + WARN_ON_ONCE(1); | |
| 311 | + break; | |
| 219 | 312 | |
| 220 | −static void netfs_cache_expand_readahead(struct netfs_io_request *rreq, | |
| 221 | − unsigned long long *_start, | |
| 222 | − unsigned long long *_len, | |
| 223 | − unsigned long long i_size) | |
| 224 | −{ | |
| 225 | − struct netfs_cache_resources *cres = &rreq->cache_resources; | |
| 313 | + done: | |
| 314 | + size -= slice; | |
| 315 | + start += slice; | |
| 316 | + cond_resched(); | |
| 317 | + } while (size > 0); | |
| 226 | 318 | |
| 227 | − if (cres->ops && cres->ops->expand_readahead) | |
| 228 | − cres->ops->expand_readahead(cres, _start, _len, i_size); | |
| 319 | + if (atomic_dec_and_test(&rreq->nr_outstanding)) | |
| 320 | + netfs_rreq_terminated(rreq, false); | |
| 321 | + | |
| 322 | + /* Defer error return as we may need to wait for outstanding I/O. */ | |
| 323 | + cmpxchg(&rreq->error, 0, ret); | |
| 229 | 324 | } |
| 230 | 325 | |
| 231 | −static void netfs_rreq_expand(struct netfs_io_request *rreq, | |
| 232 | − struct readahead_control *ractl) | |
| 326 | +/* | |
| 327 | + * Wait for the read operation to complete, successfully or otherwise. | |
| 328 | + */ | |
| 329 | +static int netfs_wait_for_read(struct netfs_io_request *rreq) | |
| 233 | 330 | { |
| 234 | − /* Give the cache a chance to change the request parameters. The | |
| 235 | − * resultant request must contain the original region. | |
| 236 | − */ | |
| 237 | − netfs_cache_expand_readahead(rreq, &rreq->start, &rreq->len, rreq->i_size); | |
| 331 | + int ret; | |
| 238 | 332 | |
| 239 | − /* Give the netfs a chance to change the request parameters. The | |
| 240 | − * resultant request must contain the original region. | |
| 241 | − */ | |
| 242 | − if (rreq->netfs_ops->expand_readahead) | |
| 243 | − rreq->netfs_ops->expand_readahead(rreq); | |
| 333 | + trace_netfs_rreq(rreq, netfs_rreq_trace_wait_ip); | |
| 334 | + wait_on_bit(&rreq->flags, NETFS_RREQ_IN_PROGRESS, TASK_UNINTERRUPTIBLE); | |
| 335 | + ret = rreq->error; | |
| 336 | + if (ret == 0 && rreq->submitted < rreq->len) { | |
| 337 | + trace_netfs_failure(rreq, NULL, ret, netfs_fail_short_read); | |
| 338 | + ret = -EIO; | |
| 339 | + } | |
| 244 | 340 | |
| 245 | − /* Expand the request if the cache wants it to start earlier. Note | |
| 246 | − * that the expansion may get further extended if the VM wishes to | |
| 247 | − * insert THPs and the preferred start and/or end wind up in the middle | |
| 248 | − * of THPs. | |
| 249 | − * | |
| 250 | − * If this is the case, however, the THP size should be an integer | |
| 251 | − * multiple of the cache granule size, so we get a whole number of | |
| 252 | − * granules to deal with. | |
| 253 | − */ | |
| 254 | − if (rreq->start != readahead_pos(ractl) || | |
| 255 | − rreq->len != readahead_length(ractl)) { | |
| 256 | − readahead_expand(ractl, rreq->start, rreq->len); | |
| 257 | − rreq->start = readahead_pos(ractl); | |
| 258 | − rreq->len = readahead_length(ractl); | |
| 341 | + return ret; | |
| 342 | +} | |
| 259 | 343 | |
| 260 | − trace_netfs_read(rreq, readahead_pos(ractl), readahead_length(ractl), | |
| 261 | − netfs_read_trace_expanded); | |
| 262 | − } | |
| 344 | +/* | |
| 345 | + * Set up the initial folioq of buffer folios in the rolling buffer and set the | |
| 346 | + * iterator to refer to it. | |
| 347 | + */ | |
| 348 | +static int netfs_prime_buffer(struct netfs_io_request *rreq) | |
| 349 | +{ | |
| 350 | + struct folio_queue *folioq; | |
| 351 | + size_t added; | |
| 352 | + | |
| 353 | + folioq = kmalloc(sizeof(*folioq), GFP_KERNEL); | |
| 354 | + if (!folioq) | |
| 355 | + return -ENOMEM; | |
| 356 | + netfs_stat(&netfs_n_folioq); | |
| 357 | + folioq_init(folioq); | |
| 358 | + rreq->buffer = folioq; | |
| 359 | + rreq->buffer_tail = folioq; | |
| 360 | + rreq->submitted = rreq->start; | |
| 361 | + iov_iter_folio_queue(&rreq->iter, ITER_DEST, folioq, 0, 0, 0); | |
| 362 | + | |
| 363 | + added = netfs_load_buffer_from_ra(rreq, folioq); | |
| 364 | + rreq->iter.count += added; | |
| 365 | + rreq->submitted += added; | |
| 366 | + return 0; | |
| 263 | 367 | } |
| 264 | 368 | |
| 265 | 369 | /* |
| 266 | − * Begin an operation, and fetch the stored zero point value from the cookie if | |
| 267 | − * available. | |
| 370 | + * Drop the ref on each folio that we inherited from the VM readahead code. We | |
| 371 | + * still have the folio locks to pin the page until we complete the I/O. | |
| 372 | + * | |
| 373 | + * Note that we can't just release the batch in each queue struct as we use the | |
| 374 | + * occupancy count in other places. | |
| 268 | 375 | */ |
| 269 | −static int netfs_begin_cache_read(struct netfs_io_request *rreq, struct netfs_inode *ctx) | |
| 376 | +static void netfs_put_ra_refs(struct folio_queue *folioq) | |
| 270 | 377 | { |
| 271 | − return fscache_begin_read_operation(&rreq->cache_resources, netfs_i_cookie(ctx)); | |
| 378 | + struct folio_batch fbatch; | |
| 379 | + | |
| 380 | + folio_batch_init(&fbatch); | |
| 381 | + while (folioq) { | |
| 382 | + for (unsigned int slot = 0; slot < folioq_count(folioq); slot++) { | |
| 383 | + struct folio *folio = folioq_folio(folioq, slot); | |
| 384 | + if (!folio) | |
| 385 | + continue; | |
| 386 | + trace_netfs_folio(folio, netfs_folio_trace_read_put); | |
| 387 | + if (!folio_batch_add(&fbatch, folio)) | |
| 388 | + folio_batch_release(&fbatch); | |
| 389 | + } | |
| 390 | + folioq = folioq->next; | |
| 391 | + } | |
| 392 | + | |
| 393 | + folio_batch_release(&fbatch); | |
| 272 | 394 | } |
| 273 | 395 | |
| 274 | 396 | /** |
| @@ -289,22 +411,17 @@ static int netfs_begin_cache_read(struct netfs_io_request *rreq, struct netfs_in · 14 unmodified lines | ||
| 289 | 411 | void netfs_readahead(struct readahead_control *ractl) |
| 290 | 412 | { |
| 291 | 413 | struct netfs_io_request *rreq; |
| 292 | − struct netfs_inode *ctx = netfs_inode(ractl->mapping->host); | |
| 414 | + struct netfs_inode *ictx = netfs_inode(ractl->mapping->host); | |
| 415 | + unsigned long long start = readahead_pos(ractl); | |
| 416 | + size_t size = readahead_length(ractl); | |
| 293 | 417 | int ret; |
| 294 | 418 | |
| 295 | − _enter("%lx,%x", readahead_index(ractl), readahead_count(ractl)); | |
| 296 | − | |
| 297 | − if (readahead_count(ractl) == 0) | |
| 298 | − return; | |
| 299 | − | |
| 300 | − rreq = netfs_alloc_request(ractl->mapping, ractl->file, | |
| 301 | − readahead_pos(ractl), | |
| 302 | − readahead_length(ractl), | |
| 419 | + rreq = netfs_alloc_request(ractl->mapping, ractl->file, start, size, | |
| 303 | 420 | NETFS_READAHEAD); |
| 304 | 421 | if (IS_ERR(rreq)) |
| 305 | 422 | return; |
| 306 | 423 | |
| 307 | − ret = netfs_begin_cache_read(rreq, ctx); | |
| 424 | + ret = netfs_begin_cache_read(rreq, ictx); | |
| 308 | 425 | if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS) |
| 309 | 426 | goto cleanup_free; |
| 310 | 427 | |
| @@ -314,18 +431,15 @@ void netfs_readahead(struct readahead_control *ractl) · 3 unmodified lines | ||
| 314 | 431 | |
| 315 | 432 | netfs_rreq_expand(rreq, ractl); |
| 316 | 433 | |
| 317 | − /* Set up the output buffer */ | |
| 318 | − iov_iter_xarray(&rreq->iter, ITER_DEST, &ractl->mapping->i_pages, | |
| 319 | − rreq->start, rreq->len); | |
| 434 | + rreq->ractl = ractl; | |
| 435 | + if (netfs_prime_buffer(rreq) < 0) | |
| 436 | + goto cleanup_free; | |
| 437 | + netfs_read_to_pagecache(rreq); | |
| 320 | 438 | |
| 321 | − /* Drop the refs on the folios here rather than in the cache or | |
| 322 | − * filesystem. The locks will be dropped in netfs_rreq_unlock(). | |
| 323 | − */ | |
| 324 | − while (readahead_folio(ractl)) | |
| 325 | − ; | |
| 439 | + /* Release the folio refs whilst we're waiting for the I/O. */ | |
| 440 | + netfs_put_ra_refs(rreq->buffer); | |
| 326 | 441 | |
| 327 | − netfs_begin_read(rreq, false); | |
| 328 | − netfs_put_request(rreq, false, netfs_rreq_trace_put_return); | |
| 442 | + netfs_put_request(rreq, true, netfs_rreq_trace_put_return); | |
| 329 | 443 | return; |
| 330 | 444 | |
| 331 | 445 | cleanup_free: |
| @@ -334,6 +448,117 @@ cleanup_free: · 2 unmodified lines | ||
| 334 | 448 | } |
| 335 | 449 | EXPORT_SYMBOL(netfs_readahead); |
| 336 | 450 | |
| 451 | +/* | |
| 452 | + * Create a rolling buffer with a single occupying folio. | |
| 453 | + */ | |
| 454 | +static int netfs_create_singular_buffer(struct netfs_io_request *rreq, struct folio *folio) | |
| 455 | +{ | |
| 456 | + struct folio_queue *folioq; | |
| 457 | + | |
| 458 | + folioq = kmalloc(sizeof(*folioq), GFP_KERNEL); | |
| 459 | + if (!folioq) | |
| 460 | + return -ENOMEM; | |
| 461 | + | |
| 462 | + netfs_stat(&netfs_n_folioq); | |
| 463 | + folioq_init(folioq); | |
| 464 | + folioq_append(folioq, folio); | |
| 465 | + BUG_ON(folioq_folio(folioq, 0) != folio); | |
| 466 | + BUG_ON(folioq_folio_order(folioq, 0) != folio_order(folio)); | |
| 467 | + rreq->buffer = folioq; | |
| 468 | + rreq->buffer_tail = folioq; | |
| 469 | + rreq->submitted = rreq->start + rreq->len; | |
| 470 | + iov_iter_folio_queue(&rreq->iter, ITER_DEST, folioq, 0, 0, rreq->len); | |
| 471 | + rreq->ractl = (struct readahead_control *)1UL; | |
| 472 | + return 0; | |
| 473 | +} | |
| 474 | + | |
| 475 | +/* | |
| 476 | + * Read into gaps in a folio partially filled by a streaming write. | |
| 477 | + */ | |
| 478 | +static int netfs_read_gaps(struct file *file, struct folio *folio) | |
| 479 | +{ | |
| 480 | + struct netfs_io_request *rreq; | |
| 481 | + struct address_space *mapping = folio->mapping; | |
| 482 | + struct netfs_folio *finfo = netfs_folio_info(folio); | |
| 483 | + struct netfs_inode *ctx = netfs_inode(mapping->host); | |
| 484 | + struct folio *sink = NULL; | |
| 485 | + struct bio_vec *bvec; | |
| 486 | + unsigned int from = finfo->dirty_offset; | |
| 487 | + unsigned int to = from + finfo->dirty_len; | |
| 488 | + unsigned int off = 0, i = 0; | |
| 489 | + size_t flen = folio_size(folio); | |
| 490 | + size_t nr_bvec = flen / PAGE_SIZE + 2; | |
| 491 | + size_t part; | |
| 492 | + int ret; | |
| 493 | + | |
| 494 | + _enter("%lx", folio->index); | |
| 495 | + | |
| 496 | + rreq = netfs_alloc_request(mapping, file, folio_pos(folio), flen, NETFS_READ_GAPS); | |
| 497 | + if (IS_ERR(rreq)) { | |
| 498 | + ret = PTR_ERR(rreq); | |
| 499 | + goto alloc_error; | |
| 500 | + } | |
| 501 | + | |
| 502 | + ret = netfs_begin_cache_read(rreq, ctx); | |
| 503 | + if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS) | |
| 504 | + goto discard; | |
| 505 | + | |
| 506 | + netfs_stat(&netfs_n_rh_read_folio); | |
| 507 | + trace_netfs_read(rreq, rreq->start, rreq->len, netfs_read_trace_read_gaps); | |
| 508 | + | |
| 509 | + /* Fiddle the buffer so that a gap at the beginning and/or a gap at the | |
| 510 | + * end get copied to, but the middle is discarded. | |
| 511 | + */ | |
| 512 | + ret = -ENOMEM; | |
| 513 | + bvec = kmalloc_array(nr_bvec, sizeof(*bvec), GFP_KERNEL); | |
| 514 | + if (!bvec) | |
| 515 | + goto discard; | |
| 516 | + | |
| 517 | + sink = folio_alloc(GFP_KERNEL, 0); | |
| 518 | + if (!sink) { | |
| 519 | + kfree(bvec); | |
| 520 | + goto discard; | |
| 521 | + } | |
| 522 | + | |
| 523 | + trace_netfs_folio(folio, netfs_folio_trace_read_gaps); | |
| 524 | + | |
| 525 | + rreq->direct_bv = bvec; | |
| 526 | + rreq->direct_bv_count = nr_bvec; | |
| 527 | + if (from > 0) { | |
| 528 | + bvec_set_folio(&bvec[i++], folio, from, 0); | |
| 529 | + off = from; | |
| 530 | + } | |
| 531 | + while (off < to) { | |
| 532 | + part = min_t(size_t, to - off, PAGE_SIZE); | |
| 533 | + bvec_set_folio(&bvec[i++], sink, part, 0); | |
| 534 | + off += part; | |
| 535 | + } | |
| 536 | + if (to < flen) | |
| 537 | + bvec_set_folio(&bvec[i++], folio, flen - to, to); | |
| 538 | + iov_iter_bvec(&rreq->iter, ITER_DEST, bvec, i, rreq->len); | |
| 539 | + rreq->submitted = rreq->start + flen; | |
| 540 | + | |
| 541 | + netfs_read_to_pagecache(rreq); | |
| 542 | + | |
| 543 | + if (sink) | |
| 544 | + folio_put(sink); | |
| 545 | + | |
| 546 | + ret = netfs_wait_for_read(rreq); | |
| 547 | + if (ret == 0) { | |
| 548 | + flush_dcache_folio(folio); | |
| 549 | + folio_mark_uptodate(folio); | |
| 550 | + } | |
| 551 | + folio_unlock(folio); | |
| 552 | + netfs_put_request(rreq, false, netfs_rreq_trace_put_return); | |
| 553 | + return ret < 0 ? ret : 0; | |
| 554 | + | |
| 555 | +discard: | |
| 556 | + netfs_put_request(rreq, false, netfs_rreq_trace_put_discard); | |
| 557 | +alloc_error: | |
| 558 | + folio_unlock(folio); | |
| 559 | + return ret; | |
| 560 | +} | |
| 561 | + | |
| 337 | 562 | /** |
| 338 | 563 | * netfs_read_folio - Helper to manage a read_folio request |
| 339 | 564 | * @file: The file to read from |
| @@ -353,9 +578,13 @@ int netfs_read_folio(struct file *file, struct folio *folio) · 13 unmodified lines | ||
| 353 | 578 | struct address_space *mapping = folio->mapping; |
| 354 | 579 | struct netfs_io_request *rreq; |
| 355 | 580 | struct netfs_inode *ctx = netfs_inode(mapping->host); |
| 356 | − struct folio *sink = NULL; | |
| 357 | 581 | int ret; |
| 358 | 582 | |
| 583 | + if (folio_test_dirty(folio)) { | |
| 584 | + trace_netfs_folio(folio, netfs_folio_trace_read_gaps); | |
| 585 | + return netfs_read_gaps(file, folio); | |
| 586 | + } | |
| 587 | + | |
| 359 | 588 | _enter("%lx", folio->index); |
| 360 | 589 | |
| 361 | 590 | rreq = netfs_alloc_request(mapping, file, |
| @@ -374,54 +603,12 @@ int netfs_read_folio(struct file *file, struct folio *folio) · 12 unmodified lines | ||
| 374 | 603 | trace_netfs_read(rreq, rreq->start, rreq->len, netfs_read_trace_readpage); |
| 375 | 604 | |
| 376 | 605 | /* Set up the output buffer */ |
| 377 | − if (folio_test_dirty(folio)) { | |
| 378 | − /* Handle someone trying to read from an unflushed streaming | |
| 379 | − * write. We fiddle the buffer so that a gap at the beginning | |
| 380 | − * and/or a gap at the end get copied to, but the middle is | |
| 381 | − * discarded. | |
| 382 | − */ | |
| 383 | − struct netfs_folio *finfo = netfs_folio_info(folio); | |
| 384 | − struct bio_vec *bvec; | |
| 385 | − unsigned int from = finfo->dirty_offset; | |
| 386 | − unsigned int to = from + finfo->dirty_len; | |
| 387 | − unsigned int off = 0, i = 0; | |
| 388 | − size_t flen = folio_size(folio); | |
| 389 | − size_t nr_bvec = flen / PAGE_SIZE + 2; | |
| 390 | − size_t part; | |
| 391 | − | |
| 392 | − ret = -ENOMEM; | |
| 393 | − bvec = kmalloc_array(nr_bvec, sizeof(*bvec), GFP_KERNEL); | |
| 394 | − if (!bvec) | |
| 395 | − goto discard; | |
| 396 | − | |
| 397 | − sink = folio_alloc(GFP_KERNEL, 0); | |
| 398 | − if (!sink) | |
| 399 | − goto discard; | |
| 400 | − | |
| 401 | − trace_netfs_folio(folio, netfs_folio_trace_read_gaps); | |
| 402 | − | |
| 403 | − rreq->direct_bv = bvec; | |
| 404 | − rreq->direct_bv_count = nr_bvec; | |
| 405 | − if (from > 0) { | |
| 406 | − bvec_set_folio(&bvec[i++], folio, from, 0); | |
| 407 | − off = from; | |
| 408 | − } | |
| 409 | − while (off < to) { | |
| 410 | − part = min_t(size_t, to - off, PAGE_SIZE); | |
| 411 | − bvec_set_folio(&bvec[i++], sink, part, 0); | |
| 412 | − off += part; | |
| 413 | − } | |
| 414 | − if (to < flen) | |
| 415 | − bvec_set_folio(&bvec[i++], folio, flen - to, to); | |
| 416 | − iov_iter_bvec(&rreq->iter, ITER_DEST, bvec, i, rreq->len); | |
| 417 | − } else { | |
| 418 | − iov_iter_xarray(&rreq->iter, ITER_DEST, &mapping->i_pages, | |
| 419 | − rreq->start, rreq->len); | |
| 420 | − } | |
| 606 | + ret = netfs_create_singular_buffer(rreq, folio); | |
| 607 | + if (ret < 0) | |
| 608 | + goto discard; | |
| 421 | 609 | |
| 422 | − ret = netfs_begin_read(rreq, true); | |
| 423 | − if (sink) | |
| 424 | − folio_put(sink); | |
| 610 | + netfs_read_to_pagecache(rreq); | |
| 611 | + ret = netfs_wait_for_read(rreq); | |
| 425 | 612 | netfs_put_request(rreq, false, netfs_rreq_trace_put_return); |
| 426 | 613 | return ret < 0 ? ret : 0; |
| 427 | 614 | |
| @@ -494,13 +681,10 @@ zero_out: · 66 unmodified lines | ||
| 494 | 681 | * |
| 495 | 682 | * Pre-read data for a write-begin request by drawing data from the cache if |
| 496 | 683 | * possible, or the netfs if not. Space beyond the EOF is zero-filled. |
| 497 | − * Multiple I/O requests from different sources will get munged together. If | |
| 498 | − * necessary, the readahead window can be expanded in either direction to a | |
| 499 | − * more convenient alighment for RPC efficiency or to make storage in the cache | |
| 500 | − * feasible. | |
| 684 | + * Multiple I/O requests from different sources will get munged together. | |
| 501 | 685 | * |
| 502 | 686 | * The calling netfs must provide a table of operations, only one of which, |
| 503 | − * issue_op, is mandatory. | |
| 687 | + * issue_read, is mandatory. | |
| 504 | 688 | * |
| 505 | 689 | * The check_write_begin() operation can be provided to check for and flush |
| 506 | 690 | * conflicting writes once the folio is grabbed and locked. It is passed a |
| @@ -528,8 +712,6 @@ int netfs_write_begin(struct netfs_inode *ctx, · 21 unmodified lines | ||
| 528 | 712 | pgoff_t index = pos >> PAGE_SHIFT; |
| 529 | 713 | int ret; |
| 530 | 714 | |
| 531 | − DEFINE_READAHEAD(ractl, file, NULL, mapping, index); | |
| 532 | − | |
| 533 | 715 | retry: |
| 534 | 716 | folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN, |
| 535 | 717 | mapping_gfp_mask(mapping)); |
| @@ -577,22 +759,13 @@ retry: · 41 unmodified lines | ||
| 577 | 759 | netfs_stat(&netfs_n_rh_write_begin); |
| 578 | 760 | trace_netfs_read(rreq, pos, len, netfs_read_trace_write_begin); |
| 579 | 761 | |
| 580 | − /* Expand the request to meet caching requirements and download | |
| 581 | − * preferences. | |
| 582 | − */ | |
| 583 | − ractl._nr_pages = folio_nr_pages(folio); | |
| 584 | − netfs_rreq_expand(rreq, &ractl); | |
| 585 | − | |
| 586 | 762 | /* Set up the output buffer */ |
| 587 | − iov_iter_xarray(&rreq->iter, ITER_DEST, &mapping->i_pages, | |
| 588 | − rreq->start, rreq->len); | |
| 589 | − | |
| 590 | − /* We hold the folio locks, so we can drop the references */ | |
| 591 | − folio_get(folio); | |
| 592 | − while (readahead_folio(&ractl)) | |
| 593 | − ; | |
| 763 | + ret = netfs_create_singular_buffer(rreq, folio); | |
| 764 | + if (ret < 0) | |
| 765 | + goto error_put; | |
| 594 | 766 | |
| 595 | − ret = netfs_begin_read(rreq, true); | |
| 767 | + netfs_read_to_pagecache(rreq); | |
| 768 | + ret = netfs_wait_for_read(rreq); | |
| 596 | 769 | if (ret < 0) |
| 597 | 770 | goto error; |
| 598 | 771 | netfs_put_request(rreq, false, netfs_rreq_trace_put_return); |
| @@ -652,10 +825,13 @@ int netfs_prefetch_for_write(struct file *file, struct folio *folio, · 53 unmodified lines | ||
| 652 | 825 | trace_netfs_read(rreq, start, flen, netfs_read_trace_prefetch_for_write); |
| 653 | 826 | |
| 654 | 827 | /* Set up the output buffer */ |
| 655 | − iov_iter_xarray(&rreq->iter, ITER_DEST, &mapping->i_pages, | |
| 656 | − rreq->start, rreq->len); | |
| 828 | + ret = netfs_create_singular_buffer(rreq, folio); | |
| 829 | + if (ret < 0) | |
| 830 | + goto error_put; | |
| 657 | 831 | |
| 658 | − ret = netfs_begin_read(rreq, true); | |
| 832 | + folioq_mark2(rreq->buffer, 0); | |
| 833 | + netfs_read_to_pagecache(rreq); | |
| 834 | + ret = netfs_wait_for_read(rreq); | |
| 659 | 835 | netfs_put_request(rreq, false, netfs_rreq_trace_put_return); |
| 660 | 836 | return ret; |
| 661 | 837 | |
| @@ -16,6 +16,143 @@ · 15 unmodified lines | ||
| 16 | 16 | #include <linux/netfs.h> |
| 17 | 17 | #include "internal.h" |
| 18 | 18 | |
| 19 | +static void netfs_prepare_dio_read_iterator(struct netfs_io_subrequest *subreq) | |
| 20 | +{ | |
| 21 | + struct netfs_io_request *rreq = subreq->rreq; | |
| 22 | + size_t rsize; | |
| 23 | + | |
| 24 | + rsize = umin(subreq->len, rreq->io_streams[0].sreq_max_len); | |
| 25 | + subreq->len = rsize; | |
| 26 | + | |
| 27 | + if (unlikely(rreq->io_streams[0].sreq_max_segs)) { | |
| 28 | + size_t limit = netfs_limit_iter(&rreq->iter, 0, rsize, | |
| 29 | + rreq->io_streams[0].sreq_max_segs); | |
| 30 | + | |
| 31 | + if (limit < rsize) { | |
| 32 | + subreq->len = limit; | |
| 33 | + trace_netfs_sreq(subreq, netfs_sreq_trace_limited); | |
| 34 | + } | |
| 35 | + } | |
| 36 | + | |
| 37 | + trace_netfs_sreq(subreq, netfs_sreq_trace_prepare); | |
| 38 | + | |
| 39 | + subreq->io_iter = rreq->iter; | |
| 40 | + iov_iter_truncate(&subreq->io_iter, subreq->len); | |
| 41 | + iov_iter_advance(&rreq->iter, subreq->len); | |
| 42 | +} | |
| 43 | + | |
| 44 | +/* | |
| 45 | + * Perform a read to a buffer from the server, slicing up the region to be read | |
| 46 | + * according to the network rsize. | |
| 47 | + */ | |
| 48 | +static int netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq) | |
| 49 | +{ | |
| 50 | + unsigned long long start = rreq->start; | |
| 51 | + ssize_t size = rreq->len; | |
| 52 | + int ret = 0; | |
| 53 | + | |
| 54 | + atomic_set(&rreq->nr_outstanding, 1); | |
| 55 | + | |
| 56 | + do { | |
| 57 | + struct netfs_io_subrequest *subreq; | |
| 58 | + ssize_t slice; | |
| 59 | + | |
| 60 | + subreq = netfs_alloc_subrequest(rreq); | |
| 61 | + if (!subreq) { | |
| 62 | + ret = -ENOMEM; | |
| 63 | + break; | |
| 64 | + } | |
| 65 | + | |
| 66 | + subreq->source = NETFS_DOWNLOAD_FROM_SERVER; | |
| 67 | + subreq->start = start; | |
| 68 | + subreq->len = size; | |
| 69 | + | |
| 70 | + atomic_inc(&rreq->nr_outstanding); | |
| 71 | + spin_lock_bh(&rreq->lock); | |
| 72 | + list_add_tail(&subreq->rreq_link, &rreq->subrequests); | |
| 73 | + subreq->prev_donated = rreq->prev_donated; | |
| 74 | + rreq->prev_donated = 0; | |
| 75 | + trace_netfs_sreq(subreq, netfs_sreq_trace_added); | |
| 76 | + spin_unlock_bh(&rreq->lock); | |
| 77 | + | |
| 78 | + netfs_stat(&netfs_n_rh_download); | |
| 79 | + if (rreq->netfs_ops->prepare_read) { | |
| 80 | + ret = rreq->netfs_ops->prepare_read(subreq); | |
| 81 | + if (ret < 0) { | |
| 82 | + atomic_dec(&rreq->nr_outstanding); | |
| 83 | + netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_cancel); | |
| 84 | + break; | |
| 85 | + } | |
| 86 | + } | |
| 87 | + | |
| 88 | + netfs_prepare_dio_read_iterator(subreq); | |
| 89 | + slice = subreq->len; | |
| 90 | + rreq->netfs_ops->issue_read(subreq); | |
| 91 | + | |
| 92 | + size -= slice; | |
| 93 | + start += slice; | |
| 94 | + rreq->submitted += slice; | |
| 95 | + | |
| 96 | + if (test_bit(NETFS_RREQ_BLOCKED, &rreq->flags) && | |
| 97 | + test_bit(NETFS_RREQ_NONBLOCK, &rreq->flags)) | |
| 98 | + break; | |
| 99 | + cond_resched(); | |
| 100 | + } while (size > 0); | |
| 101 | + | |
| 102 | + if (atomic_dec_and_test(&rreq->nr_outstanding)) | |
| 103 | + netfs_rreq_terminated(rreq, false); | |
| 104 | + return ret; | |
| 105 | +} | |
| 106 | + | |
| 107 | +/* | |
| 108 | + * Perform a read to an application buffer, bypassing the pagecache and the | |
| 109 | + * local disk cache. | |
| 110 | + */ | |
| 111 | +static int netfs_unbuffered_read(struct netfs_io_request *rreq, bool sync) | |
| 112 | +{ | |
| 113 | + int ret; | |
| 114 | + | |
| 115 | + _enter("R=%x %llx-%llx", | |
| 116 | + rreq->debug_id, rreq->start, rreq->start + rreq->len - 1); | |
| 117 | + | |
| 118 | + if (rreq->len == 0) { | |
| 119 | + pr_err("Zero-sized read [R=%x]\n", rreq->debug_id); | |
| 120 | + return -EIO; | |
| 121 | + } | |
| 122 | + | |
| 123 | + // TODO: Use bounce buffer if requested | |
| 124 | + | |
| 125 | + inode_dio_begin(rreq->inode); | |
| 126 | + | |
| 127 | + ret = netfs_dispatch_unbuffered_reads(rreq); | |
| 128 | + | |
| 129 | + if (!rreq->submitted) { | |
| 130 | + netfs_put_request(rreq, false, netfs_rreq_trace_put_no_submit); | |
| 131 | + inode_dio_end(rreq->inode); | |
| 132 | + ret = 0; | |
| 133 | + goto out; | |
| 134 | + } | |
| 135 | + | |
| 136 | + if (sync) { | |
| 137 | + trace_netfs_rreq(rreq, netfs_rreq_trace_wait_ip); | |
| 138 | + wait_on_bit(&rreq->flags, NETFS_RREQ_IN_PROGRESS, | |
| 139 | + TASK_UNINTERRUPTIBLE); | |
| 140 | + | |
| 141 | + ret = rreq->error; | |
| 142 | + if (ret == 0 && rreq->submitted < rreq->len && | |
| 143 | + rreq->origin != NETFS_DIO_READ) { | |
| 144 | + trace_netfs_failure(rreq, NULL, ret, netfs_fail_short_read); | |
| 145 | + ret = -EIO; | |
| 146 | + } | |
| 147 | + } else { | |
| 148 | + ret = -EIOCBQUEUED; | |
| 149 | + } | |
| 150 | + | |
| 151 | +out: | |
| 152 | + _leave(" = %d", ret); | |
| 153 | + return ret; | |
| 154 | +} | |
| 155 | + | |
| 19 | 156 | /** |
| 20 | 157 | * netfs_unbuffered_read_iter_locked - Perform an unbuffered or direct I/O read |
| 21 | 158 | * @iocb: The I/O control descriptor describing the read |
| @@ -31,7 +168,7 @@ ssize_t netfs_unbuffered_read_iter_locked(struct kiocb *iocb, struct iov_iter *i · 9 unmodified lines | ||
| 31 | 168 | struct netfs_io_request *rreq; |
| 32 | 169 | ssize_t ret; |
| 33 | 170 | size_t orig_count = iov_iter_count(iter); |
| 34 | − bool async = !is_sync_kiocb(iocb); | |
| 171 | + bool sync = is_sync_kiocb(iocb); | |
| 35 | 172 | |
| 36 | 173 | _enter(""); |
| 37 | 174 | |
| @@ -78,13 +215,13 @@ ssize_t netfs_unbuffered_read_iter_locked(struct kiocb *iocb, struct iov_iter *i · 40 unmodified lines | ||
| 78 | 215 | |
| 79 | 216 | // TODO: Set up bounce buffer if needed |
| 80 | 217 | |
| 81 | − if (async) | |
| 218 | + if (!sync) | |
| 82 | 219 | rreq->iocb = iocb; |
| 83 | 220 | |
| 84 | − ret = netfs_begin_read(rreq, is_sync_kiocb(iocb)); | |
| 221 | + ret = netfs_unbuffered_read(rreq, sync); | |
| 85 | 222 | if (ret < 0) |
| 86 | 223 | goto out; /* May be -EIOCBQUEUED */ |
| 87 | − if (!async) { | |
| 224 | + if (sync) { | |
| 88 | 225 | // TODO: Copy from bounce buffer |
| 89 | 226 | iocb->ki_pos += rreq->transferred; |
| 90 | 227 | ret = rreq->transferred; |
| @@ -94,8 +231,6 @@ out: · 3 unmodified lines | ||
| 94 | 231 | netfs_put_request(rreq, false, netfs_rreq_trace_put_return); |
| 95 | 232 | if (ret > 0) |
| 96 | 233 | orig_count -= ret; |
| 97 | − if (ret != -EIOCBQUEUED) | |
| 98 | − iov_iter_revert(iter, orig_count - iov_iter_count(iter)); | |
| 99 | 234 | return ret; |
| 100 | 235 | } |
| 101 | 236 | EXPORT_SYMBOL(netfs_unbuffered_read_iter_locked); |
| @@ -23,16 +23,9 @@ · 22 unmodified lines | ||
| 23 | 23 | /* |
| 24 | 24 | * buffered_read.c |
| 25 | 25 | */ |
| 26 | −void netfs_rreq_unlock_folios(struct netfs_io_request *rreq); | |
| 27 | 26 | int netfs_prefetch_for_write(struct file *file, struct folio *folio, |
| 28 | 27 | size_t offset, size_t len); |
| 29 | 28 | |
| 30 | −/* | |
| 31 | − * io.c | |
| 32 | − */ | |
| 33 | −void netfs_rreq_work(struct work_struct *work); | |
| 34 | −int netfs_begin_read(struct netfs_io_request *rreq, bool sync); | |
| 35 | − | |
| 36 | 29 | /* |
| 37 | 30 | * main.c |
| 38 | 31 | */ |
| @@ -90,6 +83,28 @@ static inline void netfs_see_request(struct netfs_io_request *rreq, · 51 unmodified lines | ||
| 90 | 83 | trace_netfs_rreq_ref(rreq->debug_id, refcount_read(&rreq->ref), what); |
| 91 | 84 | } |
| 92 | 85 | |
| 86 | +/* | |
| 87 | + * read_collect.c | |
| 88 | + */ | |
| 89 | +void netfs_read_termination_worker(struct work_struct *work); | |
| 90 | +void netfs_rreq_terminated(struct netfs_io_request *rreq, bool was_async); | |
| 91 | + | |
| 92 | +/* | |
| 93 | + * read_pgpriv2.c | |
| 94 | + */ | |
| 95 | +void netfs_pgpriv2_mark_copy_to_cache(struct netfs_io_subrequest *subreq, | |
| 96 | + struct netfs_io_request *rreq, | |
| 97 | + struct folio_queue *folioq, | |
| 98 | + int slot); | |
| 99 | +void netfs_pgpriv2_write_to_the_cache(struct netfs_io_request *rreq); | |
| 100 | +bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *wreq); | |
| 101 | + | |
| 102 | +/* | |
| 103 | + * read_retry.c | |
| 104 | + */ | |
| 105 | +void netfs_retry_reads(struct netfs_io_request *rreq); | |
| 106 | +void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq); | |
| 107 | + | |
| 93 | 108 | /* |
| 94 | 109 | * stats.c |
| 95 | 110 | */ |
| @@ -117,6 +132,7 @@ extern atomic_t netfs_n_wh_buffered_write; · 21 unmodified lines | ||
| 117 | 132 | extern atomic_t netfs_n_wh_writethrough; |
| 118 | 133 | extern atomic_t netfs_n_wh_dio_write; |
| 119 | 134 | extern atomic_t netfs_n_wh_writepages; |
| 135 | +extern atomic_t netfs_n_wh_copy_to_cache; | |
| 120 | 136 | extern atomic_t netfs_n_wh_wstream_conflict; |
| 121 | 137 | extern atomic_t netfs_n_wh_upload; |
| 122 | 138 | extern atomic_t netfs_n_wh_upload_done; |
| @@ -162,6 +178,11 @@ struct netfs_io_request *netfs_create_write_req(struct address_space *mapping, · 39 unmodified lines | ||
| 162 | 178 | void netfs_reissue_write(struct netfs_io_stream *stream, |
| 163 | 179 | struct netfs_io_subrequest *subreq, |
| 164 | 180 | struct iov_iter *source); |
| 181 | +void netfs_issue_write(struct netfs_io_request *wreq, | |
| 182 | + struct netfs_io_stream *stream); | |
| 183 | +int netfs_advance_write(struct netfs_io_request *wreq, | |
| 184 | + struct netfs_io_stream *stream, | |
| 185 | + loff_t start, size_t len, bool to_eof); | |
| 165 | 186 | struct netfs_io_request *netfs_begin_writethrough(struct kiocb *iocb, size_t len); |
| 166 | 187 | int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc, |
| 167 | 188 | struct folio *folio, size_t copied, bool to_page_end, |
| @@ -188,9 +188,59 @@ static size_t netfs_limit_xarray(const struct iov_iter *iter, size_t start_offse · 187 unmodified lines | ||
| 188 | 188 | return min(span, max_size); |
| 189 | 189 | } |
| 190 | 190 | |
| 191 | +/* | |
| 192 | + * Select the span of a folio queue iterator we're going to use. Limit it by | |
| 193 | + * both maximum size and maximum number of segments. Returns the size of the | |
| 194 | + * span in bytes. | |
| 195 | + */ | |
| 196 | +static size_t netfs_limit_folioq(const struct iov_iter *iter, size_t start_offset, | |
| 197 | + size_t max_size, size_t max_segs) | |
| 198 | +{ | |
| 199 | + const struct folio_queue *folioq = iter->folioq; | |
| 200 | + unsigned int nsegs = 0; | |
| 201 | + unsigned int slot = iter->folioq_slot; | |
| 202 | + size_t span = 0, n = iter->count; | |
| 203 | + | |
| 204 | + if (WARN_ON(!iov_iter_is_folioq(iter)) || | |
| 205 | + WARN_ON(start_offset > n) || | |
| 206 | + n == 0) | |
| 207 | + return 0; | |
| 208 | + max_size = umin(max_size, n - start_offset); | |
| 209 | + | |
| 210 | + if (slot >= folioq_nr_slots(folioq)) { | |
| 211 | + folioq = folioq->next; | |
| 212 | + slot = 0; | |
| 213 | + } | |
| 214 | + | |
| 215 | + start_offset += iter->iov_offset; | |
| 216 | + do { | |
| 217 | + size_t flen = folioq_folio_size(folioq, slot); | |
| 218 | + | |
| 219 | + if (start_offset < flen) { | |
| 220 | + span += flen - start_offset; | |
| 221 | + nsegs++; | |
| 222 | + start_offset = 0; | |
| 223 | + } else { | |
| 224 | + start_offset -= flen; | |
| 225 | + } | |
| 226 | + if (span >= max_size || nsegs >= max_segs) | |
| 227 | + break; | |
| 228 | + | |
| 229 | + slot++; | |
| 230 | + if (slot >= folioq_nr_slots(folioq)) { | |
| 231 | + folioq = folioq->next; | |
| 232 | + slot = 0; | |
| 233 | + } | |
| 234 | + } while (folioq); | |
| 235 | + | |
| 236 | + return umin(span, max_size); | |
| 237 | +} | |
| 238 | + | |
| 191 | 239 | size_t netfs_limit_iter(const struct iov_iter *iter, size_t start_offset, |
| 192 | 240 | size_t max_size, size_t max_segs) |
| 193 | 241 | { |
| 242 | + if (iov_iter_is_folioq(iter)) | |
| 243 | + return netfs_limit_folioq(iter, start_offset, max_size, max_segs); | |
| 194 | 244 | if (iov_iter_is_bvec(iter)) |
| 195 | 245 | return netfs_limit_bvec(iter, start_offset, max_size, max_segs); |
| 196 | 246 | if (iov_iter_is_xarray(iter)) |
| @@ -36,12 +36,14 @@ DEFINE_SPINLOCK(netfs_proc_lock); · 35 unmodified lines | ||
| 36 | 36 | static const char *netfs_origins[nr__netfs_io_origin] = { |
| 37 | 37 | [NETFS_READAHEAD] = "RA", |
| 38 | 38 | [NETFS_READPAGE] = "RP", |
| 39 | + [NETFS_READ_GAPS] = "RG", | |
| 39 | 40 | [NETFS_READ_FOR_WRITE] = "RW", |
| 40 | 41 | [NETFS_DIO_READ] = "DR", |
| 41 | 42 | [NETFS_WRITEBACK] = "WB", |
| 42 | 43 | [NETFS_WRITETHROUGH] = "WT", |
| 43 | 44 | [NETFS_UNBUFFERED_WRITE] = "UW", |
| 44 | 45 | [NETFS_DIO_WRITE] = "DW", |
| 46 | + [NETFS_PGPRIV2_COPY_TO_CACHE] = "2C", | |
| 45 | 47 | }; |
| 46 | 48 | |
| 47 | 49 | /* |
| @@ -61,7 +63,7 @@ static int netfs_requests_seq_show(struct seq_file *m, void *v) · 13 unmodified lines | ||
| 61 | 63 | |
| 62 | 64 | rreq = list_entry(v, struct netfs_io_request, proc_link); |
| 63 | 65 | seq_printf(m, |
| 64 | − "%08x %s %3d %2lx %4d %3d @%04llx %llx/%llx", | |
| 66 | + "%08x %s %3d %2lx %4ld %3d @%04llx %llx/%llx", | |
| 65 | 67 | rreq->debug_id, |
| 66 | 68 | netfs_origins[rreq->origin], |
| 67 | 69 | refcount_read(&rreq->ref), |
| @@ -36,7 +36,6 @@ struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, · 35 unmodified lines | ||
| 36 | 36 | memset(rreq, 0, kmem_cache_size(cache)); |
| 37 | 37 | rreq->start = start; |
| 38 | 38 | rreq->len = len; |
| 39 | − rreq->upper_len = len; | |
| 40 | 39 | rreq->origin = origin; |
| 41 | 40 | rreq->netfs_ops = ctx->ops; |
| 42 | 41 | rreq->mapping = mapping; |
| @@ -44,6 +43,8 @@ struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, · 1 unmodified line | ||
| 44 | 43 | rreq->i_size = i_size_read(inode); |
| 45 | 44 | rreq->debug_id = atomic_inc_return(&debug_ids); |
| 46 | 45 | rreq->wsize = INT_MAX; |
| 46 | + rreq->io_streams[0].sreq_max_len = ULONG_MAX; | |
| 47 | + rreq->io_streams[0].sreq_max_segs = 0; | |
| 47 | 48 | spin_lock_init(&rreq->lock); |
| 48 | 49 | INIT_LIST_HEAD(&rreq->io_streams[0].subrequests); |
| 49 | 50 | INIT_LIST_HEAD(&rreq->io_streams[1].subrequests); |
| @@ -52,9 +53,10 @@ struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, · 2 unmodified lines | ||
| 52 | 53 | |
| 53 | 54 | if (origin == NETFS_READAHEAD || |
| 54 | 55 | origin == NETFS_READPAGE || |
| 56 | + origin == NETFS_READ_GAPS || | |
| 55 | 57 | origin == NETFS_READ_FOR_WRITE || |
| 56 | 58 | origin == NETFS_DIO_READ) |
| 57 | − INIT_WORK(&rreq->work, netfs_rreq_work); | |
| 59 | + INIT_WORK(&rreq->work, netfs_read_termination_worker); | |
| 58 | 60 | else |
| 59 | 61 | INIT_WORK(&rreq->work, netfs_write_collection_worker); |
| 60 | 62 | |
| @@ -163,7 +165,7 @@ void netfs_put_request(struct netfs_io_request *rreq, bool was_async, · 102 unmodified lines | ||
| 163 | 165 | if (was_async) { |
| 164 | 166 | rreq->work.func = netfs_free_request; |
| 165 | 167 | if (!queue_work(system_unbound_wq, &rreq->work)) |
| 166 | − BUG(); | |
| 168 | + WARN_ON(1); | |
| 167 | 169 | } else { |
| 168 | 170 | netfs_free_request(&rreq->work); |
| 169 | 171 | } |
| @@ -0,0 +1,544 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0-only | |
| 2 | +/* Network filesystem read subrequest result collection, assessment and | |
| 3 | + * retrying. | |
| 4 | + * | |
| 5 | + * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. | |
| 6 | + * Written by David Howells (dhowells@redhat.com) | |
| 7 | + */ | |
| 8 | + | |
| 9 | +#include <linux/export.h> | |
| 10 | +#include <linux/fs.h> | |
| 11 | +#include <linux/mm.h> | |
| 12 | +#include <linux/pagemap.h> | |
| 13 | +#include <linux/slab.h> | |
| 14 | +#include <linux/task_io_accounting_ops.h> | |
| 15 | +#include "internal.h" | |
| 16 | + | |
| 17 | +/* | |
| 18 | + * Clear the unread part of an I/O request. | |
| 19 | + */ | |
| 20 | +static void netfs_clear_unread(struct netfs_io_subrequest *subreq) | |
| 21 | +{ | |
| 22 | + netfs_reset_iter(subreq); | |
| 23 | + WARN_ON_ONCE(subreq->len - subreq->transferred != iov_iter_count(&subreq->io_iter)); | |
| 24 | + iov_iter_zero(iov_iter_count(&subreq->io_iter), &subreq->io_iter); | |
| 25 | + if (subreq->start + subreq->transferred >= subreq->rreq->i_size) | |
| 26 | + __set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags); | |
| 27 | +} | |
| 28 | + | |
| 29 | +/* | |
| 30 | + * Flush, mark and unlock a folio that's now completely read. If we want to | |
| 31 | + * cache the folio, we set the group to NETFS_FOLIO_COPY_TO_CACHE, mark it | |
| 32 | + * dirty and let writeback handle it. | |
| 33 | + */ | |
| 34 | +static void netfs_unlock_read_folio(struct netfs_io_subrequest *subreq, | |
| 35 | + struct netfs_io_request *rreq, | |
| 36 | + struct folio_queue *folioq, | |
| 37 | + int slot) | |
| 38 | +{ | |
| 39 | + struct netfs_folio *finfo; | |
| 40 | + struct folio *folio = folioq_folio(folioq, slot); | |
| 41 | + | |
| 42 | + flush_dcache_folio(folio); | |
| 43 | + folio_mark_uptodate(folio); | |
| 44 | + | |
| 45 | + if (!test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)) { | |
| 46 | + finfo = netfs_folio_info(folio); | |
| 47 | + if (finfo) { | |
| 48 | + trace_netfs_folio(folio, netfs_folio_trace_filled_gaps); | |
| 49 | + if (finfo->netfs_group) | |
| 50 | + folio_change_private(folio, finfo->netfs_group); | |
| 51 | + else | |
| 52 | + folio_detach_private(folio); | |
| 53 | + kfree(finfo); | |
| 54 | + } | |
| 55 | + | |
| 56 | + if (test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags)) { | |
| 57 | + if (!WARN_ON_ONCE(folio_get_private(folio) != NULL)) { | |
| 58 | + trace_netfs_folio(folio, netfs_folio_trace_copy_to_cache); | |
| 59 | + folio_attach_private(folio, NETFS_FOLIO_COPY_TO_CACHE); | |
| 60 | + folio_mark_dirty(folio); | |
| 61 | + } | |
| 62 | + } else { | |
| 63 | + trace_netfs_folio(folio, netfs_folio_trace_read_done); | |
| 64 | + } | |
| 65 | + } else { | |
| 66 | + // TODO: Use of PG_private_2 is deprecated. | |
| 67 | + if (test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags)) | |
| 68 | + netfs_pgpriv2_mark_copy_to_cache(subreq, rreq, folioq, slot); | |
| 69 | + } | |
| 70 | + | |
| 71 | + if (!test_bit(NETFS_RREQ_DONT_UNLOCK_FOLIOS, &rreq->flags)) { | |
| 72 | + if (folio->index == rreq->no_unlock_folio && | |
| 73 | + test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags)) { | |
| 74 | + _debug("no unlock"); | |
| 75 | + } else { | |
| 76 | + trace_netfs_folio(folio, netfs_folio_trace_read_unlock); | |
| 77 | + folio_unlock(folio); | |
| 78 | + } | |
| 79 | + } | |
| 80 | +} | |
| 81 | + | |
| 82 | +/* | |
| 83 | + * Unlock any folios that are now completely read. Returns true if the | |
| 84 | + * subrequest is removed from the list. | |
| 85 | + */ | |
| 86 | +static bool netfs_consume_read_data(struct netfs_io_subrequest *subreq, bool was_async) | |
| 87 | +{ | |
| 88 | + struct netfs_io_subrequest *prev, *next; | |
| 89 | + struct netfs_io_request *rreq = subreq->rreq; | |
| 90 | + struct folio_queue *folioq = subreq->curr_folioq; | |
| 91 | + size_t avail, prev_donated, next_donated, fsize, part, excess; | |
| 92 | + loff_t fpos, start; | |
| 93 | + loff_t fend; | |
| 94 | + int slot = subreq->curr_folioq_slot; | |
| 95 | + | |
| 96 | + if (WARN(subreq->transferred > subreq->len, | |
| 97 | + "Subreq overread: R%x[%x] %zu > %zu", | |
| 98 | + rreq->debug_id, subreq->debug_index, | |
| 99 | + subreq->transferred, subreq->len)) | |
| 100 | + subreq->transferred = subreq->len; | |
| 101 | + | |
| 102 | +next_folio: | |
| 103 | + fsize = PAGE_SIZE << subreq->curr_folio_order; | |
| 104 | + fpos = round_down(subreq->start + subreq->consumed, fsize); | |
| 105 | + fend = fpos + fsize; | |
| 106 | + | |
| 107 | + if (WARN_ON_ONCE(!folioq) || | |
| 108 | + WARN_ON_ONCE(!folioq_folio(folioq, slot)) || | |
| 109 | + WARN_ON_ONCE(folioq_folio(folioq, slot)->index != fpos / PAGE_SIZE)) { | |
| 110 | + pr_err("R=%08x[%x] s=%llx-%llx ctl=%zx/%zx/%zx sl=%u\n", | |
| 111 | + rreq->debug_id, subreq->debug_index, | |
| 112 | + subreq->start, subreq->start + subreq->transferred - 1, | |
| 113 | + subreq->consumed, subreq->transferred, subreq->len, | |
| 114 | + slot); | |
| 115 | + if (folioq) { | |
| 116 | + struct folio *folio = folioq_folio(folioq, slot); | |
| 117 | + | |
| 118 | + pr_err("folioq: orders=%02x%02x%02x%02x\n", | |
| 119 | + folioq->orders[0], folioq->orders[1], | |
| 120 | + folioq->orders[2], folioq->orders[3]); | |
| 121 | + if (folio) | |
| 122 | + pr_err("folio: %llx-%llx ix=%llx o=%u qo=%u\n", | |
| 123 | + fpos, fend - 1, folio_pos(folio), folio_order(folio), | |
| 124 | + folioq_folio_order(folioq, slot)); | |
| 125 | + } | |
| 126 | + } | |
| 127 | + | |
| 128 | +donation_changed: | |
| 129 | + /* Try to consume the current folio if we've hit or passed the end of | |
| 130 | + * it. There's a possibility that this subreq doesn't start at the | |
| 131 | + * beginning of the folio, in which case we need to donate to/from the | |
| 132 | + * preceding subreq. | |
| 133 | + * | |
| 134 | + * We also need to include any potential donation back from the | |
| 135 | + * following subreq. | |
| 136 | + */ | |
| 137 | + prev_donated = READ_ONCE(subreq->prev_donated); | |
| 138 | + next_donated = READ_ONCE(subreq->next_donated); | |
| 139 | + if (prev_donated || next_donated) { | |
| 140 | + spin_lock_bh(&rreq->lock); | |
| 141 | + prev_donated = subreq->prev_donated; | |
| 142 | + next_donated = subreq->next_donated; | |
| 143 | + subreq->start -= prev_donated; | |
| 144 | + subreq->len += prev_donated; | |
| 145 | + subreq->transferred += prev_donated; | |
| 146 | + prev_donated = subreq->prev_donated = 0; | |
| 147 | + if (subreq->transferred == subreq->len) { | |
| 148 | + subreq->len += next_donated; | |
| 149 | + subreq->transferred += next_donated; | |
| 150 | + next_donated = subreq->next_donated = 0; | |
| 151 | + } | |
| 152 | + trace_netfs_sreq(subreq, netfs_sreq_trace_add_donations); | |
| 153 | + spin_unlock_bh(&rreq->lock); | |
| 154 | + } | |
| 155 | + | |
| 156 | + avail = subreq->transferred; | |
| 157 | + if (avail == subreq->len) | |
| 158 | + avail += next_donated; | |
| 159 | + start = subreq->start; | |
| 160 | + if (subreq->consumed == 0) { | |
| 161 | + start -= prev_donated; | |
| 162 | + avail += prev_donated; | |
| 163 | + } else { | |
| 164 | + start += subreq->consumed; | |
| 165 | + avail -= subreq->consumed; | |
| 166 | + } | |
| 167 | + part = umin(avail, fsize); | |
| 168 | + | |
| 169 | + trace_netfs_progress(subreq, start, avail, part); | |
| 170 | + | |
| 171 | + if (start + avail >= fend) { | |
| 172 | + if (fpos == start) { | |
| 173 | + /* Flush, unlock and mark for caching any folio we've just read. */ | |
| 174 | + subreq->consumed = fend - subreq->start; | |
| 175 | + netfs_unlock_read_folio(subreq, rreq, folioq, slot); | |
| 176 | + folioq_mark2(folioq, slot); | |
| 177 | + if (subreq->consumed >= subreq->len) | |
| 178 | + goto remove_subreq; | |
| 179 | + } else if (fpos < start) { | |
| 180 | + excess = fend - subreq->start; | |
| 181 | + | |
| 182 | + spin_lock_bh(&rreq->lock); | |
| 183 | + /* If we complete first on a folio split with the | |
| 184 | + * preceding subreq, donate to that subreq - otherwise | |
| 185 | + * we get the responsibility. | |
| 186 | + */ | |
| 187 | + if (subreq->prev_donated != prev_donated) { | |
| 188 | + spin_unlock_bh(&rreq->lock); | |
| 189 | + goto donation_changed; | |
| 190 | + } | |
| 191 | + | |
| 192 | + if (list_is_first(&subreq->rreq_link, &rreq->subrequests)) { | |
| 193 | + spin_unlock_bh(&rreq->lock); | |
| 194 | + pr_err("Can't donate prior to front\n"); | |
| 195 | + goto bad; | |
| 196 | + } | |
| 197 | + | |
| 198 | + prev = list_prev_entry(subreq, rreq_link); | |
| 199 | + WRITE_ONCE(prev->next_donated, prev->next_donated + excess); | |
| 200 | + subreq->start += excess; | |
| 201 | + subreq->len -= excess; | |
| 202 | + subreq->transferred -= excess; | |
| 203 | + trace_netfs_donate(rreq, subreq, prev, excess, | |
| 204 | + netfs_trace_donate_tail_to_prev); | |
| 205 | + trace_netfs_sreq(subreq, netfs_sreq_trace_donate_to_prev); | |
| 206 | + | |
| 207 | + if (subreq->consumed >= subreq->len) | |
| 208 | + goto remove_subreq_locked; | |
| 209 | + spin_unlock_bh(&rreq->lock); | |
| 210 | + } else { | |
| 211 | + pr_err("fpos > start\n"); | |
| 212 | + goto bad; | |
| 213 | + } | |
| 214 | + | |
| 215 | + /* Advance the rolling buffer to the next folio. */ | |
| 216 | + slot++; | |
| 217 | + if (slot >= folioq_nr_slots(folioq)) { | |
| 218 | + slot = 0; | |
| 219 | + folioq = folioq->next; | |
| 220 | + subreq->curr_folioq = folioq; | |
| 221 | + } | |
| 222 | + subreq->curr_folioq_slot = slot; | |
| 223 | + if (folioq && folioq_folio(folioq, slot)) | |
| 224 | + subreq->curr_folio_order = folioq->orders[slot]; | |
| 225 | + if (!was_async) | |
| 226 | + cond_resched(); | |
| 227 | + goto next_folio; | |
| 228 | + } | |
| 229 | + | |
| 230 | + /* Deal with partial progress. */ | |
| 231 | + if (subreq->transferred < subreq->len) | |
| 232 | + return false; | |
| 233 | + | |
| 234 | + /* Donate the remaining downloaded data to one of the neighbouring | |
| 235 | + * subrequests. Note that we may race with them doing the same thing. | |
| 236 | + */ | |
| 237 | + spin_lock_bh(&rreq->lock); | |
| 238 | + | |
| 239 | + if (subreq->prev_donated != prev_donated || | |
| 240 | + subreq->next_donated != next_donated) { | |
| 241 | + spin_unlock_bh(&rreq->lock); | |
| 242 | + cond_resched(); | |
| 243 | + goto donation_changed; | |
| 244 | + } | |
| 245 | + | |
| 246 | + /* Deal with the trickiest case: that this subreq is in the middle of a | |
| 247 | + * folio, not touching either edge, but finishes first. In such a | |
| 248 | + * case, we donate to the previous subreq, if there is one, so that the | |
| 249 | + * donation is only handled when that completes - and remove this | |
| 250 | + * subreq from the list. | |
| 251 | + * | |
| 252 | + * If the previous subreq finished first, we will have acquired their | |
| 253 | + * donation and should be able to unlock folios and/or donate nextwards. | |
| 254 | + */ | |
| 255 | + if (!subreq->consumed && | |
| 256 | + !prev_donated && | |
| 257 | + !list_is_first(&subreq->rreq_link, &rreq->subrequests)) { | |
| 258 | + prev = list_prev_entry(subreq, rreq_link); | |
| 259 | + WRITE_ONCE(prev->next_donated, prev->next_donated + subreq->len); | |
| 260 | + subreq->start += subreq->len; | |
| 261 | + subreq->len = 0; | |
| 262 | + subreq->transferred = 0; | |
| 263 | + trace_netfs_donate(rreq, subreq, prev, subreq->len, | |
| 264 | + netfs_trace_donate_to_prev); | |
| 265 | + trace_netfs_sreq(subreq, netfs_sreq_trace_donate_to_prev); | |
| 266 | + goto remove_subreq_locked; | |
| 267 | + } | |
| 268 | + | |
| 269 | + /* If we can't donate down the chain, donate up the chain instead. */ | |
| 270 | + excess = subreq->len - subreq->consumed + next_donated; | |
| 271 | + | |
| 272 | + if (!subreq->consumed) | |
| 273 | + excess += prev_donated; | |
| 274 | + | |
| 275 | + if (list_is_last(&subreq->rreq_link, &rreq->subrequests)) { | |
| 276 | + rreq->prev_donated = excess; | |
| 277 | + trace_netfs_donate(rreq, subreq, NULL, excess, | |
| 278 | + netfs_trace_donate_to_deferred_next); | |
| 279 | + } else { | |
| 280 | + next = list_next_entry(subreq, rreq_link); | |
| 281 | + WRITE_ONCE(next->prev_donated, excess); | |
| 282 | + trace_netfs_donate(rreq, subreq, next, excess, | |
| 283 | + netfs_trace_donate_to_next); | |
| 284 | + } | |
| 285 | + trace_netfs_sreq(subreq, netfs_sreq_trace_donate_to_next); | |
| 286 | + subreq->len = subreq->consumed; | |
| 287 | + subreq->transferred = subreq->consumed; | |
| 288 | + goto remove_subreq_locked; | |
| 289 | + | |
| 290 | +remove_subreq: | |
| 291 | + spin_lock_bh(&rreq->lock); | |
| 292 | +remove_subreq_locked: | |
| 293 | + subreq->consumed = subreq->len; | |
| 294 | + list_del(&subreq->rreq_link); | |
| 295 | + spin_unlock_bh(&rreq->lock); | |
| 296 | + netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_consumed); | |
| 297 | + return true; | |
| 298 | + | |
| 299 | +bad: | |
| 300 | + /* Errr... prev and next both donated to us, but insufficient to finish | |
| 301 | + * the folio. | |
| 302 | + */ | |
| 303 | + printk("R=%08x[%x] s=%llx-%llx %zx/%zx/%zx\n", | |
| 304 | + rreq->debug_id, subreq->debug_index, | |
| 305 | + subreq->start, subreq->start + subreq->transferred - 1, | |
| 306 | + subreq->consumed, subreq->transferred, subreq->len); | |
| 307 | + printk("folio: %llx-%llx\n", fpos, fend - 1); | |
| 308 | + printk("donated: prev=%zx next=%zx\n", prev_donated, next_donated); | |
| 309 | + printk("s=%llx av=%zx part=%zx\n", start, avail, part); | |
| 310 | + BUG(); | |
| 311 | +} | |
| 312 | + | |
| 313 | +/* | |
| 314 | + * Do page flushing and suchlike after DIO. | |
| 315 | + */ | |
| 316 | +static void netfs_rreq_assess_dio(struct netfs_io_request *rreq) | |
| 317 | +{ | |
| 318 | + struct netfs_io_subrequest *subreq; | |
| 319 | + unsigned int i; | |
| 320 | + | |
| 321 | + /* Collect unbuffered reads and direct reads, adding up the transfer | |
| 322 | + * sizes until we find the first short or failed subrequest. | |
| 323 | + */ | |
| 324 | + list_for_each_entry(subreq, &rreq->subrequests, rreq_link) { | |
| 325 | + rreq->transferred += subreq->transferred; | |
| 326 | + | |
| 327 | + if (subreq->transferred < subreq->len || | |
| 328 | + test_bit(NETFS_SREQ_FAILED, &subreq->flags)) { | |
| 329 | + rreq->error = subreq->error; | |
| 330 | + break; | |
| 331 | + } | |
| 332 | + } | |
| 333 | + | |
| 334 | + if (rreq->origin == NETFS_DIO_READ) { | |
| 335 | + for (i = 0; i < rreq->direct_bv_count; i++) { | |
| 336 | + flush_dcache_page(rreq->direct_bv[i].bv_page); | |
| 337 | + // TODO: cifs marks pages in the destination buffer | |
| 338 | + // dirty under some circumstances after a read. Do we | |
| 339 | + // need to do that too? | |
| 340 | + set_page_dirty(rreq->direct_bv[i].bv_page); | |
| 341 | + } | |
| 342 | + } | |
| 343 | + | |
| 344 | + if (rreq->iocb) { | |
| 345 | + rreq->iocb->ki_pos += rreq->transferred; | |
| 346 | + if (rreq->iocb->ki_complete) | |
| 347 | + rreq->iocb->ki_complete( | |
| 348 | + rreq->iocb, rreq->error ? rreq->error : rreq->transferred); | |
| 349 | + } | |
| 350 | + if (rreq->netfs_ops->done) | |
| 351 | + rreq->netfs_ops->done(rreq); | |
| 352 | + if (rreq->origin == NETFS_DIO_READ) | |
| 353 | + inode_dio_end(rreq->inode); | |
| 354 | +} | |
| 355 | + | |
| 356 | +/* | |
| 357 | + * Assess the state of a read request and decide what to do next. | |
| 358 | + * | |
| 359 | + * Note that we're in normal kernel thread context at this point, possibly | |
| 360 | + * running on a workqueue. | |
| 361 | + */ | |
| 362 | +static void netfs_rreq_assess(struct netfs_io_request *rreq) | |
| 363 | +{ | |
| 364 | + trace_netfs_rreq(rreq, netfs_rreq_trace_assess); | |
| 365 | + | |
| 366 | + //netfs_rreq_is_still_valid(rreq); | |
| 367 | + | |
| 368 | + if (test_and_clear_bit(NETFS_RREQ_NEED_RETRY, &rreq->flags)) { | |
| 369 | + netfs_retry_reads(rreq); | |
| 370 | + return; | |
| 371 | + } | |
| 372 | + | |
| 373 | + if (rreq->origin == NETFS_DIO_READ || | |
| 374 | + rreq->origin == NETFS_READ_GAPS) | |
| 375 | + netfs_rreq_assess_dio(rreq); | |
| 376 | + task_io_account_read(rreq->transferred); | |
| 377 | + | |
| 378 | + trace_netfs_rreq(rreq, netfs_rreq_trace_wake_ip); | |
| 379 | + clear_bit_unlock(NETFS_RREQ_IN_PROGRESS, &rreq->flags); | |
| 380 | + wake_up_bit(&rreq->flags, NETFS_RREQ_IN_PROGRESS); | |
| 381 | + | |
| 382 | + trace_netfs_rreq(rreq, netfs_rreq_trace_done); | |
| 383 | + netfs_clear_subrequests(rreq, false); | |
| 384 | + netfs_unlock_abandoned_read_pages(rreq); | |
| 385 | + if (unlikely(test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags))) | |
| 386 | + netfs_pgpriv2_write_to_the_cache(rreq); | |
| 387 | +} | |
| 388 | + | |
| 389 | +void netfs_read_termination_worker(struct work_struct *work) | |
| 390 | +{ | |
| 391 | + struct netfs_io_request *rreq = | |
| 392 | + container_of(work, struct netfs_io_request, work); | |
| 393 | + netfs_see_request(rreq, netfs_rreq_trace_see_work); | |
| 394 | + netfs_rreq_assess(rreq); | |
| 395 | + netfs_put_request(rreq, false, netfs_rreq_trace_put_work_complete); | |
| 396 | +} | |
| 397 | + | |
| 398 | +/* | |
| 399 | + * Handle the completion of all outstanding I/O operations on a read request. | |
| 400 | + * We inherit a ref from the caller. | |
| 401 | + */ | |
| 402 | +void netfs_rreq_terminated(struct netfs_io_request *rreq, bool was_async) | |
| 403 | +{ | |
| 404 | + if (!was_async) | |
| 405 | + return netfs_rreq_assess(rreq); | |
| 406 | + if (!work_pending(&rreq->work)) { | |
| 407 | + netfs_get_request(rreq, netfs_rreq_trace_get_work); | |
| 408 | + if (!queue_work(system_unbound_wq, &rreq->work)) | |
| 409 | + netfs_put_request(rreq, was_async, netfs_rreq_trace_put_work_nq); | |
| 410 | + } | |
| 411 | +} | |
| 412 | + | |
| 413 | +/** | |
| 414 | + * netfs_read_subreq_progress - Note progress of a read operation. | |
| 415 | + * @subreq: The read request that has terminated. | |
| 416 | + * @was_async: True if we're in an asynchronous context. | |
| 417 | + * | |
| 418 | + * This tells the read side of netfs lib that a contributory I/O operation has | |
| 419 | + * made some progress and that it may be possible to unlock some folios. | |
| 420 | + * | |
| 421 | + * Before calling, the filesystem should update subreq->transferred to track | |
| 422 | + * the amount of data copied into the output buffer. | |
| 423 | + * | |
| 424 | + * If @was_async is true, the caller might be running in softirq or interrupt | |
| 425 | + * context and we can't sleep. | |
| 426 | + */ | |
| 427 | +void netfs_read_subreq_progress(struct netfs_io_subrequest *subreq, | |
| 428 | + bool was_async) | |
| 429 | +{ | |
| 430 | + struct netfs_io_request *rreq = subreq->rreq; | |
| 431 | + | |
| 432 | + trace_netfs_sreq(subreq, netfs_sreq_trace_progress); | |
| 433 | + | |
| 434 | + if (subreq->transferred > subreq->consumed && | |
| 435 | + (rreq->origin == NETFS_READAHEAD || | |
| 436 | + rreq->origin == NETFS_READPAGE || | |
| 437 | + rreq->origin == NETFS_READ_FOR_WRITE)) { | |
| 438 | + netfs_consume_read_data(subreq, was_async); | |
| 439 | + __clear_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags); | |
| 440 | + } | |
| 441 | +} | |
| 442 | +EXPORT_SYMBOL(netfs_read_subreq_progress); | |
| 443 | + | |
| 444 | +/** | |
| 445 | + * netfs_read_subreq_terminated - Note the termination of an I/O operation. | |
| 446 | + * @subreq: The I/O request that has terminated. | |
| 447 | + * @error: Error code indicating type of completion. | |
| 448 | + * @was_async: The termination was asynchronous | |
| 449 | + * | |
| 450 | + * This tells the read helper that a contributory I/O operation has terminated, | |
| 451 | + * one way or another, and that it should integrate the results. | |
| 452 | + * | |
| 453 | + * The caller indicates the outcome of the operation through @error, supplying | |
| 454 | + * 0 to indicate a successful or retryable transfer (if NETFS_SREQ_NEED_RETRY | |
| 455 | + * is set) or a negative error code. The helper will look after reissuing I/O | |
| 456 | + * operations as appropriate and writing downloaded data to the cache. | |
| 457 | + * | |
| 458 | + * Before calling, the filesystem should update subreq->transferred to track | |
| 459 | + * the amount of data copied into the output buffer. | |
| 460 | + * | |
| 461 | + * If @was_async is true, the caller might be running in softirq or interrupt | |
| 462 | + * context and we can't sleep. | |
| 463 | + */ | |
| 464 | +void netfs_read_subreq_terminated(struct netfs_io_subrequest *subreq, | |
| 465 | + int error, bool was_async) | |
| 466 | +{ | |
| 467 | + struct netfs_io_request *rreq = subreq->rreq; | |
| 468 | + | |
| 469 | + switch (subreq->source) { | |
| 470 | + case NETFS_READ_FROM_CACHE: | |
| 471 | + netfs_stat(&netfs_n_rh_read_done); | |
| 472 | + break; | |
| 473 | + case NETFS_DOWNLOAD_FROM_SERVER: | |
| 474 | + netfs_stat(&netfs_n_rh_download_done); | |
| 475 | + break; | |
| 476 | + default: | |
| 477 | + break; | |
| 478 | + } | |
| 479 | + | |
| 480 | + if (rreq->origin != NETFS_DIO_READ) { | |
| 481 | + /* Collect buffered reads. | |
| 482 | + * | |
| 483 | + * If the read completed validly short, then we can clear the | |
| 484 | + * tail before going on to unlock the folios. | |
| 485 | + */ | |
| 486 | + if (error == 0 && subreq->transferred < subreq->len && | |
| 487 | + (test_bit(NETFS_SREQ_HIT_EOF, &subreq->flags) || | |
| 488 | + test_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags))) { | |
| 489 | + netfs_clear_unread(subreq); | |
| 490 | + subreq->transferred = subreq->len; | |
| 491 | + trace_netfs_sreq(subreq, netfs_sreq_trace_clear); | |
| 492 | + } | |
| 493 | + if (subreq->transferred > subreq->consumed && | |
| 494 | + (rreq->origin == NETFS_READAHEAD || | |
| 495 | + rreq->origin == NETFS_READPAGE || | |
| 496 | + rreq->origin == NETFS_READ_FOR_WRITE)) { | |
| 497 | + netfs_consume_read_data(subreq, was_async); | |
| 498 | + __clear_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags); | |
| 499 | + } | |
| 500 | + rreq->transferred += subreq->transferred; | |
| 501 | + } | |
| 502 | + | |
| 503 | + /* Deal with retry requests, short reads and errors. If we retry | |
| 504 | + * but don't make progress, we abandon the attempt. | |
| 505 | + */ | |
| 506 | + if (!error && subreq->transferred < subreq->len) { | |
| 507 | + if (test_bit(NETFS_SREQ_HIT_EOF, &subreq->flags)) { | |
| 508 | + trace_netfs_sreq(subreq, netfs_sreq_trace_hit_eof); | |
| 509 | + } else { | |
| 510 | + trace_netfs_sreq(subreq, netfs_sreq_trace_short); | |
| 511 | + if (subreq->transferred > subreq->consumed) { | |
| 512 | + __set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); | |
| 513 | + __clear_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags); | |
| 514 | + set_bit(NETFS_RREQ_NEED_RETRY, &rreq->flags); | |
| 515 | + } else if (!__test_and_set_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags)) { | |
| 516 | + __set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); | |
| 517 | + set_bit(NETFS_RREQ_NEED_RETRY, &rreq->flags); | |
| 518 | + } else { | |
| 519 | + __set_bit(NETFS_SREQ_FAILED, &subreq->flags); | |
| 520 | + error = -ENODATA; | |
| 521 | + } | |
| 522 | + } | |
| 523 | + } | |
| 524 | + | |
| 525 | + subreq->error = error; | |
| 526 | + trace_netfs_sreq(subreq, netfs_sreq_trace_terminated); | |
| 527 | + | |
| 528 | + if (unlikely(error < 0)) { | |
| 529 | + trace_netfs_failure(rreq, subreq, error, netfs_fail_read); | |
| 530 | + if (subreq->source == NETFS_READ_FROM_CACHE) { | |
| 531 | + netfs_stat(&netfs_n_rh_read_failed); | |
| 532 | + } else { | |
| 533 | + netfs_stat(&netfs_n_rh_download_failed); | |
| 534 | + set_bit(NETFS_RREQ_FAILED, &rreq->flags); | |
| 535 | + rreq->error = subreq->error; | |
| 536 | + } | |
| 537 | + } | |
| 538 | + | |
| 539 | + if (atomic_dec_and_test(&rreq->nr_outstanding)) | |
| 540 | + netfs_rreq_terminated(rreq, was_async); | |
| 541 | + | |
| 542 | + netfs_put_subrequest(subreq, was_async, netfs_sreq_trace_put_terminated); | |
| 543 | +} | |
| 544 | +EXPORT_SYMBOL(netfs_read_subreq_terminated); | |
| @@ -0,0 +1,264 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0-only | |
| 2 | +/* Read with PG_private_2 [DEPRECATED]. | |
| 3 | + * | |
| 4 | + * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. | |
| 5 | + * Written by David Howells (dhowells@redhat.com) | |
| 6 | + */ | |
| 7 | + | |
| 8 | +#include <linux/export.h> | |
| 9 | +#include <linux/fs.h> | |
| 10 | +#include <linux/mm.h> | |
| 11 | +#include <linux/pagemap.h> | |
| 12 | +#include <linux/slab.h> | |
| 13 | +#include <linux/task_io_accounting_ops.h> | |
| 14 | +#include "internal.h" | |
| 15 | + | |
| 16 | +/* | |
| 17 | + * [DEPRECATED] Mark page as requiring copy-to-cache using PG_private_2. The | |
| 18 | + * third mark in the folio queue is used to indicate that this folio needs | |
| 19 | + * writing. | |
| 20 | + */ | |
| 21 | +void netfs_pgpriv2_mark_copy_to_cache(struct netfs_io_subrequest *subreq, | |
| 22 | + struct netfs_io_request *rreq, | |
| 23 | + struct folio_queue *folioq, | |
| 24 | + int slot) | |
| 25 | +{ | |
| 26 | + struct folio *folio = folioq_folio(folioq, slot); | |
| 27 | + | |
| 28 | + trace_netfs_folio(folio, netfs_folio_trace_copy_to_cache); | |
| 29 | + folio_start_private_2(folio); | |
| 30 | + folioq_mark3(folioq, slot); | |
| 31 | +} | |
| 32 | + | |
| 33 | +/* | |
| 34 | + * [DEPRECATED] Cancel PG_private_2 on all marked folios in the event of an | |
| 35 | + * unrecoverable error. | |
| 36 | + */ | |
| 37 | +static void netfs_pgpriv2_cancel(struct folio_queue *folioq) | |
| 38 | +{ | |
| 39 | + struct folio *folio; | |
| 40 | + int slot; | |
| 41 | + | |
| 42 | + while (folioq) { | |
| 43 | + if (!folioq->marks3) { | |
| 44 | + folioq = folioq->next; | |
| 45 | + continue; | |
| 46 | + } | |
| 47 | + | |
| 48 | + slot = __ffs(folioq->marks3); | |
| 49 | + folio = folioq_folio(folioq, slot); | |
| 50 | + | |
| 51 | + trace_netfs_folio(folio, netfs_folio_trace_cancel_copy); | |
| 52 | + folio_end_private_2(folio); | |
| 53 | + folioq_unmark3(folioq, slot); | |
| 54 | + } | |
| 55 | +} | |
| 56 | + | |
| 57 | +/* | |
| 58 | + * [DEPRECATED] Copy a folio to the cache with PG_private_2 set. | |
| 59 | + */ | |
| 60 | +static int netfs_pgpriv2_copy_folio(struct netfs_io_request *wreq, struct folio *folio) | |
| 61 | +{ | |
| 62 | + struct netfs_io_stream *cache = &wreq->io_streams[1]; | |
| 63 | + size_t fsize = folio_size(folio), flen = fsize; | |
| 64 | + loff_t fpos = folio_pos(folio), i_size; | |
| 65 | + bool to_eof = false; | |
| 66 | + | |
| 67 | + _enter(""); | |
| 68 | + | |
| 69 | + /* netfs_perform_write() may shift i_size around the page or from out | |
| 70 | + * of the page to beyond it, but cannot move i_size into or through the | |
| 71 | + * page since we have it locked. | |
| 72 | + */ | |
| 73 | + i_size = i_size_read(wreq->inode); | |
| 74 | + | |
| 75 | + if (fpos >= i_size) { | |
| 76 | + /* mmap beyond eof. */ | |
| 77 | + _debug("beyond eof"); | |
| 78 | + folio_end_private_2(folio); | |
| 79 | + return 0; | |
| 80 | + } | |
| 81 | + | |
| 82 | + if (fpos + fsize > wreq->i_size) | |
| 83 | + wreq->i_size = i_size; | |
| 84 | + | |
| 85 | + if (flen > i_size - fpos) { | |
| 86 | + flen = i_size - fpos; | |
| 87 | + to_eof = true; | |
| 88 | + } else if (flen == i_size - fpos) { | |
| 89 | + to_eof = true; | |
| 90 | + } | |
| 91 | + | |
| 92 | + _debug("folio %zx %zx", flen, fsize); | |
| 93 | + | |
| 94 | + trace_netfs_folio(folio, netfs_folio_trace_store_copy); | |
| 95 | + | |
| 96 | + /* Attach the folio to the rolling buffer. */ | |
| 97 | + if (netfs_buffer_append_folio(wreq, folio, false) < 0) | |
| 98 | + return -ENOMEM; | |
| 99 | + | |
| 100 | + cache->submit_max_len = fsize; | |
| 101 | + cache->submit_off = 0; | |
| 102 | + cache->submit_len = flen; | |
| 103 | + | |
| 104 | + /* Attach the folio to one or more subrequests. For a big folio, we | |
| 105 | + * could end up with thousands of subrequests if the wsize is small - | |
| 106 | + * but we might need to wait during the creation of subrequests for | |
| 107 | + * network resources (eg. SMB credits). | |
| 108 | + */ | |
| 109 | + do { | |
| 110 | + ssize_t part; | |
| 111 | + | |
| 112 | + wreq->io_iter.iov_offset = cache->submit_off; | |
| 113 | + | |
| 114 | + atomic64_set(&wreq->issued_to, fpos + cache->submit_off); | |
| 115 | + part = netfs_advance_write(wreq, cache, fpos + cache->submit_off, | |
| 116 | + cache->submit_len, to_eof); | |
| 117 | + cache->submit_off += part; | |
| 118 | + cache->submit_max_len -= part; | |
| 119 | + if (part > cache->submit_len) | |
| 120 | + cache->submit_len = 0; | |
| 121 | + else | |
| 122 | + cache->submit_len -= part; | |
| 123 | + } while (cache->submit_len > 0); | |
| 124 | + | |
| 125 | + wreq->io_iter.iov_offset = 0; | |
| 126 | + iov_iter_advance(&wreq->io_iter, fsize); | |
| 127 | + atomic64_set(&wreq->issued_to, fpos + fsize); | |
| 128 | + | |
| 129 | + if (flen < fsize) | |
| 130 | + netfs_issue_write(wreq, cache); | |
| 131 | + | |
| 132 | + _leave(" = 0"); | |
| 133 | + return 0; | |
| 134 | +} | |
| 135 | + | |
| 136 | +/* | |
| 137 | + * [DEPRECATED] Go through the buffer and write any folios that are marked with | |
| 138 | + * the third mark to the cache. | |
| 139 | + */ | |
| 140 | +void netfs_pgpriv2_write_to_the_cache(struct netfs_io_request *rreq) | |
| 141 | +{ | |
| 142 | + struct netfs_io_request *wreq; | |
| 143 | + struct folio_queue *folioq; | |
| 144 | + struct folio *folio; | |
| 145 | + int error = 0; | |
| 146 | + int slot = 0; | |
| 147 | + | |
| 148 | + _enter(""); | |
| 149 | + | |
| 150 | + if (!fscache_resources_valid(&rreq->cache_resources)) | |
| 151 | + goto couldnt_start; | |
| 152 | + | |
| 153 | + /* Need the first folio to be able to set up the op. */ | |
| 154 | + for (folioq = rreq->buffer; folioq; folioq = folioq->next) { | |
| 155 | + if (folioq->marks3) { | |
| 156 | + slot = __ffs(folioq->marks3); | |
| 157 | + break; | |
| 158 | + } | |
| 159 | + } | |
| 160 | + if (!folioq) | |
| 161 | + return; | |
| 162 | + folio = folioq_folio(folioq, slot); | |
| 163 | + | |
| 164 | + wreq = netfs_create_write_req(rreq->mapping, NULL, folio_pos(folio), | |
| 165 | + NETFS_PGPRIV2_COPY_TO_CACHE); | |
| 166 | + if (IS_ERR(wreq)) { | |
| 167 | + kleave(" [create %ld]", PTR_ERR(wreq)); | |
| 168 | + goto couldnt_start; | |
| 169 | + } | |
| 170 | + | |
| 171 | + trace_netfs_write(wreq, netfs_write_trace_copy_to_cache); | |
| 172 | + netfs_stat(&netfs_n_wh_copy_to_cache); | |
| 173 | + | |
| 174 | + for (;;) { | |
| 175 | + error = netfs_pgpriv2_copy_folio(wreq, folio); | |
| 176 | + if (error < 0) | |
| 177 | + break; | |
| 178 | + | |
| 179 | + folioq_unmark3(folioq, slot); | |
| 180 | + if (!folioq->marks3) { | |
| 181 | + folioq = folioq->next; | |
| 182 | + if (!folioq) | |
| 183 | + break; | |
| 184 | + } | |
| 185 | + | |
| 186 | + slot = __ffs(folioq->marks3); | |
| 187 | + folio = folioq_folio(folioq, slot); | |
| 188 | + } | |
| 189 | + | |
| 190 | + netfs_issue_write(wreq, &wreq->io_streams[1]); | |
| 191 | + smp_wmb(); /* Write lists before ALL_QUEUED. */ | |
| 192 | + set_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags); | |
| 193 | + | |
| 194 | + netfs_put_request(wreq, false, netfs_rreq_trace_put_return); | |
| 195 | + _leave(" = %d", error); | |
| 196 | +couldnt_start: | |
| 197 | + netfs_pgpriv2_cancel(rreq->buffer); | |
| 198 | +} | |
| 199 | + | |
| 200 | +/* | |
| 201 | + * [DEPRECATED] Remove the PG_private_2 mark from any folios we've finished | |
| 202 | + * copying. | |
| 203 | + */ | |
| 204 | +bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *wreq) | |
| 205 | +{ | |
| 206 | + struct folio_queue *folioq = wreq->buffer; | |
| 207 | + unsigned long long collected_to = wreq->collected_to; | |
| 208 | + unsigned int slot = wreq->buffer_head_slot; | |
| 209 | + bool made_progress = false; | |
| 210 | + | |
| 211 | + if (slot >= folioq_nr_slots(folioq)) { | |
| 212 | + folioq = netfs_delete_buffer_head(wreq); | |
| 213 | + slot = 0; | |
| 214 | + } | |
| 215 | + | |
| 216 | + for (;;) { | |
| 217 | + struct folio *folio; | |
| 218 | + unsigned long long fpos, fend; | |
| 219 | + size_t fsize, flen; | |
| 220 | + | |
| 221 | + folio = folioq_folio(folioq, slot); | |
| 222 | + if (WARN_ONCE(!folio_test_private_2(folio), | |
| 223 | + "R=%08x: folio %lx is not marked private_2\n", | |
| 224 | + wreq->debug_id, folio->index)) | |
| 225 | + trace_netfs_folio(folio, netfs_folio_trace_not_under_wback); | |
| 226 | + | |
| 227 | + fpos = folio_pos(folio); | |
| 228 | + fsize = folio_size(folio); | |
| 229 | + flen = fsize; | |
| 230 | + | |
| 231 | + fend = min_t(unsigned long long, fpos + flen, wreq->i_size); | |
| 232 | + | |
| 233 | + trace_netfs_collect_folio(wreq, folio, fend, collected_to); | |
| 234 | + | |
| 235 | + /* Unlock any folio we've transferred all of. */ | |
| 236 | + if (collected_to < fend) | |
| 237 | + break; | |
| 238 | + | |
| 239 | + trace_netfs_folio(folio, netfs_folio_trace_end_copy); | |
| 240 | + folio_end_private_2(folio); | |
| 241 | + wreq->cleaned_to = fpos + fsize; | |
| 242 | + made_progress = true; | |
| 243 | + | |
| 244 | + /* Clean up the head folioq. If we clear an entire folioq, then | |
| 245 | + * we can get rid of it provided it's not also the tail folioq | |
| 246 | + * being filled by the issuer. | |
| 247 | + */ | |
| 248 | + folioq_clear(folioq, slot); | |
| 249 | + slot++; | |
| 250 | + if (slot >= folioq_nr_slots(folioq)) { | |
| 251 | + if (READ_ONCE(wreq->buffer_tail) == folioq) | |
| 252 | + break; | |
| 253 | + folioq = netfs_delete_buffer_head(wreq); | |
| 254 | + slot = 0; | |
| 255 | + } | |
| 256 | + | |
| 257 | + if (fpos + fsize >= collected_to) | |
| 258 | + break; | |
| 259 | + } | |
| 260 | + | |
| 261 | + wreq->buffer = folioq; | |
| 262 | + wreq->buffer_head_slot = slot; | |
| 263 | + return made_progress; | |
| 264 | +} | |
| @@ -0,0 +1,256 @@ | ||
| 1 | +// SPDX-License-Identifier: GPL-2.0-only | |
| 2 | +/* Network filesystem read subrequest retrying. | |
| 3 | + * | |
| 4 | + * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. | |
| 5 | + * Written by David Howells (dhowells@redhat.com) | |
| 6 | + */ | |
| 7 | + | |
| 8 | +#include <linux/fs.h> | |
| 9 | +#include <linux/slab.h> | |
| 10 | +#include "internal.h" | |
| 11 | + | |
| 12 | +static void netfs_reissue_read(struct netfs_io_request *rreq, | |
| 13 | + struct netfs_io_subrequest *subreq) | |
| 14 | +{ | |
| 15 | + struct iov_iter *io_iter = &subreq->io_iter; | |
| 16 | + | |
| 17 | + if (iov_iter_is_folioq(io_iter)) { | |
| 18 | + subreq->curr_folioq = (struct folio_queue *)io_iter->folioq; | |
| 19 | + subreq->curr_folioq_slot = io_iter->folioq_slot; | |
| 20 | + subreq->curr_folio_order = subreq->curr_folioq->orders[subreq->curr_folioq_slot]; | |
| 21 | + } | |
| 22 | + | |
| 23 | + atomic_inc(&rreq->nr_outstanding); | |
| 24 | + __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); | |
| 25 | + netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); | |
| 26 | + subreq->rreq->netfs_ops->issue_read(subreq); | |
| 27 | +} | |
| 28 | + | |
| 29 | +/* | |
| 30 | + * Go through the list of failed/short reads, retrying all retryable ones. We | |
| 31 | + * need to switch failed cache reads to network downloads. | |
| 32 | + */ | |
| 33 | +static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) | |
| 34 | +{ | |
| 35 | + struct netfs_io_subrequest *subreq; | |
| 36 | + struct netfs_io_stream *stream0 = &rreq->io_streams[0]; | |
| 37 | + LIST_HEAD(sublist); | |
| 38 | + LIST_HEAD(queue); | |
| 39 | + | |
| 40 | + _enter("R=%x", rreq->debug_id); | |
| 41 | + | |
| 42 | + if (list_empty(&rreq->subrequests)) | |
| 43 | + return; | |
| 44 | + | |
| 45 | + if (rreq->netfs_ops->retry_request) | |
| 46 | + rreq->netfs_ops->retry_request(rreq, NULL); | |
| 47 | + | |
| 48 | + /* If there's no renegotiation to do, just resend each retryable subreq | |
| 49 | + * up to the first permanently failed one. | |
| 50 | + */ | |
| 51 | + if (!rreq->netfs_ops->prepare_read && | |
| 52 | + !test_bit(NETFS_RREQ_COPY_TO_CACHE, &rreq->flags)) { | |
| 53 | + struct netfs_io_subrequest *subreq; | |
| 54 | + | |
| 55 | + list_for_each_entry(subreq, &rreq->subrequests, rreq_link) { | |
| 56 | + if (test_bit(NETFS_SREQ_FAILED, &subreq->flags)) | |
| 57 | + break; | |
| 58 | + if (__test_and_clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { | |
| 59 | + netfs_reset_iter(subreq); | |
| 60 | + netfs_reissue_read(rreq, subreq); | |
| 61 | + } | |
| 62 | + } | |
| 63 | + return; | |
| 64 | + } | |
| 65 | + | |
| 66 | + /* Okay, we need to renegotiate all the download requests and flip any | |
| 67 | + * failed cache reads over to being download requests and negotiate | |
| 68 | + * those also. All fully successful subreqs have been removed from the | |
| 69 | + * list and any spare data from those has been donated. | |
| 70 | + * | |
| 71 | + * What we do is decant the list and rebuild it one subreq at a time so | |
| 72 | + * that we don't end up with donations jumping over a gap we're busy | |
| 73 | + * populating with smaller subrequests. In the event that the subreq | |
| 74 | + * we just launched finishes before we insert the next subreq, it'll | |
| 75 | + * fill in rreq->prev_donated instead. | |
| 76 | + | |
| 77 | + * Note: Alternatively, we could split the tail subrequest right before | |
| 78 | + * we reissue it and fix up the donations under lock. | |
| 79 | + */ | |
| 80 | + list_splice_init(&rreq->subrequests, &queue); | |
| 81 | + | |
| 82 | + do { | |
| 83 | + struct netfs_io_subrequest *from; | |
| 84 | + struct iov_iter source; | |
| 85 | + unsigned long long start, len; | |
| 86 | + size_t part, deferred_next_donated = 0; | |
| 87 | + bool boundary = false; | |
| 88 | + | |
| 89 | + /* Go through the subreqs and find the next span of contiguous | |
| 90 | + * buffer that we then rejig (cifs, for example, needs the | |
| 91 | + * rsize renegotiating) and reissue. | |
| 92 | + */ | |
| 93 | + from = list_first_entry(&queue, struct netfs_io_subrequest, rreq_link); | |
| 94 | + list_move_tail(&from->rreq_link, &sublist); | |
| 95 | + start = from->start + from->transferred; | |
| 96 | + len = from->len - from->transferred; | |
| 97 | + | |
| 98 | + _debug("from R=%08x[%x] s=%llx ctl=%zx/%zx/%zx", | |
| 99 | + rreq->debug_id, from->debug_index, | |
| 100 | + from->start, from->consumed, from->transferred, from->len); | |
| 101 | + | |
| 102 | + if (test_bit(NETFS_SREQ_FAILED, &from->flags) || | |
| 103 | + !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags)) | |
| 104 | + goto abandon; | |
| 105 | + | |
| 106 | + deferred_next_donated = from->next_donated; | |
| 107 | + while ((subreq = list_first_entry_or_null( | |
| 108 | + &queue, struct netfs_io_subrequest, rreq_link))) { | |
| 109 | + if (subreq->start != start + len || | |
| 110 | + subreq->transferred > 0 || | |
| 111 | + !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) | |
| 112 | + break; | |
| 113 | + list_move_tail(&subreq->rreq_link, &sublist); | |
| 114 | + len += subreq->len; | |
| 115 | + deferred_next_donated = subreq->next_donated; | |
| 116 | + if (test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags)) | |
| 117 | + break; | |
| 118 | + } | |
| 119 | + | |
| 120 | + _debug(" - range: %llx-%llx %llx", start, start + len - 1, len); | |
| 121 | + | |
| 122 | + /* Determine the set of buffers we're going to use. Each | |
| 123 | + * subreq gets a subset of a single overall contiguous buffer. | |
| 124 | + */ | |
| 125 | + netfs_reset_iter(from); | |
| 126 | + source = from->io_iter; | |
| 127 | + source.count = len; | |
| 128 | + | |
| 129 | + /* Work through the sublist. */ | |
| 130 | + while ((subreq = list_first_entry_or_null( | |
| 131 | + &sublist, struct netfs_io_subrequest, rreq_link))) { | |
| 132 | + list_del(&subreq->rreq_link); | |
| 133 | + | |
| 134 | + subreq->source = NETFS_DOWNLOAD_FROM_SERVER; | |
| 135 | + subreq->start = start - subreq->transferred; | |
| 136 | + subreq->len = len + subreq->transferred; | |
| 137 | + stream0->sreq_max_len = subreq->len; | |
| 138 | + | |
| 139 | + __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); | |
| 140 | + __set_bit(NETFS_SREQ_RETRYING, &subreq->flags); | |
| 141 | + | |
| 142 | + spin_lock_bh(&rreq->lock); | |
| 143 | + list_add_tail(&subreq->rreq_link, &rreq->subrequests); | |
| 144 | + subreq->prev_donated += rreq->prev_donated; | |
| 145 | + rreq->prev_donated = 0; | |
| 146 | + trace_netfs_sreq(subreq, netfs_sreq_trace_retry); | |
| 147 | + spin_unlock_bh(&rreq->lock); | |
| 148 | + | |
| 149 | + BUG_ON(!len); | |
| 150 | + | |
| 151 | + /* Renegotiate max_len (rsize) */ | |
| 152 | + if (rreq->netfs_ops->prepare_read(subreq) < 0) { | |
| 153 | + trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed); | |
| 154 | + __set_bit(NETFS_SREQ_FAILED, &subreq->flags); | |
| 155 | + } | |
| 156 | + | |
| 157 | + part = umin(len, stream0->sreq_max_len); | |
| 158 | + if (unlikely(rreq->io_streams[0].sreq_max_segs)) | |
| 159 | + part = netfs_limit_iter(&source, 0, part, stream0->sreq_max_segs); | |
| 160 | + subreq->len = subreq->transferred + part; | |
| 161 | + subreq->io_iter = source; | |
| 162 | + iov_iter_truncate(&subreq->io_iter, part); | |
| 163 | + iov_iter_advance(&source, part); | |
| 164 | + len -= part; | |
| 165 | + start += part; | |
| 166 | + if (!len) { | |
| 167 | + if (boundary) | |
| 168 | + __set_bit(NETFS_SREQ_BOUNDARY, &subreq->flags); | |
| 169 | + subreq->next_donated = deferred_next_donated; | |
| 170 | + } else { | |
| 171 | + __clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags); | |
| 172 | + subreq->next_donated = 0; | |
| 173 | + } | |
| 174 | + | |
| 175 | + netfs_reissue_read(rreq, subreq); | |
| 176 | + if (!len) | |
| 177 | + break; | |
| 178 | + | |
| 179 | + /* If we ran out of subrequests, allocate another. */ | |
| 180 | + if (list_empty(&sublist)) { | |
| 181 | + subreq = netfs_alloc_subrequest(rreq); | |
| 182 | + if (!subreq) | |
| 183 | + goto abandon; | |
| 184 | + subreq->source = NETFS_DOWNLOAD_FROM_SERVER; | |
| 185 | + subreq->start = start; | |
| 186 | + | |
| 187 | + /* We get two refs, but need just one. */ | |
| 188 | + netfs_put_subrequest(subreq, false, netfs_sreq_trace_new); | |
| 189 | + trace_netfs_sreq(subreq, netfs_sreq_trace_split); | |
| 190 | + list_add_tail(&subreq->rreq_link, &sublist); | |
| 191 | + } | |
| 192 | + } | |
| 193 | + | |
| 194 | + /* If we managed to use fewer subreqs, we can discard the | |
| 195 | + * excess. | |
| 196 | + */ | |
| 197 | + while ((subreq = list_first_entry_or_null( | |
| 198 | + &sublist, struct netfs_io_subrequest, rreq_link))) { | |
| 199 | + trace_netfs_sreq(subreq, netfs_sreq_trace_discard); | |
| 200 | + list_del(&subreq->rreq_link); | |
| 201 | + netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_done); | |
| 202 | + } | |
| 203 | + | |
| 204 | + } while (!list_empty(&queue)); | |
| 205 | + | |
| 206 | + return; | |
| 207 | + | |
| 208 | + /* If we hit ENOMEM, fail all remaining subrequests */ | |
| 209 | +abandon: | |
| 210 | + list_splice_init(&sublist, &queue); | |
| 211 | + list_for_each_entry(subreq, &queue, rreq_link) { | |
| 212 | + if (!subreq->error) | |
| 213 | + subreq->error = -ENOMEM; | |
| 214 | + __clear_bit(NETFS_SREQ_FAILED, &subreq->flags); | |
| 215 | + __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); | |
| 216 | + __clear_bit(NETFS_SREQ_RETRYING, &subreq->flags); | |
| 217 | + } | |
| 218 | + spin_lock_bh(&rreq->lock); | |
| 219 | + list_splice_tail_init(&queue, &rreq->subrequests); | |
| 220 | + spin_unlock_bh(&rreq->lock); | |
| 221 | +} | |
| 222 | + | |
| 223 | +/* | |
| 224 | + * Retry reads. | |
| 225 | + */ | |
| 226 | +void netfs_retry_reads(struct netfs_io_request *rreq) | |
| 227 | +{ | |
| 228 | + trace_netfs_rreq(rreq, netfs_rreq_trace_resubmit); | |
| 229 | + | |
| 230 | + atomic_inc(&rreq->nr_outstanding); | |
| 231 | + | |
| 232 | + netfs_retry_read_subrequests(rreq); | |
| 233 | + | |
| 234 | + if (atomic_dec_and_test(&rreq->nr_outstanding)) | |
| 235 | + netfs_rreq_terminated(rreq, false); | |
| 236 | +} | |
| 237 | + | |
| 238 | +/* | |
| 239 | + * Unlock any the pages that haven't been unlocked yet due to abandoned | |
| 240 | + * subrequests. | |
| 241 | + */ | |
| 242 | +void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq) | |
| 243 | +{ | |
| 244 | + struct folio_queue *p; | |
| 245 | + | |
| 246 | + for (p = rreq->buffer; p; p = p->next) { | |
| 247 | + for (int slot = 0; slot < folioq_count(p); slot++) { | |
| 248 | + struct folio *folio = folioq_folio(p, slot); | |
| 249 | + | |
| 250 | + if (folio && !folioq_is_marked2(p, slot)) { | |
| 251 | + trace_netfs_folio(folio, netfs_folio_trace_abandon); | |
| 252 | + folio_unlock(folio); | |
| 253 | + } | |
| 254 | + } | |
| 255 | + } | |
| 256 | +} | |
| @@ -32,6 +32,7 @@ atomic_t netfs_n_wh_buffered_write; · 31 unmodified lines | ||
| 32 | 32 | atomic_t netfs_n_wh_writethrough; |
| 33 | 33 | atomic_t netfs_n_wh_dio_write; |
| 34 | 34 | atomic_t netfs_n_wh_writepages; |
| 35 | +atomic_t netfs_n_wh_copy_to_cache; | |
| 35 | 36 | atomic_t netfs_n_wh_wstream_conflict; |
| 36 | 37 | atomic_t netfs_n_wh_upload; |
| 37 | 38 | atomic_t netfs_n_wh_upload_done; |
| @@ -51,11 +52,12 @@ int netfs_stats_show(struct seq_file *m, void *v) · 13 unmodified lines | ||
| 51 | 52 | atomic_read(&netfs_n_rh_read_folio), |
| 52 | 53 | atomic_read(&netfs_n_rh_write_begin), |
| 53 | 54 | atomic_read(&netfs_n_rh_write_zskip)); |
| 54 | − seq_printf(m, "Writes : BW=%u WT=%u DW=%u WP=%u\n", | |
| 55 | + seq_printf(m, "Writes : BW=%u WT=%u DW=%u WP=%u 2C=%u\n", | |
| 55 | 56 | atomic_read(&netfs_n_wh_buffered_write), |
| 56 | 57 | atomic_read(&netfs_n_wh_writethrough), |
| 57 | 58 | atomic_read(&netfs_n_wh_dio_write), |
| 58 | − atomic_read(&netfs_n_wh_writepages)); | |
| 59 | + atomic_read(&netfs_n_wh_writepages), | |
| 60 | + atomic_read(&netfs_n_wh_copy_to_cache)); | |
| 59 | 61 | seq_printf(m, "ZeroOps: ZR=%u sh=%u sk=%u\n", |
| 60 | 62 | atomic_read(&netfs_n_rh_zero), |
| 61 | 63 | atomic_read(&netfs_n_rh_short_read), |
| @@ -87,6 +87,12 @@ static void netfs_writeback_unlock_folios(struct netfs_io_request *wreq, · 86 unmodified lines | ||
| 87 | 87 | unsigned long long collected_to = wreq->collected_to; |
| 88 | 88 | unsigned int slot = wreq->buffer_head_slot; |
| 89 | 89 | |
| 90 | + if (wreq->origin == NETFS_PGPRIV2_COPY_TO_CACHE) { | |
| 91 | + if (netfs_pgpriv2_unlock_copied_folios(wreq)) | |
| 92 | + *notes |= MADE_PROGRESS; | |
| 93 | + return; | |
| 94 | + } | |
| 95 | + | |
| 90 | 96 | if (slot >= folioq_nr_slots(folioq)) { |
| 91 | 97 | folioq = netfs_delete_buffer_head(wreq); |
| 92 | 98 | slot = 0; |
| @@ -383,7 +389,8 @@ reassess_streams: · 290 unmodified lines | ||
| 383 | 389 | smp_rmb(); |
| 384 | 390 | collected_to = ULLONG_MAX; |
| 385 | 391 | if (wreq->origin == NETFS_WRITEBACK || |
| 386 | − wreq->origin == NETFS_WRITETHROUGH) | |
| 392 | + wreq->origin == NETFS_WRITETHROUGH || | |
| 393 | + wreq->origin == NETFS_PGPRIV2_COPY_TO_CACHE) | |
| 387 | 394 | notes = BUFFERED; |
| 388 | 395 | else |
| 389 | 396 | notes = 0; |
| @@ -95,7 +95,8 @@ struct netfs_io_request *netfs_create_write_req(struct address_space *mapping, · 94 unmodified lines | ||
| 95 | 95 | struct netfs_io_request *wreq; |
| 96 | 96 | struct netfs_inode *ictx; |
| 97 | 97 | bool is_buffered = (origin == NETFS_WRITEBACK || |
| 98 | − origin == NETFS_WRITETHROUGH); | |
| 98 | + origin == NETFS_WRITETHROUGH || | |
| 99 | + origin == NETFS_PGPRIV2_COPY_TO_CACHE); | |
| 99 | 100 | |
| 100 | 101 | wreq = netfs_alloc_request(mapping, file, start, 0, origin); |
| 101 | 102 | if (IS_ERR(wreq)) |
| @@ -161,10 +162,6 @@ static void netfs_prepare_write(struct netfs_io_request *wreq, · 59 unmodified lines | ||
| 161 | 162 | |
| 162 | 163 | _enter("R=%x[%x]", wreq->debug_id, subreq->debug_index); |
| 163 | 164 | |
| 164 | − trace_netfs_sreq_ref(wreq->debug_id, subreq->debug_index, | |
| 165 | − refcount_read(&subreq->ref), | |
| 166 | − netfs_sreq_trace_new); | |
| 167 | − | |
| 168 | 165 | trace_netfs_sreq(subreq, netfs_sreq_trace_prepare); |
| 169 | 166 | |
| 170 | 167 | stream->sreq_max_len = UINT_MAX; |
| @@ -241,8 +238,8 @@ void netfs_reissue_write(struct netfs_io_stream *stream, · 70 unmodified lines | ||
| 241 | 238 | netfs_do_issue_write(stream, subreq); |
| 242 | 239 | } |
| 243 | 240 | |
| 244 | −static void netfs_issue_write(struct netfs_io_request *wreq, | |
| 245 | − struct netfs_io_stream *stream) | |
| 241 | +void netfs_issue_write(struct netfs_io_request *wreq, | |
| 242 | + struct netfs_io_stream *stream) | |
| 246 | 243 | { |
| 247 | 244 | struct netfs_io_subrequest *subreq = stream->construct; |
| 248 | 245 | |
| @@ -259,9 +256,9 @@ static void netfs_issue_write(struct netfs_io_request *wreq, · 10 unmodified lines | ||
| 259 | 256 | * we can avoid overrunning the credits obtained (cifs) and try to parallelise |
| 260 | 257 | * content-crypto preparation with network writes. |
| 261 | 258 | */ |
| 262 | −static int netfs_advance_write(struct netfs_io_request *wreq, | |
| 263 | − struct netfs_io_stream *stream, | |
| 264 | − loff_t start, size_t len, bool to_eof) | |
| 259 | +int netfs_advance_write(struct netfs_io_request *wreq, | |
| 260 | + struct netfs_io_stream *stream, | |
| 261 | + loff_t start, size_t len, bool to_eof) | |
| 265 | 262 | { |
| 266 | 263 | struct netfs_io_subrequest *subreq = stream->construct; |
| 267 | 264 | size_t part; |
| @@ -267,6 +267,7 @@ static int nfs_netfs_init_request(struct netfs_io_request *rreq, struct file *fi · 266 unmodified lines | ||
| 267 | 267 | rreq->debug_id = atomic_inc_return(&nfs_netfs_debug_id); |
| 268 | 268 | /* [DEPRECATED] Use PG_private_2 to mark folio being written to the cache. */ |
| 269 | 269 | __set_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags); |
| 270 | + rreq->io_streams[0].sreq_max_len = NFS_SB(rreq->inode->i_sb)->rsize; | |
| 270 | 271 | |
| 271 | 272 | return 0; |
| 272 | 273 | } |
| @@ -288,14 +289,6 @@ static struct nfs_netfs_io_data *nfs_netfs_alloc(struct netfs_io_subrequest *sre · 15 unmodified lines | ||
| 288 | 289 | return netfs; |
| 289 | 290 | } |
| 290 | 291 | |
| 291 | −static bool nfs_netfs_clamp_length(struct netfs_io_subrequest *sreq) | |
| 292 | −{ | |
| 293 | − size_t rsize = NFS_SB(sreq->rreq->inode->i_sb)->rsize; | |
| 294 | − | |
| 295 | − sreq->len = min(sreq->len, rsize); | |
| 296 | − return true; | |
| 297 | −} | |
| 298 | − | |
| 299 | 292 | static void nfs_netfs_issue_read(struct netfs_io_subrequest *sreq) |
| 300 | 293 | { |
| 301 | 294 | struct nfs_netfs_io_data *netfs; |
| @@ -304,17 +297,18 @@ static void nfs_netfs_issue_read(struct netfs_io_subrequest *sreq) · 2 unmodified lines | ||
| 304 | 297 | struct nfs_open_context *ctx = sreq->rreq->netfs_priv; |
| 305 | 298 | struct page *page; |
| 306 | 299 | unsigned long idx; |
| 300 | + pgoff_t start, last; | |
| 307 | 301 | int err; |
| 308 | − pgoff_t start = (sreq->start + sreq->transferred) >> PAGE_SHIFT; | |
| 309 | − pgoff_t last = ((sreq->start + sreq->len - | |
| 310 | − sreq->transferred - 1) >> PAGE_SHIFT); | |
| 302 | + | |
| 303 | + start = (sreq->start + sreq->transferred) >> PAGE_SHIFT; | |
| 304 | + last = ((sreq->start + sreq->len - sreq->transferred - 1) >> PAGE_SHIFT); | |
| 311 | 305 | |
| 312 | 306 | nfs_pageio_init_read(&pgio, inode, false, |
| 313 | 307 | &nfs_async_read_completion_ops); |
| 314 | 308 | |
| 315 | 309 | netfs = nfs_netfs_alloc(sreq); |
| 316 | 310 | if (!netfs) |
| 317 | − return netfs_subreq_terminated(sreq, -ENOMEM, false); | |
| 311 | + return netfs_read_subreq_terminated(sreq, -ENOMEM, false); | |
| 318 | 312 | |
| 319 | 313 | pgio.pg_netfs = netfs; /* used in completion */ |
| 320 | 314 | |
| @@ -380,5 +374,4 @@ const struct netfs_request_ops nfs_netfs_ops = { · 59 unmodified lines | ||
| 380 | 374 | .init_request = nfs_netfs_init_request, |
| 381 | 375 | .free_request = nfs_netfs_free_request, |
| 382 | 376 | .issue_read = nfs_netfs_issue_read, |
| 383 | − .clamp_length = nfs_netfs_clamp_length | |
| 384 | 377 | }; |
| @@ -60,8 +60,6 @@ static inline void nfs_netfs_get(struct nfs_netfs_io_data *netfs) · 59 unmodified lines | ||
| 60 | 60 | |
| 61 | 61 | static inline void nfs_netfs_put(struct nfs_netfs_io_data *netfs) |
| 62 | 62 | { |
| 63 | − ssize_t final_len; | |
| 64 | − | |
| 65 | 63 | /* Only the last RPC completion should call netfs_subreq_terminated() */ |
| 66 | 64 | if (!refcount_dec_and_test(&netfs->refcount)) |
| 67 | 65 | return; |
| @@ -74,8 +72,9 @@ static inline void nfs_netfs_put(struct nfs_netfs_io_data *netfs) · 6 unmodified lines | ||
| 74 | 72 | * Correct the final length here to be no larger than the netfs subrequest |
| 75 | 73 | * length, and thus avoid netfs's "Subreq overread" warning message. |
| 76 | 74 | */ |
| 77 | − final_len = min_t(s64, netfs->sreq->len, atomic64_read(&netfs->transferred)); | |
| 78 | − netfs_subreq_terminated(netfs->sreq, netfs->error ?: final_len, false); | |
| 75 | + netfs->sreq->transferred = min_t(s64, netfs->sreq->len, | |
| 76 | + atomic64_read(&netfs->transferred)); | |
| 77 | + netfs_read_subreq_terminated(netfs->sreq, netfs->error, false); | |
| 79 | 78 | kfree(netfs); |
| 80 | 79 | } |
| 81 | 80 | static inline void nfs_netfs_inode_init(struct nfs_inode *nfsi) |
| @@ -1485,7 +1485,6 @@ struct cifs_io_subrequest { · 1484 unmodified lines | ||
| 1485 | 1485 | struct cifs_io_request *req; |
| 1486 | 1486 | }; |
| 1487 | 1487 | ssize_t got_bytes; |
| 1488 | − size_t actual_len; | |
| 1489 | 1488 | unsigned int xid; |
| 1490 | 1489 | int result; |
| 1491 | 1490 | bool have_xid; |
| @@ -1309,10 +1309,8 @@ cifs_readv_callback(struct mid_q_entry *mid) · 1308 unmodified lines | ||
| 1309 | 1309 | if (rdata->result == 0 || rdata->result == -EAGAIN) |
| 1310 | 1310 | iov_iter_advance(&rdata->subreq.io_iter, rdata->got_bytes); |
| 1311 | 1311 | rdata->credits.value = 0; |
| 1312 | − netfs_subreq_terminated(&rdata->subreq, | |
| 1313 | − (rdata->result == 0 || rdata->result == -EAGAIN) ? | |
| 1314 | − rdata->got_bytes : rdata->result, | |
| 1315 | − false); | |
| 1312 | + rdata->subreq.transferred += rdata->got_bytes; | |
| 1313 | + netfs_read_subreq_terminated(&rdata->subreq, rdata->result, false); | |
| 1316 | 1314 | release_mid(mid); |
| 1317 | 1315 | add_credits(server, &credits, 0); |
| 1318 | 1316 | } |
| @@ -112,7 +112,6 @@ static void cifs_issue_write(struct netfs_io_subrequest *subreq) · 111 unmodified lines | ||
| 112 | 112 | goto fail; |
| 113 | 113 | } |
| 114 | 114 | |
| 115 | − wdata->actual_len = wdata->subreq.len; | |
| 116 | 115 | rc = adjust_credits(wdata->server, wdata, cifs_trace_rw_credits_issue_write_adjust); |
| 117 | 116 | if (rc) |
| 118 | 117 | goto fail; |
| @@ -141,25 +140,22 @@ static void cifs_netfs_invalidate_cache(struct netfs_io_request *wreq) · 22 unmodified lines | ||
| 141 | 140 | } |
| 142 | 141 | |
| 143 | 142 | /* |
| 144 | − * Split the read up according to how many credits we can get for each piece. | |
| 145 | − * It's okay to sleep here if we need to wait for more credit to become | |
| 146 | − * available. | |
| 147 | − * | |
| 148 | − * We also choose the server and allocate an operation ID to be cleaned up | |
| 149 | − * later. | |
| 143 | + * Negotiate the size of a read operation on behalf of the netfs library. | |
| 150 | 144 | */ |
| 151 | −static bool cifs_clamp_length(struct netfs_io_subrequest *subreq) | |
| 145 | +static int cifs_prepare_read(struct netfs_io_subrequest *subreq) | |
| 152 | 146 | { |
| 153 | 147 | struct netfs_io_request *rreq = subreq->rreq; |
| 154 | − struct netfs_io_stream *stream = &rreq->io_streams[subreq->stream_nr]; | |
| 155 | 148 | struct cifs_io_subrequest *rdata = container_of(subreq, struct cifs_io_subrequest, subreq); |
| 156 | 149 | struct cifs_io_request *req = container_of(subreq->rreq, struct cifs_io_request, rreq); |
| 157 | 150 | struct TCP_Server_Info *server = req->server; |
| 158 | 151 | struct cifs_sb_info *cifs_sb = CIFS_SB(rreq->inode->i_sb); |
| 159 | − int rc; | |
| 152 | + size_t size; | |
| 153 | + int rc = 0; | |
| 160 | 154 | |
| 161 | − rdata->xid = get_xid(); | |
| 162 | − rdata->have_xid = true; | |
| 155 | + if (!rdata->have_xid) { | |
| 156 | + rdata->xid = get_xid(); | |
| 157 | + rdata->have_xid = true; | |
| 158 | + } | |
| 163 | 159 | rdata->server = server; |
| 164 | 160 | |
| 165 | 161 | if (cifs_sb->ctx->rsize == 0) |
| @@ -167,13 +163,12 @@ static bool cifs_clamp_length(struct netfs_io_subrequest *subreq) · 1 unmodified line | ||
| 167 | 163 | server->ops->negotiate_rsize(tlink_tcon(req->cfile->tlink), |
| 168 | 164 | cifs_sb->ctx); |
| 169 | 165 | |
| 170 | − | |
| 171 | 166 | rc = server->ops->wait_mtu_credits(server, cifs_sb->ctx->rsize, |
| 172 | − &stream->sreq_max_len, &rdata->credits); | |
| 173 | − if (rc) { | |
| 174 | − subreq->error = rc; | |
| 175 | − return false; | |
| 176 | − } | |
| 167 | + &size, &rdata->credits); | |
| 168 | + if (rc) | |
| 169 | + return rc; | |
| 170 | + | |
| 171 | + rreq->io_streams[0].sreq_max_len = size; | |
| 177 | 172 | |
| 178 | 173 | rdata->credits.in_flight_check = 1; |
| 179 | 174 | rdata->credits.rreq_debug_id = rreq->debug_id; |
| @@ -185,14 +180,11 @@ static bool cifs_clamp_length(struct netfs_io_subrequest *subreq) · 5 unmodified lines | ||
| 185 | 180 | server->credits, server->in_flight, 0, |
| 186 | 181 | cifs_trace_rw_credits_read_submit); |
| 187 | 182 | |
| 188 | − subreq->len = umin(subreq->len, stream->sreq_max_len); | |
| 189 | − rdata->actual_len = subreq->len; | |
| 190 | − | |
| 191 | 183 | #ifdef CONFIG_CIFS_SMB_DIRECT |
| 192 | 184 | if (server->smbd_conn) |
| 193 | − stream->sreq_max_segs = server->smbd_conn->max_frmr_depth; | |
| 185 | + rreq->io_streams[0].sreq_max_segs = server->smbd_conn->max_frmr_depth; | |
| 194 | 186 | #endif |
| 195 | − return true; | |
| 187 | + return 0; | |
| 196 | 188 | } |
| 197 | 189 | |
| 198 | 190 | /* |
| @@ -201,59 +193,41 @@ static bool cifs_clamp_length(struct netfs_io_subrequest *subreq) · 2 unmodified lines | ||
| 201 | 193 | * to only read a portion of that, but as long as we read something, the netfs |
| 202 | 194 | * helper will call us again so that we can issue another read. |
| 203 | 195 | */ |
| 204 | −static void cifs_req_issue_read(struct netfs_io_subrequest *subreq) | |
| 196 | +static void cifs_issue_read(struct netfs_io_subrequest *subreq) | |
| 205 | 197 | { |
| 206 | 198 | struct netfs_io_request *rreq = subreq->rreq; |
| 207 | 199 | struct cifs_io_subrequest *rdata = container_of(subreq, struct cifs_io_subrequest, subreq); |
| 208 | 200 | struct cifs_io_request *req = container_of(subreq->rreq, struct cifs_io_request, rreq); |
| 209 | 201 | struct TCP_Server_Info *server = req->server; |
| 210 | − struct cifs_sb_info *cifs_sb = CIFS_SB(rreq->inode->i_sb); | |
| 211 | 202 | int rc = 0; |
| 212 | 203 | |
| 213 | 204 | cifs_dbg(FYI, "%s: op=%08x[%x] mapping=%p len=%zu/%zu\n", |
| 214 | 205 | __func__, rreq->debug_id, subreq->debug_index, rreq->mapping, |
| 215 | 206 | subreq->transferred, subreq->len); |
| 216 | 207 | |
| 217 | − if (test_bit(NETFS_SREQ_RETRYING, &subreq->flags)) { | |
| 218 | − /* | |
| 219 | − * As we're issuing a retry, we need to negotiate some new | |
| 220 | − * credits otherwise the server may reject the op with | |
| 221 | − * INVALID_PARAMETER. Note, however, we may get back less | |
| 222 | − * credit than we need to complete the op, in which case, we | |
| 223 | − * shorten the op and rely on additional rounds of retry. | |
| 224 | − */ | |
| 225 | − size_t rsize = umin(subreq->len - subreq->transferred, | |
| 226 | − cifs_sb->ctx->rsize); | |
| 227 | − | |
| 228 | − rc = server->ops->wait_mtu_credits(server, rsize, &rdata->actual_len, | |
| 229 | − &rdata->credits); | |
| 230 | − if (rc) | |
| 231 | − goto out; | |
| 232 | − | |
| 233 | − rdata->credits.in_flight_check = 1; | |
| 234 | − | |
| 235 | − trace_smb3_rw_credits(rdata->rreq->debug_id, | |
| 236 | − rdata->subreq.debug_index, | |
| 237 | − rdata->credits.value, | |
| 238 | − server->credits, server->in_flight, 0, | |
| 239 | − cifs_trace_rw_credits_read_resubmit); | |
| 240 | − } | |
| 208 | + rc = adjust_credits(server, rdata, cifs_trace_rw_credits_issue_read_adjust); | |
| 209 | + if (rc) | |
| 210 | + goto failed; | |
| 241 | 211 | |
| 242 | 212 | if (req->cfile->invalidHandle) { |
| 243 | 213 | do { |
| 244 | 214 | rc = cifs_reopen_file(req->cfile, true); |
| 245 | 215 | } while (rc == -EAGAIN); |
| 246 | 216 | if (rc) |
| 247 | − goto out; | |
| 217 | + goto failed; | |
| 248 | 218 | } |
| 249 | 219 | |
| 250 | 220 | if (subreq->rreq->origin != NETFS_DIO_READ) |
| 251 | 221 | __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags); |
| 252 | 222 | |
| 223 | + trace_netfs_sreq(subreq, netfs_sreq_trace_submit); | |
| 253 | 224 | rc = rdata->server->ops->async_readv(rdata); |
| 254 | −out: | |
| 255 | 225 | if (rc) |
| 256 | − netfs_subreq_terminated(subreq, rc, false); | |
| 226 | + goto failed; | |
| 227 | + return; | |
| 228 | + | |
| 229 | +failed: | |
| 230 | + netfs_read_subreq_terminated(subreq, rc, false); | |
| 257 | 231 | } |
| 258 | 232 | |
| 259 | 233 | /* |
| @@ -364,8 +338,8 @@ const struct netfs_request_ops cifs_req_ops = { · 104 unmodified lines | ||
| 364 | 338 | .init_request = cifs_init_request, |
| 365 | 339 | .free_request = cifs_free_request, |
| 366 | 340 | .free_subrequest = cifs_free_subrequest, |
| 367 | − .clamp_length = cifs_clamp_length, | |
| 368 | − .issue_read = cifs_req_issue_read, | |
| 341 | + .prepare_read = cifs_prepare_read, | |
| 342 | + .issue_read = cifs_issue_read, | |
| 369 | 343 | .done = cifs_rreq_done, |
| 370 | 344 | .begin_writeback = cifs_begin_writeback, |
| 371 | 345 | .prepare_write = cifs_prepare_write, |
| @@ -301,7 +301,8 @@ smb2_adjust_credits(struct TCP_Server_Info *server, · 300 unmodified lines | ||
| 301 | 301 | unsigned int /*enum smb3_rw_credits_trace*/ trace) |
| 302 | 302 | { |
| 303 | 303 | struct cifs_credits *credits = &subreq->credits; |
| 304 | − int new_val = DIV_ROUND_UP(subreq->actual_len, SMB2_MAX_BUFFER_SIZE); | |
| 304 | + int new_val = DIV_ROUND_UP(subreq->subreq.len - subreq->subreq.transferred, | |
| 305 | + SMB2_MAX_BUFFER_SIZE); | |
| 305 | 306 | int scredits, in_flight; |
| 306 | 307 | |
| 307 | 308 | if (!credits->value || credits->value == new_val) |
| @@ -4498,9 +4498,7 @@ static void smb2_readv_worker(struct work_struct *work) · 4497 unmodified lines | ||
| 4498 | 4498 | struct cifs_io_subrequest *rdata = |
| 4499 | 4499 | container_of(work, struct cifs_io_subrequest, subreq.work); |
| 4500 | 4500 | |
| 4501 | − netfs_subreq_terminated(&rdata->subreq, | |
| 4502 | − (rdata->result == 0 || rdata->result == -EAGAIN) ? | |
| 4503 | − rdata->got_bytes : rdata->result, true); | |
| 4501 | + netfs_read_subreq_terminated(&rdata->subreq, rdata->result, false); | |
| 4504 | 4502 | } |
| 4505 | 4503 | |
| 4506 | 4504 | static void |
| @@ -4532,7 +4530,7 @@ smb2_readv_callback(struct mid_q_entry *mid) · 25 unmodified lines | ||
| 4532 | 4530 | |
| 4533 | 4531 | cifs_dbg(FYI, "%s: mid=%llu state=%d result=%d bytes=%zu/%zu\n", |
| 4534 | 4532 | __func__, mid->mid, mid->mid_state, rdata->result, |
| 4535 | − rdata->actual_len, rdata->subreq.len - rdata->subreq.transferred); | |
| 4533 | + rdata->got_bytes, rdata->subreq.len - rdata->subreq.transferred); | |
| 4536 | 4534 | |
| 4537 | 4535 | switch (mid->mid_state) { |
| 4538 | 4536 | case MID_RESPONSE_RECEIVED: |
| @@ -4554,6 +4552,7 @@ smb2_readv_callback(struct mid_q_entry *mid) · 15 unmodified lines | ||
| 4554 | 4552 | break; |
| 4555 | 4553 | case MID_REQUEST_SUBMITTED: |
| 4556 | 4554 | case MID_RETRY_NEEDED: |
| 4555 | + __set_bit(NETFS_SREQ_NEED_RETRY, &rdata->subreq.flags); | |
| 4557 | 4556 | rdata->result = -EAGAIN; |
| 4558 | 4557 | if (server->sign && rdata->got_bytes) |
| 4559 | 4558 | /* reset bytes number since we can not check a sign */ |
| @@ -4588,7 +4587,7 @@ smb2_readv_callback(struct mid_q_entry *mid) · 28 unmodified lines | ||
| 4588 | 4587 | rdata->req->cfile->fid.persistent_fid, |
| 4589 | 4588 | tcon->tid, tcon->ses->Suid, |
| 4590 | 4589 | rdata->subreq.start + rdata->subreq.transferred, |
| 4591 | − rdata->actual_len, | |
| 4590 | + rdata->subreq.len - rdata->subreq.transferred, | |
| 4592 | 4591 | rdata->result); |
| 4593 | 4592 | } else |
| 4594 | 4593 | trace_smb3_read_done(rdata->rreq->debug_id, |
| @@ -4603,9 +4602,9 @@ smb2_readv_callback(struct mid_q_entry *mid) · 8 unmodified lines | ||
| 4603 | 4602 | __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags); |
| 4604 | 4603 | rdata->result = 0; |
| 4605 | 4604 | } else { |
| 4606 | − if (rdata->got_bytes < rdata->actual_len && | |
| 4607 | − rdata->subreq.start + rdata->subreq.transferred + rdata->got_bytes == | |
| 4608 | − ictx->remote_i_size) { | |
| 4605 | + size_t trans = rdata->subreq.transferred + rdata->got_bytes; | |
| 4606 | + if (trans < rdata->subreq.len && | |
| 4607 | + rdata->subreq.start + trans == ictx->remote_i_size) { | |
| 4609 | 4608 | __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags); |
| 4610 | 4609 | rdata->result = 0; |
| 4611 | 4610 | } |
| @@ -4614,6 +4613,10 @@ smb2_readv_callback(struct mid_q_entry *mid) · 2 unmodified lines | ||
| 4614 | 4613 | server->credits, server->in_flight, |
| 4615 | 4614 | 0, cifs_trace_rw_credits_read_response_clear); |
| 4616 | 4615 | rdata->credits.value = 0; |
| 4616 | + rdata->subreq.transferred += rdata->got_bytes; | |
| 4617 | + if (rdata->subreq.start + rdata->subreq.transferred >= rdata->subreq.rreq->i_size) | |
| 4618 | + __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags); | |
| 4619 | + trace_netfs_sreq(&rdata->subreq, netfs_sreq_trace_io_progress); | |
| 4617 | 4620 | INIT_WORK(&rdata->subreq.work, smb2_readv_worker); |
| 4618 | 4621 | queue_work(cifsiod_wq, &rdata->subreq.work); |
| 4619 | 4622 | release_mid(mid); |
| @@ -4648,7 +4651,7 @@ smb2_async_readv(struct cifs_io_subrequest *rdata) · 28 unmodified lines | ||
| 4648 | 4651 | io_parms.tcon = tlink_tcon(rdata->req->cfile->tlink); |
| 4649 | 4652 | io_parms.server = server = rdata->server; |
| 4650 | 4653 | io_parms.offset = subreq->start + subreq->transferred; |
| 4651 | − io_parms.length = rdata->actual_len; | |
| 4654 | + io_parms.length = subreq->len - subreq->transferred; | |
| 4652 | 4655 | io_parms.persistent_fid = rdata->req->cfile->fid.persistent_fid; |
| 4653 | 4656 | io_parms.volatile_fid = rdata->req->cfile->fid.volatile_fid; |
| 4654 | 4657 | io_parms.pid = rdata->req->pid; |
| @@ -4669,7 +4672,7 @@ smb2_async_readv(struct cifs_io_subrequest *rdata) · 14 unmodified lines | ||
| 4669 | 4672 | shdr = (struct smb2_hdr *)buf; |
| 4670 | 4673 | |
| 4671 | 4674 | if (rdata->credits.value > 0) { |
| 4672 | − shdr->CreditCharge = cpu_to_le16(DIV_ROUND_UP(rdata->actual_len, | |
| 4675 | + shdr->CreditCharge = cpu_to_le16(DIV_ROUND_UP(io_parms.length, | |
| 4673 | 4676 | SMB2_MAX_BUFFER_SIZE)); |
| 4674 | 4677 | credit_request = le16_to_cpu(shdr->CreditCharge) + 8; |
| 4675 | 4678 | if (server->credits >= server->max_credits) |
| @@ -4697,7 +4700,8 @@ smb2_async_readv(struct cifs_io_subrequest *rdata) · 21 unmodified lines | ||
| 4697 | 4700 | rdata->xid, io_parms.persistent_fid, |
| 4698 | 4701 | io_parms.tcon->tid, |
| 4699 | 4702 | io_parms.tcon->ses->Suid, |
| 4700 | − io_parms.offset, rdata->actual_len, rc); | |
| 4703 | + io_parms.offset, | |
| 4704 | + subreq->len - subreq->transferred, rc); | |
| 4701 | 4705 | } |
| 4702 | 4706 | |
| 4703 | 4707 | async_readv_out: |
| @@ -4880,6 +4884,7 @@ smb2_writev_callback(struct mid_q_entry *mid) · 176 unmodified lines | ||
| 4880 | 4884 | server->credits, server->in_flight, |
| 4881 | 4885 | 0, cifs_trace_rw_credits_write_response_clear); |
| 4882 | 4886 | wdata->credits.value = 0; |
| 4887 | + trace_netfs_sreq(&wdata->subreq, netfs_sreq_trace_io_progress); | |
| 4883 | 4888 | cifs_write_subrequest_terminated(wdata, result ?: written, true); |
| 4884 | 4889 | release_mid(mid); |
| 4885 | 4890 | trace_smb3_rw_credits(rreq_debug_id, subreq_debug_index, 0, |
| @@ -27,6 +27,7 @@ struct folio_queue { · 26 unmodified lines | ||
| 27 | 27 | struct folio_queue *prev; /* Previous queue segment of NULL */ |
| 28 | 28 | unsigned long marks; /* 1-bit mark per folio */ |
| 29 | 29 | unsigned long marks2; /* Second 1-bit mark per folio */ |
| 30 | + unsigned long marks3; /* Third 1-bit mark per folio */ | |
| 30 | 31 | #if PAGEVEC_SIZE > BITS_PER_LONG |
| 31 | 32 | #error marks is not big enough |
| 32 | 33 | #endif |
| @@ -39,6 +40,7 @@ static inline void folioq_init(struct folio_queue *folioq) · 6 unmodified lines | ||
| 39 | 40 | folioq->prev = NULL; |
| 40 | 41 | folioq->marks = 0; |
| 41 | 42 | folioq->marks2 = 0; |
| 43 | + folioq->marks3 = 0; | |
| 42 | 44 | } |
| 43 | 45 | |
| 44 | 46 | static inline unsigned int folioq_nr_slots(const struct folio_queue *folioq) |
| @@ -87,6 +89,21 @@ static inline void folioq_unmark2(struct folio_queue *folioq, unsigned int slot) · 42 unmodified lines | ||
| 87 | 89 | clear_bit(slot, &folioq->marks2); |
| 88 | 90 | } |
| 89 | 91 | |
| 92 | +static inline bool folioq_is_marked3(const struct folio_queue *folioq, unsigned int slot) | |
| 93 | +{ | |
| 94 | + return test_bit(slot, &folioq->marks3); | |
| 95 | +} | |
| 96 | + | |
| 97 | +static inline void folioq_mark3(struct folio_queue *folioq, unsigned int slot) | |
| 98 | +{ | |
| 99 | + set_bit(slot, &folioq->marks3); | |
| 100 | +} | |
| 101 | + | |
| 102 | +static inline void folioq_unmark3(struct folio_queue *folioq, unsigned int slot) | |
| 103 | +{ | |
| 104 | + clear_bit(slot, &folioq->marks3); | |
| 105 | +} | |
| 106 | + | |
| 90 | 107 | static inline unsigned int __folio_order(struct folio *folio) |
| 91 | 108 | { |
| 92 | 109 | if (!folio_test_large(folio)) |
| @@ -133,6 +150,7 @@ static inline void folioq_clear(struct folio_queue *folioq, unsigned int slot) · 40 unmodified lines | ||
| 133 | 150 | folioq->vec.folios[slot] = NULL; |
| 134 | 151 | folioq_unmark(folioq, slot); |
| 135 | 152 | folioq_unmark2(folioq, slot); |
| 153 | + folioq_unmark3(folioq, slot); | |
| 136 | 154 | } |
| 137 | 155 | |
| 138 | 156 | #endif /* _LINUX_FOLIO_QUEUE_H */ |
| @@ -178,36 +178,43 @@ struct netfs_io_subrequest { · 177 unmodified lines | ||
| 178 | 178 | unsigned long long start; /* Where to start the I/O */ |
| 179 | 179 | size_t len; /* Size of the I/O */ |
| 180 | 180 | size_t transferred; /* Amount of data transferred */ |
| 181 | + size_t consumed; /* Amount of read data consumed */ | |
| 182 | + size_t prev_donated; /* Amount of data donated from previous subreq */ | |
| 183 | + size_t next_donated; /* Amount of data donated from next subreq */ | |
| 181 | 184 | refcount_t ref; |
| 182 | 185 | short error; /* 0 or error that occurred */ |
| 183 | 186 | unsigned short debug_index; /* Index in list (for debugging output) */ |
| 184 | 187 | unsigned int nr_segs; /* Number of segs in io_iter */ |
| 185 | 188 | enum netfs_io_source source; /* Where to read from/write to */ |
| 186 | 189 | unsigned char stream_nr; /* I/O stream this belongs to */ |
| 190 | + unsigned char curr_folioq_slot; /* Folio currently being read */ | |
| 191 | + unsigned char curr_folio_order; /* Order of folio */ | |
| 192 | + struct folio_queue *curr_folioq; /* Queue segment in which current folio resides */ | |
| 187 | 193 | unsigned long flags; |
| 188 | 194 | #define NETFS_SREQ_COPY_TO_CACHE 0 /* Set if should copy the data to the cache */ |
| 189 | 195 | #define NETFS_SREQ_CLEAR_TAIL 1 /* Set if the rest of the read should be cleared */ |
| 190 | −#define NETFS_SREQ_SHORT_IO 2 /* Set if the I/O was short */ | |
| 191 | 196 | #define NETFS_SREQ_SEEK_DATA_READ 3 /* Set if ->read() should SEEK_DATA first */ |
| 192 | 197 | #define NETFS_SREQ_NO_PROGRESS 4 /* Set if we didn't manage to read any data */ |
| 193 | 198 | #define NETFS_SREQ_ONDEMAND 5 /* Set if it's from on-demand read mode */ |
| 194 | 199 | #define NETFS_SREQ_BOUNDARY 6 /* Set if ends on hard boundary (eg. ceph object) */ |
| 200 | +#define NETFS_SREQ_HIT_EOF 7 /* Set if short due to EOF */ | |
| 195 | 201 | #define NETFS_SREQ_IN_PROGRESS 8 /* Unlocked when the subrequest completes */ |
| 196 | 202 | #define NETFS_SREQ_NEED_RETRY 9 /* Set if the filesystem requests a retry */ |
| 197 | 203 | #define NETFS_SREQ_RETRYING 10 /* Set if we're retrying */ |
| 198 | 204 | #define NETFS_SREQ_FAILED 11 /* Set if the subreq failed unretryably */ |
| 199 | −#define NETFS_SREQ_HIT_EOF 12 /* Set if we hit the EOF */ | |
| 200 | 205 | }; |
| 201 | 206 | |
| 202 | 207 | enum netfs_io_origin { |
| 203 | 208 | NETFS_READAHEAD, /* This read was triggered by readahead */ |
| 204 | 209 | NETFS_READPAGE, /* This read is a synchronous read */ |
| 210 | + NETFS_READ_GAPS, /* This read is a synchronous read to fill gaps */ | |
| 205 | 211 | NETFS_READ_FOR_WRITE, /* This read is to prepare a write */ |
| 206 | 212 | NETFS_DIO_READ, /* This is a direct I/O read */ |
| 207 | 213 | NETFS_WRITEBACK, /* This write was triggered by writepages */ |
| 208 | 214 | NETFS_WRITETHROUGH, /* This write was made by netfs_perform_write() */ |
| 209 | 215 | NETFS_UNBUFFERED_WRITE, /* This is an unbuffered write */ |
| 210 | 216 | NETFS_DIO_WRITE, /* This is a direct I/O write */ |
| 217 | + NETFS_PGPRIV2_COPY_TO_CACHE, /* [DEPRECATED] This is writing read data to the cache */ | |
| 211 | 218 | nr__netfs_io_origin |
| 212 | 219 | } __mode(byte); |
| 213 | 220 | |
| @@ -224,6 +231,7 @@ struct netfs_io_request { · 10 unmodified lines | ||
| 224 | 231 | struct address_space *mapping; /* The mapping being accessed */ |
| 225 | 232 | struct kiocb *iocb; /* AIO completion vector */ |
| 226 | 233 | struct netfs_cache_resources cache_resources; |
| 234 | + struct readahead_control *ractl; /* Readahead descriptor */ | |
| 227 | 235 | struct list_head proc_link; /* Link in netfs_iorequests */ |
| 228 | 236 | struct list_head subrequests; /* Contributory I/O operations */ |
| 229 | 237 | struct netfs_io_stream io_streams[2]; /* Streams of parallel I/O operations */ |
| @@ -244,12 +252,10 @@ struct netfs_io_request { · 14 unmodified lines | ||
| 244 | 252 | unsigned int nr_group_rel; /* Number of refs to release on ->group */ |
| 245 | 253 | spinlock_t lock; /* Lock for queuing subreqs */ |
| 246 | 254 | atomic_t nr_outstanding; /* Number of ops in progress */ |
| 247 | − atomic_t nr_copy_ops; /* Number of copy-to-cache ops in progress */ | |
| 248 | − size_t upper_len; /* Length can be extended to here */ | |
| 249 | 255 | unsigned long long submitted; /* Amount submitted for I/O so far */ |
| 250 | 256 | unsigned long long len; /* Length of the request */ |
| 251 | 257 | size_t transferred; /* Amount to be indicated as transferred */ |
| 252 | − short error; /* 0 or error that occurred */ | |
| 258 | + long error; /* 0 or error that occurred */ | |
| 253 | 259 | enum netfs_io_origin origin; /* Origin of the request */ |
| 254 | 260 | bool direct_bv_unpin; /* T if direct_bv[] must be unpinned */ |
| 255 | 261 | u8 buffer_head_slot; /* First slot in ->buffer */ |
| @@ -260,9 +266,9 @@ struct netfs_io_request { · 4 unmodified lines | ||
| 260 | 266 | unsigned long long collected_to; /* Point we've collected to */ |
| 261 | 267 | unsigned long long cleaned_to; /* Position we've cleaned folios to */ |
| 262 | 268 | pgoff_t no_unlock_folio; /* Don't unlock this folio after read */ |
| 269 | + size_t prev_donated; /* Fallback for subreq->prev_donated */ | |
| 263 | 270 | refcount_t ref; |
| 264 | 271 | unsigned long flags; |
| 265 | −#define NETFS_RREQ_INCOMPLETE_IO 0 /* Some ioreqs terminated short or with error */ | |
| 266 | 272 | #define NETFS_RREQ_COPY_TO_CACHE 1 /* Need to write to the cache */ |
| 267 | 273 | #define NETFS_RREQ_NO_UNLOCK_FOLIO 2 /* Don't unlock no_unlock_folio on completion */ |
| 268 | 274 | #define NETFS_RREQ_DONT_UNLOCK_FOLIOS 3 /* Don't unlock the folios on completion */ |
| @@ -274,6 +280,7 @@ struct netfs_io_request { · 5 unmodified lines | ||
| 274 | 280 | #define NETFS_RREQ_PAUSE 11 /* Pause subrequest generation */ |
| 275 | 281 | #define NETFS_RREQ_USE_IO_ITER 12 /* Use ->io_iter rather than ->i_pages */ |
| 276 | 282 | #define NETFS_RREQ_ALL_QUEUED 13 /* All subreqs are now queued */ |
| 283 | +#define NETFS_RREQ_NEED_RETRY 14 /* Need to try retrying */ | |
| 277 | 284 | #define NETFS_RREQ_USE_PGPRIV2 31 /* [DEPRECATED] Use PG_private_2 to mark |
| 278 | 285 | * write to cache on read */ |
| 279 | 286 | const struct netfs_request_ops *netfs_ops; |
| @@ -292,7 +299,7 @@ struct netfs_request_ops { · 12 unmodified lines | ||
| 292 | 299 | |
| 293 | 300 | /* Read request handling */ |
| 294 | 301 | void (*expand_readahead)(struct netfs_io_request *rreq); |
| 295 | − bool (*clamp_length)(struct netfs_io_subrequest *subreq); | |
| 302 | + int (*prepare_read)(struct netfs_io_subrequest *subreq); | |
| 296 | 303 | void (*issue_read)(struct netfs_io_subrequest *subreq); |
| 297 | 304 | bool (*is_still_valid)(struct netfs_io_request *rreq); |
| 298 | 305 | int (*check_write_begin)(struct file *file, loff_t pos, unsigned len, |
| @@ -422,7 +429,10 @@ bool netfs_release_folio(struct folio *folio, gfp_t gfp); · 123 unmodified lines | ||
| 422 | 429 | vm_fault_t netfs_page_mkwrite(struct vm_fault *vmf, struct netfs_group *netfs_group); |
| 423 | 430 | |
| 424 | 431 | /* (Sub)request management API. */ |
| 425 | −void netfs_subreq_terminated(struct netfs_io_subrequest *, ssize_t, bool); | |
| 432 | +void netfs_read_subreq_progress(struct netfs_io_subrequest *subreq, | |
| 433 | + bool was_async); | |
| 434 | +void netfs_read_subreq_terminated(struct netfs_io_subrequest *subreq, | |
| 435 | + int error, bool was_async); | |
| 426 | 436 | void netfs_get_subrequest(struct netfs_io_subrequest *subreq, |
| 427 | 437 | enum netfs_sreq_ref_trace what); |
| 428 | 438 | void netfs_put_subrequest(struct netfs_io_subrequest *subreq, |
| @@ -20,6 +20,7 @@ · 19 unmodified lines | ||
| 20 | 20 | EM(netfs_read_trace_expanded, "EXPANDED ") \ |
| 21 | 21 | EM(netfs_read_trace_readahead, "READAHEAD") \ |
| 22 | 22 | EM(netfs_read_trace_readpage, "READPAGE ") \ |
| 23 | + EM(netfs_read_trace_read_gaps, "READ-GAPS") \ | |
| 23 | 24 | EM(netfs_read_trace_prefetch_for_write, "PREFETCHW") \ |
| 24 | 25 | E_(netfs_read_trace_write_begin, "WRITEBEGN") |
| 25 | 26 | |
| @@ -33,12 +34,14 @@ · 7 unmodified lines | ||
| 33 | 34 | #define netfs_rreq_origins \ |
| 34 | 35 | EM(NETFS_READAHEAD, "RA") \ |
| 35 | 36 | EM(NETFS_READPAGE, "RP") \ |
| 37 | + EM(NETFS_READ_GAPS, "RG") \ | |
| 36 | 38 | EM(NETFS_READ_FOR_WRITE, "RW") \ |
| 37 | 39 | EM(NETFS_DIO_READ, "DR") \ |
| 38 | 40 | EM(NETFS_WRITEBACK, "WB") \ |
| 39 | 41 | EM(NETFS_WRITETHROUGH, "WT") \ |
| 40 | 42 | EM(NETFS_UNBUFFERED_WRITE, "UW") \ |
| 41 | − E_(NETFS_DIO_WRITE, "DW") | |
| 43 | + EM(NETFS_DIO_WRITE, "DW") \ | |
| 44 | + E_(NETFS_PGPRIV2_COPY_TO_CACHE, "2C") | |
| 42 | 45 | |
| 43 | 46 | #define netfs_rreq_traces \ |
| 44 | 47 | EM(netfs_rreq_trace_assess, "ASSESS ") \ |
| @@ -69,15 +72,25 @@ · 24 unmodified lines | ||
| 69 | 72 | E_(NETFS_INVALID_WRITE, "INVL") |
| 70 | 73 | |
| 71 | 74 | #define netfs_sreq_traces \ |
| 75 | + EM(netfs_sreq_trace_add_donations, "+DON ") \ | |
| 76 | + EM(netfs_sreq_trace_added, "ADD ") \ | |
| 77 | + EM(netfs_sreq_trace_clear, "CLEAR") \ | |
| 72 | 78 | EM(netfs_sreq_trace_discard, "DSCRD") \ |
| 79 | + EM(netfs_sreq_trace_donate_to_prev, "DON-P") \ | |
| 80 | + EM(netfs_sreq_trace_donate_to_next, "DON-N") \ | |
| 73 | 81 | EM(netfs_sreq_trace_download_instead, "RDOWN") \ |
| 74 | 82 | EM(netfs_sreq_trace_fail, "FAIL ") \ |
| 75 | 83 | EM(netfs_sreq_trace_free, "FREE ") \ |
| 84 | + EM(netfs_sreq_trace_hit_eof, "EOF ") \ | |
| 85 | + EM(netfs_sreq_trace_io_progress, "IO ") \ | |
| 76 | 86 | EM(netfs_sreq_trace_limited, "LIMIT") \ |
| 77 | 87 | EM(netfs_sreq_trace_prepare, "PREP ") \ |
| 78 | 88 | EM(netfs_sreq_trace_prep_failed, "PRPFL") \ |
| 79 | − EM(netfs_sreq_trace_resubmit_short, "SHORT") \ | |
| 89 | + EM(netfs_sreq_trace_progress, "PRGRS") \ | |
| 90 | + EM(netfs_sreq_trace_reprep_failed, "REPFL") \ | |
| 80 | 91 | EM(netfs_sreq_trace_retry, "RETRY") \ |
| 92 | + EM(netfs_sreq_trace_short, "SHORT") \ | |
| 93 | + EM(netfs_sreq_trace_split, "SPLIT") \ | |
| 81 | 94 | EM(netfs_sreq_trace_submit, "SUBMT") \ |
| 82 | 95 | EM(netfs_sreq_trace_terminated, "TERM ") \ |
| 83 | 96 | EM(netfs_sreq_trace_write, "WRITE") \ |
| @@ -118,7 +131,7 @@ · 34 unmodified lines | ||
| 118 | 131 | EM(netfs_sreq_trace_new, "NEW ") \ |
| 119 | 132 | EM(netfs_sreq_trace_put_cancel, "PUT CANCEL ") \ |
| 120 | 133 | EM(netfs_sreq_trace_put_clear, "PUT CLEAR ") \ |
| 121 | − EM(netfs_sreq_trace_put_discard, "PUT DISCARD") \ | |
| 134 | + EM(netfs_sreq_trace_put_consumed, "PUT CONSUME") \ | |
| 122 | 135 | EM(netfs_sreq_trace_put_done, "PUT DONE ") \ |
| 123 | 136 | EM(netfs_sreq_trace_put_failed, "PUT FAILED ") \ |
| 124 | 137 | EM(netfs_sreq_trace_put_merged, "PUT MERGED ") \ |
| @@ -138,6 +151,7 @@ · 13 unmodified lines | ||
| 138 | 151 | EM(netfs_flush_content, "flush") \ |
| 139 | 152 | EM(netfs_streaming_filled_page, "mod-streamw-f") \ |
| 140 | 153 | EM(netfs_streaming_cont_filled_page, "mod-streamw-f+") \ |
| 154 | + EM(netfs_folio_trace_abandon, "abandon") \ | |
| 141 | 155 | EM(netfs_folio_trace_cancel_copy, "cancel-copy") \ |
| 142 | 156 | EM(netfs_folio_trace_clear, "clear") \ |
| 143 | 157 | EM(netfs_folio_trace_clear_cc, "clear-cc") \ |
| @@ -154,7 +168,11 @@ · 10 unmodified lines | ||
| 154 | 168 | EM(netfs_folio_trace_mkwrite_plus, "mkwrite+") \ |
| 155 | 169 | EM(netfs_folio_trace_not_under_wback, "!wback") \ |
| 156 | 170 | EM(netfs_folio_trace_put, "put") \ |
| 171 | + EM(netfs_folio_trace_read, "read") \ | |
| 172 | + EM(netfs_folio_trace_read_done, "read-done") \ | |
| 157 | 173 | EM(netfs_folio_trace_read_gaps, "read-gaps") \ |
| 174 | + EM(netfs_folio_trace_read_put, "read-put") \ | |
| 175 | + EM(netfs_folio_trace_read_unlock, "read-unlock") \ | |
| 158 | 176 | EM(netfs_folio_trace_redirtied, "redirtied") \ |
| 159 | 177 | EM(netfs_folio_trace_store, "store") \ |
| 160 | 178 | EM(netfs_folio_trace_store_copy, "store-copy") \ |
| @@ -167,6 +185,12 @@ · 6 unmodified lines | ||
| 167 | 185 | EM(netfs_contig_trace_jump, "-->JUMP-->") \ |
| 168 | 186 | E_(netfs_contig_trace_unlock, "Unlock") |
| 169 | 187 | |
| 188 | +#define netfs_donate_traces \ | |
| 189 | + EM(netfs_trace_donate_tail_to_prev, "tail-to-prev") \ | |
| 190 | + EM(netfs_trace_donate_to_prev, "to-prev") \ | |
| 191 | + EM(netfs_trace_donate_to_next, "to-next") \ | |
| 192 | + E_(netfs_trace_donate_to_deferred_next, "defer-next") | |
| 193 | + | |
| 170 | 194 | #ifndef __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY |
| 171 | 195 | #define __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY |
| 172 | 196 | |
| @@ -184,6 +208,7 @@ enum netfs_rreq_ref_trace { netfs_rreq_ref_traces } __mode(byte); · 11 unmodified lines | ||
| 184 | 208 | enum netfs_sreq_ref_trace { netfs_sreq_ref_traces } __mode(byte); |
| 185 | 209 | enum netfs_folio_trace { netfs_folio_traces } __mode(byte); |
| 186 | 210 | enum netfs_collect_contig_trace { netfs_collect_contig_traces } __mode(byte); |
| 211 | +enum netfs_donate_trace { netfs_donate_traces } __mode(byte); | |
| 187 | 212 | |
| 188 | 213 | #endif |
| 189 | 214 | |
| @@ -206,6 +231,7 @@ netfs_rreq_ref_traces; · 16 unmodified lines | ||
| 206 | 231 | netfs_sreq_ref_traces; |
| 207 | 232 | netfs_folio_traces; |
| 208 | 233 | netfs_collect_contig_traces; |
| 234 | +netfs_donate_traces; | |
| 209 | 235 | |
| 210 | 236 | /* |
| 211 | 237 | * Now redefine the EM() and E_() macros to map the enums to the strings that |
| @@ -226,6 +252,7 @@ TRACE_EVENT(netfs_read, · 14 unmodified lines | ||
| 226 | 252 | TP_STRUCT__entry( |
| 227 | 253 | __field(unsigned int, rreq ) |
| 228 | 254 | __field(unsigned int, cookie ) |
| 255 | + __field(loff_t, i_size ) | |
| 229 | 256 | __field(loff_t, start ) |
| 230 | 257 | __field(size_t, len ) |
| 231 | 258 | __field(enum netfs_read_trace, what ) |
| @@ -235,18 +262,19 @@ TRACE_EVENT(netfs_read, · 3 unmodified lines | ||
| 235 | 262 | TP_fast_assign( |
| 236 | 263 | __entry->rreq = rreq->debug_id; |
| 237 | 264 | __entry->cookie = rreq->cache_resources.debug_id; |
| 265 | + __entry->i_size = rreq->i_size; | |
| 238 | 266 | __entry->start = start; |
| 239 | 267 | __entry->len = len; |
| 240 | 268 | __entry->what = what; |
| 241 | 269 | __entry->netfs_inode = rreq->inode->i_ino; |
| 242 | 270 | ), |
| 243 | 271 | |
| 244 | − TP_printk("R=%08x %s c=%08x ni=%x s=%llx %zx", | |
| 272 | + TP_printk("R=%08x %s c=%08x ni=%x s=%llx l=%zx sz=%llx", | |
| 245 | 273 | __entry->rreq, |
| 246 | 274 | __print_symbolic(__entry->what, netfs_read_traces), |
| 247 | 275 | __entry->cookie, |
| 248 | 276 | __entry->netfs_inode, |
| 249 | − __entry->start, __entry->len) | |
| 277 | + __entry->start, __entry->len, __entry->i_size) | |
| 250 | 278 | ); |
| 251 | 279 | |
| 252 | 280 | TRACE_EVENT(netfs_rreq, |
| @@ -651,6 +679,71 @@ TRACE_EVENT(netfs_collect_stream, · 398 unmodified lines | ||
| 651 | 679 | __entry->collected_to, __entry->front) |
| 652 | 680 | ); |
| 653 | 681 | |
| 682 | +TRACE_EVENT(netfs_progress, | |
| 683 | + TP_PROTO(const struct netfs_io_subrequest *subreq, | |
| 684 | + unsigned long long start, size_t avail, size_t part), | |
| 685 | + | |
| 686 | + TP_ARGS(subreq, start, avail, part), | |
| 687 | + | |
| 688 | + TP_STRUCT__entry( | |
| 689 | + __field(unsigned int, rreq) | |
| 690 | + __field(unsigned int, subreq) | |
| 691 | + __field(unsigned int, consumed) | |
| 692 | + __field(unsigned int, transferred) | |
| 693 | + __field(unsigned long long, f_start) | |
| 694 | + __field(unsigned int, f_avail) | |
| 695 | + __field(unsigned int, f_part) | |
| 696 | + __field(unsigned char, slot) | |
| 697 | + ), | |
| 698 | + | |
| 699 | + TP_fast_assign( | |
| 700 | + __entry->rreq = subreq->rreq->debug_id; | |
| 701 | + __entry->subreq = subreq->debug_index; | |
| 702 | + __entry->consumed = subreq->consumed; | |
| 703 | + __entry->transferred = subreq->transferred; | |
| 704 | + __entry->f_start = start; | |
| 705 | + __entry->f_avail = avail; | |
| 706 | + __entry->f_part = part; | |
| 707 | + __entry->slot = subreq->curr_folioq_slot; | |
| 708 | + ), | |
| 709 | + | |
| 710 | + TP_printk("R=%08x[%02x] s=%llx ct=%x/%x pa=%x/%x sl=%x", | |
| 711 | + __entry->rreq, __entry->subreq, __entry->f_start, | |
| 712 | + __entry->consumed, __entry->transferred, | |
| 713 | + __entry->f_part, __entry->f_avail, __entry->slot) | |
| 714 | + ); | |
| 715 | + | |
| 716 | +TRACE_EVENT(netfs_donate, | |
| 717 | + TP_PROTO(const struct netfs_io_request *rreq, | |
| 718 | + const struct netfs_io_subrequest *from, | |
| 719 | + const struct netfs_io_subrequest *to, | |
| 720 | + size_t amount, | |
| 721 | + enum netfs_donate_trace trace), | |
| 722 | + | |
| 723 | + TP_ARGS(rreq, from, to, amount, trace), | |
| 724 | + | |
| 725 | + TP_STRUCT__entry( | |
| 726 | + __field(unsigned int, rreq) | |
| 727 | + __field(unsigned int, from) | |
| 728 | + __field(unsigned int, to) | |
| 729 | + __field(unsigned int, amount) | |
| 730 | + __field(enum netfs_donate_trace, trace) | |
| 731 | + ), | |
| 732 | + | |
| 733 | + TP_fast_assign( | |
| 734 | + __entry->rreq = rreq->debug_id; | |
| 735 | + __entry->from = from->debug_index; | |
| 736 | + __entry->to = to ? to->debug_index : -1; | |
| 737 | + __entry->amount = amount; | |
| 738 | + __entry->trace = trace; | |
| 739 | + ), | |
| 740 | + | |
| 741 | + TP_printk("R=%08x[%02x] -> [%02x] %s am=%x", | |
| 742 | + __entry->rreq, __entry->from, __entry->to, | |
| 743 | + __print_symbolic(__entry->trace, netfs_donate_traces), | |
| 744 | + __entry->amount) | |
| 745 | + ); | |
| 746 | + | |
| 654 | 747 | #undef EM |
| 655 | 748 | #undef E_ |
| 656 | 749 | #endif /* _TRACE_NETFS_H */ |