History

netfs: Simplify the writeback code

Use the new folio_queue structures to simplify the writeback code. The problem with referring to the i_pages xarray directly is that we may have gaps in the sequence of folios we're writing from that we need to skip when we're removing the writeback mark from the folios we're writing back from. At the moment the code tries to deal with this by carefully tracking the gaps in each writeback stream (eg. write to server and write to cache) and divining when there's a gap that spans folios (something that's not helped by folios not being a consistent size). Instead, the folio_queue buffer contains pointers only the folios we're dealing with, has them in ascending order and indicates a gap by placing non-consequitive folios next to each other. This makes it possible to track where we need to clean up to by just keeping track of where we've processed to on each stream and taking the minimum. Note that the I/O iterator is always rounded up to the end of the folio, even if that is beyond the EOF position, so that the cache can do DIO from the page. The excess space is cleared, though mmapped writes clobber it.

David Howells committed 2 years ago

Commit 983cdcf1 parent bfaa33b4 files · +45 −171Browse files

Unified · Split

4 files
  1. fs/netfs/write_collect.c +24 −122
  2. fs/netfs/write_issue.c +19 −17
  3. include/linux/netfs.h +0 −1
  4. include/trace/events/netfs.h +2 −31
fs/netfs/write_collect.c+24−122
@@ -15,15 +15,11 @@ · 14 unmodified lines
1515
1616 /* Notes made in the collector */
1717 #define HIT_PENDING 0x01 /* A front op was still pending */
18−#define SOME_EMPTY 0x02 /* One of more streams are empty */
19−#define ALL_EMPTY 0x04 /* All streams are empty */
20−#define MAYBE_DISCONTIG 0x08 /* A front op may be discontiguous (rounded to PAGE_SIZE) */
21−#define NEED_REASSESS 0x10 /* Need to loop round and reassess */
22−#define REASSESS_DISCONTIG 0x20 /* Reassess discontiguity if contiguity advances */
23−#define MADE_PROGRESS 0x40 /* Made progress cleaning up a stream or the folio set */
24−#define BUFFERED 0x80 /* The pagecache needs cleaning up */
25−#define NEED_RETRY 0x100 /* A front op requests retrying */
26−#define SAW_FAILURE 0x200 /* One stream or hit a permanent failure */
18+#define NEED_REASSESS 0x02 /* Need to loop round and reassess */
19+#define MADE_PROGRESS 0x04 /* Made progress cleaning up a stream or the folio set */
20+#define BUFFERED 0x08 /* The pagecache needs cleaning up */
21+#define NEED_RETRY 0x10 /* A front op requests retrying */
22+#define SAW_FAILURE 0x20 /* One stream or hit a permanent failure */
2723
2824 /*
2925 * Successful completion of write of a folio to the server and/or cache. Note
@@ -85,10 +81,10 @@ end_wb: · 55 unmodified lines
8581 * Unlock any folios we've finished with.
8682 */
8783 static void netfs_writeback_unlock_folios(struct netfs_io_request *wreq,
88− unsigned long long collected_to,
8984 unsigned int *notes)
9085 {
9186 struct folio_queue *folioq = wreq->buffer;
87+ unsigned long long collected_to = wreq->collected_to;
9288 unsigned int slot = wreq->buffer_head_slot;
9389
9490 if (slot >= folioq_nr_slots(folioq)) {
@@ -117,12 +113,6 @@ static void netfs_writeback_unlock_folios(struct netfs_io_request *wreq, · 22 unmodified lines
117113
118114 trace_netfs_collect_folio(wreq, folio, fend, collected_to);
119115
120− if (fpos + fsize > wreq->contiguity) {
121− trace_netfs_collect_contig(wreq, fpos + fsize,
122− netfs_contig_trace_unlock);
123− wreq->contiguity = fpos + fsize;
124− }
125−
126116 /* Unlock any folio we've transferred all of. */
127117 if (collected_to < fend)
128118 break;
@@ -380,7 +370,7 @@ static void netfs_collect_write_results(struct netfs_io_request *wreq) · 251 unmodified lines
380370 {
381371 struct netfs_io_subrequest *front, *remove;
382372 struct netfs_io_stream *stream;
383− unsigned long long collected_to;
373+ unsigned long long collected_to, issued_to;
384374 unsigned int notes;
385375 int s;
386376
@@ -389,28 +379,21 @@ static void netfs_collect_write_results(struct netfs_io_request *wreq) · 2 unmodified lines
389379 trace_netfs_rreq(wreq, netfs_rreq_trace_collect);
390380
391381 reassess_streams:
382+ issued_to = atomic64_read(&wreq->issued_to);
392383 smp_rmb();
393384 collected_to = ULLONG_MAX;
394− if (wreq->origin == NETFS_WRITEBACK)
395− notes = ALL_EMPTY | BUFFERED | MAYBE_DISCONTIG;
396− else if (wreq->origin == NETFS_WRITETHROUGH)
397− notes = ALL_EMPTY | BUFFERED;
385+ if (wreq->origin == NETFS_WRITEBACK ||
386+ wreq->origin == NETFS_WRITETHROUGH)
387+ notes = BUFFERED;
398388 else
399− notes = ALL_EMPTY;
389+ notes = 0;
400390
401391 /* Remove completed subrequests from the front of the streams and
402392 * advance the completion point on each stream. We stop when we hit
403393 * something that's in progress. The issuer thread may be adding stuff
404394 * to the tail whilst we're doing this.
405− *
406− * We must not, however, merge in discontiguities that span whole
407− * folios that aren't under writeback. This is made more complicated
408− * by the folios in the gap being of unpredictable sizes - if they even
409− * exist - but we don't want to look them up.
410395 */
411396 for (s = 0; s < NR_IO_STREAMS; s++) {
412− loff_t rstart, rend;
413−
414397 stream = &wreq->io_streams[s];
415398 /* Read active flag before list pointers */
416399 if (!smp_load_acquire(&stream->active))
@@ -422,26 +405,10 @@ reassess_streams: · 5 unmodified lines
422405 //_debug("sreq [%x] %llx %zx/%zx",
423406 // front->debug_index, front->start, front->transferred, front->len);
424407
425− /* Stall if there may be a discontinuity. */
426− rstart = round_down(front->start, PAGE_SIZE);
427− if (rstart > wreq->contiguity) {
428− if (wreq->contiguity > stream->collected_to) {
429− trace_netfs_collect_gap(wreq, stream,
430− wreq->contiguity, 'D');
431− stream->collected_to = wreq->contiguity;
432− }
433− notes |= REASSESS_DISCONTIG;
434− break;
408+ if (stream->collected_to < front->start) {
409+ trace_netfs_collect_gap(wreq, stream, issued_to, 'F');
410+ stream->collected_to = front->start;
435411 }
436− rend = round_up(front->start + front->len, PAGE_SIZE);
437− if (rend > wreq->contiguity) {
438− trace_netfs_collect_contig(wreq, rend,
439− netfs_contig_trace_collect);
440− wreq->contiguity = rend;
441− if (notes & REASSESS_DISCONTIG)
442− notes |= NEED_REASSESS;
443− }
444− notes &= ~MAYBE_DISCONTIG;
445412
446413 /* Stall if the front is still undergoing I/O. */
447414 if (test_bit(NETFS_SREQ_IN_PROGRESS, &front->flags)) {
@@ -483,15 +450,6 @@ reassess_streams: · 35 unmodified lines
483450 front = list_first_entry_or_null(&stream->subrequests,
484451 struct netfs_io_subrequest, rreq_link);
485452 stream->front = front;
486− if (!front) {
487− unsigned long long jump_to = atomic64_read(&wreq->issued_to);
488−
489− if (stream->collected_to < jump_to) {
490− trace_netfs_collect_gap(wreq, stream, jump_to, 'A');
491− stream->collected_to = jump_to;
492− }
493− }
494−
495453 spin_unlock_bh(&wreq->lock);
496454 netfs_put_subrequest(remove, false,
497455 notes & SAW_FAILURE ?
@@ -499,10 +457,13 @@ reassess_streams: · 1 unmodified line
499457 netfs_sreq_trace_put_done);
500458 }
501459
502− if (front)
503− notes &= ~ALL_EMPTY;
504− else
505− notes |= SOME_EMPTY;
460+ /* If we have an empty stream, we need to jump it forward
461+ * otherwise the collection point will never advance.
462+ */
463+ if (!front && issued_to > stream->collected_to) {
464+ trace_netfs_collect_gap(wreq, stream, issued_to, 'E');
465+ stream->collected_to = issued_to;
466+ }
506467
507468 if (stream->collected_to < collected_to)
508469 collected_to = stream->collected_to;
@@ -511,36 +472,6 @@ reassess_streams: · 2 unmodified lines
511472 if (collected_to != ULLONG_MAX && collected_to > wreq->collected_to)
512473 wreq->collected_to = collected_to;
513474
514− /* If we have an empty stream, we need to jump it forward over any gap
515− * otherwise the collection point will never advance.
516− *
517− * Note that the issuer always adds to the stream with the lowest
518− * so-far submitted start, so if we see two consecutive subreqs in one
519− * stream with nothing between then in another stream, then the second
520− * stream has a gap that can be jumped.
521− */
522− if (notes & SOME_EMPTY) {
523− unsigned long long jump_to = wreq->start + READ_ONCE(wreq->submitted);
524−
525− for (s = 0; s < NR_IO_STREAMS; s++) {
526− stream = &wreq->io_streams[s];
527− if (stream->active &&
528− stream->front &&
529− stream->front->start < jump_to)
530− jump_to = stream->front->start;
531− }
532−
533− for (s = 0; s < NR_IO_STREAMS; s++) {
534− stream = &wreq->io_streams[s];
535− if (stream->active &&
536− !stream->front &&
537− stream->collected_to < jump_to) {
538− trace_netfs_collect_gap(wreq, stream, jump_to, 'B');
539− stream->collected_to = jump_to;
540− }
541− }
542− }
543−
544475 for (s = 0; s < NR_IO_STREAMS; s++) {
545476 stream = &wreq->io_streams[s];
546477 if (stream->active)
@@ -551,43 +482,14 @@ reassess_streams: · 4 unmodified lines
551482
552483 /* Unlock any folios that we have now finished with. */
553484 if (notes & BUFFERED) {
554− unsigned long long clean_to = min(wreq->collected_to, wreq->contiguity);
555−
556− if (wreq->cleaned_to < clean_to)
557− netfs_writeback_unlock_folios(wreq, clean_to, &notes);
485+ if (wreq->cleaned_to < wreq->collected_to)
486+ netfs_writeback_unlock_folios(wreq, &notes);
558487 } else {
559488 wreq->cleaned_to = wreq->collected_to;
560489 }
561490
562491 // TODO: Discard encryption buffers
563492
564− /* If all streams are discontiguous with the last folio we cleared, we
565− * may need to skip a set of folios.
566− */
567− if ((notes & (MAYBE_DISCONTIG | ALL_EMPTY)) == MAYBE_DISCONTIG) {
568− unsigned long long jump_to = ULLONG_MAX;
569−
570− for (s = 0; s < NR_IO_STREAMS; s++) {
571− stream = &wreq->io_streams[s];
572− if (stream->active && stream->front &&
573− stream->front->start < jump_to)
574− jump_to = stream->front->start;
575− }
576−
577− trace_netfs_collect_contig(wreq, jump_to, netfs_contig_trace_jump);
578− wreq->contiguity = jump_to;
579− wreq->cleaned_to = jump_to;
580− wreq->collected_to = jump_to;
581− for (s = 0; s < NR_IO_STREAMS; s++) {
582− stream = &wreq->io_streams[s];
583− if (stream->collected_to < jump_to)
584− stream->collected_to = jump_to;
585− }
586− //cond_resched();
587− notes |= MADE_PROGRESS;
588− goto reassess_streams;
589− }
590−
591493 if (notes & NEED_RETRY)
592494 goto need_retry;
593495 if ((notes & MADE_PROGRESS) && test_bit(NETFS_RREQ_PAUSE, &wreq->flags)) {
fs/netfs/write_issue.c+19−17
@@ -107,7 +107,6 @@ struct netfs_io_request *netfs_create_write_req(struct address_space *mapping, · 106 unmodified lines
107107 if (is_buffered && netfs_is_cache_enabled(ictx))
108108 fscache_begin_write_operation(&wreq->cache_resources, netfs_i_cookie(ictx));
109109
110− wreq->contiguity = wreq->start;
111110 wreq->cleaned_to = wreq->start;
112111
113112 wreq->io_streams[0].stream_nr = 0;
@@ -158,6 +157,7 @@ static void netfs_prepare_write(struct netfs_io_request *wreq, · 44 unmodified lines
158157 subreq->source = stream->source;
159158 subreq->start = start;
160159 subreq->stream_nr = stream->stream_nr;
160+ subreq->io_iter = wreq->io_iter;
161161
162162 _enter("R=%x[%x]", wreq->debug_id, subreq->debug_index);
163163
@@ -213,22 +213,15 @@ static void netfs_prepare_write(struct netfs_io_request *wreq, · 49 unmodified lines
213213 * netfs_write_subrequest_terminated() when complete.
214214 */
215215 static void netfs_do_issue_write(struct netfs_io_stream *stream,
216− struct netfs_io_subrequest *subreq,
217− struct iov_iter *source)
216+ struct netfs_io_subrequest *subreq)
218217 {
219218 struct netfs_io_request *wreq = subreq->rreq;
220− size_t size = subreq->len - subreq->transferred;
221219
222220 _enter("R=%x[%x],%zx", wreq->debug_id, subreq->debug_index, subreq->len);
223221
224222 if (test_bit(NETFS_SREQ_FAILED, &subreq->flags))
225223 return netfs_write_subrequest_terminated(subreq, subreq->error, false);
226224
227− // TODO: Use encrypted buffer
228− subreq->io_iter = *source;
229− iov_iter_advance(source, size);
230− iov_iter_truncate(&subreq->io_iter, size);
231−
232225 trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
233226 stream->issue_write(subreq);
234227 }
@@ -237,8 +230,15 @@ void netfs_reissue_write(struct netfs_io_stream *stream, · 2 unmodified lines
237230 struct netfs_io_subrequest *subreq,
238231 struct iov_iter *source)
239232 {
233+ size_t size = subreq->len - subreq->transferred;
234+
235+ // TODO: Use encrypted buffer
236+ subreq->io_iter = *source;
237+ iov_iter_advance(source, size);
238+ iov_iter_truncate(&subreq->io_iter, size);
239+
240240 __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
241− netfs_do_issue_write(stream, subreq, source);
241+ netfs_do_issue_write(stream, subreq);
242242 }
243243
244244 static void netfs_issue_write(struct netfs_io_request *wreq,
@@ -249,10 +249,8 @@ static void netfs_issue_write(struct netfs_io_request *wreq, · 4 unmodified lines
249249 if (!subreq)
250250 return;
251251 stream->construct = NULL;
252−
253− if (subreq->start + subreq->len > wreq->start + wreq->submitted)
254− WRITE_ONCE(wreq->submitted, subreq->start + subreq->len - wreq->start);
255− netfs_do_issue_write(stream, subreq, &wreq->io_iter);
252+ subreq->io_iter.count = subreq->len;
253+ netfs_do_issue_write(stream, subreq);
256254 }
257255
258256 /*
@@ -464,10 +462,11 @@ static int netfs_write_folio(struct netfs_io_request *wreq, · 205 unmodified lines
464462 if (choose_s < 0)
465463 break;
466464 stream = &wreq->io_streams[choose_s];
465+ wreq->io_iter.iov_offset = stream->submit_off;
467466
467+ atomic64_set(&wreq->issued_to, fpos + stream->submit_off);
468468 part = netfs_advance_write(wreq, stream, fpos + stream->submit_off,
469469 stream->submit_len, to_eof);
470− atomic64_set(&wreq->issued_to, fpos + stream->submit_off);
471470 stream->submit_off += part;
472471 stream->submit_max_len -= part;
473472 if (part > stream->submit_len)
@@ -478,6 +477,8 @@ static int netfs_write_folio(struct netfs_io_request *wreq, · 4 unmodified lines
478477 debug = true;
479478 }
480479
480+ wreq->io_iter.iov_offset = 0;
481+ iov_iter_advance(&wreq->io_iter, fsize);
481482 atomic64_set(&wreq->issued_to, fpos + fsize);
482483
483484 if (!debug)
@@ -526,10 +527,10 @@ int netfs_writepages(struct address_space *mapping, · 42 unmodified lines
526527 netfs_stat(&netfs_n_wh_writepages);
527528
528529 do {
529− _debug("wbiter %lx %llx", folio->index, wreq->start + wreq->submitted);
530+ _debug("wbiter %lx %llx", folio->index, atomic64_read(&wreq->issued_to));
530531
531532 /* It appears we don't have to handle cyclic writeback wrapping. */
532− WARN_ON_ONCE(wreq && folio_pos(folio) < wreq->start + wreq->submitted);
533+ WARN_ON_ONCE(wreq && folio_pos(folio) < atomic64_read(&wreq->issued_to));
533534
534535 if (netfs_folio_group(folio) != NETFS_FOLIO_COPY_TO_CACHE &&
535536 unlikely(!test_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags))) {
@@ -673,6 +674,7 @@ int netfs_unbuffered_write(struct netfs_io_request *wreq, bool may_wait, size_t · 137 unmodified lines
673674 part = netfs_advance_write(wreq, upload, start, len, false);
674675 start += part;
675676 len -= part;
677+ iov_iter_advance(&wreq->io_iter, part);
676678 if (test_bit(NETFS_RREQ_PAUSE, &wreq->flags)) {
677679 trace_netfs_rreq(wreq, netfs_rreq_trace_wait_pause);
678680 wait_on_bit(&wreq->flags, NETFS_RREQ_PAUSE, TASK_UNINTERRUPTIBLE);
include/linux/netfs.h+0−1
@@ -257,7 +257,6 @@ struct netfs_io_request { · 256 unmodified lines
257257 unsigned long long i_size; /* Size of the file */
258258 unsigned long long start; /* Start position */
259259 atomic64_t issued_to; /* Write issuer folio cursor */
260− unsigned long long contiguity; /* Tracking for gaps in the writeback sequence */
261260 unsigned long long collected_to; /* Point we've collected to */
262261 unsigned long long cleaned_to; /* Position we've cleaned folios to */
263262 pgoff_t no_unlock_folio; /* Don't unlock this folio after read */
include/trace/events/netfs.h+2−31
@@ -512,33 +512,6 @@ TRACE_EVENT(netfs_collect, · 511 unmodified lines
512512 __entry->start + __entry->len)
513513 );
514514
515−TRACE_EVENT(netfs_collect_contig,
516− TP_PROTO(const struct netfs_io_request *wreq, unsigned long long to,
517− enum netfs_collect_contig_trace type),
518−
519− TP_ARGS(wreq, to, type),
520−
521− TP_STRUCT__entry(
522− __field(unsigned int, wreq)
523− __field(enum netfs_collect_contig_trace, type)
524− __field(unsigned long long, contiguity)
525− __field(unsigned long long, to)
526− ),
527−
528− TP_fast_assign(
529− __entry->wreq = wreq->debug_id;
530− __entry->type = type;
531− __entry->contiguity = wreq->contiguity;
532− __entry->to = to;
533− ),
534−
535− TP_printk("R=%08x %llx -> %llx %s",
536− __entry->wreq,
537− __entry->contiguity,
538− __entry->to,
539− __print_symbolic(__entry->type, netfs_collect_contig_traces))
540− );
541−
542515 TRACE_EVENT(netfs_collect_sreq,
543516 TP_PROTO(const struct netfs_io_request *wreq,
544517 const struct netfs_io_subrequest *subreq),
@@ -610,7 +583,6 @@ TRACE_EVENT(netfs_collect_state, · 65 unmodified lines
610583 __field(unsigned int, notes )
611584 __field(unsigned long long, collected_to )
612585 __field(unsigned long long, cleaned_to )
613− __field(unsigned long long, contiguity )
614586 ),
615587
616588 TP_fast_assign(
@@ -618,12 +590,11 @@ TRACE_EVENT(netfs_collect_state, · 1 unmodified line
618590 __entry->notes = notes;
619591 __entry->collected_to = collected_to;
620592 __entry->cleaned_to = wreq->cleaned_to;
621− __entry->contiguity = wreq->contiguity;
622593 ),
623594
624− TP_printk("R=%08x cto=%llx fto=%llx ctg=%llx n=%x",
595+ TP_printk("R=%08x col=%llx cln=%llx n=%x",
625596 __entry->wreq, __entry->collected_to,
626− __entry->cleaned_to, __entry->contiguity,
597+ __entry->cleaned_to,
627598 __entry->notes)
628599 );
629600

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t