History

netfs: Speed up buffered reading

Improve the efficiency of buffered reads in a number of ways: (1) Overhaul the algorithm in general so that it's a lot more compact and split the read submission code between buffered and unbuffered versions. The unbuffered version can be vastly simplified. (2) Read-result collection is handed off to a work queue rather than being done in the I/O thread. Multiple subrequests can be processes simultaneously. (3) When a subrequest is collected, any folios it fully spans are collected and "spare" data on either side is donated to either the previous or the next subrequest in the sequence. Notes: (*) Readahead expansion is massively slows down fio, presumably because it causes a load of extra allocations, both folio and xarray, up front before RPC requests can be transmitted. (*) RDMA with cifs does appear to work, both with SIW and RXE. (*) PG_private_2-based reading and copy-to-cache is split out into its own file and altered to use folio_queue. Note that the copy to the cache now creates a new write transaction against the cache and adds the folios to be copied into it. This allows it to use part of the writeback I/O code.

David Howells committed 2 years ago

Commit ee4cdf71 parent 2e45b9228 files · +2058 −470Browse files

Showing the changes to include/trace (1 of 28 files). Show all

Unified · Split

include/trace/events/netfs.h+98−5
@@ -20,6 +20,7 @@ · 19 unmodified lines
20 EM(netfs_read_trace_expanded, "EXPANDED ") \20 EM(netfs_read_trace_expanded, "EXPANDED ") \
21 EM(netfs_read_trace_readahead, "READAHEAD") \21 EM(netfs_read_trace_readahead, "READAHEAD") \
22 EM(netfs_read_trace_readpage, "READPAGE ") \22 EM(netfs_read_trace_readpage, "READPAGE ") \
23 EM(netfs_read_trace_read_gaps, "READ-GAPS") \
23 EM(netfs_read_trace_prefetch_for_write, "PREFETCHW") \24 EM(netfs_read_trace_prefetch_for_write, "PREFETCHW") \
24 E_(netfs_read_trace_write_begin, "WRITEBEGN")25 E_(netfs_read_trace_write_begin, "WRITEBEGN")
25 26
@@ -33,12 +34,14 @@ · 7 unmodified lines
33#define netfs_rreq_origins \34#define netfs_rreq_origins \
34 EM(NETFS_READAHEAD, "RA") \35 EM(NETFS_READAHEAD, "RA") \
35 EM(NETFS_READPAGE, "RP") \36 EM(NETFS_READPAGE, "RP") \
37 EM(NETFS_READ_GAPS, "RG") \
36 EM(NETFS_READ_FOR_WRITE, "RW") \38 EM(NETFS_READ_FOR_WRITE, "RW") \
37 EM(NETFS_DIO_READ, "DR") \39 EM(NETFS_DIO_READ, "DR") \
38 EM(NETFS_WRITEBACK, "WB") \40 EM(NETFS_WRITEBACK, "WB") \
39 EM(NETFS_WRITETHROUGH, "WT") \41 EM(NETFS_WRITETHROUGH, "WT") \
40 EM(NETFS_UNBUFFERED_WRITE, "UW") \42 EM(NETFS_UNBUFFERED_WRITE, "UW") \
41 E_(NETFS_DIO_WRITE, "DW")43 EM(NETFS_DIO_WRITE, "DW") \
44 E_(NETFS_PGPRIV2_COPY_TO_CACHE, "2C")
42 45
43#define netfs_rreq_traces \46#define netfs_rreq_traces \
44 EM(netfs_rreq_trace_assess, "ASSESS ") \47 EM(netfs_rreq_trace_assess, "ASSESS ") \
@@ -69,15 +72,25 @@ · 24 unmodified lines
69 E_(NETFS_INVALID_WRITE, "INVL")72 E_(NETFS_INVALID_WRITE, "INVL")
70 73
71#define netfs_sreq_traces \74#define netfs_sreq_traces \
75 EM(netfs_sreq_trace_add_donations, "+DON ") \
76 EM(netfs_sreq_trace_added, "ADD ") \
77 EM(netfs_sreq_trace_clear, "CLEAR") \
72 EM(netfs_sreq_trace_discard, "DSCRD") \78 EM(netfs_sreq_trace_discard, "DSCRD") \
79 EM(netfs_sreq_trace_donate_to_prev, "DON-P") \
80 EM(netfs_sreq_trace_donate_to_next, "DON-N") \
73 EM(netfs_sreq_trace_download_instead, "RDOWN") \81 EM(netfs_sreq_trace_download_instead, "RDOWN") \
74 EM(netfs_sreq_trace_fail, "FAIL ") \82 EM(netfs_sreq_trace_fail, "FAIL ") \
75 EM(netfs_sreq_trace_free, "FREE ") \83 EM(netfs_sreq_trace_free, "FREE ") \
84 EM(netfs_sreq_trace_hit_eof, "EOF ") \
85 EM(netfs_sreq_trace_io_progress, "IO ") \
76 EM(netfs_sreq_trace_limited, "LIMIT") \86 EM(netfs_sreq_trace_limited, "LIMIT") \
77 EM(netfs_sreq_trace_prepare, "PREP ") \87 EM(netfs_sreq_trace_prepare, "PREP ") \
78 EM(netfs_sreq_trace_prep_failed, "PRPFL") \88 EM(netfs_sreq_trace_prep_failed, "PRPFL") \
79 EM(netfs_sreq_trace_resubmit_short, "SHORT") \89 EM(netfs_sreq_trace_progress, "PRGRS") \
90 EM(netfs_sreq_trace_reprep_failed, "REPFL") \
80 EM(netfs_sreq_trace_retry, "RETRY") \91 EM(netfs_sreq_trace_retry, "RETRY") \
92 EM(netfs_sreq_trace_short, "SHORT") \
93 EM(netfs_sreq_trace_split, "SPLIT") \
81 EM(netfs_sreq_trace_submit, "SUBMT") \94 EM(netfs_sreq_trace_submit, "SUBMT") \
82 EM(netfs_sreq_trace_terminated, "TERM ") \95 EM(netfs_sreq_trace_terminated, "TERM ") \
83 EM(netfs_sreq_trace_write, "WRITE") \96 EM(netfs_sreq_trace_write, "WRITE") \
@@ -118,7 +131,7 @@ · 34 unmodified lines
118 EM(netfs_sreq_trace_new, "NEW ") \131 EM(netfs_sreq_trace_new, "NEW ") \
119 EM(netfs_sreq_trace_put_cancel, "PUT CANCEL ") \132 EM(netfs_sreq_trace_put_cancel, "PUT CANCEL ") \
120 EM(netfs_sreq_trace_put_clear, "PUT CLEAR ") \133 EM(netfs_sreq_trace_put_clear, "PUT CLEAR ") \
121 EM(netfs_sreq_trace_put_discard, "PUT DISCARD") \134 EM(netfs_sreq_trace_put_consumed, "PUT CONSUME") \
122 EM(netfs_sreq_trace_put_done, "PUT DONE ") \135 EM(netfs_sreq_trace_put_done, "PUT DONE ") \
123 EM(netfs_sreq_trace_put_failed, "PUT FAILED ") \136 EM(netfs_sreq_trace_put_failed, "PUT FAILED ") \
124 EM(netfs_sreq_trace_put_merged, "PUT MERGED ") \137 EM(netfs_sreq_trace_put_merged, "PUT MERGED ") \
@@ -138,6 +151,7 @@ · 13 unmodified lines
138 EM(netfs_flush_content, "flush") \151 EM(netfs_flush_content, "flush") \
139 EM(netfs_streaming_filled_page, "mod-streamw-f") \152 EM(netfs_streaming_filled_page, "mod-streamw-f") \
140 EM(netfs_streaming_cont_filled_page, "mod-streamw-f+") \153 EM(netfs_streaming_cont_filled_page, "mod-streamw-f+") \
154 EM(netfs_folio_trace_abandon, "abandon") \
141 EM(netfs_folio_trace_cancel_copy, "cancel-copy") \155 EM(netfs_folio_trace_cancel_copy, "cancel-copy") \
142 EM(netfs_folio_trace_clear, "clear") \156 EM(netfs_folio_trace_clear, "clear") \
143 EM(netfs_folio_trace_clear_cc, "clear-cc") \157 EM(netfs_folio_trace_clear_cc, "clear-cc") \
@@ -154,7 +168,11 @@ · 10 unmodified lines
154 EM(netfs_folio_trace_mkwrite_plus, "mkwrite+") \168 EM(netfs_folio_trace_mkwrite_plus, "mkwrite+") \
155 EM(netfs_folio_trace_not_under_wback, "!wback") \169 EM(netfs_folio_trace_not_under_wback, "!wback") \
156 EM(netfs_folio_trace_put, "put") \170 EM(netfs_folio_trace_put, "put") \
171 EM(netfs_folio_trace_read, "read") \
172 EM(netfs_folio_trace_read_done, "read-done") \
157 EM(netfs_folio_trace_read_gaps, "read-gaps") \173 EM(netfs_folio_trace_read_gaps, "read-gaps") \
174 EM(netfs_folio_trace_read_put, "read-put") \
175 EM(netfs_folio_trace_read_unlock, "read-unlock") \
158 EM(netfs_folio_trace_redirtied, "redirtied") \176 EM(netfs_folio_trace_redirtied, "redirtied") \
159 EM(netfs_folio_trace_store, "store") \177 EM(netfs_folio_trace_store, "store") \
160 EM(netfs_folio_trace_store_copy, "store-copy") \178 EM(netfs_folio_trace_store_copy, "store-copy") \
@@ -167,6 +185,12 @@ · 6 unmodified lines
167 EM(netfs_contig_trace_jump, "-->JUMP-->") \185 EM(netfs_contig_trace_jump, "-->JUMP-->") \
168 E_(netfs_contig_trace_unlock, "Unlock")186 E_(netfs_contig_trace_unlock, "Unlock")
169 187
188#define netfs_donate_traces \
189 EM(netfs_trace_donate_tail_to_prev, "tail-to-prev") \
190 EM(netfs_trace_donate_to_prev, "to-prev") \
191 EM(netfs_trace_donate_to_next, "to-next") \
192 E_(netfs_trace_donate_to_deferred_next, "defer-next")
193
170#ifndef __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY194#ifndef __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY
171#define __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY195#define __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY
172 196
@@ -184,6 +208,7 @@ enum netfs_rreq_ref_trace { netfs_rreq_ref_traces } __mode(byte); · 11 unmodified lines
184enum netfs_sreq_ref_trace { netfs_sreq_ref_traces } __mode(byte);208enum netfs_sreq_ref_trace { netfs_sreq_ref_traces } __mode(byte);
185enum netfs_folio_trace { netfs_folio_traces } __mode(byte);209enum netfs_folio_trace { netfs_folio_traces } __mode(byte);
186enum netfs_collect_contig_trace { netfs_collect_contig_traces } __mode(byte);210enum netfs_collect_contig_trace { netfs_collect_contig_traces } __mode(byte);
211enum netfs_donate_trace { netfs_donate_traces } __mode(byte);
187 212
188#endif213#endif
189 214
@@ -206,6 +231,7 @@ netfs_rreq_ref_traces; · 16 unmodified lines
206netfs_sreq_ref_traces;231netfs_sreq_ref_traces;
207netfs_folio_traces;232netfs_folio_traces;
208netfs_collect_contig_traces;233netfs_collect_contig_traces;
234netfs_donate_traces;
209 235
210/*236/*
211 * Now redefine the EM() and E_() macros to map the enums to the strings that237 * Now redefine the EM() and E_() macros to map the enums to the strings that
@@ -226,6 +252,7 @@ TRACE_EVENT(netfs_read, · 14 unmodified lines
226 TP_STRUCT__entry(252 TP_STRUCT__entry(
227 __field(unsigned int, rreq )253 __field(unsigned int, rreq )
228 __field(unsigned int, cookie )254 __field(unsigned int, cookie )
255 __field(loff_t, i_size )
229 __field(loff_t, start )256 __field(loff_t, start )
230 __field(size_t, len )257 __field(size_t, len )
231 __field(enum netfs_read_trace, what )258 __field(enum netfs_read_trace, what )
@@ -235,18 +262,19 @@ TRACE_EVENT(netfs_read, · 3 unmodified lines
235 TP_fast_assign(262 TP_fast_assign(
236 __entry->rreq = rreq->debug_id;263 __entry->rreq = rreq->debug_id;
237 __entry->cookie = rreq->cache_resources.debug_id;264 __entry->cookie = rreq->cache_resources.debug_id;
265 __entry->i_size = rreq->i_size;
238 __entry->start = start;266 __entry->start = start;
239 __entry->len = len;267 __entry->len = len;
240 __entry->what = what;268 __entry->what = what;
241 __entry->netfs_inode = rreq->inode->i_ino;269 __entry->netfs_inode = rreq->inode->i_ino;
242 ),270 ),
243 271
244 TP_printk("R=%08x %s c=%08x ni=%x s=%llx %zx",272 TP_printk("R=%08x %s c=%08x ni=%x s=%llx l=%zx sz=%llx",
245 __entry->rreq,273 __entry->rreq,
246 __print_symbolic(__entry->what, netfs_read_traces),274 __print_symbolic(__entry->what, netfs_read_traces),
247 __entry->cookie,275 __entry->cookie,
248 __entry->netfs_inode,276 __entry->netfs_inode,
249 __entry->start, __entry->len)277 __entry->start, __entry->len, __entry->i_size)
250 );278 );
251 279
252TRACE_EVENT(netfs_rreq,280TRACE_EVENT(netfs_rreq,
@@ -651,6 +679,71 @@ TRACE_EVENT(netfs_collect_stream, · 398 unmodified lines
651 __entry->collected_to, __entry->front)679 __entry->collected_to, __entry->front)
652 );680 );
653 681
682TRACE_EVENT(netfs_progress,
683 TP_PROTO(const struct netfs_io_subrequest *subreq,
684 unsigned long long start, size_t avail, size_t part),
685
686 TP_ARGS(subreq, start, avail, part),
687
688 TP_STRUCT__entry(
689 __field(unsigned int, rreq)
690 __field(unsigned int, subreq)
691 __field(unsigned int, consumed)
692 __field(unsigned int, transferred)
693 __field(unsigned long long, f_start)
694 __field(unsigned int, f_avail)
695 __field(unsigned int, f_part)
696 __field(unsigned char, slot)
697 ),
698
699 TP_fast_assign(
700 __entry->rreq = subreq->rreq->debug_id;
701 __entry->subreq = subreq->debug_index;
702 __entry->consumed = subreq->consumed;
703 __entry->transferred = subreq->transferred;
704 __entry->f_start = start;
705 __entry->f_avail = avail;
706 __entry->f_part = part;
707 __entry->slot = subreq->curr_folioq_slot;
708 ),
709
710 TP_printk("R=%08x[%02x] s=%llx ct=%x/%x pa=%x/%x sl=%x",
711 __entry->rreq, __entry->subreq, __entry->f_start,
712 __entry->consumed, __entry->transferred,
713 __entry->f_part, __entry->f_avail, __entry->slot)
714 );
715
716TRACE_EVENT(netfs_donate,
717 TP_PROTO(const struct netfs_io_request *rreq,
718 const struct netfs_io_subrequest *from,
719 const struct netfs_io_subrequest *to,
720 size_t amount,
721 enum netfs_donate_trace trace),
722
723 TP_ARGS(rreq, from, to, amount, trace),
724
725 TP_STRUCT__entry(
726 __field(unsigned int, rreq)
727 __field(unsigned int, from)
728 __field(unsigned int, to)
729 __field(unsigned int, amount)
730 __field(enum netfs_donate_trace, trace)
731 ),
732
733 TP_fast_assign(
734 __entry->rreq = rreq->debug_id;
735 __entry->from = from->debug_index;
736 __entry->to = to ? to->debug_index : -1;
737 __entry->amount = amount;
738 __entry->trace = trace;
739 ),
740
741 TP_printk("R=%08x[%02x] -> [%02x] %s am=%x",
742 __entry->rreq, __entry->from, __entry->to,
743 __print_symbolic(__entry->trace, netfs_donate_traces),
744 __entry->amount)
745 );
746
654#undef EM747#undef EM
655#undef E_748#undef E_
656#endif /* _TRACE_NETFS_H */749#endif /* _TRACE_NETFS_H */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t