History

netfs: Speed up buffered reading

Improve the efficiency of buffered reads in a number of ways: (1) Overhaul the algorithm in general so that it's a lot more compact and split the read submission code between buffered and unbuffered versions. The unbuffered version can be vastly simplified. (2) Read-result collection is handed off to a work queue rather than being done in the I/O thread. Multiple subrequests can be processes simultaneously. (3) When a subrequest is collected, any folios it fully spans are collected and "spare" data on either side is donated to either the previous or the next subrequest in the sequence. Notes: (*) Readahead expansion is massively slows down fio, presumably because it causes a load of extra allocations, both folio and xarray, up front before RPC requests can be transmitted. (*) RDMA with cifs does appear to work, both with SIW and RXE. (*) PG_private_2-based reading and copy-to-cache is split out into its own file and altered to use folio_queue. Note that the copy to the cache now creates a new write transaction against the cache and adds the folios to be copied into it. This allows it to use part of the writeback I/O code.

David Howells committed 2 years ago

Commit ee4cdf71 parent 2e45b9228 files · +2058 −470Browse files

Showing the changes to include/trace (1 of 28 files). Show all

Unified · Split

include/trace/events/netfs.h+98−5
@@ -20,6 +20,7 @@ · 19 unmodified lines
2020 EM(netfs_read_trace_expanded, "EXPANDED ") \
2121 EM(netfs_read_trace_readahead, "READAHEAD") \
2222 EM(netfs_read_trace_readpage, "READPAGE ") \
23+ EM(netfs_read_trace_read_gaps, "READ-GAPS") \
2324 EM(netfs_read_trace_prefetch_for_write, "PREFETCHW") \
2425 E_(netfs_read_trace_write_begin, "WRITEBEGN")
2526
@@ -33,12 +34,14 @@ · 7 unmodified lines
3334 #define netfs_rreq_origins \
3435 EM(NETFS_READAHEAD, "RA") \
3536 EM(NETFS_READPAGE, "RP") \
37+ EM(NETFS_READ_GAPS, "RG") \
3638 EM(NETFS_READ_FOR_WRITE, "RW") \
3739 EM(NETFS_DIO_READ, "DR") \
3840 EM(NETFS_WRITEBACK, "WB") \
3941 EM(NETFS_WRITETHROUGH, "WT") \
4042 EM(NETFS_UNBUFFERED_WRITE, "UW") \
41− E_(NETFS_DIO_WRITE, "DW")
43+ EM(NETFS_DIO_WRITE, "DW") \
44+ E_(NETFS_PGPRIV2_COPY_TO_CACHE, "2C")
4245
4346 #define netfs_rreq_traces \
4447 EM(netfs_rreq_trace_assess, "ASSESS ") \
@@ -69,15 +72,25 @@ · 24 unmodified lines
6972 E_(NETFS_INVALID_WRITE, "INVL")
7073
7174 #define netfs_sreq_traces \
75+ EM(netfs_sreq_trace_add_donations, "+DON ") \
76+ EM(netfs_sreq_trace_added, "ADD ") \
77+ EM(netfs_sreq_trace_clear, "CLEAR") \
7278 EM(netfs_sreq_trace_discard, "DSCRD") \
79+ EM(netfs_sreq_trace_donate_to_prev, "DON-P") \
80+ EM(netfs_sreq_trace_donate_to_next, "DON-N") \
7381 EM(netfs_sreq_trace_download_instead, "RDOWN") \
7482 EM(netfs_sreq_trace_fail, "FAIL ") \
7583 EM(netfs_sreq_trace_free, "FREE ") \
84+ EM(netfs_sreq_trace_hit_eof, "EOF ") \
85+ EM(netfs_sreq_trace_io_progress, "IO ") \
7686 EM(netfs_sreq_trace_limited, "LIMIT") \
7787 EM(netfs_sreq_trace_prepare, "PREP ") \
7888 EM(netfs_sreq_trace_prep_failed, "PRPFL") \
79− EM(netfs_sreq_trace_resubmit_short, "SHORT") \
89+ EM(netfs_sreq_trace_progress, "PRGRS") \
90+ EM(netfs_sreq_trace_reprep_failed, "REPFL") \
8091 EM(netfs_sreq_trace_retry, "RETRY") \
92+ EM(netfs_sreq_trace_short, "SHORT") \
93+ EM(netfs_sreq_trace_split, "SPLIT") \
8194 EM(netfs_sreq_trace_submit, "SUBMT") \
8295 EM(netfs_sreq_trace_terminated, "TERM ") \
8396 EM(netfs_sreq_trace_write, "WRITE") \
@@ -118,7 +131,7 @@ · 34 unmodified lines
118131 EM(netfs_sreq_trace_new, "NEW ") \
119132 EM(netfs_sreq_trace_put_cancel, "PUT CANCEL ") \
120133 EM(netfs_sreq_trace_put_clear, "PUT CLEAR ") \
121− EM(netfs_sreq_trace_put_discard, "PUT DISCARD") \
134+ EM(netfs_sreq_trace_put_consumed, "PUT CONSUME") \
122135 EM(netfs_sreq_trace_put_done, "PUT DONE ") \
123136 EM(netfs_sreq_trace_put_failed, "PUT FAILED ") \
124137 EM(netfs_sreq_trace_put_merged, "PUT MERGED ") \
@@ -138,6 +151,7 @@ · 13 unmodified lines
138151 EM(netfs_flush_content, "flush") \
139152 EM(netfs_streaming_filled_page, "mod-streamw-f") \
140153 EM(netfs_streaming_cont_filled_page, "mod-streamw-f+") \
154+ EM(netfs_folio_trace_abandon, "abandon") \
141155 EM(netfs_folio_trace_cancel_copy, "cancel-copy") \
142156 EM(netfs_folio_trace_clear, "clear") \
143157 EM(netfs_folio_trace_clear_cc, "clear-cc") \
@@ -154,7 +168,11 @@ · 10 unmodified lines
154168 EM(netfs_folio_trace_mkwrite_plus, "mkwrite+") \
155169 EM(netfs_folio_trace_not_under_wback, "!wback") \
156170 EM(netfs_folio_trace_put, "put") \
171+ EM(netfs_folio_trace_read, "read") \
172+ EM(netfs_folio_trace_read_done, "read-done") \
157173 EM(netfs_folio_trace_read_gaps, "read-gaps") \
174+ EM(netfs_folio_trace_read_put, "read-put") \
175+ EM(netfs_folio_trace_read_unlock, "read-unlock") \
158176 EM(netfs_folio_trace_redirtied, "redirtied") \
159177 EM(netfs_folio_trace_store, "store") \
160178 EM(netfs_folio_trace_store_copy, "store-copy") \
@@ -167,6 +185,12 @@ · 6 unmodified lines
167185 EM(netfs_contig_trace_jump, "-->JUMP-->") \
168186 E_(netfs_contig_trace_unlock, "Unlock")
169187
188+#define netfs_donate_traces \
189+ EM(netfs_trace_donate_tail_to_prev, "tail-to-prev") \
190+ EM(netfs_trace_donate_to_prev, "to-prev") \
191+ EM(netfs_trace_donate_to_next, "to-next") \
192+ E_(netfs_trace_donate_to_deferred_next, "defer-next")
193+
170194 #ifndef __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY
171195 #define __NETFS_DECLARE_TRACE_ENUMS_ONCE_ONLY
172196
@@ -184,6 +208,7 @@ enum netfs_rreq_ref_trace { netfs_rreq_ref_traces } __mode(byte); · 11 unmodified lines
184208 enum netfs_sreq_ref_trace { netfs_sreq_ref_traces } __mode(byte);
185209 enum netfs_folio_trace { netfs_folio_traces } __mode(byte);
186210 enum netfs_collect_contig_trace { netfs_collect_contig_traces } __mode(byte);
211+enum netfs_donate_trace { netfs_donate_traces } __mode(byte);
187212
188213 #endif
189214
@@ -206,6 +231,7 @@ netfs_rreq_ref_traces; · 16 unmodified lines
206231 netfs_sreq_ref_traces;
207232 netfs_folio_traces;
208233 netfs_collect_contig_traces;
234+netfs_donate_traces;
209235
210236 /*
211237 * Now redefine the EM() and E_() macros to map the enums to the strings that
@@ -226,6 +252,7 @@ TRACE_EVENT(netfs_read, · 14 unmodified lines
226252 TP_STRUCT__entry(
227253 __field(unsigned int, rreq )
228254 __field(unsigned int, cookie )
255+ __field(loff_t, i_size )
229256 __field(loff_t, start )
230257 __field(size_t, len )
231258 __field(enum netfs_read_trace, what )
@@ -235,18 +262,19 @@ TRACE_EVENT(netfs_read, · 3 unmodified lines
235262 TP_fast_assign(
236263 __entry->rreq = rreq->debug_id;
237264 __entry->cookie = rreq->cache_resources.debug_id;
265+ __entry->i_size = rreq->i_size;
238266 __entry->start = start;
239267 __entry->len = len;
240268 __entry->what = what;
241269 __entry->netfs_inode = rreq->inode->i_ino;
242270 ),
243271
244− TP_printk("R=%08x %s c=%08x ni=%x s=%llx %zx",
272+ TP_printk("R=%08x %s c=%08x ni=%x s=%llx l=%zx sz=%llx",
245273 __entry->rreq,
246274 __print_symbolic(__entry->what, netfs_read_traces),
247275 __entry->cookie,
248276 __entry->netfs_inode,
249− __entry->start, __entry->len)
277+ __entry->start, __entry->len, __entry->i_size)
250278 );
251279
252280 TRACE_EVENT(netfs_rreq,
@@ -651,6 +679,71 @@ TRACE_EVENT(netfs_collect_stream, · 398 unmodified lines
651679 __entry->collected_to, __entry->front)
652680 );
653681
682+TRACE_EVENT(netfs_progress,
683+ TP_PROTO(const struct netfs_io_subrequest *subreq,
684+ unsigned long long start, size_t avail, size_t part),
685+
686+ TP_ARGS(subreq, start, avail, part),
687+
688+ TP_STRUCT__entry(
689+ __field(unsigned int, rreq)
690+ __field(unsigned int, subreq)
691+ __field(unsigned int, consumed)
692+ __field(unsigned int, transferred)
693+ __field(unsigned long long, f_start)
694+ __field(unsigned int, f_avail)
695+ __field(unsigned int, f_part)
696+ __field(unsigned char, slot)
697+ ),
698+
699+ TP_fast_assign(
700+ __entry->rreq = subreq->rreq->debug_id;
701+ __entry->subreq = subreq->debug_index;
702+ __entry->consumed = subreq->consumed;
703+ __entry->transferred = subreq->transferred;
704+ __entry->f_start = start;
705+ __entry->f_avail = avail;
706+ __entry->f_part = part;
707+ __entry->slot = subreq->curr_folioq_slot;
708+ ),
709+
710+ TP_printk("R=%08x[%02x] s=%llx ct=%x/%x pa=%x/%x sl=%x",
711+ __entry->rreq, __entry->subreq, __entry->f_start,
712+ __entry->consumed, __entry->transferred,
713+ __entry->f_part, __entry->f_avail, __entry->slot)
714+ );
715+
716+TRACE_EVENT(netfs_donate,
717+ TP_PROTO(const struct netfs_io_request *rreq,
718+ const struct netfs_io_subrequest *from,
719+ const struct netfs_io_subrequest *to,
720+ size_t amount,
721+ enum netfs_donate_trace trace),
722+
723+ TP_ARGS(rreq, from, to, amount, trace),
724+
725+ TP_STRUCT__entry(
726+ __field(unsigned int, rreq)
727+ __field(unsigned int, from)
728+ __field(unsigned int, to)
729+ __field(unsigned int, amount)
730+ __field(enum netfs_donate_trace, trace)
731+ ),
732+
733+ TP_fast_assign(
734+ __entry->rreq = rreq->debug_id;
735+ __entry->from = from->debug_index;
736+ __entry->to = to ? to->debug_index : -1;
737+ __entry->amount = amount;
738+ __entry->trace = trace;
739+ ),
740+
741+ TP_printk("R=%08x[%02x] -> [%02x] %s am=%x",
742+ __entry->rreq, __entry->from, __entry->to,
743+ __print_symbolic(__entry->trace, netfs_donate_traces),
744+ __entry->amount)
745+ );
746+
654747 #undef EM
655748 #undef E_
656749 #endif /* _TRACE_NETFS_H */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t